diff options
Diffstat (limited to 'v1.49/development')
| -rw-r--r-- | v1.49/development/cas.html | 317 | ||||
| -rw-r--r-- | v1.49/development/contributing_guide.html | 604 | ||||
| -rw-r--r-- | v1.49/development/database_schema.html | 376 | ||||
| -rw-r--r-- | v1.49/development/experimental_features.html | 288 | ||||
| -rw-r--r-- | v1.49/development/git.html | 376 | ||||
| -rw-r--r-- | v1.49/development/img/git/branches.jpg | bin | 0 -> 72228 bytes | |||
| -rw-r--r-- | v1.49/development/img/git/clean.png | bin | 0 -> 110840 bytes | |||
| -rw-r--r-- | v1.49/development/img/git/squash.png | bin | 0 -> 29667 bytes | |||
| -rw-r--r-- | v1.49/development/internal_documentation/index.html | 264 | ||||
| -rw-r--r-- | v1.49/development/room-dag-concepts.html | 305 | ||||
| -rw-r--r-- | v1.49/development/saml.html | 293 | ||||
| -rw-r--r-- | v1.49/development/url_previews.html | 331 |
12 files changed, 3154 insertions, 0 deletions
diff --git a/v1.49/development/cas.html b/v1.49/development/cas.html new file mode 100644 index 0000000000..e72fed5ab8 --- /dev/null +++ b/v1.49/development/cas.html @@ -0,0 +1,317 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>CAS - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html" class="active">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/cas.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="how-to-test-cas-as-a-developer-without-a-server"><a class="header" href="#how-to-test-cas-as-a-developer-without-a-server">How to test CAS as a developer without a server</a></h1> +<p>The <a href="https://github.com/jbittel/django-mama-cas">django-mama-cas</a> project is an +easy to run CAS implementation built on top of Django.</p> +<h2 id="prerequisites"><a class="header" href="#prerequisites">Prerequisites</a></h2> +<ol> +<li>Create a new virtualenv: <code>python3 -m venv <your virtualenv></code></li> +<li>Activate your virtualenv: <code>source /path/to/your/virtualenv/bin/activate</code></li> +<li>Install Django and django-mama-cas: +<pre><code class="language-sh">python -m pip install "django<3" "django-mama-cas==2.4.0" +</code></pre> +</li> +<li>Create a Django project in the current directory: +<pre><code class="language-sh">django-admin startproject cas_test . +</code></pre> +</li> +<li>Follow the <a href="https://django-mama-cas.readthedocs.io/en/latest/installation.html#configuring">install directions</a> for django-mama-cas</li> +<li>Setup the SQLite database: <code>python manage.py migrate</code></li> +<li>Create a user: +<pre><code class="language-sh">python manage.py createsuperuser +</code></pre> +<ol> +<li>Use whatever you want as the username and password.</li> +<li>Leave the other fields blank.</li> +</ol> +</li> +<li>Use the built-in Django test server to serve the CAS endpoints on port 8000: +<pre><code class="language-sh">python manage.py runserver +</code></pre> +</li> +</ol> +<p>You should now have a Django project configured to serve CAS authentication with +a single user created.</p> +<h2 id="configure-synapse-and-element-to-use-cas"><a class="header" href="#configure-synapse-and-element-to-use-cas">Configure Synapse (and Element) to use CAS</a></h2> +<ol> +<li>Modify your <code>homeserver.yaml</code> to enable CAS and point it to your locally +running Django test server: +<pre><code class="language-yaml">cas_config: + enabled: true + server_url: "http://localhost:8000" + service_url: "http://localhost:8081" + #displayname_attribute: name + #required_attributes: + # name: value +</code></pre> +</li> +<li>Restart Synapse.</li> +</ol> +<p>Note that the above configuration assumes the homeserver is running on port 8081 +and that the CAS server is on port 8000, both on localhost.</p> +<h2 id="testing-the-configuration"><a class="header" href="#testing-the-configuration">Testing the configuration</a></h2> +<p>Then in Element:</p> +<ol> +<li>Visit the login page with a Element pointing at your homeserver.</li> +<li>Click the Single Sign-On button.</li> +<li>Login using the credentials created with <code>createsuperuser</code>.</li> +<li>You should be logged in.</li> +</ol> +<p>If you want to repeat this process you'll need to manually logout first:</p> +<ol> +<li>http://localhost:8000/admin/</li> +<li>Click "logout" in the top right.</li> +</ol> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../development/saml.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/room-dag-concepts.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../development/saml.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/room-dag-concepts.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/contributing_guide.html b/v1.49/development/contributing_guide.html new file mode 100644 index 0000000000..9fa46d9014 --- /dev/null +++ b/v1.49/development/contributing_guide.html @@ -0,0 +1,604 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Contributing Guide - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html" class="active">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/contributing_guide.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="contributing"><a class="header" href="#contributing">Contributing</a></h1> +<p>This document aims to get you started with contributing to Synapse!</p> +<h1 id="1-who-can-contribute-to-synapse"><a class="header" href="#1-who-can-contribute-to-synapse">1. Who can contribute to Synapse?</a></h1> +<p>Everyone is welcome to contribute code to <a href="https://github.com/matrix-org">matrix.org +projects</a>, provided that they are willing to +license their contributions under the same license as the project itself. We +follow a simple 'inbound=outbound' model for contributions: the act of +submitting an 'inbound' contribution means that the contributor agrees to +license the code under the same terms as the project's overall 'outbound' +license - in our case, this is almost always Apache Software License v2 (see +<a href="https://github.com/matrix-org/synapse/blob/develop/LICENSE">LICENSE</a>).</p> +<h1 id="2-what-do-i-need"><a class="header" href="#2-what-do-i-need">2. What do I need?</a></h1> +<p>If you are running Windows, the Windows Subsystem for Linux (WSL) is strongly +recommended for development. More information about WSL can be found at +<a href="https://docs.microsoft.com/en-us/windows/wsl/install">https://docs.microsoft.com/en-us/windows/wsl/install</a>. Running Synapse natively +on Windows is not officially supported.</p> +<p>The code of Synapse is written in Python 3. To do pretty much anything, you'll need <a href="https://wiki.python.org/moin/BeginnersGuide/Download">a recent version of Python 3</a>.</p> +<p>The source code of Synapse is hosted on GitHub. You will also need <a href="https://github.com/git-guides/install-git">a recent version of git</a>.</p> +<p>For some tests, you will need <a href="https://docs.docker.com/get-docker/">a recent version of Docker</a>.</p> +<h1 id="3-get-the-source"><a class="header" href="#3-get-the-source">3. Get the source.</a></h1> +<p>The preferred and easiest way to contribute changes is to fork the relevant +project on GitHub, and then <a href="https://help.github.com/articles/using-pull-requests/">create a pull request</a> to ask us to pull your +changes into our repo.</p> +<p>Please base your changes on the <code>develop</code> branch.</p> +<pre><code class="language-sh">git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git +git checkout develop +</code></pre> +<p>If you need help getting started with git, this is beyond the scope of the document, but you +can find many good git tutorials on the web.</p> +<h1 id="4-install-the-dependencies"><a class="header" href="#4-install-the-dependencies">4. Install the dependencies</a></h1> +<p>Once you have installed Python 3 and added the source, please open a terminal and +setup a <em>virtualenv</em>, as follows:</p> +<pre><code class="language-sh">cd path/where/you/have/cloned/the/repository +python3 -m venv ./env +source ./env/bin/activate +pip install -e ".[all,dev]" +pip install tox +</code></pre> +<p>This will install the developer dependencies for the project.</p> +<h1 id="5-get-in-touch"><a class="header" href="#5-get-in-touch">5. Get in touch.</a></h1> +<p>Join our developer community on Matrix: <a href="https://matrix.to/#/#synapse-dev:matrix.org">#synapse-dev:matrix.org</a>!</p> +<h1 id="6-pick-an-issue"><a class="header" href="#6-pick-an-issue">6. Pick an issue.</a></h1> +<p>Fix your favorite problem or perhaps find a <a href="https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22">Good First Issue</a> +to work on.</p> +<h1 id="7-turn-coffee-into-code-and-documentation"><a class="header" href="#7-turn-coffee-into-code-and-documentation">7. Turn coffee into code and documentation!</a></h1> +<p>There is a growing amount of documentation located in the +<a href="https://github.com/matrix-org/synapse/tree/develop/docs"><code>docs</code></a> +directory, with a rendered version <a href="https://matrix-org.github.io/synapse">available online</a>. +This documentation is intended primarily for sysadmins running their +own Synapse instance, as well as developers interacting externally with +Synapse. +<a href="https://github.com/matrix-org/synapse/tree/develop/docs/development"><code>docs/development</code></a> +exists primarily to house documentation for +Synapse developers. +<a href="https://github.com/matrix-org/synapse/tree/develop/docs/admin_api"><code>docs/admin_api</code></a> houses documentation +regarding Synapse's Admin API, which is used mostly by sysadmins and external +service developers.</p> +<p>Synapse's code style is documented <a href="../code_style.html">here</a>. Please follow +it, including the conventions for the <a href="../code_style.html#configuration-file-format">sample configuration +file</a>.</p> +<p>We welcome improvements and additions to our documentation itself! When +writing new pages, please +<a href="https://github.com/matrix-org/synapse/tree/develop/docs#adding-to-the-documentation">build <code>docs</code> to a book</a> +to check that your contributions render correctly. The docs are written in +<a href="https://guides.github.com/features/mastering-markdown/">GitHub-Flavoured Markdown</a>.</p> +<p>Some documentation also exists in <a href="https://github.com/matrix-org/synapse/wiki">Synapse's GitHub +Wiki</a>, although this is primarily +contributed to by community authors.</p> +<h1 id="8-test-test-test"><a class="header" href="#8-test-test-test">8. Test, test, test!</a></h1> +<p><a name="test-test-test"></a></p> +<p>While you're developing and before submitting a patch, you'll +want to test your code.</p> +<h2 id="run-the-linters"><a class="header" href="#run-the-linters">Run the linters.</a></h2> +<p>The linters look at your code and do two things:</p> +<ul> +<li>ensure that your code follows the coding style adopted by the project;</li> +<li>catch a number of errors in your code.</li> +</ul> +<p>They're pretty fast, don't hesitate!</p> +<pre><code class="language-sh">source ./env/bin/activate +./scripts-dev/lint.sh +</code></pre> +<p>Note that this script <em>will modify your files</em> to fix styling errors. +Make sure that you have saved all your files.</p> +<p>If you wish to restrict the linters to only the files changed since the last commit +(much faster!), you can instead run:</p> +<pre><code class="language-sh">source ./env/bin/activate +./scripts-dev/lint.sh -d +</code></pre> +<p>Or if you know exactly which files you wish to lint, you can instead run:</p> +<pre><code class="language-sh">source ./env/bin/activate +./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder +</code></pre> +<h2 id="run-the-unit-tests-twisted-trial"><a class="header" href="#run-the-unit-tests-twisted-trial">Run the unit tests (Twisted trial).</a></h2> +<p>The unit tests run parts of Synapse, including your changes, to see if anything +was broken. They are slower than the linters but will typically catch more errors.</p> +<pre><code class="language-sh">source ./env/bin/activate +trial tests +</code></pre> +<p>If you wish to only run <em>some</em> unit tests, you may specify +another module instead of <code>tests</code> - or a test class or a method:</p> +<pre><code class="language-sh">source ./env/bin/activate +trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite +</code></pre> +<p>If your tests fail, you may wish to look at the logs (the default log level is <code>ERROR</code>):</p> +<pre><code class="language-sh">less _trial_temp/test.log +</code></pre> +<p>To increase the log level for the tests, set <code>SYNAPSE_TEST_LOG_LEVEL</code>:</p> +<pre><code class="language-sh">SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +</code></pre> +<h3 id="running-tests-under-postgresql"><a class="header" href="#running-tests-under-postgresql">Running tests under PostgreSQL</a></h3> +<p>Invoking <code>trial</code> as above will use an in-memory SQLite database. This is great for +quick development and testing. However, we recommend using a PostgreSQL database +in production (and indeed, we have some code paths specific to each database). +This means that we need to run our unit tests against PostgreSQL too. Our CI does +this automatically for pull requests and release candidates, but it's sometimes +useful to reproduce this locally.</p> +<p>To do so, <a href="../postgres.html">configure Postgres</a> and run <code>trial</code> with the +following environment variables matching your configuration:</p> +<ul> +<li><code>SYNAPSE_POSTGRES</code> to anything nonempty</li> +<li><code>SYNAPSE_POSTGRES_HOST</code></li> +<li><code>SYNAPSE_POSTGRES_USER</code></li> +<li><code>SYNAPSE_POSTGRES_PASSWORD</code></li> +</ul> +<p>For example:</p> +<pre><code class="language-shell">export SYNAPSE_POSTGRES=1 +export SYNAPSE_POSTGRES_HOST=localhost +export SYNAPSE_POSTGRES_USER=postgres +export SYNAPSE_POSTGRES_PASSWORD=mydevenvpassword +trial +</code></pre> +<h4 id="prebuilt-container"><a class="header" href="#prebuilt-container">Prebuilt container</a></h4> +<p>Since configuring PostgreSQL can be fiddly, we can make use of a pre-made +Docker container to set up PostgreSQL and run our tests for us. To do so, run</p> +<pre><code class="language-shell">scripts-dev/test_postgresql.sh +</code></pre> +<p>Any extra arguments to the script will be passed to <code>tox</code> and then to <code>trial</code>, +so we can run a specific test in this container with e.g.</p> +<pre><code class="language-shell">scripts-dev/test_postgresql.sh tests.replication.test_sharded_event_persister.EventPersisterShardTestCase +</code></pre> +<p>The container creates a folder in your Synapse checkout called +<code>.tox-pg-container</code> and uses this as a tox environment. The output of any +<code>trial</code> runs goes into <code>_trial_temp</code> in your synapse source directory — the same +as running <code>trial</code> directly on your host machine.</p> +<h2 id="run-the-integration-tests-a-hrefhttpsgithubcommatrix-orgsytestsytesta"><a class="header" href="#run-the-integration-tests-a-hrefhttpsgithubcommatrix-orgsytestsytesta">Run the integration tests (<a href="https://github.com/matrix-org/sytest">Sytest</a>).</a></h2> +<p>The integration tests are a more comprehensive suite of tests. They +run a full version of Synapse, including your changes, to check if +anything was broken. They are slower than the unit tests but will +typically catch more errors.</p> +<p>The following command will let you run the integration test with the most common +configuration:</p> +<pre><code class="language-sh">$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster +</code></pre> +<p>This configuration should generally cover your needs. For more details about other configurations, see <a href="https://github.com/matrix-org/sytest/blob/develop/docker/README.md">documentation in the SyTest repo</a>.</p> +<h2 id="run-the-integration-tests-a-hrefhttpsgithubcommatrix-orgcomplementcomplementa"><a class="header" href="#run-the-integration-tests-a-hrefhttpsgithubcommatrix-orgcomplementcomplementa">Run the integration tests (<a href="https://github.com/matrix-org/complement">Complement</a>).</a></h2> +<p><a href="https://github.com/matrix-org/complement">Complement</a> is a suite of black box tests that can be run on any homeserver implementation. It can also be thought of as end-to-end (e2e) tests.</p> +<p>It's often nice to develop on Synapse and write Complement tests at the same time. +Here is how to run your local Synapse checkout against your local Complement checkout.</p> +<p>(checkout <a href="https://github.com/matrix-org/complement"><code>complement</code></a> alongside your <code>synapse</code> checkout)</p> +<pre><code class="language-sh">COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh +</code></pre> +<p>To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output:</p> +<pre><code class="language-sh">COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory +</code></pre> +<p>To run a specific test, you can specify the whole name structure:</p> +<pre><code class="language-sh">COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order +</code></pre> +<h3 id="access-database-for-homeserver-after-complement-test-runs"><a class="header" href="#access-database-for-homeserver-after-complement-test-runs">Access database for homeserver after Complement test runs.</a></h3> +<p>If you're curious what the database looks like after you run some tests, here are some steps to get you going in Synapse:</p> +<ol> +<li>In your Complement test comment out <code>defer deployment.Destroy(t)</code> and replace with <code>defer time.Sleep(2 * time.Hour)</code> to keep the homeserver running after the tests complete</li> +<li>Start the Complement tests</li> +<li>Find the name of the container, <code>docker ps -f name=complement_</code> (this will filter for just the Compelement related Docker containers)</li> +<li>Access the container replacing the name with what you found in the previous step: <code>docker exec -it complement_1_hs_with_application_service.hs1_2 /bin/bash</code></li> +<li>Install sqlite (database driver), <code>apt-get update && apt-get install -y sqlite3</code></li> +<li>Then run <code>sqlite3</code> and open the database <code>.open /conf/homeserver.db</code> (this db path comes from the Synapse homeserver.yaml)</li> +</ol> +<h1 id="9-submit-your-patch"><a class="header" href="#9-submit-your-patch">9. Submit your patch.</a></h1> +<p>Once you're happy with your patch, it's time to prepare a Pull Request.</p> +<p>To prepare a Pull Request, please:</p> +<ol> +<li>verify that <a href="#test-test-test">all the tests pass</a>, including the coding style;</li> +<li><a href="#sign-off">sign off</a> your contribution;</li> +<li><code>git push</code> your commit to your fork of Synapse;</li> +<li>on GitHub, <a href="https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request">create the Pull Request</a>;</li> +<li>add a <a href="#changelog">changelog entry</a> and push it to your Pull Request;</li> +<li>for most contributors, that's all - however, if you are a member of the organization <code>matrix-org</code>, on GitHub, please request a review from <code>matrix.org / Synapse Core</code>.</li> +<li>if you need to update your PR, please avoid rebasing and just add new commits to your branch.</li> +</ol> +<h2 id="changelog"><a class="header" href="#changelog">Changelog</a></h2> +<p>All changes, even minor ones, need a corresponding changelog / newsfragment +entry. These are managed by <a href="https://github.com/hawkowl/towncrier">Towncrier</a>.</p> +<p>To create a changelog entry, make a new file in the <code>changelog.d</code> directory named +in the format of <code>PRnumber.type</code>. The type can be one of the following:</p> +<ul> +<li><code>feature</code></li> +<li><code>bugfix</code></li> +<li><code>docker</code> (for updates to the Docker image)</li> +<li><code>doc</code> (for updates to the documentation)</li> +<li><code>removal</code> (also used for deprecations)</li> +<li><code>misc</code> (for internal-only changes)</li> +</ul> +<p>This file will become part of our <a href="https://github.com/matrix-org/synapse/blob/master/CHANGES.md">changelog</a> at the next +release, so the content of the file should be a short description of your +change in the same style as the rest of the changelog. The file can contain Markdown +formatting, and should end with a full stop (.) or an exclamation mark (!) for +consistency.</p> +<p>Adding credits to the changelog is encouraged, we value your +contributions and would like to have you shouted out in the release notes!</p> +<p>For example, a fix in PR #1234 would have its changelog entry in +<code>changelog.d/1234.bugfix</code>, and contain content like:</p> +<blockquote> +<p>The security levels of Florbs are now validated when received +via the <code>/federation/florb</code> endpoint. Contributed by Jane Matrix.</p> +</blockquote> +<p>If there are multiple pull requests involved in a single bugfix/feature/etc, +then the content for each <code>changelog.d</code> file should be the same. Towncrier will +merge the matching files together into a single changelog entry when we come to +release.</p> +<h3 id="how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr"><a class="header" href="#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr">How do I know what to call the changelog file before I create the PR?</a></h3> +<p>Obviously, you don't know if you should call your newsfile +<code>1234.bugfix</code> or <code>5678.bugfix</code> until you create the PR, which leads to a +chicken-and-egg problem.</p> +<p>There are two options for solving this:</p> +<ol> +<li> +<p>Open the PR without a changelog file, see what number you got, and <em>then</em> +add the changelog file to your branch (see <a href="#updating-your-pull-request">Updating your pull +request</a>), or:</p> +</li> +<li> +<p>Look at the <a href="https://github.com/matrix-org/synapse/issues?q=">list of all +issues/PRs</a>, add one to the +highest number you see, and quickly open the PR before somebody else claims +your number.</p> +<p><a href="https://github.com/richvdh/scripts/blob/master/next_github_number.sh">This +script</a> +might be helpful if you find yourself doing this a lot.</p> +</li> +</ol> +<p>Sorry, we know it's a bit fiddly, but it's <em>really</em> helpful for us when we come +to put together a release!</p> +<h3 id="debian-changelog"><a class="header" href="#debian-changelog">Debian changelog</a></h3> +<p>Changes which affect the debian packaging files (in <code>debian</code>) are an +exception to the rule that all changes require a <code>changelog.d</code> file.</p> +<p>In this case, you will need to add an entry to the debian changelog for the +next release. For this, run the following command:</p> +<pre><code>dch +</code></pre> +<p>This will make up a new version number (if there isn't already an unreleased +version in flight), and open an editor where you can add a new changelog entry. +(Our release process will ensure that the version number and maintainer name is +corrected for the release.)</p> +<p>If your change affects both the debian packaging <em>and</em> files outside the debian +directory, you will need both a regular newsfragment <em>and</em> an entry in the +debian changelog. (Though typically such changes should be submitted as two +separate pull requests.)</p> +<h2 id="sign-off"><a class="header" href="#sign-off">Sign off</a></h2> +<p>In order to have a concrete record that your contribution is intentional +and you agree to license it under the same terms as the project's license, we've adopted the +same lightweight approach that the Linux Kernel +<a href="https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin%3E">submitting patches process</a>, +<a href="https://github.com/docker/docker/blob/master/CONTRIBUTING.md">Docker</a>, and many other +projects use: the DCO (Developer Certificate of Origin: +http://developercertificate.org/). This is a simple declaration that you wrote +the contribution or otherwise have the right to contribute it to Matrix:</p> +<pre><code>Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +</code></pre> +<p>If you agree to this for your contribution, then all that's needed is to +include the line in your commit or pull request comment:</p> +<pre><code>Signed-off-by: Your Name <your@email.example.org> +</code></pre> +<p>We accept contributions under a legally identifiable name, such as +your name on government documentation or common-law names (names +claimed by legitimate usage or repute). Unfortunately, we cannot +accept anonymous contributions at this time.</p> +<p>Git allows you to add this signoff automatically when using the <code>-s</code> +flag to <code>git commit</code>, which uses the name and email set in your +<code>user.name</code> and <code>user.email</code> git configs.</p> +<h1 id="10-turn-feedback-into-better-code"><a class="header" href="#10-turn-feedback-into-better-code">10. Turn feedback into better code.</a></h1> +<p>Once the Pull Request is opened, you will see a few things:</p> +<ol> +<li>our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;</li> +<li>one or more of the developers will take a look at your Pull Request and offer feedback.</li> +</ol> +<p>From this point, you should:</p> +<ol> +<li>Look at the results of the CI pipeline. +<ul> +<li>If there is any error, fix the error.</li> +</ul> +</li> +<li>If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.</li> +<li>Create a new commit with the changes. +<ul> +<li>Please do NOT overwrite the history. New commits make the reviewer's life easier.</li> +<li>Push this commits to your Pull Request.</li> +</ul> +</li> +<li>Back to 1.</li> +</ol> +<p>Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!</p> +<h1 id="11-find-a-new-issue"><a class="header" href="#11-find-a-new-issue">11. Find a new issue.</a></h1> +<p>By now, you know the drill!</p> +<h1 id="notes-for-maintainers-on-merging-prs-etc"><a class="header" href="#notes-for-maintainers-on-merging-prs-etc">Notes for maintainers on merging PRs etc</a></h1> +<p>There are some notes for those with commit access to the project on how we +manage git <a href="git.html">here</a>.</p> +<h1 id="conclusion"><a class="header" href="#conclusion">Conclusion</a></h1> +<p>That's it! Matrix is a very open and collaborative project as you might expect +given our obsession with open communication. If we're going to successfully +matrix together all the fragmented communication technologies out there we are +reliant on contributions and collaboration from the community to do so. So +please get involved - and we hope you have as much fun hacking on Matrix as we +do!</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../usage/administration/admin_faq.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../code_style.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../usage/administration/admin_faq.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../code_style.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/database_schema.html b/v1.49/development/database_schema.html new file mode 100644 index 0000000000..53a5dcd24a --- /dev/null +++ b/v1.49/development/database_schema.html @@ -0,0 +1,376 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Database Schemas - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html" class="active">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/database_schema.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="synapse-database-schema-files"><a class="header" href="#synapse-database-schema-files">Synapse database schema files</a></h1> +<p>Synapse's database schema is stored in the <code>synapse.storage.schema</code> module.</p> +<h2 id="logical-databases"><a class="header" href="#logical-databases">Logical databases</a></h2> +<p>Synapse supports splitting its datastore across multiple physical databases (which can +be useful for large installations), and the schema files are therefore split according +to the logical database they apply to.</p> +<p>At the time of writing, the following "logical" databases are supported:</p> +<ul> +<li><code>state</code> - used to store Matrix room state (more specifically, <code>state_groups</code>, +their relationships and contents).</li> +<li><code>main</code> - stores everything else.</li> +</ul> +<p>Additionally, the <code>common</code> directory contains schema files for tables which must be +present on <em>all</em> physical databases.</p> +<h2 id="synapse-schema-versions"><a class="header" href="#synapse-schema-versions">Synapse schema versions</a></h2> +<p>Synapse manages its database schema via "schema versions". These are mainly used to +help avoid confusion if the Synapse codebase is rolled back after the database is +updated. They work as follows:</p> +<ul> +<li> +<p>The Synapse codebase defines a constant <code>synapse.storage.schema.SCHEMA_VERSION</code> +which represents the expectations made about the database by that version. For +example, as of Synapse v1.36, this is <code>59</code>.</p> +</li> +<li> +<p>The database stores a "compatibility version" in +<code>schema_compat_version.compat_version</code> which defines the <code>SCHEMA_VERSION</code> of the +oldest version of Synapse which will work with the database. On startup, if +<code>compat_version</code> is found to be newer than <code>SCHEMA_VERSION</code>, Synapse will refuse to +start.</p> +<p>Synapse automatically updates this field from +<code>synapse.storage.schema.SCHEMA_COMPAT_VERSION</code>.</p> +</li> +<li> +<p>Whenever a backwards-incompatible change is made to the database format (normally +via a <code>delta</code> file), <code>synapse.storage.schema.SCHEMA_COMPAT_VERSION</code> is also updated +so that administrators can not accidentally roll back to a too-old version of Synapse.</p> +</li> +</ul> +<p>Generally, the goal is to maintain compatibility with at least one or two previous +releases of Synapse, so any substantial change tends to require multiple releases and a +bit of forward-planning to get right.</p> +<p>As a worked example: we want to remove the <code>room_stats_historical</code> table. Here is how it +might pan out.</p> +<ol> +<li> +<p>Replace any code that <em>reads</em> from <code>room_stats_historical</code> with alternative +implementations, but keep writing to it in case of rollback to an earlier version. +Also, increase <code>synapse.storage.schema.SCHEMA_VERSION</code>. In this +instance, there is no existing code which reads from <code>room_stats_historical</code>, so +our starting point is:</p> +<p>v1.36.0: <code>SCHEMA_VERSION=59</code>, <code>SCHEMA_COMPAT_VERSION=59</code></p> +</li> +<li> +<p>Next (say in Synapse v1.37.0): remove the code that <em>writes</em> to +<code>room_stats_historical</code>, but don’t yet remove the table in case of rollback to +v1.36.0. Again, we increase <code>synapse.storage.schema.SCHEMA_VERSION</code>, but +because we have not broken compatibility with v1.36, we do not yet update +<code>SCHEMA_COMPAT_VERSION</code>. We now have:</p> +<p>v1.37.0: <code>SCHEMA_VERSION=60</code>, <code>SCHEMA_COMPAT_VERSION=59</code>.</p> +</li> +<li> +<p>Later (say in Synapse v1.38.0): we can remove the table altogether. This will +break compatibility with v1.36.0, so we must update <code>SCHEMA_COMPAT_VERSION</code> accordingly. +There is no need to update <code>synapse.storage.schema.SCHEMA_VERSION</code>, since there is no +change to the Synapse codebase here. So we end up with:</p> +<p>v1.38.0: <code>SCHEMA_VERSION=60</code>, <code>SCHEMA_COMPAT_VERSION=60</code>.</p> +</li> +</ol> +<p>If in doubt about whether to update <code>SCHEMA_VERSION</code> or not, it is generally best to +lean towards doing so.</p> +<h2 id="full-schema-dumps"><a class="header" href="#full-schema-dumps">Full schema dumps</a></h2> +<p>In the <code>full_schemas</code> directories, only the most recently-numbered snapshot is used +(<code>54</code> at the time of writing). Older snapshots (eg, <code>16</code>) are present for historical +reference only.</p> +<h3 id="building-full-schema-dumps"><a class="header" href="#building-full-schema-dumps">Building full schema dumps</a></h3> +<p>If you want to recreate these schemas, they need to be made from a database that +has had all background updates run.</p> +<p>To do so, use <code>scripts-dev/make_full_schema.sh</code>. This will produce new +<code>full.sql.postgres</code> and <code>full.sql.sqlite</code> files.</p> +<p>Ensure postgres is installed, then run:</p> +<pre><code class="language-sh">./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ +</code></pre> +<p>NB at the time of writing, this script predates the split into separate <code>state</code>/<code>main</code> +databases so will require updates to handle that correctly.</p> +<h2 id="boolean-columns"><a class="header" href="#boolean-columns">Boolean columns</a></h2> +<p>Boolean columns require special treatment, since SQLite treats booleans the +same as integers.</p> +<p>There are three separate aspects to this:</p> +<ul> +<li> +<p>Any new boolean column must be added to the <code>BOOLEAN_COLUMNS</code> list in +<code>scripts/synapse_port_db</code>. This tells the port script to cast the integer +value from SQLite to a boolean before writing the value to the postgres +database.</p> +</li> +<li> +<p>Before SQLite 3.23, <code>TRUE</code> and <code>FALSE</code> were not recognised as constants by +SQLite, and the <code>IS [NOT] TRUE</code>/<code>IS [NOT] FALSE</code> operators were not +supported. This makes it necessary to avoid using <code>TRUE</code> and <code>FALSE</code> +constants in SQL commands.</p> +<p>For example, to insert a <code>TRUE</code> value into the database, write:</p> +<pre><code class="language-python">txn.execute("INSERT INTO tbl(col) VALUES (?)", (True, )) +</code></pre> +</li> +<li> +<p>Default values for new boolean columns present a particular +difficulty. Generally it is best to create separate schema files for +Postgres and SQLite. For example:</p> +<pre><code class="language-sql"># in 00delta.sql.postgres: +ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT FALSE; +</code></pre> +<pre><code class="language-sql"># in 00delta.sql.sqlite: +ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT 0; +</code></pre> +<p>Note that there is a particularly insidious failure mode here: the Postgres +flavour will be accepted by SQLite 3.22, but will give a column whose +default value is the <strong>string</strong> <code>"FALSE"</code> - which, when cast back to a boolean +in Python, evaluates to <code>True</code>.</p> +</li> +</ul> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../opentracing.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/experimental_features.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../opentracing.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/experimental_features.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/experimental_features.html b/v1.49/development/experimental_features.html new file mode 100644 index 0000000000..569244d466 --- /dev/null +++ b/v1.49/development/experimental_features.html @@ -0,0 +1,288 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Experimental features - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html" class="active">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/experimental_features.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="implementing-experimental-features-in-synapse"><a class="header" href="#implementing-experimental-features-in-synapse">Implementing experimental features in Synapse</a></h1> +<p>It can be desirable to implement "experimental" features which are disabled by +default and must be explicitly enabled via the Synapse configuration. This is +applicable for features which:</p> +<ul> +<li>Are unstable in the Matrix spec (e.g. those defined by an MSC that has not yet been merged).</li> +<li>Developers are not confident in their use by general Synapse administrators/users +(e.g. a feature is incomplete, buggy, performs poorly, or needs further testing).</li> +</ul> +<p>Note that this only really applies to features which are expected to be desirable +to a broad audience. The <a href="../modules/index.html">module infrastructure</a> should +instead be investigated for non-standard features.</p> +<p>Guarding experimental features behind configuration flags should help with some +of the following scenarios:</p> +<ul> +<li>Ensure that clients do not assume that unstable features exist (failing +gracefully if they do not).</li> +<li>Unstable features do not become de-facto standards and can be removed +aggressively (since only those who have opted-in will be affected).</li> +<li>Ease finding the implementation of unstable features in Synapse (for future +removal or stabilization).</li> +<li>Ease testing a feature (or removal of feature) due to enabling/disabling without +code changes. It also becomes possible to ask for wider testing, if desired.</li> +</ul> +<p>Experimental configuration flags should be disabled by default (requiring Synapse +administrators to explicitly opt-in), although there are situations where it makes +sense (from a product point-of-view) to enable features by default. This is +expected and not an issue.</p> +<p>It is not a requirement for experimental features to be behind a configuration flag, +but one should be used if unsure.</p> +<p>New experimental configuration flags should be added under the <code>experimental</code> +configuration key (see the <code>synapse.config.experimental</code> file) and either explain +(briefly) what is being enabled, or include the MSC number.</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../development/database_schema.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../log_contexts.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../development/database_schema.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../log_contexts.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/git.html b/v1.49/development/git.html new file mode 100644 index 0000000000..1e661fb3fe --- /dev/null +++ b/v1.49/development/git.html @@ -0,0 +1,376 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Git Usage - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html" class="active">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/git.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="some-notes-on-how-we-use-git"><a class="header" href="#some-notes-on-how-we-use-git">Some notes on how we use git</a></h1> +<h2 id="on-keeping-the-commit-history-clean"><a class="header" href="#on-keeping-the-commit-history-clean">On keeping the commit history clean</a></h2> +<p>In an ideal world, our git commit history would be a linear progression of +commits each of which contains a single change building on what came +before. Here, by way of an arbitrary example, is the top of <code>git log --graph b2dba0607</code>:</p> +<img src="img/git/clean.png" alt="clean git graph" width="500px"> +<p>Note how the commit comment explains clearly what is changing and why. Also +note the <em>absence</em> of merge commits, as well as the absence of commits called +things like (to pick a few culprits): +<a href="https://github.com/matrix-org/synapse/commit/84691da6c">“pep8”</a>, <a href="https://github.com/matrix-org/synapse/commit/474810d9d">“fix broken +test”</a>, +<a href="https://github.com/matrix-org/synapse/commit/c9d72e457">“oops”</a>, +<a href="https://github.com/matrix-org/synapse/commit/836358823">“typo”</a>, or <a href="https://github.com/matrix-org/synapse/commit/707374d5d">“Who's +the president?”</a>.</p> +<p>There are a number of reasons why keeping a clean commit history is a good +thing:</p> +<ul> +<li> +<p>From time to time, after a change lands, it turns out to be necessary to +revert it, or to backport it to a release branch. Those operations are +<em>much</em> easier when the change is contained in a single commit.</p> +</li> +<li> +<p>Similarly, it's much easier to answer questions like “is the fix for +<code>/publicRooms</code> on the release branch?” if that change consists of a single +commit.</p> +</li> +<li> +<p>Likewise: “what has changed on this branch in the last week?” is much +clearer without merges and “pep8” commits everywhere.</p> +</li> +<li> +<p>Sometimes we need to figure out where a bug got introduced, or some +behaviour changed. One way of doing that is with <code>git bisect</code>: pick an +arbitrary commit between the known good point and the known bad point, and +see how the code behaves. However, that strategy fails if the commit you +chose is the middle of someone's epic branch in which they broke the world +before putting it back together again.</p> +</li> +</ul> +<p>One counterargument is that it is sometimes useful to see how a PR evolved as +it went through review cycles. This is true, but that information is always +available via the GitHub UI (or via the little-known <a href="https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally">refs/pull +namespace</a>).</p> +<p>Of course, in reality, things are more complicated than that. We have release +branches as well as <code>develop</code> and <code>master</code>, and we deliberately merge changes +between them. Bugs often slip through and have to be fixed later. That's all +fine: this not a cast-iron rule which must be obeyed, but an ideal to aim +towards.</p> +<h2 id="merges-squashes-rebases-wtf"><a class="header" href="#merges-squashes-rebases-wtf">Merges, squashes, rebases: wtf?</a></h2> +<p>Ok, so that's what we'd like to achieve. How do we achieve it?</p> +<p>The TL;DR is: when you come to merge a pull request, you <em>probably</em> want to +“squash and merge”:</p> +<p><img src="img/git/squash.png" alt="squash and merge" />.</p> +<p>(This applies whether you are merging your own PR, or that of another +contributor.)</p> +<p>“Squash and merge”<sup id="a1"><a href="#f1">1</a></sup> takes all of the changes in the +PR, and bundles them into a single commit. GitHub gives you the opportunity to +edit the commit message before you confirm, and normally you should do so, +because the default will be useless (again: <code>* woops typo</code> is not a useful +thing to keep in the historical record).</p> +<p>The main problem with this approach comes when you have a series of pull +requests which build on top of one another: as soon as you squash-merge the +first PR, you'll end up with a stack of conflicts to resolve in all of the +others. In general, it's best to avoid this situation in the first place by +trying not to have multiple related PRs in flight at the same time. Still, +sometimes that's not possible and doing a regular merge is the lesser evil.</p> +<p>Another occasion in which a regular merge makes more sense is a PR where you've +deliberately created a series of commits each of which makes sense in its own +right. For example: <a href="https://github.com/matrix-org/synapse/pull/6837">a PR which gradually propagates a refactoring operation +through the codebase</a>, or <a href="https://github.com/matrix-org/synapse/pull/5987">a +PR which is the culmination of several other +PRs</a>. In this case the ability +to figure out when a particular change/bug was introduced could be very useful.</p> +<p>Ultimately: <strong>this is not a hard-and-fast-rule</strong>. If in doubt, ask yourself “do +each of the commits I am about to merge make sense in their own right”, but +remember that we're just doing our best to balance “keeping the commit history +clean” with other factors.</p> +<h2 id="git-branching-model"><a class="header" href="#git-branching-model">Git branching model</a></h2> +<p>A <a href="https://nvie.com/posts/a-successful-git-branching-model/">lot</a> +<a href="http://scottchacon.com/2011/08/31/github-flow.html">of</a> +<a href="https://www.endoflineblog.com/gitflow-considered-harmful">words</a> have been +written in the past about git branching models (no really, <a href="https://martinfowler.com/articles/branching-patterns.html">a +lot</a>). I tend to +think the whole thing is overblown. Fundamentally, it's not that +complicated. Here's how we do it.</p> +<p>Let's start with a picture:</p> +<p><img src="img/git/branches.jpg" alt="branching model" /></p> +<p>It looks complicated, but it's really not. There's one basic rule: <em>anyone</em> is +free to merge from <em>any</em> more-stable branch to <em>any</em> less-stable branch at +<em>any</em> time<sup id="a2"><a href="#f2">2</a></sup>. (The principle behind this is that if a +change is good enough for the more-stable branch, then it's also good enough go +put in a less-stable branch.)</p> +<p>Meanwhile, merging (or squashing, as per the above) from a less-stable to a +more-stable branch is a deliberate action in which you want to publish a change +or a set of changes to (some subset of) the world: for example, this happens +when a PR is landed, or as part of our release process.</p> +<p>So, what counts as a more- or less-stable branch? A little reflection will show +that our active branches are ordered thus, from more-stable to less-stable:</p> +<ul> +<li><code>master</code> (tracks our last release).</li> +<li><code>release-vX.Y</code> (the branch where we prepare the next release)<sup + id="a3"><a href="#f3">3</a></sup>.</li> +<li>PR branches which are targeting the release.</li> +<li><code>develop</code> (our "mainline" branch containing our bleeding-edge).</li> +<li>regular PR branches.</li> +</ul> +<p>The corollary is: if you have a bugfix that needs to land in both +<code>release-vX.Y</code> <em>and</em> <code>develop</code>, then you should base your PR on +<code>release-vX.Y</code>, get it merged there, and then merge from <code>release-vX.Y</code> to +<code>develop</code>. (If a fix lands in <code>develop</code> and we later need it in a +release-branch, we can of course cherry-pick it, but landing it in the release +branch first helps reduce the chance of annoying conflicts.)</p> +<hr /> +<p><b id="f1">[1]</b>: “Squash and merge” is GitHub's term for this +operation. Given that there is no merge involved, I'm not convinced it's the +most intuitive name. <a href="#a1">^</a></p> +<p><b id="f2">[2]</b>: Well, anyone with commit access.<a href="#a2">^</a></p> +<p><b id="f3">[3]</b>: Very, very occasionally (I think this has happened once in +the history of Synapse), we've had two releases in flight at once. Obviously, +<code>release-v1.2</code> is more-stable than <code>release-v1.3</code>. <a href="#a3">^</a></p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../code_style.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../opentracing.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../code_style.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../opentracing.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/img/git/branches.jpg b/v1.49/development/img/git/branches.jpg new file mode 100644 index 0000000000..715ecc8cd0 --- /dev/null +++ b/v1.49/development/img/git/branches.jpg Binary files differdiff --git a/v1.49/development/img/git/clean.png b/v1.49/development/img/git/clean.png new file mode 100644 index 0000000000..3accd7ccef --- /dev/null +++ b/v1.49/development/img/git/clean.png Binary files differdiff --git a/v1.49/development/img/git/squash.png b/v1.49/development/img/git/squash.png new file mode 100644 index 0000000000..234caca3e4 --- /dev/null +++ b/v1.49/development/img/git/squash.png Binary files differdiff --git a/v1.49/development/internal_documentation/index.html b/v1.49/development/internal_documentation/index.html new file mode 100644 index 0000000000..82dd02bac4 --- /dev/null +++ b/v1.49/development/internal_documentation/index.html @@ -0,0 +1,264 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Internal Documentation - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../../favicon.svg"> + + + <link rel="shortcut icon" href="../../favicon.png"> + + <link rel="stylesheet" href="../../css/variables.css"> + <link rel="stylesheet" href="../../css/general.css"> + <link rel="stylesheet" href="../../css/chrome.css"> + + <link rel="stylesheet" href="../../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../../highlight.css"> + <link rel="stylesheet" href="../../tomorrow-night.css"> + <link rel="stylesheet" href="../../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../../development/internal_documentation/index.html" class="active">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/internal_documentation/README.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="internal-documentation"><a class="header" href="#internal-documentation">Internal Documentation</a></h1> +<p>This section covers implementation documentation for various parts of Synapse.</p> +<p>If a developer is planning to make a change to a feature of Synapse, it can be useful for +general documentation of how that feature is implemented to be available. This saves the +developer time in place of needing to understand how the feature works by reading the +code.</p> +<p>Documentation that would be more useful for the perspective of a system administrator, +rather than a developer who's intending to change to code, should instead be placed +under the Usage section of the documentation.</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../../tcp_replication.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../../development/saml.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../../tcp_replication.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../../development/saml.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/room-dag-concepts.html b/v1.49/development/room-dag-concepts.html new file mode 100644 index 0000000000..724ea1ea93 --- /dev/null +++ b/v1.49/development/room-dag-concepts.html @@ -0,0 +1,305 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>Room DAG concepts - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html" class="active">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/room-dag-concepts.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="room-dag-concepts"><a class="header" href="#room-dag-concepts">Room DAG concepts</a></h1> +<h2 id="edges"><a class="header" href="#edges">Edges</a></h2> +<p>The word "edge" comes from graph theory lingo. An edge is just a connection +between two events. In Synapse, we connect events by specifying their +<code>prev_events</code>. A subsequent event points back at a previous event.</p> +<pre><code>A (oldest) <---- B <---- C (most recent) +</code></pre> +<h2 id="depth-and-stream-ordering"><a class="header" href="#depth-and-stream-ordering">Depth and stream ordering</a></h2> +<p>Events are normally sorted by <code>(topological_ordering, stream_ordering)</code> where +<code>topological_ordering</code> is just <code>depth</code>. In other words, we first sort by <code>depth</code> +and then tie-break based on <code>stream_ordering</code>. <code>depth</code> is incremented as new +messages are added to the DAG. Normally, <code>stream_ordering</code> is an auto +incrementing integer, but backfilled events start with <code>stream_ordering=-1</code> and decrement.</p> +<hr /> +<ul> +<li><code>/sync</code> returns things in the order they arrive at the server (<code>stream_ordering</code>).</li> +<li><code>/messages</code> (and <code>/backfill</code> in the federation API) return them in the order determined by the event graph <code>(topological_ordering, stream_ordering)</code>.</li> +</ul> +<p>The general idea is that, if you're following a room in real-time (i.e. +<code>/sync</code>), you probably want to see the messages as they arrive at your server, +rather than skipping any that arrived late; whereas if you're looking at a +historical section of timeline (i.e. <code>/messages</code>), you want to see the best +representation of the state of the room as others were seeing it at the time.</p> +<h2 id="forward-extremity"><a class="header" href="#forward-extremity">Forward extremity</a></h2> +<p>Most-recent-in-time events in the DAG which are not referenced by any other events' <code>prev_events</code> yet.</p> +<p>The forward extremities of a room are used as the <code>prev_events</code> when the next event is sent.</p> +<h2 id="backward-extremity"><a class="header" href="#backward-extremity">Backward extremity</a></h2> +<p>The current marker of where we have backfilled up to and will generally be the +<code>prev_events</code> of the oldest-in-time events we have in the DAG. This gives a starting point when +backfilling history.</p> +<p>When we persist a non-outlier event, we clear it as a backward extremity and set +all of its <code>prev_events</code> as the new backward extremities if they aren't already +persisted in the <code>events</code> table.</p> +<h2 id="outliers"><a class="header" href="#outliers">Outliers</a></h2> +<p>We mark an event as an <code>outlier</code> when we haven't figured out the state for the +room at that point in the DAG yet.</p> +<p>We won't <em>necessarily</em> have the <code>prev_events</code> of an <code>outlier</code> in the database, +but it's entirely possible that we <em>might</em>.</p> +<p>For example, when we fetch the event auth chain or state for a given event, we +mark all of those claimed auth events as outliers because we haven't done the +state calculation ourself.</p> +<h2 id="state-groups"><a class="header" href="#state-groups">State groups</a></h2> +<p>For every non-outlier event we need to know the state at that event. Instead of +storing the full state for each event in the DB (i.e. a <code>event_id -> state</code> +mapping), which is <em>very</em> space inefficient when state doesn't change, we +instead assign each different set of state a "state group" and then have +mappings of <code>event_id -> state_group</code> and <code>state_group -> state</code>.</p> +<h3 id="stage-group-edges"><a class="header" href="#stage-group-edges">Stage group edges</a></h3> +<p>TODO: <code>state_group_edges</code> is a further optimization... +notes from @Azrenbeth, https://pastebin.com/seUGVGeT</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../development/cas.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../auth_chain_difference_algorithm.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../development/cas.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../auth_chain_difference_algorithm.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/saml.html b/v1.49/development/saml.html new file mode 100644 index 0000000000..40925936a9 --- /dev/null +++ b/v1.49/development/saml.html @@ -0,0 +1,293 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>SAML - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html" class="active">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/saml.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="how-to-test-saml-as-a-developer-without-a-server"><a class="header" href="#how-to-test-saml-as-a-developer-without-a-server">How to test SAML as a developer without a server</a></h1> +<p>https://fujifish.github.io/samling/samling.html (https://github.com/fujifish/samling) is a great resource for being able to tinker with the +SAML options within Synapse without needing to deploy and configure a complicated software stack.</p> +<p>To make Synapse (and therefore Element) use it:</p> +<ol> +<li>Use the samling.html URL above or deploy your own and visit the IdP Metadata tab.</li> +<li>Copy the XML to your clipboard.</li> +<li>On your Synapse server, create a new file <code>samling.xml</code> next to your <code>homeserver.yaml</code> with +the XML from step 2 as the contents.</li> +<li>Edit your <code>homeserver.yaml</code> to include: +<pre><code class="language-yaml">saml2_config: + sp_config: + allow_unknown_attributes: true # Works around a bug with AVA Hashes: https://github.com/IdentityPython/pysaml2/issues/388 + metadata: + local: ["samling.xml"] +</code></pre> +</li> +<li>Ensure that your <code>homeserver.yaml</code> has a setting for <code>public_baseurl</code>: +<pre><code class="language-yaml">public_baseurl: http://localhost:8080/ +</code></pre> +</li> +<li>Run <code>apt-get install xmlsec1</code> and <code>pip install --upgrade --force 'pysaml2>=4.5.0'</code> to ensure +the dependencies are installed and ready to go.</li> +<li>Restart Synapse.</li> +</ol> +<p>Then in Element:</p> +<ol> +<li>Visit the login page and point Element towards your homeserver using the <code>public_baseurl</code> above.</li> +<li>Click the Single Sign-On button.</li> +<li>On the samling page, enter a Name Identifier and add a SAML Attribute for <code>uid=your_localpart</code>. +The response must also be signed.</li> +<li>Click "Next".</li> +<li>Click "Post Response" (change nothing).</li> +<li>You should be logged in.</li> +</ol> +<p>If you try and repeat this process, you may be automatically logged in using the information you +gave previously. To fix this, open your developer console (<code>F12</code> or <code>Ctrl+Shift+I</code>) while on the +samling page and clear the site data. In Chrome, this will be a button on the Application tab.</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../development/internal_documentation/index.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/cas.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../development/internal_documentation/index.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../development/cas.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file diff --git a/v1.49/development/url_previews.html b/v1.49/development/url_previews.html new file mode 100644 index 0000000000..a4cd400709 --- /dev/null +++ b/v1.49/development/url_previews.html @@ -0,0 +1,331 @@ +<!DOCTYPE HTML> +<html lang="en" class="sidebar-visible no-js light"> + <head> + <!-- Book generated using mdBook --> + <meta charset="UTF-8"> + <title>URL Previews - Synapse</title> + + + + + <!-- Custom HTML head --> + + + + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <meta name="description" content=""> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="theme-color" content="#ffffff" /> + + + <link rel="icon" href="../favicon.svg"> + + + <link rel="shortcut icon" href="../favicon.png"> + + <link rel="stylesheet" href="../css/variables.css"> + <link rel="stylesheet" href="../css/general.css"> + <link rel="stylesheet" href="../css/chrome.css"> + + <link rel="stylesheet" href="../css/print.css" media="print"> + + + <!-- Fonts --> + <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> + + <link rel="stylesheet" href="../fonts/fonts.css"> + + + <!-- Highlight.js Stylesheets --> + <link rel="stylesheet" href="../highlight.css"> + <link rel="stylesheet" href="../tomorrow-night.css"> + <link rel="stylesheet" href="../ayu-highlight.css"> + + <!-- Custom theme stylesheets --> + + <link rel="stylesheet" href="../docs/website_files/table-of-contents.css"> + + <link rel="stylesheet" href="../docs/website_files/remove-nav-buttons.css"> + + <link rel="stylesheet" href="../docs/website_files/indent-section-headers.css"> + + + + </head> + <body> + <!-- Provide site root to javascript --> + <script type="text/javascript"> + var path_to_root = "../"; + var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light"; + </script> + + <!-- Work around some values being stored in localStorage wrapped in quotes --> + <script type="text/javascript"> + try { + var theme = localStorage.getItem('mdbook-theme'); + var sidebar = localStorage.getItem('mdbook-sidebar'); + if (theme.startsWith('"') && theme.endsWith('"')) { + localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); + } + if (sidebar.startsWith('"') && sidebar.endsWith('"')) { + localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); + } + } catch (e) { } + </script> + + <!-- Set the theme before any content is loaded, prevents flash --> + <script type="text/javascript"> + var theme; + try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } + if (theme === null || theme === undefined) { theme = default_theme; } + var html = document.querySelector('html'); + html.classList.remove('no-js') + html.classList.remove('light') + html.classList.add(theme); + html.classList.add('js'); + </script> + + <!-- Hide / unhide sidebar before it is displayed --> + <script type="text/javascript"> + var html = document.querySelector('html'); + var sidebar = 'hidden'; + if (document.body.clientWidth >= 1080) { + try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } + sidebar = sidebar || 'visible'; + } + html.classList.remove('sidebar-visible'); + html.classList.add("sidebar-" + sidebar); + </script> + + <nav id="sidebar" class="sidebar" aria-label="Table of contents"> + <div class="sidebar-scrollbox"> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="../welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="../setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="../postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="../reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="../setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="../turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="../delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="../upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded "><a href="../MSC1711_certificates_FAQ.html">Upgrading from pre-Synapse 1.0</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="../federate.html">Federation</a></li><li class="chapter-item expanded "><a href="../usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="../usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="../structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="../templates.html">Templates</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="../sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="../password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="../jwt.html">JSON Web Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="../CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="../application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="../server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="../development/url_previews.html" class="active">URL Previews</a></li><li class="chapter-item expanded "><a href="../user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="../message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="../modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="../modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="../modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="../modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="../modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="../modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="../modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="../systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="../usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="../admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="../admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="../admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="../admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="../admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="../admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="../admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="../admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="../admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="../admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="../admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="../manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="../metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="../usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="../usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="../usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="../usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="../usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="../usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="../development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="../code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="../development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li class="chapter-item expanded "><a href="../opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="../development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="../development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="../replication.html">Replication</a></li><li class="chapter-item expanded "><a href="../tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="../development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="../development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="../development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="../auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="../room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="../deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="../other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + </div> + <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> + </nav> + + <div id="page-wrapper" class="page-wrapper"> + + <div class="page"> + + <div id="menu-bar-hover-placeholder"></div> + <div id="menu-bar" class="menu-bar sticky bordered"> + <div class="left-buttons"> + <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> + <i class="fa fa-bars"></i> + </button> + <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> + <i class="fa fa-paint-brush"></i> + </button> + <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> + <li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li> + <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> + <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> + <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> + <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> + </ul> + + <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> + <i class="fa fa-search"></i> + </button> + + </div> + + <h1 class="menu-title">Synapse</h1> + + <div class="right-buttons"> + + <a href="../print.html" title="Print this book" aria-label="Print this book"> + <i id="print-button" class="fa fa-print"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse" title="Git repository" aria-label="Git repository"> + <i id="git-repository-button" class="fa fa-github"></i> + </a> + + + <a href="https://github.com/matrix-org/synapse/edit/develop/docs/development/url_previews.md" title="Suggest an edit" aria-label="Suggest an edit"> + <i id="git-edit-button" class="fa fa-edit"></i> + </a> + + + </div> + </div> + + + <div id="search-wrapper" class="hidden"> + <form id="searchbar-outer" class="searchbar-outer"> + <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> + </form> + <div id="searchresults-outer" class="searchresults-outer hidden"> + <div id="searchresults-header" class="searchresults-header"></div> + <ul id="searchresults"> + </ul> + </div> + </div> + + + <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> + <script type="text/javascript"> + document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); + document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); + Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { + link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); + }); + </script> + + <div id="content" class="content"> + <main> + <!-- Page table of contents --> + <div class="sidetoc"> + <nav class="pagetoc"></nav> + </div> + + <h1 id="url-previews"><a class="header" href="#url-previews">URL Previews</a></h1> +<p>The <code>GET /_matrix/media/r0/preview_url</code> endpoint provides a generic preview API +for URLs which outputs <a href="https://ogp.me/">Open Graph</a> responses (with some Matrix +specific additions).</p> +<p>This does have trade-offs compared to other designs:</p> +<ul> +<li>Pros: +<ul> +<li>Simple and flexible; can be used by any clients at any point</li> +</ul> +</li> +<li>Cons: +<ul> +<li>If each homeserver provides one of these independently, all the HSes in a +room may needlessly DoS the target URI</li> +<li>The URL metadata must be stored somewhere, rather than just using Matrix +itself to store the media.</li> +<li>Matrix cannot be used to distribute the metadata between homeservers.</li> +</ul> +</li> +</ul> +<p>When Synapse is asked to preview a URL it does the following:</p> +<ol> +<li>Checks against a URL blacklist (defined as <code>url_preview_url_blacklist</code> in the +config).</li> +<li>Checks the in-memory cache by URLs and returns the result if it exists. (This +is also used to de-duplicate processing of multiple in-flight requests at once.)</li> +<li>Kicks off a background process to generate a preview: +<ol> +<li>Checks the database cache by URL and timestamp and returns the result if it +has not expired and was successful (a 2xx return code).</li> +<li>Checks if the URL matches an <a href="https://oembed.com/">oEmbed</a> pattern. If it +does, update the URL to download.</li> +<li>Downloads the URL and stores it into a file via the media storage provider +and saves the local media metadata.</li> +<li>If the media is an image: +<ol> +<li>Generates thumbnails.</li> +<li>Generates an Open Graph response based on image properties.</li> +</ol> +</li> +<li>If the media is HTML: +<ol> +<li>Decodes the HTML via the stored file.</li> +<li>Generates an Open Graph response from the HTML.</li> +<li>If an image exists in the Open Graph response: +<ol> +<li>Downloads the URL and stores it into a file via the media storage +provider and saves the local media metadata.</li> +<li>Generates thumbnails.</li> +<li>Updates the Open Graph response based on image properties.</li> +</ol> +</li> +</ol> +</li> +<li>If the media is JSON and an oEmbed URL was found: +<ol> +<li>Convert the oEmbed response to an Open Graph response.</li> +<li>If a thumbnail or image is in the oEmbed response: +<ol> +<li>Downloads the URL and stores it into a file via the media storage +provider and saves the local media metadata.</li> +<li>Generates thumbnails.</li> +<li>Updates the Open Graph response based on image properties.</li> +</ol> +</li> +</ol> +</li> +<li>Stores the result in the database cache.</li> +</ol> +</li> +<li>Returns the result.</li> +</ol> +<p>The in-memory cache expires after 1 hour.</p> +<p>Expired entries in the database cache (and their associated media files) are +deleted every 10 seconds. The default expiration time is 1 hour from download.</p> + + </main> + + <nav class="nav-wrapper" aria-label="Page navigation"> + <!-- Mobile navigation buttons --> + + <a rel="prev" href="../consent_tracking.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../user_directory.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + + <div style="clear: both"></div> + </nav> + </div> + </div> + + <nav class="nav-wide-wrapper" aria-label="Page navigation"> + + <a rel="prev" href="../consent_tracking.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> + <i class="fa fa-angle-left"></i> + </a> + + + + <a rel="next" href="../user_directory.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> + <i class="fa fa-angle-right"></i> + </a> + + </nav> + + </div> + + + + + + + + + <script type="text/javascript"> + window.playground_copyable = true; + </script> + + + + + + <script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../mark.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../searcher.js" type="text/javascript" charset="utf-8"></script> + + + <script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script> + <script src="../highlight.js" type="text/javascript" charset="utf-8"></script> + <script src="../book.js" type="text/javascript" charset="utf-8"></script> + + <!-- Custom JS scripts --> + + <script type="text/javascript" src="../docs/website_files/table-of-contents.js"></script> + + + + + </body> +</html> \ No newline at end of file |
