diff --git a/CHANGES.md b/CHANGES.md
index dc8c74fe58..bb6bcb75ed 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,84 @@
+Synapse 1.2.0rc1 (2019-07-22)
+=============================
+
+Features
+--------
+
+- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
+- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
+- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
+- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
+- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
+- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
+- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
+- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
+- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
+
+
+Bugfixes
+--------
+
+- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
+- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
+- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
+- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
+- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
+- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
+- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
+- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
+- Ignore redactions of m.room.create events. ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
+
+
+Updates to the Docker image
+---------------------------
+
+- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
+- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
+
+
+Improved Documentation
+----------------------
+
+- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
+- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
+- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
+- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
+
+
+Internal Changes
+----------------
+
+- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
+- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
+- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
+- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
+- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
+- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
+- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
+- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
+- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
+- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
+- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
+- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
+- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
+- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
+- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
+- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
+- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
+- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
+- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
+- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
+- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
+- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
+- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
+
+
Synapse 1.1.0 (2019-07-04)
==========================
diff --git a/changelog.d/5397.doc b/changelog.d/5397.doc
deleted file mode 100644
index c2b500b482..0000000000
--- a/changelog.d/5397.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks!
diff --git a/changelog.d/5544.feature b/changelog.d/5544.feature
deleted file mode 100644
index 7d3459129d..0000000000
--- a/changelog.d/5544.feature
+++ /dev/null
@@ -1,2 +0,0 @@
-Add support for opentracing.
-
diff --git a/changelog.d/5589.feature b/changelog.d/5589.feature
deleted file mode 100644
index a87e669dd4..0000000000
--- a/changelog.d/5589.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add ability to pull all locally stored events out of synapse that a particular user can see.
diff --git a/changelog.d/5597.feature b/changelog.d/5597.feature
deleted file mode 100644
index 6f92748885..0000000000
--- a/changelog.d/5597.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance.
diff --git a/changelog.d/5606.misc b/changelog.d/5606.misc
deleted file mode 100644
index bb3c028167..0000000000
--- a/changelog.d/5606.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move logging code out of `synapse.util` and into `synapse.logging`.
diff --git a/changelog.d/5609.bugfix b/changelog.d/5609.bugfix
deleted file mode 100644
index 534ee22a1b..0000000000
--- a/changelog.d/5609.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'utime went backwards' errors on daemonization.
diff --git a/changelog.d/5611.misc b/changelog.d/5611.misc
deleted file mode 100644
index a2d1695139..0000000000
--- a/changelog.d/5611.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a blacklist file to the repo to blacklist certain sytests from failing CI.
diff --git a/changelog.d/5613.feature b/changelog.d/5613.feature
deleted file mode 100644
index 4b7bb2745c..0000000000
--- a/changelog.d/5613.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add `sender` and `origin_server_ts` fields to `m.replace`.
diff --git a/changelog.d/5616.misc b/changelog.d/5616.misc
deleted file mode 100644
index 9f94be6778..0000000000
--- a/changelog.d/5616.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make runtime errors surrounding password reset emails much clearer.
diff --git a/changelog.d/5617.misc b/changelog.d/5617.misc
deleted file mode 100644
index bb3c028167..0000000000
--- a/changelog.d/5617.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move logging code out of `synapse.util` and into `synapse.logging`.
diff --git a/changelog.d/5619.docker b/changelog.d/5619.docker
deleted file mode 100644
index b69e5cc57c..0000000000
--- a/changelog.d/5619.docker
+++ /dev/null
@@ -1 +0,0 @@
-Base Docker image on a newer Alpine Linux version (3.8 -> 3.10).
diff --git a/changelog.d/5620.docker b/changelog.d/5620.docker
deleted file mode 100644
index cbb5a75d6a..0000000000
--- a/changelog.d/5620.docker
+++ /dev/null
@@ -1 +0,0 @@
-Add missing space in default logging file format generated by the Docker image.
diff --git a/changelog.d/5621.bugfix b/changelog.d/5621.bugfix
deleted file mode 100644
index f1a2851f45..0000000000
--- a/changelog.d/5621.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Various minor fixes to the federation request rate limiter.
diff --git a/changelog.d/5622.misc b/changelog.d/5622.misc
deleted file mode 100644
index 9f0a87311c..0000000000
--- a/changelog.d/5622.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove dead code for persiting outgoing federation transactions.
diff --git a/changelog.d/5623.feature b/changelog.d/5623.feature
deleted file mode 100644
index b73080e88d..0000000000
--- a/changelog.d/5623.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add default push rule to ignore reactions.
diff --git a/changelog.d/5625.removal b/changelog.d/5625.removal
deleted file mode 100644
index d33a778d69..0000000000
--- a/changelog.d/5625.removal
+++ /dev/null
@@ -1 +0,0 @@
-Remove support for the `invite_3pid_guest` configuration setting.
diff --git a/changelog.d/5626.feature b/changelog.d/5626.feature
deleted file mode 100644
index 5ef793b943..0000000000
--- a/changelog.d/5626.feature
+++ /dev/null
@@ -1 +0,0 @@
-Include the original event when asking for its relations.
diff --git a/changelog.d/5627.misc b/changelog.d/5627.misc
deleted file mode 100644
index 730721b5ef..0000000000
--- a/changelog.d/5627.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI.
diff --git a/changelog.d/5628.misc b/changelog.d/5628.misc
deleted file mode 100644
index fec8446793..0000000000
--- a/changelog.d/5628.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move RegistrationHandler.get_or_create_user to test code.
diff --git a/changelog.d/5629.bugfix b/changelog.d/5629.bugfix
deleted file mode 100644
index 672eabad40..0000000000
--- a/changelog.d/5629.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Forbid viewing relations on an event once it has been redacted.
diff --git a/changelog.d/5630.misc b/changelog.d/5630.misc
deleted file mode 100644
index f112d873eb..0000000000
--- a/changelog.d/5630.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add some more common python virtual-environment paths to the black exclusion list.
diff --git a/changelog.d/5636.misc b/changelog.d/5636.misc
deleted file mode 100644
index 3add990283..0000000000
--- a/changelog.d/5636.misc
+++ /dev/null
@@ -1 +0,0 @@
-Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details.
\ No newline at end of file
diff --git a/changelog.d/5637.misc b/changelog.d/5637.misc
deleted file mode 100644
index f18d6197e5..0000000000
--- a/changelog.d/5637.misc
+++ /dev/null
@@ -1 +0,0 @@
-Unblacklist some user_directory sytests.
diff --git a/changelog.d/5638.bugfix b/changelog.d/5638.bugfix
deleted file mode 100644
index 66781ad9e6..0000000000
--- a/changelog.d/5638.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format.
diff --git a/changelog.d/5639.misc b/changelog.d/5639.misc
deleted file mode 100644
index 413b13128c..0000000000
--- a/changelog.d/5639.misc
+++ /dev/null
@@ -1 +0,0 @@
-Factor out some redundant code in the login implementation.
diff --git a/changelog.d/5640.misc b/changelog.d/5640.misc
deleted file mode 100644
index 7d69a1b3b6..0000000000
--- a/changelog.d/5640.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update ModuleApi to avoid register(generate_token=True).
diff --git a/changelog.d/5641.misc b/changelog.d/5641.misc
deleted file mode 100644
index 1899bc963d..0000000000
--- a/changelog.d/5641.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove access-token support from RegistrationHandler.register, and rename it.
diff --git a/changelog.d/5642.misc b/changelog.d/5642.misc
deleted file mode 100644
index e7f8e214a4..0000000000
--- a/changelog.d/5642.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove access-token support from `RegistrationStore.register`, and rename it.
diff --git a/changelog.d/5643.misc b/changelog.d/5643.misc
deleted file mode 100644
index 2b2316469e..0000000000
--- a/changelog.d/5643.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging for auto-join when a new user is created.
diff --git a/changelog.d/5644.bugfix b/changelog.d/5644.bugfix
deleted file mode 100644
index f6302fd08d..0000000000
--- a/changelog.d/5644.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix newly-registered users not being able to lookup their own profile without joining a room.
diff --git a/changelog.d/5645.misc b/changelog.d/5645.misc
deleted file mode 100644
index 4fa9699e4f..0000000000
--- a/changelog.d/5645.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure.
\ No newline at end of file
diff --git a/changelog.d/5651.doc b/changelog.d/5651.doc
deleted file mode 100644
index e2d5a8dc8a..0000000000
--- a/changelog.d/5651.doc
+++ /dev/null
@@ -1 +0,0 @@
---no-pep517 should be --no-use-pep517 in the documentation to setup the development environment.
diff --git a/changelog.d/5654.bugfix b/changelog.d/5654.bugfix
deleted file mode 100644
index 5f76b041cd..0000000000
--- a/changelog.d/5654.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`.
\ No newline at end of file
diff --git a/changelog.d/5655.misc b/changelog.d/5655.misc
deleted file mode 100644
index acab6aee92..0000000000
--- a/changelog.d/5655.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a small typo in a code comment.
\ No newline at end of file
diff --git a/changelog.d/5656.misc b/changelog.d/5656.misc
deleted file mode 100644
index a8de20a7d0..0000000000
--- a/changelog.d/5656.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up exception handling around client access tokens.
diff --git a/changelog.d/5657.misc b/changelog.d/5657.misc
deleted file mode 100644
index bdec9ae4c0..0000000000
--- a/changelog.d/5657.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a mechanism for per-test homeserver configuration in the unit tests.
diff --git a/changelog.d/5658.bugfix b/changelog.d/5658.bugfix
deleted file mode 100644
index f6ae906a9a..0000000000
--- a/changelog.d/5658.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated.
diff --git a/changelog.d/5659.misc b/changelog.d/5659.misc
deleted file mode 100644
index 686001295c..0000000000
--- a/changelog.d/5659.misc
+++ /dev/null
@@ -1 +0,0 @@
-Inline issue_access_token.
diff --git a/changelog.d/5660.feature b/changelog.d/5660.feature
deleted file mode 100644
index 82889fdaf1..0000000000
--- a/changelog.d/5660.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement `session_lifetime` configuration option, after which access tokens will expire.
diff --git a/changelog.d/5661.doc b/changelog.d/5661.doc
deleted file mode 100644
index c70e62014e..0000000000
--- a/changelog.d/5661.doc
+++ /dev/null
@@ -1 +0,0 @@
-Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks!
diff --git a/changelog.d/5664.misc b/changelog.d/5664.misc
deleted file mode 100644
index 0ca7a0fbd0..0000000000
--- a/changelog.d/5664.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update the sytest BuildKite configuration to checkout Synapse in `/src`.
diff --git a/changelog.d/5673.misc b/changelog.d/5673.misc
deleted file mode 100644
index 1942256358..0000000000
--- a/changelog.d/5673.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a `docker` type to the towncrier configuration.
diff --git a/changelog.d/5674.feature b/changelog.d/5674.feature
deleted file mode 100644
index 04bdfa4ad5..0000000000
--- a/changelog.d/5674.feature
+++ /dev/null
@@ -1 +0,0 @@
-Return "This account has been deactivated" when a deactivated user tries to login.
diff --git a/changelog.d/5675.doc b/changelog.d/5675.doc
deleted file mode 100644
index 4cd4d0be1a..0000000000
--- a/changelog.d/5675.doc
+++ /dev/null
@@ -1 +0,0 @@
-Minor tweaks to postgres documentation.
diff --git a/changelog.d/5689.misc b/changelog.d/5689.misc
deleted file mode 100644
index 8aa3e3f6a2..0000000000
--- a/changelog.d/5689.misc
+++ /dev/null
@@ -1 +0,0 @@
-Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces.
\ No newline at end of file
diff --git a/changelog.d/5695.misc b/changelog.d/5695.misc
new file mode 100644
index 0000000000..4741d32e25
--- /dev/null
+++ b/changelog.d/5695.misc
@@ -0,0 +1 @@
+Add precautionary measures to prevent future abuse of `window.opener` in default welcome page.
diff --git a/changelog.d/5699.bugfix b/changelog.d/5699.bugfix
deleted file mode 100644
index 30d5e67f67..0000000000
--- a/changelog.d/5699.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some problems with authenticating redactions in recent room versions.
\ No newline at end of file
diff --git a/changelog.d/5700.bugfix b/changelog.d/5700.bugfix
deleted file mode 100644
index 51bce8d441..0000000000
--- a/changelog.d/5700.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix some problems with authenticating redactions in recent room versions.
-
diff --git a/changelog.d/5701.bugfix b/changelog.d/5701.bugfix
deleted file mode 100644
index fd2866e16a..0000000000
--- a/changelog.d/5701.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Ignore redactions of m.room.create events.
diff --git a/changelog.d/5706.misc b/changelog.d/5706.misc
new file mode 100644
index 0000000000..5e15dfd5fa
--- /dev/null
+++ b/changelog.d/5706.misc
@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.
diff --git a/changelog.d/5707.bugfix b/changelog.d/5707.bugfix
deleted file mode 100644
index aa3046c5e1..0000000000
--- a/changelog.d/5707.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some problems with authenticating redactions in recent room versions.
diff --git a/changelog.d/5712.feature b/changelog.d/5712.feature
deleted file mode 100644
index 7d3459129d..0000000000
--- a/changelog.d/5712.feature
+++ /dev/null
@@ -1,2 +0,0 @@
-Add support for opentracing.
-
diff --git a/changelog.d/5713.misc b/changelog.d/5713.misc
new file mode 100644
index 0000000000..01ea1cf8d7
--- /dev/null
+++ b/changelog.d/5713.misc
@@ -0,0 +1 @@
+Improve caching when fetching `get_filtered_current_state_ids`.
diff --git a/changelog.d/5714.feature b/changelog.d/5714.feature
deleted file mode 100644
index 2fd32e5e38..0000000000
--- a/changelog.d/5714.feature
+++ /dev/null
@@ -1 +0,0 @@
-Enable aggregations support by default
diff --git a/changelog.d/5715.misc b/changelog.d/5715.misc
new file mode 100644
index 0000000000..a77366e0c0
--- /dev/null
+++ b/changelog.d/5715.misc
@@ -0,0 +1 @@
+Don't accept opentracing data from clients.
diff --git a/changelog.d/5719.misc b/changelog.d/5719.misc
new file mode 100644
index 0000000000..6d5294724c
--- /dev/null
+++ b/changelog.d/5719.misc
@@ -0,0 +1 @@
+Update the coding style document.
diff --git a/changelog.d/5720.misc b/changelog.d/5720.misc
new file mode 100644
index 0000000000..590f64f19d
--- /dev/null
+++ b/changelog.d/5720.misc
@@ -0,0 +1 @@
+Improve database query performance when recording retry intervals for remote hosts.
diff --git a/changelog.d/5725.bugfix b/changelog.d/5725.bugfix
new file mode 100644
index 0000000000..73ef419727
--- /dev/null
+++ b/changelog.d/5725.bugfix
@@ -0,0 +1 @@
+start.sh no longer uses deprecated cli option.
diff --git a/demo/start.sh b/demo/start.sh
index 1c4f12d0bb..eccaa2abeb 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -29,7 +29,7 @@ for port in 8080 8081 8082; do
if ! grep -F "Customisation made by demo/start.sh" -q $DIR/etc/$port.config; then
printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-
+
echo 'enable_registration: true' >> $DIR/etc/$port.config
# Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
@@ -43,7 +43,7 @@ for port in 8080 8081 8082; do
tls: true
resources:
- names: [client, federation]
-
+
- port: $port
tls: false
bind_addresses: ['::1', '127.0.0.1']
@@ -68,7 +68,7 @@ for port in 8080 8081 8082; do
# Generate tls keys
openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-
+
# Ignore keys from the trusted keys server
echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
echo 'trusted_key_servers:' >> $DIR/etc/$port.config
@@ -120,7 +120,6 @@ for port in 8080 8081 8082; do
python3 -m synapse.app.homeserver \
--config-path "$DIR/etc/$port.config" \
-D \
- -vv \
popd
done
diff --git a/docs/code_style.rst b/docs/code_style.rst
index e3ca626bfd..39ac4ebedc 100644
--- a/docs/code_style.rst
+++ b/docs/code_style.rst
@@ -1,4 +1,8 @@
-# Code Style
+Code Style
+==========
+
+Formatting tools
+----------------
The Synapse codebase uses a number of code formatting tools in order to
quickly and automatically check for formatting (and sometimes logical) errors
@@ -6,20 +10,20 @@ in code.
The necessary tools are detailed below.
-## Formatting tools
+- **black**
-The Synapse codebase uses [black](https://pypi.org/project/black/) as an
-opinionated code formatter, ensuring all comitted code is properly
-formatted.
+ The Synapse codebase uses `black <https://pypi.org/project/black/>`_ as an
+ opinionated code formatter, ensuring all comitted code is properly
+ formatted.
-First install ``black`` with::
+ First install ``black`` with::
- pip install --upgrade black
+ pip install --upgrade black
-Have ``black`` auto-format your code (it shouldn't change any
-functionality) with::
+ Have ``black`` auto-format your code (it shouldn't change any functionality)
+ with::
- black . --exclude="\.tox|build|env"
+ black . --exclude="\.tox|build|env"
- **flake8**
@@ -54,17 +58,16 @@ functionality is supported in your editor for a more convenient development
workflow. It is not, however, recommended to run ``flake8`` on save as it
takes a while and is very resource intensive.
-## General rules
+General rules
+-------------
- **Naming**:
- Use camel case for class and type names
- Use underscores for functions and variables.
-- Use double quotes ``"foo"`` rather than single quotes ``'foo'``.
-
-- **Comments**: should follow the `google code style
- <http://google.github.io/styleguide/pyguide.html?showone=Comments#Comments>`_.
+- **Docstrings**: should follow the `google code style
+ <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
This is so that we can generate documentation with `sphinx
<http://sphinxcontrib-napoleon.readthedocs.org/en/latest/>`_. See the
`examples
@@ -73,6 +76,8 @@ takes a while and is very resource intensive.
- **Imports**:
+ - Imports should be sorted by ``isort`` as described above.
+
- Prefer to import classes and functions rather than packages or modules.
Example::
@@ -92,25 +97,84 @@ takes a while and is very resource intensive.
This goes against the advice in the Google style guide, but it means that
errors in the name are caught early (at import time).
- - Multiple imports from the same package can be combined onto one line::
+ - Avoid wildcard imports (``from synapse.types import *``) and relative
+ imports (``from .types import UserID``).
- from synapse.types import GroupID, RoomID, UserID
+Configuration file format
+-------------------------
- An effort should be made to keep the individual imports in alphabetical
- order.
+The `sample configuration file <./sample_config.yaml>`_ acts as a reference to
+Synapse's configuration options for server administrators. Remember that many
+readers will be unfamiliar with YAML and server administration in general, so
+that it is important that the file be as easy to understand as possible, which
+includes following a consistent format.
- If the list becomes long, wrap it with parentheses and split it over
- multiple lines.
+Some guidelines follow:
- - As per `PEP-8 <https://www.python.org/dev/peps/pep-0008/#imports>`_,
- imports should be grouped in the following order, with a blank line between
- each group:
+* Sections should be separated with a heading consisting of a single line
+ prefixed and suffixed with ``##``. There should be **two** blank lines
+ before the section header, and **one** after.
- 1. standard library imports
- 2. related third party imports
- 3. local application/library specific imports
+* Each option should be listed in the file with the following format:
- - Imports within each group should be sorted alphabetically by module name.
+ * A comment describing the setting. Each line of this comment should be
+ prefixed with a hash (``#``) and a space.
- - Avoid wildcard imports (``from synapse.types import *``) and relative
- imports (``from .types import UserID``).
+ The comment should describe the default behaviour (ie, what happens if
+ the setting is omitted), as well as what the effect will be if the
+ setting is changed.
+
+ Often, the comment end with something like "uncomment the
+ following to \<do action>".
+
+ * A line consisting of only ``#``.
+
+ * A commented-out example setting, prefixed with only ``#``.
+
+ For boolean (on/off) options, convention is that this example should be
+ the *opposite* to the default (so the comment will end with "Uncomment
+ the following to enable [or disable] \<feature\>." For other options,
+ the example should give some non-default value which is likely to be
+ useful to the reader.
+
+* There should be a blank line between each option.
+
+* Where several settings are grouped into a single dict, *avoid* the
+ convention where the whole block is commented out, resulting in comment
+ lines starting ``# #``, as this is hard to read and confusing to
+ edit. Instead, leave the top-level config option uncommented, and follow
+ the conventions above for sub-options. Ensure that your code correctly
+ handles the top-level option being set to ``None`` (as it will be if no
+ sub-options are enabled).
+
+* Lines should be wrapped at 80 characters.
+
+Example::
+
+ ## Frobnication ##
+
+ # The frobnicator will ensure that all requests are fully frobnicated.
+ # To enable it, uncomment the following.
+ #
+ #frobnicator_enabled: true
+
+ # By default, the frobnicator will frobnicate with the default frobber.
+ # The following will make it use an alternative frobber.
+ #
+ #frobincator_frobber: special_frobber
+
+ # Settings for the frobber
+ #
+ frobber:
+ # frobbing speed. Defaults to 1.
+ #
+ #speed: 10
+
+ # frobbing distance. Defaults to 1000.
+ #
+ #distance: 100
+
+Note that the sample configuration is generated from the synapse code and is
+maintained by a script, ``scripts-dev/generate_sample_config``. Making sure
+that the output from this script matches the desired format is left as an
+exercise for the reader!
diff --git a/docs/opentracing.rst b/docs/opentracing.rst
new file mode 100644
index 0000000000..b91a2208a8
--- /dev/null
+++ b/docs/opentracing.rst
@@ -0,0 +1,100 @@
+===========
+OpenTracing
+===========
+
+Background
+----------
+
+OpenTracing is a semi-standard being adopted by a number of distributed tracing
+platforms. It is a common api for facilitating vendor-agnostic tracing
+instrumentation. That is, we can use the OpenTracing api and select one of a
+number of tracer implementations to do the heavy lifting in the background.
+Our current selected implementation is Jaeger.
+
+OpenTracing is a tool which gives an insight into the causal relationship of
+work done in and between servers. The servers each track events and report them
+to a centralised server - in Synapse's case: Jaeger. The basic unit used to
+represent events is the span. The span roughly represents a single piece of work
+that was done and the time at which it occurred. A span can have child spans,
+meaning that the work of the child had to be completed for the parent span to
+complete, or it can have follow-on spans which represent work that is undertaken
+as a result of the parent but is not depended on by the parent to in order to
+finish.
+
+Since this is undertaken in a distributed environment a request to another
+server, such as an RPC or a simple GET, can be considered a span (a unit or
+work) for the local server. This causal link is what OpenTracing aims to
+capture and visualise. In order to do this metadata about the local server's
+span, i.e the 'span context', needs to be included with the request to the
+remote.
+
+It is up to the remote server to decide what it does with the spans
+it creates. This is called the sampling policy and it can be configured
+through Jaeger's settings.
+
+For OpenTracing concepts see
+https://opentracing.io/docs/overview/what-is-tracing/.
+
+For more information about Jaeger's implementation see
+https://www.jaegertracing.io/docs/
+
+=====================
+Seting up OpenTracing
+=====================
+
+To receive OpenTracing spans, start up a Jaeger server. This can be done
+using docker like so:
+
+.. code-block:: bash
+
+ docker run -d --name jaeger
+ -p 6831:6831/udp \
+ -p 6832:6832/udp \
+ -p 5778:5778 \
+ -p 16686:16686 \
+ -p 14268:14268 \
+ jaegertracing/all-in-one:1.13
+
+Latest documentation is probably at
+https://www.jaegertracing.io/docs/1.13/getting-started/
+
+
+Enable OpenTracing in Synapse
+-----------------------------
+
+OpenTracing is not enabled by default. It must be enabled in the homeserver
+config by uncommenting the config options under ``opentracing`` as shown in
+the `sample config <./sample_config.yaml>`_. For example:
+
+.. code-block:: yaml
+
+ opentracing:
+ tracer_enabled: true
+ homeserver_whitelist:
+ - "mytrustedhomeserver.org"
+ - "*.myotherhomeservers.com"
+
+Homeserver whitelisting
+-----------------------
+
+The homeserver whitelist is configured using regular expressions. A list of regular
+expressions can be given and their union will be compared when propagating any
+spans contexts to another homeserver.
+
+Though it's mostly safe to send and receive span contexts to and from
+untrusted users since span contexts are usually opaque ids it can lead to
+two problems, namely:
+
+- If the span context is marked as sampled by the sending homeserver the receiver will
+ sample it. Therefore two homeservers with wildly different sampling policies
+ could incur higher sampling counts than intended.
+- Sending servers can attach arbitrary data to spans, known as 'baggage'. For safety this has been disabled in Synapse
+ but that doesn't prevent another server sending you baggage which will be logged
+ to OpenTracing's logs.
+
+==================
+Configuring Jaeger
+==================
+
+Sampling strategies can be set as in this document:
+https://www.jaegertracing.io/docs/1.13/sampling/
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index 5b804d16a4..0a96197ca6 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -1422,18 +1422,8 @@ opentracing:
#enabled: true
# The list of homeservers we wish to send and receive span contexts and span baggage.
- #
- # Though it's mostly safe to send and receive span contexts to and from
- # untrusted users since span contexts are usually opaque ids it can lead to
- # two problems, namely:
- # - If the span context is marked as sampled by the sending homeserver the receiver will
- # sample it. Therefore two homeservers with wildly disparaging sampling policies
- # could incur higher sampling counts than intended.
- # - Span baggage can be arbitrary data. For safety this has been disabled in synapse
- # but that doesn't prevent another server sending you baggage which will be logged
- # to opentracing logs.
- #
- # This a list of regexes which are matched against the server_name of the
+ # See docs/opentracing.rst
+ # This is a list of regexes which are matched against the server_name of the
# homeserver.
#
# By defult, it is empty, so no servers are matched.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index cf22fabd61..f26e49da36 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -35,4 +35,4 @@ try:
except ImportError:
pass
-__version__ = "1.1.0"
+__version__ = "1.2.0rc1"
diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py
index a2ce9ab3f6..4479454415 100644
--- a/synapse/config/tracer.py
+++ b/synapse/config/tracer.py
@@ -48,18 +48,8 @@ class TracerConfig(Config):
#enabled: true
# The list of homeservers we wish to send and receive span contexts and span baggage.
- #
- # Though it's mostly safe to send and receive span contexts to and from
- # untrusted users since span contexts are usually opaque ids it can lead to
- # two problems, namely:
- # - If the span context is marked as sampled by the sending homeserver the receiver will
- # sample it. Therefore two homeservers with wildly disparaging sampling policies
- # could incur higher sampling counts than intended.
- # - Span baggage can be arbitrary data. For safety this has been disabled in synapse
- # but that doesn't prevent another server sending you baggage which will be logged
- # to opentracing logs.
- #
- # This a list of regexes which are matched against the server_name of the
+ # See docs/opentracing.rst
+ # This is a list of regexes which are matched against the server_name of the
# homeserver.
#
# By defult, it is empty, so no servers are matched.
diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py
index 415040f5ee..04393697c0 100644
--- a/synapse/logging/opentracing.py
+++ b/synapse/logging/opentracing.py
@@ -24,6 +24,131 @@
# this move the methods have work very similarly to opentracing's and it should only
# be a matter of few regexes to move over to opentracing's access patterns proper.
+"""
+============================
+Using OpenTracing in Synapse
+============================
+
+Python-specific tracing concepts are at https://opentracing.io/guides/python/.
+Note that Synapse wraps OpenTracing in a small module (this one) in order to make the
+OpenTracing dependency optional. That means that the access patterns are
+different to those demonstrated in the OpenTracing guides. However, it is
+still useful to know, especially if OpenTracing is included as a full dependency
+in the future or if you are modifying this module.
+
+
+OpenTracing is encapsulated so that
+no span objects from OpenTracing are exposed in Synapse's code. This allows
+OpenTracing to be easily disabled in Synapse and thereby have OpenTracing as
+an optional dependency. This does however limit the number of modifiable spans
+at any point in the code to one. From here out references to `opentracing`
+in the code snippets refer to the Synapses module.
+
+Tracing
+-------
+
+In Synapse it is not possible to start a non-active span. Spans can be started
+using the ``start_active_span`` method. This returns a scope (see
+OpenTracing docs) which is a context manager that needs to be entered and
+exited. This is usually done by using ``with``.
+
+.. code-block:: python
+
+ from synapse.logging.opentracing import start_active_span
+
+ with start_active_span("operation name"):
+ # Do something we want to tracer
+
+Forgetting to enter or exit a scope will result in some mysterious and grievous log
+context errors.
+
+At anytime where there is an active span ``opentracing.set_tag`` can be used to
+set a tag on the current active span.
+
+Tracing functions
+-----------------
+
+Functions can be easily traced using decorators. There is a decorator for
+'normal' function and for functions which are actually deferreds. The name of
+the function becomes the operation name for the span.
+
+.. code-block:: python
+
+ from synapse.logging.opentracing import trace, trace_deferred
+
+ # Start a span using 'normal_function' as the operation name
+ @trace
+ def normal_function(*args, **kwargs):
+ # Does all kinds of cool and expected things
+ return something_usual_and_useful
+
+ # Start a span using 'deferred_function' as the operation name
+ @trace_deferred
+ @defer.inlineCallbacks
+ def deferred_function(*args, **kwargs):
+ # We start
+ yield we_wait
+ # we finish
+ defer.returnValue(something_usual_and_useful)
+
+Operation names can be explicitly set for functions by using
+``trace_using_operation_name`` and
+``trace_deferred_using_operation_name``
+
+.. code-block:: python
+
+ from synapse.logging.opentracing import (
+ trace_using_operation_name,
+ trace_deferred_using_operation_name
+ )
+
+ @trace_using_operation_name("A *much* better operation name")
+ def normal_function(*args, **kwargs):
+ # Does all kinds of cool and expected things
+ return something_usual_and_useful
+
+ @trace_deferred_using_operation_name("Another exciting operation name!")
+ @defer.inlineCallbacks
+ def deferred_function(*args, **kwargs):
+ # We start
+ yield we_wait
+ # we finish
+ defer.returnValue(something_usual_and_useful)
+
+Contexts and carriers
+---------------------
+
+There are a selection of wrappers for injecting and extracting contexts from
+carriers provided. Unfortunately OpenTracing's three context injection
+techniques are not adequate for our inject of OpenTracing span-contexts into
+Twisted's http headers, EDU contents and our database tables. Also note that
+the binary encoding format mandated by OpenTracing is not actually implemented
+by jaeger_client v4.0.0 - it will silently noop.
+Please refer to the end of ``logging/opentracing.py`` for the available
+injection and extraction methods.
+
+Homeserver whitelisting
+-----------------------
+
+Most of the whitelist checks are encapsulated in the modules's injection
+and extraction method but be aware that using custom carriers or crossing
+unchartered waters will require the enforcement of the whitelist.
+``logging/opentracing.py`` has a ``whitelisted_homeserver`` method which takes
+in a destination and compares it to the whitelist.
+
+=======
+Gotchas
+=======
+
+- Checking whitelists on span propagation
+- Inserting pii
+- Forgetting to enter or exit a scope
+- Span source: make sure that the span you expect to be active across a
+ function call really will be that one. Does the current function have more
+ than one caller? Will all of those calling functions have be in a context
+ with an active span?
+"""
+
import contextlib
import logging
import re
@@ -340,8 +465,7 @@ def trace_servlet(servlet_name, func):
@wraps(func)
@defer.inlineCallbacks
def _trace_servlet_inner(request, *args, **kwargs):
- with start_active_span_from_context(
- request.requestHeaders,
+ with start_active_span(
"incoming-client-request",
tags={
"request_id": request.get_request_id(),
diff --git a/synapse/static/index.html b/synapse/static/index.html
index d3f1c7dce0..bf46df9097 100644
--- a/synapse/static/index.html
+++ b/synapse/static/index.html
@@ -48,13 +48,13 @@
</div>
<h1>It works! Synapse is running</h1>
<p>Your Synapse server is listening on this port and is ready for messages.</p>
- <p>To use this server you'll need <a href="https://matrix.org/docs/projects/try-matrix-now.html#clients" target="_blank">a Matrix client</a>.
+ <p>To use this server you'll need <a href="https://matrix.org/docs/projects/try-matrix-now.html#clients" target="_blank" rel="noopener noreferrer">a Matrix client</a>.
</p>
<p>Welcome to the Matrix universe :)</p>
<hr>
<p>
<small>
- <a href="https://matrix.org" target="_blank">
+ <a href="https://matrix.org" target="_blank" rel="noopener noreferrer">
matrix.org
</a>
</small>
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2f940dbae6..a7c93efa46 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -86,7 +86,21 @@ _CURRENT_STATE_CACHE_NAME = "cs_cache_fake"
class LoggingTransaction(object):
"""An object that almost-transparently proxies for the 'txn' object
passed to the constructor. Adds logging and metrics to the .execute()
- method."""
+ method.
+
+ Args:
+ txn: The database transcation object to wrap.
+ name (str): The name of this transactions for logging.
+ database_engine (Sqlite3Engine|PostgresEngine)
+ after_callbacks(list|None): A list that callbacks will be appended to
+ that have been added by `call_after` which should be run on
+ successful completion of the transaction. None indicates that no
+ callbacks should be allowed to be scheduled to run.
+ exception_callbacks(list|None): A list that callbacks will be appended
+ to that have been added by `call_on_exception` which should be run
+ if transaction ends with an error. None indicates that no callbacks
+ should be allowed to be scheduled to run.
+ """
__slots__ = [
"txn",
@@ -97,7 +111,7 @@ class LoggingTransaction(object):
]
def __init__(
- self, txn, name, database_engine, after_callbacks, exception_callbacks
+ self, txn, name, database_engine, after_callbacks=None, exception_callbacks=None
):
object.__setattr__(self, "txn", txn)
object.__setattr__(self, "name", name)
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index eca77069fd..dcfb67e029 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -79,8 +79,6 @@ class EventPushActionsWorkerStore(SQLBaseStore):
db_conn.cursor(),
name="_find_stream_orderings_for_times_txn",
database_engine=self.database_engine,
- after_callbacks=[],
- exception_callbacks=[],
)
self._find_stream_orderings_for_times_txn(cur)
cur.close()
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index b486ca50eb..b70457bfc6 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -918,8 +918,6 @@ class EventsStore(
min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering
max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering
- self._update_current_state_txn(txn, state_delta_for_room, min_stream_order)
-
self._update_forward_extremities_txn(
txn,
new_forward_extremities=new_forward_extremeties,
@@ -993,6 +991,10 @@ class EventsStore(
backfilled=backfilled,
)
+ # We call this last as it assumes we've inserted the events into
+ # room_memberships, where applicable.
+ self._update_current_state_txn(txn, state_delta_for_room, min_stream_order)
+
def _update_current_state_txn(self, txn, state_delta_by_room, stream_id):
for room_id, current_state_tuple in iteritems(state_delta_by_room):
to_delete, to_insert = current_state_tuple
@@ -1062,16 +1064,16 @@ class EventsStore(
),
)
- self._simple_insert_many_txn(
- txn,
- table="current_state_events",
- values=[
- {
- "event_id": ev_id,
- "room_id": room_id,
- "type": key[0],
- "state_key": key[1],
- }
+ # We include the membership in the current state table, hence we do
+ # a lookup when we insert. This assumes that all events have already
+ # been inserted into room_memberships.
+ txn.executemany(
+ """INSERT INTO current_state_events
+ (room_id, type, state_key, event_id, membership)
+ VALUES (?, ?, ?, ?, (SELECT membership FROM room_memberships WHERE event_id = ?))
+ """,
+ [
+ (room_id, key[0], key[1], ev_id, ev_id)
for key, ev_id in iteritems(to_insert)
],
)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 7c4e1dc7ec..d20eacda59 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 55
+SCHEMA_VERSION = 56
dir_path = os.path.abspath(os.path.dirname(__file__))
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 32cfd010a5..257bcdb2f8 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -24,6 +24,8 @@ from canonicaljson import json
from twisted.internet import defer
from synapse.api.constants import EventTypes, Membership
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.storage._base import LoggingTransaction
from synapse.storage.events_worker import EventsWorkerStore
from synapse.types import get_domain_from_id
from synapse.util.async_helpers import Linearizer
@@ -53,9 +55,51 @@ ProfileInfo = namedtuple("ProfileInfo", ("avatar_url", "display_name"))
MemberSummary = namedtuple("MemberSummary", ("members", "count"))
_MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
+_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership"
class RoomMemberWorkerStore(EventsWorkerStore):
+ def __init__(self, db_conn, hs):
+ super(RoomMemberWorkerStore, self).__init__(db_conn, hs)
+
+ # Is the current_state_events.membership up to date? Or is the
+ # background update still running?
+ self._current_state_events_membership_up_to_date = False
+
+ txn = LoggingTransaction(
+ db_conn.cursor(),
+ name="_check_safe_current_state_events_membership_updated",
+ database_engine=self.database_engine,
+ )
+ self._check_safe_current_state_events_membership_updated_txn(txn)
+ txn.close()
+
+ def _check_safe_current_state_events_membership_updated_txn(self, txn):
+ """Checks if it is safe to assume the new current_state_events
+ membership column is up to date
+ """
+
+ pending_update = self._simple_select_one_txn(
+ txn,
+ table="background_updates",
+ keyvalues={"update_name": _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME},
+ retcols=["update_name"],
+ allow_none=True,
+ )
+
+ self._current_state_events_membership_up_to_date = not pending_update
+
+ # If the update is still running, reschedule to run.
+ if pending_update:
+ self._clock.call_later(
+ 15.0,
+ run_as_background_process,
+ "_check_safe_current_state_events_membership_updated",
+ self.runInteraction,
+ "_check_safe_current_state_events_membership_updated",
+ self._check_safe_current_state_events_membership_updated_txn,
+ )
+
@cachedInlineCallbacks(max_entries=100000, iterable=True, cache_context=True)
def get_hosts_in_room(self, room_id, cache_context):
"""Returns the set of all hosts currently in the room
@@ -69,14 +113,23 @@ class RoomMemberWorkerStore(EventsWorkerStore):
@cached(max_entries=100000, iterable=True)
def get_users_in_room(self, room_id):
def f(txn):
- sql = (
- "SELECT m.user_id FROM room_memberships as m"
- " INNER JOIN current_state_events as c"
- " ON m.event_id = c.event_id "
- " AND m.room_id = c.room_id "
- " AND m.user_id = c.state_key"
- " WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?"
- )
+ # If we can assume current_state_events.membership is up to date
+ # then we can avoid a join, which is a Very Good Thing given how
+ # frequently this function gets called.
+ if self._current_state_events_membership_up_to_date:
+ sql = """
+ SELECT state_key FROM current_state_events
+ WHERE type = 'm.room.member' AND room_id = ? AND membership = ?
+ """
+ else:
+ sql = """
+ SELECT state_key FROM room_memberships as m
+ INNER JOIN current_state_events as c
+ ON m.event_id = c.event_id
+ AND m.room_id = c.room_id
+ AND m.user_id = c.state_key
+ WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?
+ """
txn.execute(sql, (room_id, Membership.JOIN))
return [to_ascii(r[0]) for r in txn]
@@ -98,15 +151,26 @@ class RoomMemberWorkerStore(EventsWorkerStore):
# first get counts.
# We do this all in one transaction to keep the cache small.
# FIXME: get rid of this when we have room_stats
- sql = """
- SELECT count(*), m.membership FROM room_memberships as m
- INNER JOIN current_state_events as c
- ON m.event_id = c.event_id
- AND m.room_id = c.room_id
- AND m.user_id = c.state_key
- WHERE c.type = 'm.room.member' AND c.room_id = ?
- GROUP BY m.membership
- """
+
+ # If we can assume current_state_events.membership is up to date
+ # then we can avoid a join, which is a Very Good Thing given how
+ # frequently this function gets called.
+ if self._current_state_events_membership_up_to_date:
+ sql = """
+ SELECT count(*), membership FROM current_state_events
+ WHERE type = 'm.room.member' AND room_id = ?
+ GROUP BY membership
+ """
+ else:
+ sql = """
+ SELECT count(*), m.membership FROM room_memberships as m
+ INNER JOIN current_state_events as c
+ ON m.event_id = c.event_id
+ AND m.room_id = c.room_id
+ AND m.user_id = c.state_key
+ WHERE c.type = 'm.room.member' AND c.room_id = ?
+ GROUP BY m.membership
+ """
txn.execute(sql, (room_id,))
res = {}
@@ -224,7 +288,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
results = []
if membership_list:
where_clause = "user_id = ? AND (%s) AND forgotten = 0" % (
- " OR ".join(["membership = ?" for _ in membership_list]),
+ " OR ".join(["m.membership = ?" for _ in membership_list]),
)
args = [user_id]
@@ -453,8 +517,8 @@ class RoomMemberWorkerStore(EventsWorkerStore):
sql = """
SELECT state_key FROM current_state_events AS c
- INNER JOIN room_memberships USING (event_id)
- WHERE membership = 'join'
+ INNER JOIN room_memberships AS m USING (event_id)
+ WHERE m.membership = 'join'
AND type = 'm.room.member'
AND c.room_id = ?
AND state_key LIKE ?
@@ -602,6 +666,10 @@ class RoomMemberStore(RoomMemberWorkerStore):
self.register_background_update_handler(
_MEMBERSHIP_PROFILE_UPDATE_NAME, self._background_add_membership_profile
)
+ self.register_background_update_handler(
+ _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME,
+ self._background_current_state_membership,
+ )
def _store_room_members_txn(self, txn, events, backfilled):
"""Store a room member in the database.
@@ -781,6 +849,52 @@ class RoomMemberStore(RoomMemberWorkerStore):
defer.returnValue(result)
+ @defer.inlineCallbacks
+ def _background_current_state_membership(self, progress, batch_size):
+ """Update the new membership column on current_state_events.
+ """
+
+ if "rooms" not in progress:
+ rooms = yield self._simple_select_onecol(
+ table="current_state_events",
+ keyvalues={},
+ retcol="DISTINCT room_id",
+ desc="_background_current_state_membership_get_rooms",
+ )
+ progress["rooms"] = rooms
+
+ rooms = progress["rooms"]
+
+ def _background_current_state_membership_txn(txn):
+ processed = 0
+ while rooms and processed < batch_size:
+ sql = """
+ UPDATE current_state_events AS c
+ SET membership = (
+ SELECT membership FROM room_memberships
+ WHERE event_id = c.event_id
+ )
+ WHERE room_id = ?
+ """
+ txn.execute(sql, (rooms.pop(),))
+ processed += txn.rowcount
+
+ self._background_update_progress_txn(
+ txn, _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME, progress
+ )
+
+ return processed
+
+ result = yield self.runInteraction(
+ "_background_current_state_membership_update",
+ _background_current_state_membership_txn,
+ )
+
+ if not rooms:
+ yield self._end_background_update(_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME)
+
+ defer.returnValue(result)
+
class _JoinedHostsCache(object):
"""Cache for joined hosts in a room that is optimised to handle updates
diff --git a/synapse/storage/schema/delta/56/current_state_events_membership.sql b/synapse/storage/schema/delta/56/current_state_events_membership.sql
new file mode 100644
index 0000000000..b2e08cd85d
--- /dev/null
+++ b/synapse/storage/schema/delta/56/current_state_events_membership.sql
@@ -0,0 +1,25 @@
+/* Copyright 2019 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- We add membership to current state so that we don't need to join against
+-- room_memberships, which can be surprisingly costly (we do such queries
+-- very frequently).
+-- This will be null for non-membership events and the content.membership key
+-- for membership events. (Will also be null for membership events until the
+-- background update job has finished).
+ALTER TABLE current_state_events ADD membership TEXT;
+
+INSERT INTO background_updates (update_name, progress_json) VALUES
+ ('current_state_events_membership', '{}');
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 0bfe1b4550..a35289876d 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -510,6 +510,12 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
event ID.
"""
+ where_clause, where_args = state_filter.make_sql_filter_clause()
+
+ if not where_clause:
+ # We delegate to the cached version
+ return self.get_current_state_ids(room_id)
+
def _get_filtered_current_state_ids_txn(txn):
results = {}
sql = """
@@ -517,8 +523,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
WHERE room_id = ?
"""
- where_clause, where_args = state_filter.make_sql_filter_clause()
-
if where_clause:
sql += " AND (%s)" % (where_clause,)
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index fd18619178..c585cf6cf7 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -196,6 +196,26 @@ class TransactionStore(SQLBaseStore):
def _set_destination_retry_timings(
self, txn, destination, retry_last_ts, retry_interval
):
+
+ if self.database_engine.can_native_upsert:
+ # Upsert retry time interval if retry_interval is zero (i.e. we're
+ # resetting it) or greater than the existing retry interval.
+
+ sql = """
+ INSERT INTO destinations (destination, retry_last_ts, retry_interval)
+ VALUES (?, ?, ?)
+ ON CONFLICT (destination) DO UPDATE SET
+ retry_last_ts = EXCLUDED.retry_last_ts,
+ retry_interval = EXCLUDED.retry_interval
+ WHERE
+ EXCLUDED.retry_interval = 0
+ OR destinations.retry_interval < EXCLUDED.retry_interval
+ """
+
+ txn.execute(sql, (destination, retry_last_ts, retry_interval))
+
+ return
+
self.database_engine.lock_table(txn, "destinations")
# We need to be careful here as the data may have changed from under us
diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index 83466e25d9..7fd16fe65e 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -618,15 +618,15 @@ class UserDirectoryStore(StateDeltasStore, BackgroundUpdateStore):
sql = """
SELECT room_id FROM (
SELECT c.room_id FROM current_state_events AS c
- INNER JOIN room_memberships USING (event_id)
+ INNER JOIN room_memberships AS m USING (event_id)
WHERE type = 'm.room.member'
- AND membership = 'join'
+ AND m.membership = 'join'
AND state_key = ?
) AS f1 INNER JOIN (
SELECT c.room_id FROM current_state_events AS c
- INNER JOIN room_memberships USING (event_id)
+ INNER JOIN room_memberships AS m USING (event_id)
WHERE type = 'm.room.member'
- AND membership = 'join'
+ AND m.membership = 'join'
AND state_key = ?
) f2 USING (room_id)
"""
|
