diff options
70 files changed, 222 insertions, 161 deletions
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d5e5aeec6b..c8ae1a44be 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -220,8 +220,10 @@ steps: env: POSTGRES: "1" WORKERS: "1" + BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" + - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: @@ -229,7 +231,6 @@ steps: propagate-environment: true always-pull: true workdir: "/src" - soft_fail: true retry: automatic: - exit_status: -1 diff --git a/.buildkite/worker-blacklist b/.buildkite/worker-blacklist new file mode 100644 index 0000000000..8ed8eef1a3 --- /dev/null +++ b/.buildkite/worker-blacklist @@ -0,0 +1,34 @@ +# This file serves as a blacklist for SyTest tests that we expect will fail in +# Synapse when run under worker mode. For more details, see sytest-blacklist. + +Message history can be paginated + +m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users + +m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users + +Can re-join room if re-invited + +/upgrade creates a new room + +The only membership state included in an initial sync is for all the senders in the timeline + +Local device key changes get to remote servers + +If remote user leaves room we no longer receive device updates + +Forgotten room messages cannot be paginated + +Inbound federation can get public room list + +Members from the gap are included in gappy incr LL sync + +Leaves are present in non-gapped incremental syncs + +Old leaves are present in gapped incremental syncs + +User sees updates to presence from other users in the incremental sync. + +Gapped incremental syncs include all state changes + +Old members are included in gappy incr LL sync if they start speaking diff --git a/CHANGES.md b/CHANGES.md index dc8c74fe58..bb6bcb75ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,84 @@ +Synapse 1.2.0rc1 (2019-07-22) +============================= + +Features +-------- + +- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712)) +- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589)) +- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597)) +- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613)) +- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623)) +- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626)) +- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660)) +- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674)) +- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714)) + + +Bugfixes +-------- + +- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609)) +- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621)) +- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629)) +- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638)) +- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644)) +- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654)) +- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658)) +- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707)) +- Ignore redactions of m.room.create events. ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) + + +Updates to the Docker image +--------------------------- + +- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619)) +- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620)) + + +Improved Documentation +---------------------- + +- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397)) +- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651)) +- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661)) +- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675)) + + +Deprecations and Removals +------------------------- + +- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625)) + + +Internal Changes +---------------- + +- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617)) +- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611)) +- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616)) +- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622)) +- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627)) +- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628)) +- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630)) +- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636)) +- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637)) +- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639)) +- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640)) +- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641)) +- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642)) +- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643)) +- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645)) +- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655)) +- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656)) +- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657)) +- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659)) +- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664)) +- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673)) +- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689)) +- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703)) + + Synapse 1.1.0 (2019-07-04) ========================== diff --git a/MANIFEST.in b/MANIFEST.in index 834ddfad39..919cd8a1cd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,7 +7,6 @@ include demo/README include demo/demo.tls.dh include demo/*.py include demo/*.sh -include sytest-blacklist recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.sql.postgres @@ -34,6 +33,7 @@ exclude Dockerfile exclude .dockerignore exclude test_postgresql.sh exclude .editorconfig +exclude sytest-blacklist include pyproject.toml recursive-include changelog.d * diff --git a/changelog.d/5397.doc b/changelog.d/5397.doc deleted file mode 100644 index c2b500b482..0000000000 --- a/changelog.d/5397.doc +++ /dev/null @@ -1 +0,0 @@ -Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! diff --git a/changelog.d/5544.feature b/changelog.d/5544.feature deleted file mode 100644 index 7d3459129d..0000000000 --- a/changelog.d/5544.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add support for opentracing. - diff --git a/changelog.d/5589.feature b/changelog.d/5589.feature deleted file mode 100644 index a87e669dd4..0000000000 --- a/changelog.d/5589.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to pull all locally stored events out of synapse that a particular user can see. diff --git a/changelog.d/5597.feature b/changelog.d/5597.feature deleted file mode 100644 index 6f92748885..0000000000 --- a/changelog.d/5597.feature +++ /dev/null @@ -1 +0,0 @@ -Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. diff --git a/changelog.d/5606.misc b/changelog.d/5606.misc deleted file mode 100644 index bb3c028167..0000000000 --- a/changelog.d/5606.misc +++ /dev/null @@ -1 +0,0 @@ -Move logging code out of `synapse.util` and into `synapse.logging`. diff --git a/changelog.d/5609.bugfix b/changelog.d/5609.bugfix deleted file mode 100644 index 534ee22a1b..0000000000 --- a/changelog.d/5609.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 'utime went backwards' errors on daemonization. diff --git a/changelog.d/5611.misc b/changelog.d/5611.misc deleted file mode 100644 index a2d1695139..0000000000 --- a/changelog.d/5611.misc +++ /dev/null @@ -1 +0,0 @@ -Add a blacklist file to the repo to blacklist certain sytests from failing CI. diff --git a/changelog.d/5613.feature b/changelog.d/5613.feature deleted file mode 100644 index 4b7bb2745c..0000000000 --- a/changelog.d/5613.feature +++ /dev/null @@ -1 +0,0 @@ -Add `sender` and `origin_server_ts` fields to `m.replace`. diff --git a/changelog.d/5616.misc b/changelog.d/5616.misc deleted file mode 100644 index 9f94be6778..0000000000 --- a/changelog.d/5616.misc +++ /dev/null @@ -1 +0,0 @@ -Make runtime errors surrounding password reset emails much clearer. diff --git a/changelog.d/5617.misc b/changelog.d/5617.misc deleted file mode 100644 index bb3c028167..0000000000 --- a/changelog.d/5617.misc +++ /dev/null @@ -1 +0,0 @@ -Move logging code out of `synapse.util` and into `synapse.logging`. diff --git a/changelog.d/5619.docker b/changelog.d/5619.docker deleted file mode 100644 index b69e5cc57c..0000000000 --- a/changelog.d/5619.docker +++ /dev/null @@ -1 +0,0 @@ -Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). diff --git a/changelog.d/5620.docker b/changelog.d/5620.docker deleted file mode 100644 index cbb5a75d6a..0000000000 --- a/changelog.d/5620.docker +++ /dev/null @@ -1 +0,0 @@ -Add missing space in default logging file format generated by the Docker image. diff --git a/changelog.d/5621.bugfix b/changelog.d/5621.bugfix deleted file mode 100644 index f1a2851f45..0000000000 --- a/changelog.d/5621.bugfix +++ /dev/null @@ -1 +0,0 @@ -Various minor fixes to the federation request rate limiter. diff --git a/changelog.d/5622.misc b/changelog.d/5622.misc deleted file mode 100644 index 9f0a87311c..0000000000 --- a/changelog.d/5622.misc +++ /dev/null @@ -1 +0,0 @@ -Remove dead code for persiting outgoing federation transactions. diff --git a/changelog.d/5623.feature b/changelog.d/5623.feature deleted file mode 100644 index b73080e88d..0000000000 --- a/changelog.d/5623.feature +++ /dev/null @@ -1 +0,0 @@ -Add default push rule to ignore reactions. diff --git a/changelog.d/5625.removal b/changelog.d/5625.removal deleted file mode 100644 index d33a778d69..0000000000 --- a/changelog.d/5625.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for the `invite_3pid_guest` configuration setting. diff --git a/changelog.d/5626.feature b/changelog.d/5626.feature deleted file mode 100644 index 5ef793b943..0000000000 --- a/changelog.d/5626.feature +++ /dev/null @@ -1 +0,0 @@ -Include the original event when asking for its relations. diff --git a/changelog.d/5627.misc b/changelog.d/5627.misc deleted file mode 100644 index 730721b5ef..0000000000 --- a/changelog.d/5627.misc +++ /dev/null @@ -1 +0,0 @@ -Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. diff --git a/changelog.d/5628.misc b/changelog.d/5628.misc deleted file mode 100644 index fec8446793..0000000000 --- a/changelog.d/5628.misc +++ /dev/null @@ -1 +0,0 @@ -Move RegistrationHandler.get_or_create_user to test code. diff --git a/changelog.d/5629.bugfix b/changelog.d/5629.bugfix deleted file mode 100644 index 672eabad40..0000000000 --- a/changelog.d/5629.bugfix +++ /dev/null @@ -1 +0,0 @@ -Forbid viewing relations on an event once it has been redacted. diff --git a/changelog.d/5630.misc b/changelog.d/5630.misc deleted file mode 100644 index f112d873eb..0000000000 --- a/changelog.d/5630.misc +++ /dev/null @@ -1 +0,0 @@ -Add some more common python virtual-environment paths to the black exclusion list. diff --git a/changelog.d/5636.misc b/changelog.d/5636.misc deleted file mode 100644 index 3add990283..0000000000 --- a/changelog.d/5636.misc +++ /dev/null @@ -1 +0,0 @@ -Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. \ No newline at end of file diff --git a/changelog.d/5637.misc b/changelog.d/5637.misc deleted file mode 100644 index f18d6197e5..0000000000 --- a/changelog.d/5637.misc +++ /dev/null @@ -1 +0,0 @@ -Unblacklist some user_directory sytests. diff --git a/changelog.d/5638.bugfix b/changelog.d/5638.bugfix deleted file mode 100644 index 66781ad9e6..0000000000 --- a/changelog.d/5638.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. diff --git a/changelog.d/5639.misc b/changelog.d/5639.misc deleted file mode 100644 index 413b13128c..0000000000 --- a/changelog.d/5639.misc +++ /dev/null @@ -1 +0,0 @@ -Factor out some redundant code in the login implementation. diff --git a/changelog.d/5640.misc b/changelog.d/5640.misc deleted file mode 100644 index 7d69a1b3b6..0000000000 --- a/changelog.d/5640.misc +++ /dev/null @@ -1 +0,0 @@ -Update ModuleApi to avoid register(generate_token=True). diff --git a/changelog.d/5641.misc b/changelog.d/5641.misc deleted file mode 100644 index 1899bc963d..0000000000 --- a/changelog.d/5641.misc +++ /dev/null @@ -1 +0,0 @@ -Remove access-token support from RegistrationHandler.register, and rename it. diff --git a/changelog.d/5642.misc b/changelog.d/5642.misc deleted file mode 100644 index e7f8e214a4..0000000000 --- a/changelog.d/5642.misc +++ /dev/null @@ -1 +0,0 @@ -Remove access-token support from `RegistrationStore.register`, and rename it. diff --git a/changelog.d/5643.misc b/changelog.d/5643.misc deleted file mode 100644 index 2b2316469e..0000000000 --- a/changelog.d/5643.misc +++ /dev/null @@ -1 +0,0 @@ -Improve logging for auto-join when a new user is created. diff --git a/changelog.d/5644.bugfix b/changelog.d/5644.bugfix deleted file mode 100644 index f6302fd08d..0000000000 --- a/changelog.d/5644.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix newly-registered users not being able to lookup their own profile without joining a room. diff --git a/changelog.d/5645.misc b/changelog.d/5645.misc deleted file mode 100644 index 4fa9699e4f..0000000000 --- a/changelog.d/5645.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. \ No newline at end of file diff --git a/changelog.d/5651.doc b/changelog.d/5651.doc deleted file mode 100644 index e2d5a8dc8a..0000000000 --- a/changelog.d/5651.doc +++ /dev/null @@ -1 +0,0 @@ ---no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. diff --git a/changelog.d/5654.bugfix b/changelog.d/5654.bugfix deleted file mode 100644 index 5f76b041cd..0000000000 --- a/changelog.d/5654.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. \ No newline at end of file diff --git a/changelog.d/5655.misc b/changelog.d/5655.misc deleted file mode 100644 index acab6aee92..0000000000 --- a/changelog.d/5655.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a small typo in a code comment. \ No newline at end of file diff --git a/changelog.d/5656.misc b/changelog.d/5656.misc deleted file mode 100644 index a8de20a7d0..0000000000 --- a/changelog.d/5656.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up exception handling around client access tokens. diff --git a/changelog.d/5657.misc b/changelog.d/5657.misc deleted file mode 100644 index bdec9ae4c0..0000000000 --- a/changelog.d/5657.misc +++ /dev/null @@ -1 +0,0 @@ -Add a mechanism for per-test homeserver configuration in the unit tests. diff --git a/changelog.d/5658.bugfix b/changelog.d/5658.bugfix deleted file mode 100644 index f6ae906a9a..0000000000 --- a/changelog.d/5658.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. diff --git a/changelog.d/5659.misc b/changelog.d/5659.misc deleted file mode 100644 index 686001295c..0000000000 --- a/changelog.d/5659.misc +++ /dev/null @@ -1 +0,0 @@ -Inline issue_access_token. diff --git a/changelog.d/5660.feature b/changelog.d/5660.feature deleted file mode 100644 index 82889fdaf1..0000000000 --- a/changelog.d/5660.feature +++ /dev/null @@ -1 +0,0 @@ -Implement `session_lifetime` configuration option, after which access tokens will expire. diff --git a/changelog.d/5661.doc b/changelog.d/5661.doc deleted file mode 100644 index c70e62014e..0000000000 --- a/changelog.d/5661.doc +++ /dev/null @@ -1 +0,0 @@ -Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! diff --git a/changelog.d/5664.misc b/changelog.d/5664.misc deleted file mode 100644 index 0ca7a0fbd0..0000000000 --- a/changelog.d/5664.misc +++ /dev/null @@ -1 +0,0 @@ -Update the sytest BuildKite configuration to checkout Synapse in `/src`. diff --git a/changelog.d/5673.misc b/changelog.d/5673.misc deleted file mode 100644 index 1942256358..0000000000 --- a/changelog.d/5673.misc +++ /dev/null @@ -1 +0,0 @@ -Add a `docker` type to the towncrier configuration. diff --git a/changelog.d/5674.feature b/changelog.d/5674.feature deleted file mode 100644 index 04bdfa4ad5..0000000000 --- a/changelog.d/5674.feature +++ /dev/null @@ -1 +0,0 @@ -Return "This account has been deactivated" when a deactivated user tries to login. diff --git a/changelog.d/5675.doc b/changelog.d/5675.doc deleted file mode 100644 index 4cd4d0be1a..0000000000 --- a/changelog.d/5675.doc +++ /dev/null @@ -1 +0,0 @@ -Minor tweaks to postgres documentation. diff --git a/changelog.d/5689.misc b/changelog.d/5689.misc deleted file mode 100644 index 8aa3e3f6a2..0000000000 --- a/changelog.d/5689.misc +++ /dev/null @@ -1 +0,0 @@ -Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. \ No newline at end of file diff --git a/changelog.d/5699.bugfix b/changelog.d/5699.bugfix deleted file mode 100644 index 30d5e67f67..0000000000 --- a/changelog.d/5699.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. \ No newline at end of file diff --git a/changelog.d/5700.bugfix b/changelog.d/5700.bugfix deleted file mode 100644 index 51bce8d441..0000000000 --- a/changelog.d/5700.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. - diff --git a/changelog.d/5701.bugfix b/changelog.d/5701.bugfix deleted file mode 100644 index fd2866e16a..0000000000 --- a/changelog.d/5701.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ignore redactions of m.room.create events. diff --git a/changelog.d/5703.misc b/changelog.d/5703.misc deleted file mode 100644 index 6e9b2d734e..0000000000 --- a/changelog.d/5703.misc +++ /dev/null @@ -1 +0,0 @@ -Documentation for opentracing. diff --git a/changelog.d/5707.bugfix b/changelog.d/5707.bugfix deleted file mode 100644 index aa3046c5e1..0000000000 --- a/changelog.d/5707.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. diff --git a/changelog.d/5712.feature b/changelog.d/5712.feature deleted file mode 100644 index 7d3459129d..0000000000 --- a/changelog.d/5712.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add support for opentracing. - diff --git a/changelog.d/5714.feature b/changelog.d/5714.feature deleted file mode 100644 index 2fd32e5e38..0000000000 --- a/changelog.d/5714.feature +++ /dev/null @@ -1 +0,0 @@ -Enable aggregations support by default diff --git a/changelog.d/5724.bugfix b/changelog.d/5724.bugfix new file mode 100644 index 0000000000..1b3683daf6 --- /dev/null +++ b/changelog.d/5724.bugfix @@ -0,0 +1 @@ +Fix stack overflow in server key lookup code. \ No newline at end of file diff --git a/changelog.d/5729.removal b/changelog.d/5729.removal new file mode 100644 index 0000000000..3af5198e6b --- /dev/null +++ b/changelog.d/5729.removal @@ -0,0 +1 @@ + Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. diff --git a/changelog.d/5730.misc b/changelog.d/5730.misc new file mode 100644 index 0000000000..a99677f5e7 --- /dev/null +++ b/changelog.d/5730.misc @@ -0,0 +1 @@ +Cache result of get_version_string to reduce overhead of `/version` federation requests. diff --git a/changelog.d/5731.misc b/changelog.d/5731.misc new file mode 100644 index 0000000000..dffae5d874 --- /dev/null +++ b/changelog.d/5731.misc @@ -0,0 +1 @@ +Return 'user_type' in admin API user endpoints results. diff --git a/changelog.d/5733.misc b/changelog.d/5733.misc new file mode 100644 index 0000000000..a2a8c26383 --- /dev/null +++ b/changelog.d/5733.misc @@ -0,0 +1 @@ +Don't package the sytest test blacklist file. diff --git a/changelog.d/5738.misc b/changelog.d/5738.misc new file mode 100644 index 0000000000..5e15dfd5fa --- /dev/null +++ b/changelog.d/5738.misc @@ -0,0 +1 @@ +Reduce database IO usage by optimising queries for current membership. diff --git a/changelog.d/5740.misc b/changelog.d/5740.misc new file mode 100644 index 0000000000..97a476bef5 --- /dev/null +++ b/changelog.d/5740.misc @@ -0,0 +1 @@ +Blacklist some flakey tests in worker mode. diff --git a/synapse/__init__.py b/synapse/__init__.py index cf22fabd61..f26e49da36 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -35,4 +35,4 @@ try: except ImportError: pass -__version__ = "1.1.0" +__version__ = "1.2.0rc1" diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 246d72cd61..bc0fc165e3 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -31,6 +31,7 @@ class WorkerConfig(Config): self.worker_listeners = config.get("worker_listeners", []) self.worker_daemonize = config.get("worker_daemonize") self.worker_pid_file = config.get("worker_pid_file") + self.worker_log_config = config.get("worker_log_config") # The host used to connect to the main synapse self.worker_replication_host = config.get("worker_replication_host", None) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 341c863152..e8bb420ad1 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -238,27 +238,9 @@ class Keyring(object): """ try: - # create a deferred for each server we're going to look up the keys - # for; we'll resolve them once we have completed our lookups. - # These will be passed into wait_for_previous_lookups to block - # any other lookups until we have finished. - # The deferreds are called with no logcontext. - server_to_deferred = { - rq.server_name: defer.Deferred() for rq in verify_requests - } - - # We want to wait for any previous lookups to complete before - # proceeding. - yield self.wait_for_previous_lookups(server_to_deferred) + ctx = LoggingContext.current_context() - # Actually start fetching keys. - self._get_server_verify_keys(verify_requests) - - # When we've finished fetching all the keys for a given server_name, - # resolve the deferred passed to `wait_for_previous_lookups` so that - # any lookups waiting will proceed. - # - # map from server name to a set of request ids + # map from server name to a set of outstanding request ids server_to_request_ids = {} for verify_request in verify_requests: @@ -266,40 +248,61 @@ class Keyring(object): request_id = id(verify_request) server_to_request_ids.setdefault(server_name, set()).add(request_id) - def remove_deferreds(res, verify_request): + # Wait for any previous lookups to complete before proceeding. + yield self.wait_for_previous_lookups(server_to_request_ids.keys()) + + # take out a lock on each of the servers by sticking a Deferred in + # key_downloads + for server_name in server_to_request_ids.keys(): + self.key_downloads[server_name] = defer.Deferred() + logger.debug("Got key lookup lock on %s", server_name) + + # When we've finished fetching all the keys for a given server_name, + # drop the lock by resolving the deferred in key_downloads. + def drop_server_lock(server_name): + d = self.key_downloads.pop(server_name) + d.callback(None) + + def lookup_done(res, verify_request): server_name = verify_request.server_name - request_id = id(verify_request) - server_to_request_ids[server_name].discard(request_id) - if not server_to_request_ids[server_name]: - d = server_to_deferred.pop(server_name, None) - if d: - d.callback(None) + server_requests = server_to_request_ids[server_name] + server_requests.remove(id(verify_request)) + + # if there are no more requests for this server, we can drop the lock. + if not server_requests: + with PreserveLoggingContext(ctx): + logger.debug("Releasing key lookup lock on %s", server_name) + + # ... but not immediately, as that can cause stack explosions if + # we get a long queue of lookups. + self.clock.call_later(0, drop_server_lock, server_name) + return res for verify_request in verify_requests: - verify_request.key_ready.addBoth(remove_deferreds, verify_request) + verify_request.key_ready.addBoth(lookup_done, verify_request) + + # Actually start fetching keys. + self._get_server_verify_keys(verify_requests) except Exception: logger.exception("Error starting key lookups") @defer.inlineCallbacks - def wait_for_previous_lookups(self, server_to_deferred): + def wait_for_previous_lookups(self, server_names): """Waits for any previous key lookups for the given servers to finish. Args: - server_to_deferred (dict[str, Deferred]): server_name to deferred which gets - resolved once we've finished looking up keys for that server. - The Deferreds should be regular twisted ones which call their - callbacks with no logcontext. - - Returns: a Deferred which resolves once all key lookups for the given - servers have completed. Follows the synapse rules of logcontext - preservation. + server_names (Iterable[str]): list of servers which we want to look up + + Returns: + Deferred[None]: resolves once all key lookups for the given servers have + completed. Follows the synapse rules of logcontext preservation. """ loop_count = 1 while True: wait_on = [ (server_name, self.key_downloads[server_name]) - for server_name in server_to_deferred.keys() + for server_name in server_names if server_name in self.key_downloads ] if not wait_on: @@ -314,19 +317,6 @@ class Keyring(object): loop_count += 1 - ctx = LoggingContext.current_context() - - def rm(r, server_name_): - with PreserveLoggingContext(ctx): - logger.debug("Releasing key lookup lock on %s", server_name_) - self.key_downloads.pop(server_name_, None) - return r - - for server_name, deferred in server_to_deferred.items(): - logger.debug("Got key lookup lock on %s", server_name) - self.key_downloads[server_name] = deferred - deferred.addBoth(rm, server_name) - def _get_server_verify_keys(self, verify_requests): """Tries to find at least one key for each verify request diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6b0ca80087..86a333a919 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -469,7 +469,7 @@ class DataStore( return self._simple_select_list( table="users", keyvalues={}, - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], desc="get_users", ) @@ -494,7 +494,7 @@ class DataStore( orderby=order, start=start, limit=limit, - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], ) count = yield self.runInteraction("get_users_paginate", self.get_user_count_txn) retval = {"users": users, "total": count} @@ -514,7 +514,7 @@ class DataStore( table="users", term=term, col="name", - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], desc="search_users", ) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 257bcdb2f8..b3c002b9eb 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -852,22 +852,25 @@ class RoomMemberStore(RoomMemberWorkerStore): @defer.inlineCallbacks def _background_current_state_membership(self, progress, batch_size): """Update the new membership column on current_state_events. + + This works by iterating over all rooms in alphebetical order. """ - if "rooms" not in progress: - rooms = yield self._simple_select_onecol( - table="current_state_events", - keyvalues={}, - retcol="DISTINCT room_id", - desc="_background_current_state_membership_get_rooms", - ) - progress["rooms"] = rooms + def _background_current_state_membership_txn(txn, last_processed_room): + processed = 0 + while processed < batch_size: + txn.execute( + """ + SELECT MIN(room_id) FROM rooms WHERE room_id > ? + """, + (last_processed_room,), + ) + row = txn.fetchone() + if not row or not row[0]: + return processed, True - rooms = progress["rooms"] + next_room, = row - def _background_current_state_membership_txn(txn): - processed = 0 - while rooms and processed < batch_size: sql = """ UPDATE current_state_events AS c SET membership = ( @@ -876,24 +879,33 @@ class RoomMemberStore(RoomMemberWorkerStore): ) WHERE room_id = ? """ - txn.execute(sql, (rooms.pop(),)) + txn.execute(sql, (next_room,)) processed += txn.rowcount + last_processed_room = next_room + self._background_update_progress_txn( - txn, _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME, progress + txn, + _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME, + {"last_processed_room": last_processed_room}, ) - return processed + return processed, False - result = yield self.runInteraction( + # If we haven't got a last processed room then just use the empty + # string, which will compare before all room IDs correctly. + last_processed_room = progress.get("last_processed_room", "") + + row_count, finished = yield self.runInteraction( "_background_current_state_membership_update", _background_current_state_membership_txn, + last_processed_room, ) - if not rooms: + if finished: yield self._end_background_update(_CURRENT_STATE_MEMBERSHIP_UPDATE_NAME) - defer.returnValue(result) + defer.returnValue(row_count) class _JoinedHostsCache(object): diff --git a/synapse/util/versionstring.py b/synapse/util/versionstring.py index a4d9a462f7..fa404b9d75 100644 --- a/synapse/util/versionstring.py +++ b/synapse/util/versionstring.py @@ -22,6 +22,23 @@ logger = logging.getLogger(__name__) def get_version_string(module): + """Given a module calculate a git-aware version string for it. + + If called on a module not in a git checkout will return `__verison__`. + + Args: + module (module) + + Returns: + str + """ + + cached_version = getattr(module, "_synapse_version_string_cache", None) + if cached_version: + return cached_version + + version_string = module.__version__ + try: null = open(os.devnull, "w") cwd = os.path.dirname(os.path.abspath(module.__file__)) @@ -80,8 +97,10 @@ def get_version_string(module): s for s in (git_branch, git_tag, git_commit, git_dirty) if s ) - return "%s (%s)" % (module.__version__, git_version) + version_string = "%s (%s)" % (module.__version__, git_version) except Exception as e: logger.info("Failed to check for git repository: %s", e) - return module.__version__ + module._synapse_version_string_cache = version_string + + return version_string diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 795703967d..8d94a503d6 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -86,35 +86,6 @@ class KeyringTestCase(unittest.HomeserverTestCase): getattr(LoggingContext.current_context(), "request", None), expected ) - def test_wait_for_previous_lookups(self): - kr = keyring.Keyring(self.hs) - - lookup_1_deferred = defer.Deferred() - lookup_2_deferred = defer.Deferred() - - # we run the lookup in a logcontext so that the patched inlineCallbacks can check - # it is doing the right thing with logcontexts. - wait_1_deferred = run_in_context( - kr.wait_for_previous_lookups, {"server1": lookup_1_deferred} - ) - - # there were no previous lookups, so the deferred should be ready - self.successResultOf(wait_1_deferred) - - # set off another wait. It should block because the first lookup - # hasn't yet completed. - wait_2_deferred = run_in_context( - kr.wait_for_previous_lookups, {"server1": lookup_2_deferred} - ) - - self.assertFalse(wait_2_deferred.called) - - # let the first lookup complete (in the sentinel context) - lookup_1_deferred.callback(None) - - # now the second wait should complete. - self.successResultOf(wait_2_deferred) - def test_verify_json_objects_for_server_awaits_previous_requests(self): key1 = signedjson.key.generate_signing_key(1) |