diff options
59 files changed, 128 insertions, 139 deletions
diff --git a/CHANGES.md b/CHANGES.md index dc8c74fe58..bb6bcb75ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,84 @@ +Synapse 1.2.0rc1 (2019-07-22) +============================= + +Features +-------- + +- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712)) +- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589)) +- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597)) +- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613)) +- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623)) +- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626)) +- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660)) +- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674)) +- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714)) + + +Bugfixes +-------- + +- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609)) +- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621)) +- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629)) +- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638)) +- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644)) +- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654)) +- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658)) +- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707)) +- Ignore redactions of m.room.create events. ([\#5701](https://github.com/matrix-org/synapse/issues/5701)) + + +Updates to the Docker image +--------------------------- + +- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619)) +- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620)) + + +Improved Documentation +---------------------- + +- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397)) +- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651)) +- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661)) +- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675)) + + +Deprecations and Removals +------------------------- + +- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625)) + + +Internal Changes +---------------- + +- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617)) +- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611)) +- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616)) +- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622)) +- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627)) +- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628)) +- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630)) +- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636)) +- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637)) +- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639)) +- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640)) +- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641)) +- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642)) +- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643)) +- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645)) +- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655)) +- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656)) +- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657)) +- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659)) +- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664)) +- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673)) +- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689)) +- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703)) + + Synapse 1.1.0 (2019-07-04) ========================== diff --git a/changelog.d/5397.doc b/changelog.d/5397.doc deleted file mode 100644 index c2b500b482..0000000000 --- a/changelog.d/5397.doc +++ /dev/null @@ -1 +0,0 @@ -Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! diff --git a/changelog.d/5544.feature b/changelog.d/5544.feature deleted file mode 100644 index 7d3459129d..0000000000 --- a/changelog.d/5544.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add support for opentracing. - diff --git a/changelog.d/5589.feature b/changelog.d/5589.feature deleted file mode 100644 index a87e669dd4..0000000000 --- a/changelog.d/5589.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to pull all locally stored events out of synapse that a particular user can see. diff --git a/changelog.d/5597.feature b/changelog.d/5597.feature deleted file mode 100644 index 6f92748885..0000000000 --- a/changelog.d/5597.feature +++ /dev/null @@ -1 +0,0 @@ -Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. diff --git a/changelog.d/5606.misc b/changelog.d/5606.misc deleted file mode 100644 index bb3c028167..0000000000 --- a/changelog.d/5606.misc +++ /dev/null @@ -1 +0,0 @@ -Move logging code out of `synapse.util` and into `synapse.logging`. diff --git a/changelog.d/5609.bugfix b/changelog.d/5609.bugfix deleted file mode 100644 index 534ee22a1b..0000000000 --- a/changelog.d/5609.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 'utime went backwards' errors on daemonization. diff --git a/changelog.d/5611.misc b/changelog.d/5611.misc deleted file mode 100644 index a2d1695139..0000000000 --- a/changelog.d/5611.misc +++ /dev/null @@ -1 +0,0 @@ -Add a blacklist file to the repo to blacklist certain sytests from failing CI. diff --git a/changelog.d/5613.feature b/changelog.d/5613.feature deleted file mode 100644 index 4b7bb2745c..0000000000 --- a/changelog.d/5613.feature +++ /dev/null @@ -1 +0,0 @@ -Add `sender` and `origin_server_ts` fields to `m.replace`. diff --git a/changelog.d/5616.misc b/changelog.d/5616.misc deleted file mode 100644 index 9f94be6778..0000000000 --- a/changelog.d/5616.misc +++ /dev/null @@ -1 +0,0 @@ -Make runtime errors surrounding password reset emails much clearer. diff --git a/changelog.d/5617.misc b/changelog.d/5617.misc deleted file mode 100644 index bb3c028167..0000000000 --- a/changelog.d/5617.misc +++ /dev/null @@ -1 +0,0 @@ -Move logging code out of `synapse.util` and into `synapse.logging`. diff --git a/changelog.d/5619.docker b/changelog.d/5619.docker deleted file mode 100644 index b69e5cc57c..0000000000 --- a/changelog.d/5619.docker +++ /dev/null @@ -1 +0,0 @@ -Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). diff --git a/changelog.d/5620.docker b/changelog.d/5620.docker deleted file mode 100644 index cbb5a75d6a..0000000000 --- a/changelog.d/5620.docker +++ /dev/null @@ -1 +0,0 @@ -Add missing space in default logging file format generated by the Docker image. diff --git a/changelog.d/5621.bugfix b/changelog.d/5621.bugfix deleted file mode 100644 index f1a2851f45..0000000000 --- a/changelog.d/5621.bugfix +++ /dev/null @@ -1 +0,0 @@ -Various minor fixes to the federation request rate limiter. diff --git a/changelog.d/5622.misc b/changelog.d/5622.misc deleted file mode 100644 index 9f0a87311c..0000000000 --- a/changelog.d/5622.misc +++ /dev/null @@ -1 +0,0 @@ -Remove dead code for persiting outgoing federation transactions. diff --git a/changelog.d/5623.feature b/changelog.d/5623.feature deleted file mode 100644 index b73080e88d..0000000000 --- a/changelog.d/5623.feature +++ /dev/null @@ -1 +0,0 @@ -Add default push rule to ignore reactions. diff --git a/changelog.d/5625.removal b/changelog.d/5625.removal deleted file mode 100644 index d33a778d69..0000000000 --- a/changelog.d/5625.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for the `invite_3pid_guest` configuration setting. diff --git a/changelog.d/5626.feature b/changelog.d/5626.feature deleted file mode 100644 index 5ef793b943..0000000000 --- a/changelog.d/5626.feature +++ /dev/null @@ -1 +0,0 @@ -Include the original event when asking for its relations. diff --git a/changelog.d/5627.misc b/changelog.d/5627.misc deleted file mode 100644 index 730721b5ef..0000000000 --- a/changelog.d/5627.misc +++ /dev/null @@ -1 +0,0 @@ -Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. diff --git a/changelog.d/5628.misc b/changelog.d/5628.misc deleted file mode 100644 index fec8446793..0000000000 --- a/changelog.d/5628.misc +++ /dev/null @@ -1 +0,0 @@ -Move RegistrationHandler.get_or_create_user to test code. diff --git a/changelog.d/5629.bugfix b/changelog.d/5629.bugfix deleted file mode 100644 index 672eabad40..0000000000 --- a/changelog.d/5629.bugfix +++ /dev/null @@ -1 +0,0 @@ -Forbid viewing relations on an event once it has been redacted. diff --git a/changelog.d/5630.misc b/changelog.d/5630.misc deleted file mode 100644 index f112d873eb..0000000000 --- a/changelog.d/5630.misc +++ /dev/null @@ -1 +0,0 @@ -Add some more common python virtual-environment paths to the black exclusion list. diff --git a/changelog.d/5636.misc b/changelog.d/5636.misc deleted file mode 100644 index 3add990283..0000000000 --- a/changelog.d/5636.misc +++ /dev/null @@ -1 +0,0 @@ -Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. \ No newline at end of file diff --git a/changelog.d/5637.misc b/changelog.d/5637.misc deleted file mode 100644 index f18d6197e5..0000000000 --- a/changelog.d/5637.misc +++ /dev/null @@ -1 +0,0 @@ -Unblacklist some user_directory sytests. diff --git a/changelog.d/5638.bugfix b/changelog.d/5638.bugfix deleted file mode 100644 index 66781ad9e6..0000000000 --- a/changelog.d/5638.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. diff --git a/changelog.d/5639.misc b/changelog.d/5639.misc deleted file mode 100644 index 413b13128c..0000000000 --- a/changelog.d/5639.misc +++ /dev/null @@ -1 +0,0 @@ -Factor out some redundant code in the login implementation. diff --git a/changelog.d/5640.misc b/changelog.d/5640.misc deleted file mode 100644 index 7d69a1b3b6..0000000000 --- a/changelog.d/5640.misc +++ /dev/null @@ -1 +0,0 @@ -Update ModuleApi to avoid register(generate_token=True). diff --git a/changelog.d/5641.misc b/changelog.d/5641.misc deleted file mode 100644 index 1899bc963d..0000000000 --- a/changelog.d/5641.misc +++ /dev/null @@ -1 +0,0 @@ -Remove access-token support from RegistrationHandler.register, and rename it. diff --git a/changelog.d/5642.misc b/changelog.d/5642.misc deleted file mode 100644 index e7f8e214a4..0000000000 --- a/changelog.d/5642.misc +++ /dev/null @@ -1 +0,0 @@ -Remove access-token support from `RegistrationStore.register`, and rename it. diff --git a/changelog.d/5643.misc b/changelog.d/5643.misc deleted file mode 100644 index 2b2316469e..0000000000 --- a/changelog.d/5643.misc +++ /dev/null @@ -1 +0,0 @@ -Improve logging for auto-join when a new user is created. diff --git a/changelog.d/5644.bugfix b/changelog.d/5644.bugfix deleted file mode 100644 index f6302fd08d..0000000000 --- a/changelog.d/5644.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix newly-registered users not being able to lookup their own profile without joining a room. diff --git a/changelog.d/5645.misc b/changelog.d/5645.misc deleted file mode 100644 index 4fa9699e4f..0000000000 --- a/changelog.d/5645.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. \ No newline at end of file diff --git a/changelog.d/5651.doc b/changelog.d/5651.doc deleted file mode 100644 index e2d5a8dc8a..0000000000 --- a/changelog.d/5651.doc +++ /dev/null @@ -1 +0,0 @@ ---no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. diff --git a/changelog.d/5654.bugfix b/changelog.d/5654.bugfix deleted file mode 100644 index 5f76b041cd..0000000000 --- a/changelog.d/5654.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. \ No newline at end of file diff --git a/changelog.d/5655.misc b/changelog.d/5655.misc deleted file mode 100644 index acab6aee92..0000000000 --- a/changelog.d/5655.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a small typo in a code comment. \ No newline at end of file diff --git a/changelog.d/5656.misc b/changelog.d/5656.misc deleted file mode 100644 index a8de20a7d0..0000000000 --- a/changelog.d/5656.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up exception handling around client access tokens. diff --git a/changelog.d/5657.misc b/changelog.d/5657.misc deleted file mode 100644 index bdec9ae4c0..0000000000 --- a/changelog.d/5657.misc +++ /dev/null @@ -1 +0,0 @@ -Add a mechanism for per-test homeserver configuration in the unit tests. diff --git a/changelog.d/5658.bugfix b/changelog.d/5658.bugfix deleted file mode 100644 index f6ae906a9a..0000000000 --- a/changelog.d/5658.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. diff --git a/changelog.d/5659.misc b/changelog.d/5659.misc deleted file mode 100644 index 686001295c..0000000000 --- a/changelog.d/5659.misc +++ /dev/null @@ -1 +0,0 @@ -Inline issue_access_token. diff --git a/changelog.d/5660.feature b/changelog.d/5660.feature deleted file mode 100644 index 82889fdaf1..0000000000 --- a/changelog.d/5660.feature +++ /dev/null @@ -1 +0,0 @@ -Implement `session_lifetime` configuration option, after which access tokens will expire. diff --git a/changelog.d/5661.doc b/changelog.d/5661.doc deleted file mode 100644 index c70e62014e..0000000000 --- a/changelog.d/5661.doc +++ /dev/null @@ -1 +0,0 @@ -Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! diff --git a/changelog.d/5664.misc b/changelog.d/5664.misc deleted file mode 100644 index 0ca7a0fbd0..0000000000 --- a/changelog.d/5664.misc +++ /dev/null @@ -1 +0,0 @@ -Update the sytest BuildKite configuration to checkout Synapse in `/src`. diff --git a/changelog.d/5673.misc b/changelog.d/5673.misc deleted file mode 100644 index 1942256358..0000000000 --- a/changelog.d/5673.misc +++ /dev/null @@ -1 +0,0 @@ -Add a `docker` type to the towncrier configuration. diff --git a/changelog.d/5674.feature b/changelog.d/5674.feature deleted file mode 100644 index 04bdfa4ad5..0000000000 --- a/changelog.d/5674.feature +++ /dev/null @@ -1 +0,0 @@ -Return "This account has been deactivated" when a deactivated user tries to login. diff --git a/changelog.d/5675.doc b/changelog.d/5675.doc deleted file mode 100644 index 4cd4d0be1a..0000000000 --- a/changelog.d/5675.doc +++ /dev/null @@ -1 +0,0 @@ -Minor tweaks to postgres documentation. diff --git a/changelog.d/5689.misc b/changelog.d/5689.misc deleted file mode 100644 index 8aa3e3f6a2..0000000000 --- a/changelog.d/5689.misc +++ /dev/null @@ -1 +0,0 @@ -Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. \ No newline at end of file diff --git a/changelog.d/5699.bugfix b/changelog.d/5699.bugfix deleted file mode 100644 index 30d5e67f67..0000000000 --- a/changelog.d/5699.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. \ No newline at end of file diff --git a/changelog.d/5700.bugfix b/changelog.d/5700.bugfix deleted file mode 100644 index 51bce8d441..0000000000 --- a/changelog.d/5700.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. - diff --git a/changelog.d/5701.bugfix b/changelog.d/5701.bugfix deleted file mode 100644 index fd2866e16a..0000000000 --- a/changelog.d/5701.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ignore redactions of m.room.create events. diff --git a/changelog.d/5703.misc b/changelog.d/5703.misc deleted file mode 100644 index 6e9b2d734e..0000000000 --- a/changelog.d/5703.misc +++ /dev/null @@ -1 +0,0 @@ -Documentation for opentracing. diff --git a/changelog.d/5707.bugfix b/changelog.d/5707.bugfix deleted file mode 100644 index aa3046c5e1..0000000000 --- a/changelog.d/5707.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix some problems with authenticating redactions in recent room versions. diff --git a/changelog.d/5712.feature b/changelog.d/5712.feature deleted file mode 100644 index 7d3459129d..0000000000 --- a/changelog.d/5712.feature +++ /dev/null @@ -1,2 +0,0 @@ -Add support for opentracing. - diff --git a/changelog.d/5714.feature b/changelog.d/5714.feature deleted file mode 100644 index 2fd32e5e38..0000000000 --- a/changelog.d/5714.feature +++ /dev/null @@ -1 +0,0 @@ -Enable aggregations support by default diff --git a/changelog.d/5724.bugfix b/changelog.d/5724.bugfix new file mode 100644 index 0000000000..1b3683daf6 --- /dev/null +++ b/changelog.d/5724.bugfix @@ -0,0 +1 @@ +Fix stack overflow in server key lookup code. \ No newline at end of file diff --git a/changelog.d/5731.misc b/changelog.d/5731.misc new file mode 100644 index 0000000000..dffae5d874 --- /dev/null +++ b/changelog.d/5731.misc @@ -0,0 +1 @@ +Return 'user_type' in admin API user endpoints results. diff --git a/synapse/__init__.py b/synapse/__init__.py index cf22fabd61..f26e49da36 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -35,4 +35,4 @@ try: except ImportError: pass -__version__ = "1.1.0" +__version__ = "1.2.0rc1" diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 341c863152..e8bb420ad1 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -238,27 +238,9 @@ class Keyring(object): """ try: - # create a deferred for each server we're going to look up the keys - # for; we'll resolve them once we have completed our lookups. - # These will be passed into wait_for_previous_lookups to block - # any other lookups until we have finished. - # The deferreds are called with no logcontext. - server_to_deferred = { - rq.server_name: defer.Deferred() for rq in verify_requests - } - - # We want to wait for any previous lookups to complete before - # proceeding. - yield self.wait_for_previous_lookups(server_to_deferred) + ctx = LoggingContext.current_context() - # Actually start fetching keys. - self._get_server_verify_keys(verify_requests) - - # When we've finished fetching all the keys for a given server_name, - # resolve the deferred passed to `wait_for_previous_lookups` so that - # any lookups waiting will proceed. - # - # map from server name to a set of request ids + # map from server name to a set of outstanding request ids server_to_request_ids = {} for verify_request in verify_requests: @@ -266,40 +248,61 @@ class Keyring(object): request_id = id(verify_request) server_to_request_ids.setdefault(server_name, set()).add(request_id) - def remove_deferreds(res, verify_request): + # Wait for any previous lookups to complete before proceeding. + yield self.wait_for_previous_lookups(server_to_request_ids.keys()) + + # take out a lock on each of the servers by sticking a Deferred in + # key_downloads + for server_name in server_to_request_ids.keys(): + self.key_downloads[server_name] = defer.Deferred() + logger.debug("Got key lookup lock on %s", server_name) + + # When we've finished fetching all the keys for a given server_name, + # drop the lock by resolving the deferred in key_downloads. + def drop_server_lock(server_name): + d = self.key_downloads.pop(server_name) + d.callback(None) + + def lookup_done(res, verify_request): server_name = verify_request.server_name - request_id = id(verify_request) - server_to_request_ids[server_name].discard(request_id) - if not server_to_request_ids[server_name]: - d = server_to_deferred.pop(server_name, None) - if d: - d.callback(None) + server_requests = server_to_request_ids[server_name] + server_requests.remove(id(verify_request)) + + # if there are no more requests for this server, we can drop the lock. + if not server_requests: + with PreserveLoggingContext(ctx): + logger.debug("Releasing key lookup lock on %s", server_name) + + # ... but not immediately, as that can cause stack explosions if + # we get a long queue of lookups. + self.clock.call_later(0, drop_server_lock, server_name) + return res for verify_request in verify_requests: - verify_request.key_ready.addBoth(remove_deferreds, verify_request) + verify_request.key_ready.addBoth(lookup_done, verify_request) + + # Actually start fetching keys. + self._get_server_verify_keys(verify_requests) except Exception: logger.exception("Error starting key lookups") @defer.inlineCallbacks - def wait_for_previous_lookups(self, server_to_deferred): + def wait_for_previous_lookups(self, server_names): """Waits for any previous key lookups for the given servers to finish. Args: - server_to_deferred (dict[str, Deferred]): server_name to deferred which gets - resolved once we've finished looking up keys for that server. - The Deferreds should be regular twisted ones which call their - callbacks with no logcontext. - - Returns: a Deferred which resolves once all key lookups for the given - servers have completed. Follows the synapse rules of logcontext - preservation. + server_names (Iterable[str]): list of servers which we want to look up + + Returns: + Deferred[None]: resolves once all key lookups for the given servers have + completed. Follows the synapse rules of logcontext preservation. """ loop_count = 1 while True: wait_on = [ (server_name, self.key_downloads[server_name]) - for server_name in server_to_deferred.keys() + for server_name in server_names if server_name in self.key_downloads ] if not wait_on: @@ -314,19 +317,6 @@ class Keyring(object): loop_count += 1 - ctx = LoggingContext.current_context() - - def rm(r, server_name_): - with PreserveLoggingContext(ctx): - logger.debug("Releasing key lookup lock on %s", server_name_) - self.key_downloads.pop(server_name_, None) - return r - - for server_name, deferred in server_to_deferred.items(): - logger.debug("Got key lookup lock on %s", server_name) - self.key_downloads[server_name] = deferred - deferred.addBoth(rm, server_name) - def _get_server_verify_keys(self, verify_requests): """Tries to find at least one key for each verify request diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6b0ca80087..86a333a919 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -469,7 +469,7 @@ class DataStore( return self._simple_select_list( table="users", keyvalues={}, - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], desc="get_users", ) @@ -494,7 +494,7 @@ class DataStore( orderby=order, start=start, limit=limit, - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], ) count = yield self.runInteraction("get_users_paginate", self.get_user_count_txn) retval = {"users": users, "total": count} @@ -514,7 +514,7 @@ class DataStore( table="users", term=term, col="name", - retcols=["name", "password_hash", "is_guest", "admin"], + retcols=["name", "password_hash", "is_guest", "admin", "user_type"], desc="search_users", ) diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 795703967d..8d94a503d6 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -86,35 +86,6 @@ class KeyringTestCase(unittest.HomeserverTestCase): getattr(LoggingContext.current_context(), "request", None), expected ) - def test_wait_for_previous_lookups(self): - kr = keyring.Keyring(self.hs) - - lookup_1_deferred = defer.Deferred() - lookup_2_deferred = defer.Deferred() - - # we run the lookup in a logcontext so that the patched inlineCallbacks can check - # it is doing the right thing with logcontexts. - wait_1_deferred = run_in_context( - kr.wait_for_previous_lookups, {"server1": lookup_1_deferred} - ) - - # there were no previous lookups, so the deferred should be ready - self.successResultOf(wait_1_deferred) - - # set off another wait. It should block because the first lookup - # hasn't yet completed. - wait_2_deferred = run_in_context( - kr.wait_for_previous_lookups, {"server1": lookup_2_deferred} - ) - - self.assertFalse(wait_2_deferred.called) - - # let the first lookup complete (in the sentinel context) - lookup_1_deferred.callback(None) - - # now the second wait should complete. - self.successResultOf(wait_2_deferred) - def test_verify_json_objects_for_server_awaits_previous_requests(self): key1 = signedjson.key.generate_signing_key(1) |