diff --git a/.buildkite/scripts/test_old_deps.sh b/.buildkite/scripts/test_old_deps.sh
index 9fe5b696b0..9270d55f04 100755
--- a/.buildkite/scripts/test_old_deps.sh
+++ b/.buildkite/scripts/test_old_deps.sh
@@ -1,16 +1,16 @@
#!/usr/bin/env bash
-# this script is run by buildkite in a plain `xenial` container; it installs the
-# minimal requirements for tox and hands over to the py35-old tox environment.
+# this script is run by buildkite in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
set -ex
apt-get update
-apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
export LANG="C.UTF-8"
# Prevent virtualenv from auto-updating pip to an incompatible version
export VIRTUALENV_NO_DOWNLOAD=1
-exec tox -e py35-old,combine
+exec tox -e py3-old,combine
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000000..12c82ac620
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,322 @@
+name: Tests
+
+on:
+ push:
+ branches: ["develop", "release-*"]
+ pull_request:
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ toxenv:
+ - "check-sampleconfig"
+ - "check_codestyle"
+ - "check_isort"
+ - "mypy"
+ - "packaging"
+
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v2
+ - run: pip install tox
+ - run: tox -e ${{ matrix.toxenv }}
+
+ lint-crlf:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Check line endings
+ run: scripts-dev/check_line_terminators.sh
+
+ lint-newsfile:
+ if: ${{ github.base_ref == 'develop' || contains(github.base_ref, 'release-') }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v2
+ - run: pip install tox
+ - name: Patch Buildkite-specific test script
+ run: |
+ sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
+ scripts-dev/check-newsfragment
+ - run: scripts-dev/check-newsfragment
+
+ lint-sdist:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v2
+ with:
+ python-version: "3.x"
+ - run: pip install wheel
+ - run: python setup.py sdist bdist_wheel
+ - uses: actions/upload-artifact@v2
+ with:
+ name: Python Distributions
+ path: dist/*
+
+ # Dummy step to gate other tests on without repeating the whole list
+ linting-done:
+ if: ${{ always() }} # Run this even if prior jobs were skipped
+ needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
+ runs-on: ubuntu-latest
+ steps:
+ - run: "true"
+
+ trial:
+ if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+ needs: linting-done
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["3.6", "3.7", "3.8", "3.9"]
+ database: ["sqlite"]
+ include:
+ # Newest Python without optional deps
+ - python-version: "3.9"
+ toxenv: "py-noextras,combine"
+
+ # Oldest Python with PostgreSQL
+ - python-version: "3.6"
+ database: "postgres"
+ postgres-version: "9.6"
+
+ # Newest Python with PostgreSQL
+ - python-version: "3.9"
+ database: "postgres"
+ postgres-version: "13"
+
+ steps:
+ - uses: actions/checkout@v2
+ - run: sudo apt-get -qq install xmlsec1
+ - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+ if: ${{ matrix.postgres-version }}
+ run: |
+ docker run -d -p 5432:5432 \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+ postgres:${{ matrix.postgres-version }}
+ - uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - run: pip install tox
+ - name: Await PostgreSQL
+ if: ${{ matrix.postgres-version }}
+ timeout-minutes: 2
+ run: until pg_isready -h localhost; do sleep 1; done
+ - run: tox -e py,combine
+ env:
+ TRIAL_FLAGS: "--jobs=2"
+ SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+ SYNAPSE_POSTGRES_HOST: localhost
+ SYNAPSE_POSTGRES_USER: postgres
+ SYNAPSE_POSTGRES_PASSWORD: postgres
+ - name: Dump logs
+ # Note: Dumps to workflow logs instead of using actions/upload-artifact
+ # This keeps logs colocated with failing jobs
+ # It also ignores find's exit code; this is a best effort affair
+ run: >-
+ find _trial_temp -name '*.log'
+ -exec echo "::group::{}" \;
+ -exec cat {} \;
+ -exec echo "::endgroup::" \;
+ || true
+
+ trial-olddeps:
+ if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+ needs: linting-done
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Test with old deps
+ uses: docker://ubuntu:bionic # For old python and sqlite
+ with:
+ workdir: /github/workspace
+ entrypoint: .buildkite/scripts/test_old_deps.sh
+ env:
+ TRIAL_FLAGS: "--jobs=2"
+ - name: Dump logs
+ # Note: Dumps to workflow logs instead of using actions/upload-artifact
+ # This keeps logs colocated with failing jobs
+ # It also ignores find's exit code; this is a best effort affair
+ run: >-
+ find _trial_temp -name '*.log'
+ -exec echo "::group::{}" \;
+ -exec cat {} \;
+ -exec echo "::endgroup::" \;
+ || true
+
+ trial-pypy:
+ # Very slow; only run if the branch name includes 'pypy'
+ if: ${{ contains(github.ref, 'pypy') && !failure() }}
+ needs: linting-done
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["pypy-3.6"]
+
+ steps:
+ - uses: actions/checkout@v2
+ - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
+ - uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - run: pip install tox
+ - run: tox -e py,combine
+ env:
+ TRIAL_FLAGS: "--jobs=2"
+ - name: Dump logs
+ # Note: Dumps to workflow logs instead of using actions/upload-artifact
+ # This keeps logs colocated with failing jobs
+ # It also ignores find's exit code; this is a best effort affair
+ run: >-
+ find _trial_temp -name '*.log'
+ -exec echo "::group::{}" \;
+ -exec cat {} \;
+ -exec echo "::endgroup::" \;
+ || true
+
+ sytest:
+ if: ${{ !failure() }}
+ needs: linting-done
+ runs-on: ubuntu-latest
+ container:
+ image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+ volumes:
+ - ${{ github.workspace }}:/src
+ env:
+ BUILDKITE_BRANCH: ${{ github.head_ref }}
+ POSTGRES: ${{ matrix.postgres && 1}}
+ MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+ WORKERS: ${{ matrix.workers && 1 }}
+ REDIS: ${{ matrix.redis && 1 }}
+ BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - sytest-tag: bionic
+
+ - sytest-tag: bionic
+ postgres: postgres
+
+ - sytest-tag: testing
+ postgres: postgres
+
+ - sytest-tag: bionic
+ postgres: multi-postgres
+ workers: workers
+
+ - sytest-tag: buster
+ postgres: multi-postgres
+ workers: workers
+
+ - sytest-tag: buster
+ postgres: postgres
+ workers: workers
+ redis: redis
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Prepare test blacklist
+ run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
+ - name: Run SyTest
+ run: /bootstrap.sh synapse
+ working-directory: /src
+ - name: Dump results.tap
+ if: ${{ always() }}
+ run: cat /logs/results.tap
+ - name: Upload SyTest logs
+ uses: actions/upload-artifact@v2
+ if: ${{ always() }}
+ with:
+ name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+ path: |
+ /logs/results.tap
+ /logs/**/*.log*
+
+ portdb:
+ if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+ needs: linting-done
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ include:
+ - python-version: "3.6"
+ postgres-version: "9.6"
+
+ - python-version: "3.9"
+ postgres-version: "13"
+
+ services:
+ postgres:
+ image: postgres:${{ matrix.postgres-version }}
+ ports:
+ - 5432:5432
+ env:
+ POSTGRES_PASSWORD: "postgres"
+ POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
+ steps:
+ - uses: actions/checkout@v2
+ - run: sudo apt-get -qq install xmlsec1
+ - uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Patch Buildkite-specific test scripts
+ run: |
+ sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+ sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
+ sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
+ sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
+ - run: .buildkite/scripts/test_synapse_port_db.sh
+
+ complement:
+ if: ${{ !failure() }}
+ needs: linting-done
+ runs-on: ubuntu-latest
+ container:
+ # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+ image: matrixdotorg/complement:latest
+ env:
+ CI: true
+ ports:
+ - 8448:8448
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+
+ steps:
+ - name: Run actions/checkout@v2 for synapse
+ uses: actions/checkout@v2
+ with:
+ path: synapse
+
+ - name: Run actions/checkout@v2 for complement
+ uses: actions/checkout@v2
+ with:
+ repository: "matrix-org/complement"
+ path: complement
+
+ # Build initial Synapse image
+ - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+ working-directory: synapse
+
+ # Build a ready-to-run Synapse image based on the initial image above.
+ # This new image includes a config file, keys for signing and TLS, and
+ # other settings to make it suitable for testing under Complement.
+ - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+ working-directory: complement/dockerfiles
+
+ # Run Complement
+ - run: go test -v -tags synapse_blacklist ./tests
+ env:
+ COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+ working-directory: complement
diff --git a/CHANGES.md b/CHANGES.md
index 27483532d0..41908f84be 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,75 @@
+Synapse 1.32.0rc1 (2021-04-13)
+==============================
+
+**Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+.
+
+This release removes the deprecated `GET /_synapse/admin/v1/users/<user_id>` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities.
+
+This release requires Application Services to use type `m.login.application_services` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date.
+
+Features
+--------
+
+- Add a Synapse module for routing presence updates between users. ([\#9491](https://github.com/matrix-org/synapse/issues/9491))
+- Add an admin API to manage ratelimit for a specific user. ([\#9648](https://github.com/matrix-org/synapse/issues/9648))
+- Include request information in structured logging output. ([\#9654](https://github.com/matrix-org/synapse/issues/9654))
+- Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. ([\#9691](https://github.com/matrix-org/synapse/issues/9691))
+- Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`. ([\#9700](https://github.com/matrix-org/synapse/issues/9700))
+- Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9717](https://github.com/matrix-org/synapse/issues/9717), [\#9735](https://github.com/matrix-org/synapse/issues/9735))
+- Update experimental support for Spaces: include `m.room.create` in the room state sent with room-invites. ([\#9710](https://github.com/matrix-org/synapse/issues/9710))
+- Synapse now requires Python 3.6 or later. It also requires Postgres 9.6 or later or SQLite 3.22 or later. ([\#9766](https://github.com/matrix-org/synapse/issues/9766))
+
+
+Bugfixes
+--------
+
+- Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup. ([\#8926](https://github.com/matrix-org/synapse/issues/8926))
+- Fix recently added ratelimits to correctly honour the application service `rate_limited` flag. ([\#9711](https://github.com/matrix-org/synapse/issues/9711))
+- Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors. ([\#9725](https://github.com/matrix-org/synapse/issues/9725))
+- Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU. ([\#9770](https://github.com/matrix-org/synapse/issues/9770))
+- Fix duplicate logging of exceptions thrown during federation transaction processing. ([\#9780](https://github.com/matrix-org/synapse/issues/9780))
+
+
+Updates to the Docker image
+---------------------------
+
+- Move opencontainers labels to the final Docker image such that users can inspect them. ([\#9765](https://github.com/matrix-org/synapse/issues/9765))
+
+
+Improved Documentation
+----------------------
+
+- Make the `allowed_local_3pids` regex example in the sample config stricter. ([\#9719](https://github.com/matrix-org/synapse/issues/9719))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove old admin API `GET /_synapse/admin/v1/users/<user_id>`. ([\#9401](https://github.com/matrix-org/synapse/issues/9401))
+- Make `/_matrix/client/r0/register` expect a type of `m.login.application_service` when an Application Service registers a user, to align with [the relevant spec](https://spec.matrix.org/unstable/application-service-api/#server-admin-style-permissions). ([\#9548](https://github.com/matrix-org/synapse/issues/9548))
+
+
+Internal Changes
+----------------
+
+- Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz. ([\#9718](https://github.com/matrix-org/synapse/issues/9718))
+- Experiment with GitHub Actions for CI. ([\#9661](https://github.com/matrix-org/synapse/issues/9661))
+- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9682](https://github.com/matrix-org/synapse/issues/9682))
+- Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist. ([\#9685](https://github.com/matrix-org/synapse/issues/9685))
+- Improve Jaeger tracing for `to_device` messages. ([\#9686](https://github.com/matrix-org/synapse/issues/9686))
+- Add release helper script for automating part of the Synapse release process. ([\#9713](https://github.com/matrix-org/synapse/issues/9713))
+- Add type hints to expiring cache. ([\#9730](https://github.com/matrix-org/synapse/issues/9730))
+- Convert various testcases to `HomeserverTestCase`. ([\#9736](https://github.com/matrix-org/synapse/issues/9736))
+- Start linting mypy with `no_implicit_optional`. ([\#9742](https://github.com/matrix-org/synapse/issues/9742))
+- Add missing type hints to federation handler and server. ([\#9743](https://github.com/matrix-org/synapse/issues/9743))
+- Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests. ([\#9753](https://github.com/matrix-org/synapse/issues/9753))
+- Fix incompatibility with `tox` 2.5. ([\#9769](https://github.com/matrix-org/synapse/issues/9769))
+- Enable Complement tests for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary API. ([\#9771](https://github.com/matrix-org/synapse/issues/9771))
+- Use mock from the standard library instead of a separate package. ([\#9772](https://github.com/matrix-org/synapse/issues/9772))
+- Update Black configuration to target Python 3.6. ([\#9781](https://github.com/matrix-org/synapse/issues/9781))
+- Add option to skip unit tests when building Debian packages. ([\#9793](https://github.com/matrix-org/synapse/issues/9793))
+
+
Synapse 1.31.0 (2021-04-06)
===========================
diff --git a/UPGRADE.rst b/UPGRADE.rst
index ba488e1041..665821d4ef 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -85,6 +85,19 @@ for example:
wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
+Upgrading to v1.32.0
+====================
+
+Removal of old List Accounts Admin API
+--------------------------------------
+
+The deprecated v1 "list accounts" admin API (``GET /_synapse/admin/v1/users/<user_id>``) has been removed in this version.
+
+The `v2 list accounts API <https://github.com/matrix-org/synapse/blob/master/docs/admin_api/user_admin_api.rst#list-accounts>`_
+has been available since Synapse 1.7.0 (2019-12-13), and is accessible under ``GET /_synapse/admin/v2/users``.
+
+The deprecation of the old endpoint was announced with Synapse 1.28.0 (released on 2021-02-25).
+
Upgrading to v1.29.0
====================
diff --git a/changelog.d/8926.bugfix b/changelog.d/8926.bugfix
deleted file mode 100644
index aad7bd83ce..0000000000
--- a/changelog.d/8926.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup.
diff --git a/changelog.d/9491.feature b/changelog.d/9491.feature
deleted file mode 100644
index 8b56a95a44..0000000000
--- a/changelog.d/9491.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add a Synapse module for routing presence updates between users.
diff --git a/changelog.d/9654.feature b/changelog.d/9654.feature
deleted file mode 100644
index a54c96cf19..0000000000
--- a/changelog.d/9654.feature
+++ /dev/null
@@ -1 +0,0 @@
-Include request information in structured logging output.
diff --git a/changelog.d/9685.misc b/changelog.d/9685.misc
deleted file mode 100644
index 0506d8af0c..0000000000
--- a/changelog.d/9685.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist.
\ No newline at end of file
diff --git a/changelog.d/9686.misc b/changelog.d/9686.misc
deleted file mode 100644
index bb2335acf9..0000000000
--- a/changelog.d/9686.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve Jaeger tracing for `to_device` messages.
diff --git a/changelog.d/9691.feature b/changelog.d/9691.feature
deleted file mode 100644
index 3c711db4f5..0000000000
--- a/changelog.d/9691.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel.
\ No newline at end of file
diff --git a/changelog.d/9700.feature b/changelog.d/9700.feature
deleted file mode 100644
index 037de8367f..0000000000
--- a/changelog.d/9700.feature
+++ /dev/null
@@ -1 +0,0 @@
-Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`.
diff --git a/changelog.d/9710.feature b/changelog.d/9710.feature
deleted file mode 100644
index fce308cc41..0000000000
--- a/changelog.d/9710.feature
+++ /dev/null
@@ -1 +0,0 @@
-Experimental Spaces support: include `m.room.create` in the room state sent with room-invites.
diff --git a/changelog.d/9711.bugfix b/changelog.d/9711.bugfix
deleted file mode 100644
index 4ca3438d46..0000000000
--- a/changelog.d/9711.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix recently added ratelimits to correctly honour the application service `rate_limited` flag.
diff --git a/changelog.d/9717.feature b/changelog.d/9717.feature
deleted file mode 100644
index c2c74f13d5..0000000000
--- a/changelog.d/9717.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.
diff --git a/changelog.d/9718.removal b/changelog.d/9718.removal
deleted file mode 100644
index 6de7814217..0000000000
--- a/changelog.d/9718.removal
+++ /dev/null
@@ -1 +0,0 @@
-Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz.
diff --git a/changelog.d/9719.doc b/changelog.d/9719.doc
deleted file mode 100644
index f018606dd6..0000000000
--- a/changelog.d/9719.doc
+++ /dev/null
@@ -1 +0,0 @@
-Make the allowed_local_3pids regex example in the sample config stricter.
diff --git a/changelog.d/9725.bugfix b/changelog.d/9725.bugfix
deleted file mode 100644
index 71283685c8..0000000000
--- a/changelog.d/9725.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors.
diff --git a/changelog.d/9730.misc b/changelog.d/9730.misc
deleted file mode 100644
index 8063059b0b..0000000000
--- a/changelog.d/9730.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to expiring cache.
diff --git a/changelog.d/9736.misc b/changelog.d/9736.misc
deleted file mode 100644
index 1e445e4344..0000000000
--- a/changelog.d/9736.misc
+++ /dev/null
@@ -1 +0,0 @@
-Convert various testcases to `HomeserverTestCase`.
diff --git a/changelog.d/9742.misc b/changelog.d/9742.misc
deleted file mode 100644
index 681ab04df8..0000000000
--- a/changelog.d/9742.misc
+++ /dev/null
@@ -1 +0,0 @@
-Start linting mypy with `no_implicit_optional`.
\ No newline at end of file
diff --git a/changelog.d/9743.misc b/changelog.d/9743.misc
deleted file mode 100644
index c2f75c1df9..0000000000
--- a/changelog.d/9743.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add missing type hints to federation handler and server.
diff --git a/changelog.d/9753.misc b/changelog.d/9753.misc
deleted file mode 100644
index 31184fe0bd..0000000000
--- a/changelog.d/9753.misc
+++ /dev/null
@@ -1 +0,0 @@
-Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests.
\ No newline at end of file
diff --git a/changelog.d/9770.bugfix b/changelog.d/9770.bugfix
deleted file mode 100644
index baf93138de..0000000000
--- a/changelog.d/9770.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU.
diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py
index 67e032244e..856dd437db 100755
--- a/contrib/cmdclient/console.py
+++ b/contrib/cmdclient/console.py
@@ -24,6 +24,7 @@ import sys
import time
import urllib
from http import TwistedHttpClient
+from typing import Optional
import nacl.encoding
import nacl.signing
@@ -718,7 +719,7 @@ class SynapseCmd(cmd.Cmd):
method,
path,
data=None,
- query_params={"access_token": None},
+ query_params: Optional[dict] = None,
alt_text=None,
):
"""Runs an HTTP request and pretty prints the output.
@@ -729,6 +730,8 @@ class SynapseCmd(cmd.Cmd):
data: Raw JSON data if any
query_params: dict of query parameters to add to the url
"""
+ query_params = query_params or {"access_token": None}
+
url = self._url() + path
if "access_token" in query_params:
query_params["access_token"] = self._tok()
diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py
index 851e80c25b..1cf913756e 100644
--- a/contrib/cmdclient/http.py
+++ b/contrib/cmdclient/http.py
@@ -16,6 +16,7 @@
import json
import urllib
from pprint import pformat
+from typing import Optional
from twisted.internet import defer, reactor
from twisted.web.client import Agent, readBody
@@ -85,8 +86,9 @@ class TwistedHttpClient(HttpClient):
body = yield readBody(response)
defer.returnValue(json.loads(body))
- def _create_put_request(self, url, json_data, headers_dict={}):
+ def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
"""Wrapper of _create_request to issue a PUT request"""
+ headers_dict = headers_dict or {}
if "Content-Type" not in headers_dict:
raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -95,14 +97,22 @@ class TwistedHttpClient(HttpClient):
"PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
)
- def _create_get_request(self, url, headers_dict={}):
+ def _create_get_request(self, url, headers_dict: Optional[dict] = None):
"""Wrapper of _create_request to issue a GET request"""
- return self._create_request("GET", url, headers_dict=headers_dict)
+ return self._create_request("GET", url, headers_dict=headers_dict or {})
@defer.inlineCallbacks
def do_request(
- self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+ self,
+ method,
+ url,
+ data=None,
+ qparams=None,
+ jsonreq=True,
+ headers: Optional[dict] = None,
):
+ headers = headers or {}
+
if qparams:
url = "%s?%s" % (url, urllib.urlencode(qparams, True))
@@ -123,8 +133,12 @@ class TwistedHttpClient(HttpClient):
defer.returnValue(json.loads(body))
@defer.inlineCallbacks
- def _create_request(self, method, url, producer=None, headers_dict={}):
+ def _create_request(
+ self, method, url, producer=None, headers_dict: Optional[dict] = None
+ ):
"""Creates and sends a request to the given url"""
+ headers_dict = headers_dict or {}
+
headers_dict["User-Agent"] = ["Synapse Cmd Client"]
retries_left = 5
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index cad7d16883..21caad90cc 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -50,15 +50,24 @@ PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
-# we copy the tests to a temporary directory so that we can put them on the
-# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-tmpdir=`mktemp -d`
-trap "rm -r $tmpdir" EXIT
+case "$DEB_BUILD_OPTIONS" in
+ *nocheck*)
+ # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
+ ;;
+
+ *)
+ # Copy tests to a temporary directory so that we can put them on the
+ # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+ tmpdir=`mktemp -d`
+ trap "rm -r $tmpdir" EXIT
+
+ cp -r tests "$tmpdir"
-cp -r tests "$tmpdir"
+ PYTHONPATH="$tmpdir" \
+ "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
-PYTHONPATH="$tmpdir" \
- "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+ ;;
+esac
# build the config file
"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
diff --git a/debian/changelog b/debian/changelog
index 09602ff54b..5d526316fc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.31.0+nmu1) UNRELEASED; urgency=medium
+
+ * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
+
+ -- Dan Callahan <danc@element.io> Mon, 12 Apr 2021 13:07:36 +0000
+
matrix-synapse-py3 (1.31.0) stable; urgency=medium
* New synapse release 1.31.0.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 5b7bf02776..4f5cd06d72 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -18,11 +18,6 @@ ARG PYTHON_VERSION=3.8
###
FROM docker.io/python:${PYTHON_VERSION}-slim as builder
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
# install the OS build deps
RUN apt-get update && apt-get install -y \
build-essential \
@@ -66,6 +61,11 @@ RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
FROM docker.io/python:${PYTHON_VERSION}-slim
+LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
+LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
+LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
RUN apt-get update && apt-get install -y \
curl \
gosu \
diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst
index a8a5a2628c..dbce9c90b6 100644
--- a/docs/admin_api/user_admin_api.rst
+++ b/docs/admin_api/user_admin_api.rst
@@ -202,7 +202,7 @@ The following fields are returned in the JSON response body:
- ``users`` - An array of objects, each containing information about an user.
User objects contain the following fields:
- - ``name`` - string - Fully-qualified user ID (ex. `@user:server.com`).
+ - ``name`` - string - Fully-qualified user ID (ex. ``@user:server.com``).
- ``is_guest`` - bool - Status if that user is a guest account.
- ``admin`` - bool - Status if that user is a server administrator.
- ``user_type`` - string - Type of the user. Normal users are type ``None``.
@@ -864,3 +864,118 @@ The following parameters should be set in the URL:
- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
be local.
+
+Override ratelimiting for users
+===============================
+
+This API allows to override or disable ratelimiting for a specific user.
+There are specific APIs to set, get and delete a ratelimit.
+
+Get status of ratelimit
+-----------------------
+
+The API is::
+
+ GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+ {
+ "messages_per_second": 0,
+ "burst_count": 0
+ }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+ be local.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+ be performed in a second. `0` mean that ratelimiting is disabled for this user.
+- ``burst_count`` - integer - How many actions that can be performed before
+ being limited.
+
+If **no** custom ratelimit is set, an empty JSON dict is returned.
+
+.. code:: json
+
+ {}
+
+Set ratelimit
+-------------
+
+The API is::
+
+ POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+ {
+ "messages_per_second": 0,
+ "burst_count": 0
+ }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+ be local.
+
+Body parameters:
+
+- ``messages_per_second`` - positive integer, optional. The number of actions that can
+ be performed in a second. Defaults to ``0``.
+- ``burst_count`` - positive integer, optional. How many actions that can be performed
+ before being limited. Defaults to ``0``.
+
+To disable users' ratelimit set both values to ``0``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+ be performed in a second.
+- ``burst_count`` - integer - How many actions that can be performed before
+ being limited.
+
+Delete ratelimit
+----------------
+
+The API is::
+
+ DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+.. code:: json
+
+ {}
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+ be local.
+
diff --git a/pyproject.toml b/pyproject.toml
index cd880d4e39..8bca1fa4ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@
showcontent = true
[tool.black]
-target-version = ['py35']
+target-version = ['py36']
exclude = '''
(
diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
index d0685c8b35..3bb6e2c7ea 100755
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -18,11 +18,9 @@ import threading
from concurrent.futures import ThreadPoolExecutor
DISTS = (
- "debian:stretch",
"debian:buster",
"debian:bullseye",
"debian:sid",
- "ubuntu:xenial",
"ubuntu:bionic",
"ubuntu:focal",
"ubuntu:groovy",
@@ -43,7 +41,7 @@ class Builder(object):
self._lock = threading.Lock()
self._failed = False
- def run_build(self, dist):
+ def run_build(self, dist, skip_tests=False):
"""Build deb for a single distribution"""
if self._failed:
@@ -51,13 +49,13 @@ class Builder(object):
raise Exception("failed")
try:
- self._inner_build(dist)
+ self._inner_build(dist, skip_tests)
except Exception as e:
print("build of %s failed: %s" % (dist, e), file=sys.stderr)
self._failed = True
raise
- def _inner_build(self, dist):
+ def _inner_build(self, dist, skip_tests=False):
projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
os.chdir(projdir)
@@ -101,6 +99,7 @@ class Builder(object):
"--volume=" + debsdir + ":/debs",
"-e", "TARGET_USERID=%i" % (os.getuid(), ),
"-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
+ "-e", "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
"dh-venv-builder:" + tag,
], stdout=stdout, stderr=subprocess.STDOUT)
@@ -124,7 +123,7 @@ class Builder(object):
self.active_containers.remove(c)
-def run_builds(dists, jobs=1):
+def run_builds(dists, jobs=1, skip_tests=False):
builder = Builder(redirect_stdout=(jobs > 1))
def sig(signum, _frame):
@@ -133,7 +132,7 @@ def run_builds(dists, jobs=1):
signal.signal(signal.SIGINT, sig)
with ThreadPoolExecutor(max_workers=jobs) as e:
- res = e.map(builder.run_build, dists)
+ res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
# make sure we consume the iterable so that exceptions are raised.
for r in res:
@@ -149,8 +148,12 @@ if __name__ == '__main__':
help='specify the number of builds to run in parallel',
)
parser.add_argument(
+ '--no-check', action='store_true',
+ help='skip running tests after building',
+ )
+ parser.add_argument(
'dist', nargs='*', default=DISTS,
help='a list of distributions to build for. Default: %(default)s',
)
args = parser.parse_args()
- run_builds(dists=args.dist, jobs=args.jobs)
+ run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 31cc20a826..1612ab522c 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -46,4 +46,4 @@ if [[ -n "$1" ]]; then
fi
# Run the tests!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
diff --git a/scripts-dev/release.py b/scripts-dev/release.py
new file mode 100755
index 0000000000..1042fa48bc
--- /dev/null
+++ b/scripts-dev/release.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interactive script for doing a release. See `run()` below.
+"""
+
+import subprocess
+import sys
+from typing import Optional
+
+import click
+import git
+from packaging import version
+from redbaron import RedBaron
+
+
+@click.command()
+def run():
+ """An interactive script to walk through the initial stages of creating a
+ release, including creating release branch, updating changelog and pushing to
+ GitHub.
+
+ Requires the dev dependencies be installed, which can be done via:
+
+ pip install -e .[dev]
+
+ """
+
+ # Make sure we're in a git repo.
+ try:
+ repo = git.Repo()
+ except git.InvalidGitRepositoryError:
+ raise click.ClickException("Not in Synapse repo.")
+
+ if repo.is_dirty():
+ raise click.ClickException("Uncommitted changes exist.")
+
+ click.secho("Updating git repo...")
+ repo.remote().fetch()
+
+ # Parse the AST and load the `__version__` node so that we can edit it
+ # later.
+ with open("synapse/__init__.py") as f:
+ red = RedBaron(f.read())
+
+ version_node = None
+ for node in red:
+ if node.type != "assignment":
+ continue
+
+ if node.target.type != "name":
+ continue
+
+ if node.target.value != "__version__":
+ continue
+
+ version_node = node
+ break
+
+ if not version_node:
+ print("Failed to find '__version__' definition in synapse/__init__.py")
+ sys.exit(1)
+
+ # Parse the current version.
+ current_version = version.parse(version_node.value.value.strip('"'))
+ assert isinstance(current_version, version.Version)
+
+ # Figure out what sort of release we're doing and calcuate the new version.
+ rc = click.confirm("RC", default=True)
+ if current_version.pre:
+ # If the current version is an RC we don't need to bump any of the
+ # version numbers (other than the RC number).
+ base_version = "{}.{}.{}".format(
+ current_version.major,
+ current_version.minor,
+ current_version.micro,
+ )
+
+ if rc:
+ new_version = "{}.{}.{}rc{}".format(
+ current_version.major,
+ current_version.minor,
+ current_version.micro,
+ current_version.pre[1] + 1,
+ )
+ else:
+ new_version = base_version
+ else:
+ # If this is a new release cycle then we need to know if its a major
+ # version bump or a hotfix.
+ release_type = click.prompt(
+ "Release type",
+ type=click.Choice(("major", "hotfix")),
+ show_choices=True,
+ default="major",
+ )
+
+ if release_type == "major":
+ base_version = new_version = "{}.{}.{}".format(
+ current_version.major,
+ current_version.minor + 1,
+ 0,
+ )
+ if rc:
+ new_version = "{}.{}.{}rc1".format(
+ current_version.major,
+ current_version.minor + 1,
+ 0,
+ )
+
+ else:
+ base_version = new_version = "{}.{}.{}".format(
+ current_version.major,
+ current_version.minor,
+ current_version.micro + 1,
+ )
+ if rc:
+ new_version = "{}.{}.{}rc1".format(
+ current_version.major,
+ current_version.minor,
+ current_version.micro + 1,
+ )
+
+ # Confirm the calculated version is OK.
+ if not click.confirm(f"Create new version: {new_version}?", default=True):
+ click.get_current_context().abort()
+
+ # Switch to the release branch.
+ release_branch_name = f"release-v{base_version}"
+ release_branch = find_ref(repo, release_branch_name)
+ if release_branch:
+ if release_branch.is_remote():
+ # If the release branch only exists on the remote we check it out
+ # locally.
+ repo.git.checkout(release_branch_name)
+ release_branch = repo.active_branch
+ else:
+ # If a branch doesn't exist we create one. We ask which one branch it
+ # should be based off, defaulting to sensible values depending on the
+ # release type.
+ if current_version.is_prerelease:
+ default = release_branch_name
+ elif release_type == "major":
+ default = "develop"
+ else:
+ default = "master"
+
+ branch_name = click.prompt(
+ "Which branch should the release be based on?", default=default
+ )
+
+ base_branch = find_ref(repo, branch_name)
+ if not base_branch:
+ print(f"Could not find base branch {branch_name}!")
+ click.get_current_context().abort()
+
+ # Check out the base branch and ensure it's up to date
+ repo.head.reference = base_branch
+ repo.head.reset(index=True, working_tree=True)
+ if not base_branch.is_remote():
+ update_branch(repo)
+
+ # Create the new release branch
+ release_branch = repo.create_head(release_branch_name, commit=base_branch)
+
+ # Switch to the release branch and ensure its up to date.
+ repo.git.checkout(release_branch_name)
+ update_branch(repo)
+
+ # Update the `__version__` variable and write it back to the file.
+ version_node.value = '"' + new_version + '"'
+ with open("synapse/__init__.py", "w") as f:
+ f.write(red.dumps())
+
+ # Generate changelogs
+ subprocess.run("python3 -m towncrier", shell=True)
+
+ # Generate debian changelogs if its not an RC.
+ if not rc:
+ subprocess.run(
+ f'dch -M -v {new_version} "New synapse release {new_version}."', shell=True
+ )
+ subprocess.run('dch -M -r -D stable ""', shell=True)
+
+ # Show the user the changes and ask if they want to edit the change log.
+ repo.git.add("-u")
+ subprocess.run("git diff --cached", shell=True)
+
+ if click.confirm("Edit changelog?", default=False):
+ click.edit(filename="CHANGES.md")
+
+ # Commit the changes.
+ repo.git.add("-u")
+ repo.git.commit(f"-m {new_version}")
+
+ # We give the option to bail here in case the user wants to make sure things
+ # are OK before pushing.
+ if not click.confirm("Push branch to github?", default=True):
+ print("")
+ print("Run when ready to push:")
+ print("")
+ print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
+ print("")
+ sys.exit(0)
+
+ # Otherwise, push and open the changelog in the browser.
+ repo.git.push("-u", repo.remote().name, repo.active_branch.name)
+
+ click.launch(
+ f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
+ )
+
+
+def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
+ """Find the branch/ref, looking first locally then in the remote."""
+ if ref_name in repo.refs:
+ return repo.refs[ref_name]
+ elif ref_name in repo.remote().refs:
+ return repo.remote().refs[ref_name]
+ else:
+ return None
+
+
+def update_branch(repo: git.Repo):
+ """Ensure branch is up to date if it has a remote"""
+ if repo.active_branch.tracking_branch():
+ repo.git.merge(repo.active_branch.tracking_branch().name)
+
+
+if __name__ == "__main__":
+ run()
diff --git a/setup.cfg b/setup.cfg
index 7329eed213..33601b71d5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,16 +18,15 @@ ignore =
# E203: whitespace before ':' (which is contrary to pep8?)
# E731: do not assign a lambda expression, use a def
# E501: Line too long (black enforces this for us)
-# B00*: Subsection of the bugbear suite (TODO: add in remaining fixes)
-ignore=W503,W504,E203,E731,E501,B006,B007,B008
+# B007: Subsection of the bugbear suite (TODO: add in remaining fixes)
+ignore=W503,W504,E203,E731,E501,B007
[isort]
line_length = 88
-sections=FUTURE,STDLIB,COMPAT,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
+sections=FUTURE,STDLIB,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
default_section=THIRDPARTY
known_first_party = synapse
known_tests=tests
-known_compat = mock
known_twisted=twisted,OpenSSL
multi_line_output=3
include_trailing_comma=true
diff --git a/setup.py b/setup.py
index 29e9971dc1..e2e488761d 100755
--- a/setup.py
+++ b/setup.py
@@ -103,6 +103,13 @@ CONDITIONAL_REQUIREMENTS["lint"] = [
"flake8",
]
+CONDITIONAL_REQUIREMENTS["dev"] = CONDITIONAL_REQUIREMENTS["lint"] + [
+ # The following are used by the release script
+ "click==7.1.2",
+ "redbaron==0.9.2",
+ "GitPython==3.1.14",
+]
+
CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
# Dependencies which are exclusively required by unit test code. This is
@@ -110,7 +117,7 @@ CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
# Tests assume that all optional dependencies are installed.
#
# parameterized_class decorator was introduced in parameterized 0.7.0
-CONDITIONAL_REQUIREMENTS["test"] = ["mock>=2.0", "parameterized>=0.7.0"]
+CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0"]
setup(
name="matrix-synapse",
@@ -123,13 +130,12 @@ setup(
zip_safe=False,
long_description=long_description,
long_description_content_type="text/x-rst",
- python_requires="~=3.5",
+ python_requires="~=3.6",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Topic :: Communications :: Chat",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3 :: Only",
- "Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 1d2883acf6..125a73d378 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -48,7 +48,7 @@ try:
except ImportError:
pass
-__version__ = "1.31.0"
+__version__ = "1.32.0rc1"
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
# We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 6856dab06c..a8ae41de48 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -73,6 +73,11 @@ class LoginType:
DUMMY = "m.login.dummy"
+# This is used in the `type` parameter for /register when called by
+# an appservice to register a new user.
+APP_SERVICE_REGISTRATION_TYPE = "m.login.application_service"
+
+
class EventTypes:
Member = "m.room.member"
Create = "m.room.create"
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 366c476f80..5203ffe90f 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -49,7 +49,7 @@ This is all tied together by the AppServiceScheduler which DIs the required
components.
"""
import logging
-from typing import List
+from typing import List, Optional
from synapse.appservice import ApplicationService, ApplicationServiceState
from synapse.events import EventBase
@@ -191,11 +191,11 @@ class _TransactionController:
self,
service: ApplicationService,
events: List[EventBase],
- ephemeral: List[JsonDict] = [],
+ ephemeral: Optional[List[JsonDict]] = None,
):
try:
txn = await self.store.create_appservice_txn(
- service=service, events=events, ephemeral=ephemeral
+ service=service, events=events, ephemeral=ephemeral or []
)
service_is_up = await self._is_service_up(service)
if service_is_up:
diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py
index 3f3997f4e5..7a8d5851c4 100644
--- a/synapse/config/ratelimiting.py
+++ b/synapse/config/ratelimiting.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Dict
+from typing import Dict, Optional
from ._base import Config
@@ -21,8 +21,10 @@ class RateLimitConfig:
def __init__(
self,
config: Dict[str, float],
- defaults={"per_second": 0.17, "burst_count": 3.0},
+ defaults: Optional[Dict[str, float]] = None,
):
+ defaults = defaults or {"per_second": 0.17, "burst_count": 3.0}
+
self.per_second = config.get("per_second", defaults["per_second"])
self.burst_count = int(config.get("burst_count", defaults["burst_count"]))
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index ad37b93c02..85b5db4c40 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -270,7 +270,7 @@ class TlsConfig(Config):
tls_certificate_path,
tls_private_key_path,
acme_domain,
- **kwargs
+ **kwargs,
):
"""If the acme_domain is specified acme will be enabled.
If the TLS paths are not specified the default will be certs in the
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 8f6b955d17..f9032e3697 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -330,9 +330,11 @@ class FrozenEvent(EventBase):
self,
event_dict: JsonDict,
room_version: RoomVersion,
- internal_metadata_dict: JsonDict = {},
+ internal_metadata_dict: Optional[JsonDict] = None,
rejected_reason: Optional[str] = None,
):
+ internal_metadata_dict = internal_metadata_dict or {}
+
event_dict = dict(event_dict)
# Signatures is a dict of dicts, and this is faster than doing a
@@ -386,9 +388,11 @@ class FrozenEventV2(EventBase):
self,
event_dict: JsonDict,
room_version: RoomVersion,
- internal_metadata_dict: JsonDict = {},
+ internal_metadata_dict: Optional[JsonDict] = None,
rejected_reason: Optional[str] = None,
):
+ internal_metadata_dict = internal_metadata_dict or {}
+
event_dict = dict(event_dict)
# Signatures is a dict of dicts, and this is faster than doing a
@@ -507,9 +511,11 @@ def _event_type_from_format_version(format_version: int) -> Type[EventBase]:
def make_event_from_dict(
event_dict: JsonDict,
room_version: RoomVersion = RoomVersions.V1,
- internal_metadata_dict: JsonDict = {},
+ internal_metadata_dict: Optional[JsonDict] = None,
rejected_reason: Optional[str] = None,
) -> EventBase:
"""Construct an EventBase from the given event dict"""
event_type = _event_type_from_format_version(room_version.event_format)
- return event_type(event_dict, room_version, internal_metadata_dict, rejected_reason)
+ return event_type(
+ event_dict, room_version, internal_metadata_dict or {}, rejected_reason
+ )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 5ef0556ef7..a9c1391d27 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -425,13 +425,9 @@ class FederationSendServlet(BaseFederationServlet):
logger.exception(e)
return 400, {"error": "Invalid transaction"}
- try:
- code, response = await self.handler.on_incoming_transaction(
- origin, transaction_data
- )
- except Exception:
- logger.exception("on_incoming_transaction failed")
- raise
+ code, response = await self.handler.on_incoming_transaction(
+ origin, transaction_data
+ )
return code, response
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index b662c42621..0f8bf000ac 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -18,6 +18,7 @@ server protocol.
"""
import logging
+from typing import Optional
import attr
@@ -98,7 +99,7 @@ class Transaction(JsonEncodedObject):
"pdus",
]
- def __init__(self, transaction_id=None, pdus=[], **kwargs):
+ def __init__(self, transaction_id=None, pdus: Optional[list] = None, **kwargs):
"""If we include a list of pdus then we decode then as PDU's
automatically.
"""
@@ -107,7 +108,7 @@ class Transaction(JsonEncodedObject):
if "edus" in kwargs and not kwargs["edus"]:
del kwargs["edus"]
- super().__init__(transaction_id=transaction_id, pdus=pdus, **kwargs)
+ super().__init__(transaction_id=transaction_id, pdus=pdus or [], **kwargs)
@staticmethod
def create_new(pdus, **kwargs):
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 996f9e5deb..9fb7ee335d 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -182,7 +182,7 @@ class ApplicationServicesHandler:
self,
stream_key: str,
new_token: Optional[int],
- users: Collection[Union[str, UserID]] = [],
+ users: Optional[Collection[Union[str, UserID]]] = None,
):
"""This is called by the notifier in the background
when a ephemeral event handled by the homeserver.
@@ -215,7 +215,7 @@ class ApplicationServicesHandler:
# We only start a new background process if necessary rather than
# optimistically (to cut down on overhead).
self._notify_interested_services_ephemeral(
- services, stream_key, new_token, users
+ services, stream_key, new_token, users or []
)
@wrap_as_background_process("notify_interested_services_ephemeral")
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 5ea8a7b603..67888898ff 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1790,7 +1790,7 @@ class FederationHandler(BaseHandler):
room_id: str,
user_id: str,
membership: str,
- content: JsonDict = {},
+ content: JsonDict,
params: Optional[Dict[str, Union[str, Iterable[str]]]] = None,
) -> Tuple[str, EventBase, RoomVersion]:
(
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 577de9b550..4b7b821c4e 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -137,7 +137,7 @@ class MessageHandler:
self,
user_id: str,
room_id: str,
- state_filter: StateFilter = StateFilter.all(),
+ state_filter: Optional[StateFilter] = None,
at_token: Optional[StreamToken] = None,
is_guest: bool = False,
) -> List[dict]:
@@ -164,6 +164,8 @@ class MessageHandler:
AuthError (403) if the user doesn't have permission to view
members of this room.
"""
+ state_filter = state_filter or StateFilter.all()
+
if at_token:
# FIXME this claims to get the state at a stream position, but
# get_recent_events_for_room operates by topo ordering. This therefore
@@ -874,7 +876,7 @@ class EventCreationHandler:
event: EventBase,
context: EventContext,
ratelimit: bool = True,
- extra_users: List[UserID] = [],
+ extra_users: Optional[List[UserID]] = None,
ignore_shadow_ban: bool = False,
) -> EventBase:
"""Processes a new event.
@@ -902,6 +904,7 @@ class EventCreationHandler:
Raises:
ShadowBanError if the requester has been shadow-banned.
"""
+ extra_users = extra_users or []
# we don't apply shadow-banning to membership events here. Invites are blocked
# higher up the stack, and we allow shadow-banned users to send join and leave
@@ -1071,7 +1074,7 @@ class EventCreationHandler:
event: EventBase,
context: EventContext,
ratelimit: bool = True,
- extra_users: List[UserID] = [],
+ extra_users: Optional[List[UserID]] = None,
) -> EventBase:
"""Called when we have fully built the event, have already
calculated the push actions for the event, and checked auth.
@@ -1083,6 +1086,8 @@ class EventCreationHandler:
it was de-duplicated (e.g. because we had already persisted an
event with the same transaction ID.)
"""
+ extra_users = extra_users or []
+
assert self.storage.persistence is not None
assert self._events_shard_config.should_handle(
self._instance_name, event.room_id
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index c817f2952d..0047907cd9 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -1071,7 +1071,7 @@ class PresenceEventSource:
room_ids=None,
include_offline=True,
explicit_room_id=None,
- **kwargs
+ **kwargs,
) -> Tuple[List[UserPresenceState], int]:
# The process for getting presence events are:
# 1. Get the rooms the user is in.
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 9701b76d0f..3b6660c873 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -169,7 +169,7 @@ class RegistrationHandler(BaseHandler):
user_type: Optional[str] = None,
default_display_name: Optional[str] = None,
address: Optional[str] = None,
- bind_emails: Iterable[str] = [],
+ bind_emails: Optional[Iterable[str]] = None,
by_admin: bool = False,
user_agent_ips: Optional[List[Tuple[str, str]]] = None,
auth_provider_id: Optional[str] = None,
@@ -204,6 +204,8 @@ class RegistrationHandler(BaseHandler):
Raises:
SynapseError if there was a problem registering.
"""
+ bind_emails = bind_emails or []
+
await self.check_registration_ratelimit(address)
result = await self.spam_checker.check_registration_for_spam(
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 03699b4019..08f28ca76c 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -20,7 +20,7 @@ from http import HTTPStatus
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
from synapse import types
-from synapse.api.constants import AccountDataTypes, EventTypes, Membership
+from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, Membership
from synapse.api.errors import (
AuthError,
Codes,
@@ -29,6 +29,7 @@ from synapse.api.errors import (
SynapseError,
)
from synapse.api.ratelimiting import Ratelimiter
+from synapse.api.room_versions import RoomVersion
from synapse.events import EventBase
from synapse.events.snapshot import EventContext
from synapse.types import JsonDict, Requester, RoomAlias, RoomID, StateMap, UserID
@@ -179,6 +180,62 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
await self._invites_per_user_limiter.ratelimit(requester, invitee_user_id)
+ async def _can_join_without_invite(
+ self, state_ids: StateMap[str], room_version: RoomVersion, user_id: str
+ ) -> bool:
+ """
+ Check whether a user can join a room without an invite.
+
+ When joining a room with restricted joined rules (as defined in MSC3083),
+ the membership of spaces must be checked during join.
+
+ Args:
+ state_ids: The state of the room as it currently is.
+ room_version: The room version of the room being joined.
+ user_id: The user joining the room.
+
+ Returns:
+ True if the user can join the room, false otherwise.
+ """
+ # This only applies to room versions which support the new join rule.
+ if not room_version.msc3083_join_rules:
+ return True
+
+ # If there's no join rule, then it defaults to public (so this doesn't apply).
+ join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None)
+ if not join_rules_event_id:
+ return True
+
+ # If the join rule is not restricted, this doesn't apply.
+ join_rules_event = await self.store.get_event(join_rules_event_id)
+ if join_rules_event.content.get("join_rule") != JoinRules.MSC3083_RESTRICTED:
+ return True
+
+ # If allowed is of the wrong form, then only allow invited users.
+ allowed_spaces = join_rules_event.content.get("allow", [])
+ if not isinstance(allowed_spaces, list):
+ return False
+
+ # Get the list of joined rooms and see if there's an overlap.
+ joined_rooms = await self.store.get_rooms_for_user(user_id)
+
+ # Pull out the other room IDs, invalid data gets filtered.
+ for space in allowed_spaces:
+ if not isinstance(space, dict):
+ continue
+
+ space_id = space.get("space")
+ if not isinstance(space_id, str):
+ continue
+
+ # The user was joined to one of the spaces specified, they can join
+ # this room!
+ if space_id in joined_rooms:
+ return True
+
+ # The user was not in any of the required spaces.
+ return False
+
async def _local_membership_update(
self,
requester: Requester,
@@ -236,9 +293,25 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
if event.membership == Membership.JOIN:
newly_joined = True
+ user_is_invited = False
if prev_member_event_id:
prev_member_event = await self.store.get_event(prev_member_event_id)
newly_joined = prev_member_event.membership != Membership.JOIN
+ user_is_invited = prev_member_event.membership == Membership.INVITE
+
+ # If the member is not already in the room and is not accepting an invite,
+ # check if they should be allowed access via membership in a space.
+ if (
+ newly_joined
+ and not user_is_invited
+ and not await self._can_join_without_invite(
+ prev_state_ids, event.room_version, user_id
+ )
+ ):
+ raise AuthError(
+ 403,
+ "You do not belong to any of the required spaces to join this room.",
+ )
# Only rate-limit if the user actually joined the room, otherwise we'll end
# up blocking profile updates.
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ceed40f90d..0eded88a54 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -549,7 +549,7 @@ class SyncHandler:
)
async def get_state_after_event(
- self, event: EventBase, state_filter: StateFilter = StateFilter.all()
+ self, event: EventBase, state_filter: Optional[StateFilter] = None
) -> StateMap[str]:
"""
Get the room state after the given event
@@ -559,7 +559,7 @@ class SyncHandler:
state_filter: The state filter used to fetch state from the database.
"""
state_ids = await self.state_store.get_state_ids_for_event(
- event.event_id, state_filter=state_filter
+ event.event_id, state_filter=state_filter or StateFilter.all()
)
if event.is_state():
state_ids = dict(state_ids)
@@ -570,7 +570,7 @@ class SyncHandler:
self,
room_id: str,
stream_position: StreamToken,
- state_filter: StateFilter = StateFilter.all(),
+ state_filter: Optional[StateFilter] = None,
) -> StateMap[str]:
"""Get the room state at a particular stream position
@@ -590,7 +590,7 @@ class SyncHandler:
if last_events:
last_event = last_events[-1]
state = await self.get_state_after_event(
- last_event, state_filter=state_filter
+ last_event, state_filter=state_filter or StateFilter.all()
)
else:
diff --git a/synapse/http/client.py b/synapse/http/client.py
index e691ba6d88..f7a07f0466 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -297,7 +297,7 @@ class SimpleHttpClient:
def __init__(
self,
hs: "HomeServer",
- treq_args: Dict[str, Any] = {},
+ treq_args: Optional[Dict[str, Any]] = None,
ip_whitelist: Optional[IPSet] = None,
ip_blacklist: Optional[IPSet] = None,
use_proxy: bool = False,
@@ -317,7 +317,7 @@ class SimpleHttpClient:
self._ip_whitelist = ip_whitelist
self._ip_blacklist = ip_blacklist
- self._extra_treq_args = treq_args
+ self._extra_treq_args = treq_args or {}
self.user_agent = hs.version_string
self.clock = hs.get_clock()
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 5f01ebd3d4..ab47dec8f2 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -272,7 +272,7 @@ class MatrixFederationHttpClient:
self,
request: MatrixFederationRequest,
try_trailing_slash_on_400: bool = False,
- **send_request_args
+ **send_request_args,
) -> IResponse:
"""Wrapper for _send_request which can optionally retry the request
upon receiving a combination of a 400 HTTP response code and a
diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py
index 16ec850064..ea5ad14cb0 100644
--- a/synapse/http/proxyagent.py
+++ b/synapse/http/proxyagent.py
@@ -27,7 +27,7 @@ from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase
from twisted.web.error import SchemeNotSupported
from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent
+from twisted.web.iweb import IAgent, IPolicyForHTTPS
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
@@ -88,12 +88,14 @@ class ProxyAgent(_AgentBase):
self,
reactor,
proxy_reactor=None,
- contextFactory=BrowserLikePolicyForHTTPS(),
+ contextFactory: Optional[IPolicyForHTTPS] = None,
connectTimeout=None,
bindAddress=None,
pool=None,
use_proxy=False,
):
+ contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
+
_AgentBase.__init__(self, reactor, pool)
if proxy_reactor is None:
diff --git a/synapse/http/site.py b/synapse/http/site.py
index c0c873ce32..32b5e19c09 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -497,7 +497,7 @@ class SynapseSite(Site):
resource,
server_version_string,
*args,
- **kwargs
+ **kwargs,
):
Site.__init__(self, resource, *args, **kwargs)
diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py
index b8081f197e..bfe9136fd8 100644
--- a/synapse/logging/opentracing.py
+++ b/synapse/logging/opentracing.py
@@ -486,7 +486,7 @@ def start_active_span_from_request(
def start_active_span_from_edu(
edu_content,
operation_name,
- references=[],
+ references: Optional[list] = None,
tags=None,
start_time=None,
ignore_active_span=False,
@@ -501,6 +501,7 @@ def start_active_span_from_edu(
For the other args see opentracing.tracer
"""
+ references = references or []
if opentracing is None:
return noop_context_manager()
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index 3ecd46c038..ca1bd4cdc9 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Generator, Iterable, List, Optional, Tuple
from twisted.internet import defer
@@ -127,7 +127,7 @@ class ModuleApi:
return defer.ensureDeferred(self._auth_handler.check_user_exists(user_id))
@defer.inlineCallbacks
- def register(self, localpart, displayname=None, emails=[]):
+ def register(self, localpart, displayname=None, emails: Optional[List[str]] = None):
"""Registers a new user with given localpart and optional displayname, emails.
Also returns an access token for the new user.
@@ -147,11 +147,13 @@ class ModuleApi:
logger.warning(
"Using deprecated ModuleApi.register which creates a dummy user device."
)
- user_id = yield self.register_user(localpart, displayname, emails)
+ user_id = yield self.register_user(localpart, displayname, emails or [])
_, access_token = yield self.register_device(user_id)
return user_id, access_token
- def register_user(self, localpart, displayname=None, emails=[]):
+ def register_user(
+ self, localpart, displayname=None, emails: Optional[List[str]] = None
+ ):
"""Registers a new user with given localpart and optional displayname, emails.
Args:
@@ -170,7 +172,7 @@ class ModuleApi:
self._hs.get_registration_handler().register_user(
localpart=localpart,
default_display_name=displayname,
- bind_emails=emails,
+ bind_emails=emails or [],
)
)
diff --git a/synapse/notifier.py b/synapse/notifier.py
index c178db57e3..7ce34380af 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -276,7 +276,7 @@ class Notifier:
event: EventBase,
event_pos: PersistedEventPosition,
max_room_stream_token: RoomStreamToken,
- extra_users: Collection[UserID] = [],
+ extra_users: Optional[Collection[UserID]] = None,
):
"""Unwraps event and calls `on_new_room_event_args`."""
self.on_new_room_event_args(
@@ -286,7 +286,7 @@ class Notifier:
state_key=event.get("state_key"),
membership=event.content.get("membership"),
max_room_stream_token=max_room_stream_token,
- extra_users=extra_users,
+ extra_users=extra_users or [],
)
def on_new_room_event_args(
@@ -297,7 +297,7 @@ class Notifier:
membership: Optional[str],
event_pos: PersistedEventPosition,
max_room_stream_token: RoomStreamToken,
- extra_users: Collection[UserID] = [],
+ extra_users: Optional[Collection[UserID]] = None,
):
"""Used by handlers to inform the notifier something has happened
in the room, room event wise.
@@ -313,7 +313,7 @@ class Notifier:
self.pending_new_room_events.append(
_PendingRoomEventEntry(
event_pos=event_pos,
- extra_users=extra_users,
+ extra_users=extra_users or [],
room_id=room_id,
type=event_type,
state_key=state_key,
@@ -382,14 +382,14 @@ class Notifier:
self,
stream_key: str,
new_token: Union[int, RoomStreamToken],
- users: Collection[Union[str, UserID]] = [],
+ users: Optional[Collection[Union[str, UserID]]] = None,
):
try:
stream_token = None
if isinstance(new_token, int):
stream_token = new_token
self.appservice_handler.notify_interested_services_ephemeral(
- stream_key, stream_token, users
+ stream_key, stream_token, users or []
)
except Exception:
logger.exception("Error notifying application services of event")
@@ -404,13 +404,16 @@ class Notifier:
self,
stream_key: str,
new_token: Union[int, RoomStreamToken],
- users: Collection[Union[str, UserID]] = [],
- rooms: Collection[str] = [],
+ users: Optional[Collection[Union[str, UserID]]] = None,
+ rooms: Optional[Collection[str]] = None,
):
"""Used to inform listeners that something has happened event wise.
Will wake up all listeners for the given users and rooms.
"""
+ users = users or []
+ rooms = rooms or []
+
with Measure(self.clock, "on_new_event"):
user_streams = set()
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index 8457db1e22..2dec818a5f 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -54,6 +54,7 @@ from synapse.rest.admin.users import (
AccountValidityRenewServlet,
DeactivateAccountRestServlet,
PushersRestServlet,
+ RateLimitRestServlet,
ResetPasswordRestServlet,
SearchUsersRestServlet,
ShadowBanRestServlet,
@@ -62,7 +63,6 @@ from synapse.rest.admin.users import (
UserMembershipRestServlet,
UserRegisterServlet,
UserRestServletV2,
- UsersRestServlet,
UsersRestServletV2,
UserTokenRestServlet,
WhoisRestServlet,
@@ -240,6 +240,7 @@ def register_servlets(hs, http_server):
ShadowBanRestServlet(hs).register(http_server)
ForwardExtremitiesRestServlet(hs).register(http_server)
RoomEventContextServlet(hs).register(http_server)
+ RateLimitRestServlet(hs).register(http_server)
def register_servlets_for_client_rest_resource(hs, http_server):
@@ -248,7 +249,6 @@ def register_servlets_for_client_rest_resource(hs, http_server):
PurgeHistoryStatusRestServlet(hs).register(http_server)
DeactivateAccountRestServlet(hs).register(http_server)
PurgeHistoryRestServlet(hs).register(http_server)
- UsersRestServlet(hs).register(http_server)
ResetPasswordRestServlet(hs).register(http_server)
SearchUsersRestServlet(hs).register(http_server)
ShutdownRoomRestServlet(hs).register(http_server)
diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py
index fa7804583a..04990c71fb 100644
--- a/synapse/rest/admin/users.py
+++ b/synapse/rest/admin/users.py
@@ -45,29 +45,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-class UsersRestServlet(RestServlet):
- PATTERNS = admin_patterns("/users/(?P<user_id>[^/]*)$")
-
- def __init__(self, hs: "HomeServer"):
- self.hs = hs
- self.store = hs.get_datastore()
- self.auth = hs.get_auth()
- self.admin_handler = hs.get_admin_handler()
-
- async def on_GET(
- self, request: SynapseRequest, user_id: str
- ) -> Tuple[int, List[JsonDict]]:
- target_user = UserID.from_string(user_id)
- await assert_requester_is_admin(self.auth, request)
-
- if not self.hs.is_mine(target_user):
- raise SynapseError(400, "Can only users a local user")
-
- ret = await self.store.get_users()
-
- return 200, ret
-
-
class UsersRestServletV2(RestServlet):
PATTERNS = admin_patterns("/users$", "v2")
@@ -1004,3 +981,114 @@ class ShadowBanRestServlet(RestServlet):
await self.store.set_shadow_banned(UserID.from_string(user_id), True)
return 200, {}
+
+
+class RateLimitRestServlet(RestServlet):
+ """An admin API to override ratelimiting for an user.
+
+ Example:
+ POST /_synapse/admin/v1/users/@test:example.com/override_ratelimit
+ {
+ "messages_per_second": 0,
+ "burst_count": 0
+ }
+ 200 OK
+ {
+ "messages_per_second": 0,
+ "burst_count": 0
+ }
+ """
+
+ PATTERNS = admin_patterns("/users/(?P<user_id>[^/]*)/override_ratelimit")
+
+ def __init__(self, hs: "HomeServer"):
+ self.hs = hs
+ self.store = hs.get_datastore()
+ self.auth = hs.get_auth()
+
+ async def on_GET(
+ self, request: SynapseRequest, user_id: str
+ ) -> Tuple[int, JsonDict]:
+ await assert_requester_is_admin(self.auth, request)
+
+ if not self.hs.is_mine_id(user_id):
+ raise SynapseError(400, "Can only lookup local users")
+
+ if not await self.store.get_user_by_id(user_id):
+ raise NotFoundError("User not found")
+
+ ratelimit = await self.store.get_ratelimit_for_user(user_id)
+
+ if ratelimit:
+ # convert `null` to `0` for consistency
+ # both values do the same in retelimit handler
+ ret = {
+ "messages_per_second": 0
+ if ratelimit.messages_per_second is None
+ else ratelimit.messages_per_second,
+ "burst_count": 0
+ if ratelimit.burst_count is None
+ else ratelimit.burst_count,
+ }
+ else:
+ ret = {}
+
+ return 200, ret
+
+ async def on_POST(
+ self, request: SynapseRequest, user_id: str
+ ) -> Tuple[int, JsonDict]:
+ await assert_requester_is_admin(self.auth, request)
+
+ if not self.hs.is_mine_id(user_id):
+ raise SynapseError(400, "Only local users can be ratelimited")
+
+ if not await self.store.get_user_by_id(user_id):
+ raise NotFoundError("User not found")
+
+ body = parse_json_object_from_request(request, allow_empty_body=True)
+
+ messages_per_second = body.get("messages_per_second", 0)
+ burst_count = body.get("burst_count", 0)
+
+ if not isinstance(messages_per_second, int) or messages_per_second < 0:
+ raise SynapseError(
+ 400,
+ "%r parameter must be a positive int" % (messages_per_second,),
+ errcode=Codes.INVALID_PARAM,
+ )
+
+ if not isinstance(burst_count, int) or burst_count < 0:
+ raise SynapseError(
+ 400,
+ "%r parameter must be a positive int" % (burst_count,),
+ errcode=Codes.INVALID_PARAM,
+ )
+
+ await self.store.set_ratelimit_for_user(
+ user_id, messages_per_second, burst_count
+ )
+ ratelimit = await self.store.get_ratelimit_for_user(user_id)
+ assert ratelimit is not None
+
+ ret = {
+ "messages_per_second": ratelimit.messages_per_second,
+ "burst_count": ratelimit.burst_count,
+ }
+
+ return 200, ret
+
+ async def on_DELETE(
+ self, request: SynapseRequest, user_id: str
+ ) -> Tuple[int, JsonDict]:
+ await assert_requester_is_admin(self.auth, request)
+
+ if not self.hs.is_mine_id(user_id):
+ raise SynapseError(400, "Only local users can be ratelimited")
+
+ if not await self.store.get_user_by_id(user_id):
+ raise NotFoundError("User not found")
+
+ await self.store.delete_ratelimit_for_user(user_id)
+
+ return 200, {}
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index c212da0cb2..4a064849c1 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import hmac
import logging
import random
@@ -22,7 +21,7 @@ from typing import List, Union
import synapse
import synapse.api.auth
import synapse.types
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
from synapse.api.errors import (
Codes,
InteractiveAuthIncompleteError,
@@ -430,15 +429,20 @@ class RegisterRestServlet(RestServlet):
raise SynapseError(400, "Invalid username")
desired_username = body["username"]
- appservice = None
- if self.auth.has_access_token(request):
- appservice = self.auth.get_appservice_by_req(request)
-
# fork off as soon as possible for ASes which have completely
# different registration flows to normal users
# == Application Service Registration ==
- if appservice:
+ if body.get("type") == APP_SERVICE_REGISTRATION_TYPE:
+ if not self.auth.has_access_token(request):
+ raise SynapseError(
+ 400,
+ "Appservice token must be provided when using a type of m.login.application_service",
+ )
+
+ # Verify the AS
+ self.auth.get_appservice_by_req(request)
+
# Set the desired user according to the AS API (which uses the
# 'user' key not 'username'). Since this is a new addition, we'll
# fallback to 'username' if they gave one.
@@ -459,6 +463,11 @@ class RegisterRestServlet(RestServlet):
)
return 200, result
+ elif self.auth.has_access_token(request):
+ raise SynapseError(
+ 400,
+ "An access token should not be provided on requests to /register (except if type is m.login.application_service)",
+ )
# == Normal User Registration == (everyone else)
if not self._registration_enabled:
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index 94590e7b45..77ef29ec71 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -488,7 +488,7 @@ class DatabasePool:
exception_callbacks: List[_CallbackListEntry],
func: "Callable[..., R]",
*args: Any,
- **kwargs: Any
+ **kwargs: Any,
) -> R:
"""Start a new database transaction with the given connection.
@@ -622,7 +622,7 @@ class DatabasePool:
func: "Callable[..., R]",
*args: Any,
db_autocommit: bool = False,
- **kwargs: Any
+ **kwargs: Any,
) -> R:
"""Starts a transaction on the database and runs a given function
@@ -682,7 +682,7 @@ class DatabasePool:
func: "Callable[..., R]",
*args: Any,
db_autocommit: bool = False,
- **kwargs: Any
+ **kwargs: Any,
) -> R:
"""Wraps the .runWithConnection() method on the underlying db_pool.
@@ -775,7 +775,7 @@ class DatabasePool:
desc: str,
decoder: Optional[Callable[[Cursor], R]],
query: str,
- *args: Any
+ *args: Any,
) -> R:
"""Runs a single query for a result set.
@@ -900,7 +900,7 @@ class DatabasePool:
table: str,
keyvalues: Dict[str, Any],
values: Dict[str, Any],
- insertion_values: Dict[str, Any] = {},
+ insertion_values: Optional[Dict[str, Any]] = None,
desc: str = "simple_upsert",
lock: bool = True,
) -> Optional[bool]:
@@ -927,6 +927,8 @@ class DatabasePool:
Native upserts always return None. Emulated upserts return True if a
new entry was created, False if an existing one was updated.
"""
+ insertion_values = insertion_values or {}
+
attempts = 0
while True:
try:
@@ -964,7 +966,7 @@ class DatabasePool:
table: str,
keyvalues: Dict[str, Any],
values: Dict[str, Any],
- insertion_values: Dict[str, Any] = {},
+ insertion_values: Optional[Dict[str, Any]] = None,
lock: bool = True,
) -> Optional[bool]:
"""
@@ -982,6 +984,8 @@ class DatabasePool:
Native upserts always return None. Emulated upserts return True if a
new entry was created, False if an existing one was updated.
"""
+ insertion_values = insertion_values or {}
+
if self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables:
self.simple_upsert_txn_native_upsert(
txn, table, keyvalues, values, insertion_values=insertion_values
@@ -1003,7 +1007,7 @@ class DatabasePool:
table: str,
keyvalues: Dict[str, Any],
values: Dict[str, Any],
- insertion_values: Dict[str, Any] = {},
+ insertion_values: Optional[Dict[str, Any]] = None,
lock: bool = True,
) -> bool:
"""
@@ -1017,6 +1021,8 @@ class DatabasePool:
Returns True if a new entry was created, False if an existing
one was updated.
"""
+ insertion_values = insertion_values or {}
+
# We need to lock the table :(, unless we're *really* careful
if lock:
self.engine.lock_table(txn, table)
@@ -1077,7 +1083,7 @@ class DatabasePool:
table: str,
keyvalues: Dict[str, Any],
values: Dict[str, Any],
- insertion_values: Dict[str, Any] = {},
+ insertion_values: Optional[Dict[str, Any]] = None,
) -> None:
"""
Use the native UPSERT functionality in recent PostgreSQL versions.
@@ -1090,7 +1096,7 @@ class DatabasePool:
"""
allvalues = {} # type: Dict[str, Any]
allvalues.update(keyvalues)
- allvalues.update(insertion_values)
+ allvalues.update(insertion_values or {})
if not values:
latter = "NOTHING"
@@ -1513,7 +1519,7 @@ class DatabasePool:
column: str,
iterable: Iterable[Any],
retcols: Iterable[str],
- keyvalues: Dict[str, Any] = {},
+ keyvalues: Optional[Dict[str, Any]] = None,
desc: str = "simple_select_many_batch",
batch_size: int = 100,
) -> List[Any]:
@@ -1531,6 +1537,8 @@ class DatabasePool:
desc: description of the transaction, for logging and metrics
batch_size: the number of rows for each select query
"""
+ keyvalues = keyvalues or {}
+
results = [] # type: List[Dict[str, Any]]
if not iterable:
@@ -2059,69 +2067,18 @@ def make_in_list_sql_clause(
KV = TypeVar("KV")
-def make_tuple_comparison_clause(
- database_engine: BaseDatabaseEngine, keys: List[Tuple[str, KV]]
-) -> Tuple[str, List[KV]]:
+def make_tuple_comparison_clause(keys: List[Tuple[str, KV]]) -> Tuple[str, List[KV]]:
"""Returns a tuple comparison SQL clause
- Depending what the SQL engine supports, builds a SQL clause that looks like either
- "(a, b) > (?, ?)", or "(a > ?) OR (a == ? AND b > ?)".
+ Builds a SQL clause that looks like "(a, b) > (?, ?)"
Args:
- database_engine
keys: A set of (column, value) pairs to be compared.
Returns:
A tuple of SQL query and the args
"""
- if database_engine.supports_tuple_comparison:
- return (
- "(%s) > (%s)" % (",".join(k[0] for k in keys), ",".join("?" for _ in keys)),
- [k[1] for k in keys],
- )
-
- # we want to build a clause
- # (a > ?) OR
- # (a == ? AND b > ?) OR
- # (a == ? AND b == ? AND c > ?)
- # ...
- # (a == ? AND b == ? AND ... AND z > ?)
- #
- # or, equivalently:
- #
- # (a > ? OR (a == ? AND
- # (b > ? OR (b == ? AND
- # ...
- # (y > ? OR (y == ? AND
- # z > ?
- # ))
- # ...
- # ))
- # ))
- #
- # which itself is equivalent to (and apparently easier for the query optimiser):
- #
- # (a >= ? AND (a > ? OR
- # (b >= ? AND (b > ? OR
- # ...
- # (y >= ? AND (y > ? OR
- # z > ?
- # ))
- # ...
- # ))
- # ))
- #
- #
-
- clause = ""
- args = [] # type: List[KV]
- for k, v in keys[:-1]:
- clause = clause + "(%s >= ? AND (%s > ? OR " % (k, k)
- args.extend([v, v])
-
- (k, v) = keys[-1]
- clause += "%s > ?" % (k,)
- args.append(v)
-
- clause += "))" * (len(keys) - 1)
- return clause, args
+ return (
+ "(%s) > (%s)" % (",".join(k[0] for k in keys), ",".join("?" for _ in keys)),
+ [k[1] for k in keys],
+ )
diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py
index ebf6cdfedf..e832c6c25d 100644
--- a/synapse/storage/databases/main/client_ips.py
+++ b/synapse/storage/databases/main/client_ips.py
@@ -298,7 +298,6 @@ class ClientIpBackgroundUpdateStore(SQLBaseStore):
# times, which is fine.
where_clause, where_args = make_tuple_comparison_clause(
- self.database_engine,
[("user_id", last_user_id), ("device_id", last_device_id)],
)
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index d327e9aa0b..9bf8ba888f 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -985,7 +985,7 @@ class DeviceBackgroundUpdateStore(SQLBaseStore):
def _txn(txn):
clause, args = make_tuple_comparison_clause(
- self.db_pool.engine, [(x, last_row[x]) for x in KEY_COLS]
+ [(x, last_row[x]) for x in KEY_COLS]
)
sql = """
SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 98dac19a95..ad17123915 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -320,8 +320,8 @@ class PersistEventsStore:
txn: LoggingTransaction,
events_and_contexts: List[Tuple[EventBase, EventContext]],
backfilled: bool,
- state_delta_for_room: Dict[str, DeltaState] = {},
- new_forward_extremeties: Dict[str, List[str]] = {},
+ state_delta_for_room: Optional[Dict[str, DeltaState]] = None,
+ new_forward_extremeties: Optional[Dict[str, List[str]]] = None,
):
"""Insert some number of room events into the necessary database tables.
@@ -342,6 +342,9 @@ class PersistEventsStore:
extremities.
"""
+ state_delta_for_room = state_delta_for_room or {}
+ new_forward_extremeties = new_forward_extremeties or {}
+
all_events_and_contexts = events_and_contexts
min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index 78367ea58d..79e7df6ca9 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -838,7 +838,6 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
# We want to do a `(topological_ordering, stream_ordering) > (?,?)`
# comparison, but that is not supported on older SQLite versions
tuple_clause, tuple_args = make_tuple_comparison_clause(
- self.database_engine,
[
("events.room_id", last_room_id),
("topological_ordering", last_depth),
diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py
index 8f462dfc31..bd7826f4e9 100644
--- a/synapse/storage/databases/main/group_server.py
+++ b/synapse/storage/databases/main/group_server.py
@@ -1171,7 +1171,7 @@ class GroupServerStore(GroupServerWorkerStore):
user_id: str,
membership: str,
is_admin: bool = False,
- content: JsonDict = {},
+ content: Optional[JsonDict] = None,
local_attestation: Optional[dict] = None,
remote_attestation: Optional[dict] = None,
is_publicised: bool = False,
@@ -1192,6 +1192,8 @@ class GroupServerStore(GroupServerWorkerStore):
is_publicised: Whether this should be publicised.
"""
+ content = content or {}
+
def _register_user_group_membership_txn(txn, next_id):
# TODO: Upsert?
self.db_pool.simple_delete_txn(
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 9cbcd53026..47fb12f3f6 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -521,13 +521,11 @@ class RoomWorkerStore(SQLBaseStore):
)
@cached(max_entries=10000)
- async def get_ratelimit_for_user(self, user_id):
- """Check if there are any overrides for ratelimiting for the given
- user
+ async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]:
+ """Check if there are any overrides for ratelimiting for the given user
Args:
- user_id (str)
-
+ user_id: user ID of the user
Returns:
RatelimitOverride if there is an override, else None. If the contents
of RatelimitOverride are None or 0 then ratelimitng has been
@@ -549,6 +547,62 @@ class RoomWorkerStore(SQLBaseStore):
else:
return None
+ async def set_ratelimit_for_user(
+ self, user_id: str, messages_per_second: int, burst_count: int
+ ) -> None:
+ """Sets whether a user is set an overridden ratelimit.
+ Args:
+ user_id: user ID of the user
+ messages_per_second: The number of actions that can be performed in a second.
+ burst_count: How many actions that can be performed before being limited.
+ """
+
+ def set_ratelimit_txn(txn):
+ self.db_pool.simple_upsert_txn(
+ txn,
+ table="ratelimit_override",
+ keyvalues={"user_id": user_id},
+ values={
+ "messages_per_second": messages_per_second,
+ "burst_count": burst_count,
+ },
+ )
+
+ self._invalidate_cache_and_stream(
+ txn, self.get_ratelimit_for_user, (user_id,)
+ )
+
+ await self.db_pool.runInteraction("set_ratelimit", set_ratelimit_txn)
+
+ async def delete_ratelimit_for_user(self, user_id: str) -> None:
+ """Delete an overridden ratelimit for a user.
+ Args:
+ user_id: user ID of the user
+ """
+
+ def delete_ratelimit_txn(txn):
+ row = self.db_pool.simple_select_one_txn(
+ txn,
+ table="ratelimit_override",
+ keyvalues={"user_id": user_id},
+ retcols=["user_id"],
+ allow_none=True,
+ )
+
+ if not row:
+ return
+
+ # They are there, delete them.
+ self.db_pool.simple_delete_one_txn(
+ txn, "ratelimit_override", keyvalues={"user_id": user_id}
+ )
+
+ self._invalidate_cache_and_stream(
+ txn, self.get_ratelimit_for_user, (user_id,)
+ )
+
+ await self.db_pool.runInteraction("delete_ratelimit", delete_ratelimit_txn)
+
@cached()
async def get_retention_policy_for_room(self, room_id):
"""Get the retention policy for a given room.
diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py
index a7f371732f..93431efe00 100644
--- a/synapse/storage/databases/main/state.py
+++ b/synapse/storage/databases/main/state.py
@@ -190,7 +190,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
# FIXME: how should this be cached?
async def get_filtered_current_state_ids(
- self, room_id: str, state_filter: StateFilter = StateFilter.all()
+ self, room_id: str, state_filter: Optional[StateFilter] = None
) -> StateMap[str]:
"""Get the current state event of a given type for a room based on the
current_state_events table. This may not be as up-to-date as the result
@@ -205,7 +205,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
Map from type/state_key to event ID.
"""
- where_clause, where_args = state_filter.make_sql_filter_clause()
+ where_clause, where_args = (
+ state_filter or StateFilter.all()
+ ).make_sql_filter_clause()
if not where_clause:
# We delegate to the cached version
diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py
index 1fd333b707..75c09b3687 100644
--- a/synapse/storage/databases/state/bg_updates.py
+++ b/synapse/storage/databases/state/bg_updates.py
@@ -14,6 +14,7 @@
# limitations under the License.
import logging
+from typing import Optional
from synapse.storage._base import SQLBaseStore
from synapse.storage.database import DatabasePool
@@ -73,8 +74,10 @@ class StateGroupBackgroundUpdateStore(SQLBaseStore):
return count
def _get_state_groups_from_groups_txn(
- self, txn, groups, state_filter=StateFilter.all()
+ self, txn, groups, state_filter: Optional[StateFilter] = None
):
+ state_filter = state_filter or StateFilter.all()
+
results = {group: {} for group in groups}
where_clause, where_args = state_filter.make_sql_filter_clause()
diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py
index 97ec65f757..dfcf89d91c 100644
--- a/synapse/storage/databases/state/store.py
+++ b/synapse/storage/databases/state/store.py
@@ -15,7 +15,7 @@
import logging
from collections import namedtuple
-from typing import Dict, Iterable, List, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple
from synapse.api.constants import EventTypes
from synapse.storage._base import SQLBaseStore
@@ -210,7 +210,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
return state_filter.filter_state(state_dict_ids), not missing_types
async def _get_state_for_groups(
- self, groups: Iterable[int], state_filter: StateFilter = StateFilter.all()
+ self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
) -> Dict[int, MutableStateMap[str]]:
"""Gets the state at each of a list of state groups, optionally
filtering by type/state_key
@@ -223,6 +223,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
Returns:
Dict of state group to state map.
"""
+ state_filter = state_filter or StateFilter.all()
member_filter, non_member_filter = state_filter.get_member_split()
diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py
index cca839c70f..21db1645d3 100644
--- a/synapse/storage/engines/_base.py
+++ b/synapse/storage/engines/_base.py
@@ -44,14 +44,6 @@ class BaseDatabaseEngine(Generic[ConnectionType], metaclass=abc.ABCMeta):
@property
@abc.abstractmethod
- def supports_tuple_comparison(self) -> bool:
- """
- Do we support comparing tuples, i.e. `(a, b) > (c, d)`?
- """
- ...
-
- @property
- @abc.abstractmethod
def supports_using_any_list(self) -> bool:
"""
Do we support using `a = ANY(?)` and passing a list
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 80a3558aec..dba8cc51d3 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -47,8 +47,8 @@ class PostgresEngine(BaseDatabaseEngine):
self._version = db_conn.server_version
# Are we on a supported PostgreSQL version?
- if not allow_outdated_version and self._version < 90500:
- raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.")
+ if not allow_outdated_version and self._version < 90600:
+ raise RuntimeError("Synapse requires PostgreSQL 9.6 or above.")
with db_conn.cursor() as txn:
txn.execute("SHOW SERVER_ENCODING")
@@ -130,13 +130,6 @@ class PostgresEngine(BaseDatabaseEngine):
return True
@property
- def supports_tuple_comparison(self):
- """
- Do we support comparing tuples, i.e. `(a, b) > (c, d)`?
- """
- return True
-
- @property
def supports_using_any_list(self):
"""Do we support using `a = ANY(?)` and passing a list"""
return True
diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index b87e7798da..f4f16456f2 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -57,14 +57,6 @@ class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]):
return self.module.sqlite_version_info >= (3, 24, 0)
@property
- def supports_tuple_comparison(self):
- """
- Do we support comparing tuples, i.e. `(a, b) > (c, d)`? This requires
- SQLite 3.15+.
- """
- return self.module.sqlite_version_info >= (3, 15, 0)
-
- @property
def supports_using_any_list(self):
"""Do we support using `a = ANY(?)` and passing a list"""
return False
@@ -72,8 +64,11 @@ class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]):
def check_database(self, db_conn, allow_outdated_version: bool = False):
if not allow_outdated_version:
version = self.module.sqlite_version_info
- if version < (3, 11, 0):
- raise RuntimeError("Synapse requires sqlite 3.11 or above.")
+ # Synapse is untested against older SQLite versions, and we don't want
+ # to let users upgrade to a version of Synapse with broken support for their
+ # sqlite version, because it risks leaving them with a half-upgraded db.
+ if version < (3, 22, 0):
+ raise RuntimeError("Synapse requires sqlite 3.22 or above.")
def check_new_database(self, txn):
"""Gets called when setting up a brand new database. This allows us to
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 2e277a21c4..c1c147c62a 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -449,7 +449,7 @@ class StateGroupStorage:
return self.stores.state._get_state_groups_from_groups(groups, state_filter)
async def get_state_for_events(
- self, event_ids: Iterable[str], state_filter: StateFilter = StateFilter.all()
+ self, event_ids: Iterable[str], state_filter: Optional[StateFilter] = None
) -> Dict[str, StateMap[EventBase]]:
"""Given a list of event_ids and type tuples, return a list of state
dicts for each event.
@@ -465,7 +465,7 @@ class StateGroupStorage:
groups = set(event_to_groups.values())
group_to_state = await self.stores.state._get_state_for_groups(
- groups, state_filter
+ groups, state_filter or StateFilter.all()
)
state_event_map = await self.stores.main.get_events(
@@ -485,7 +485,7 @@ class StateGroupStorage:
return {event: event_to_state[event] for event in event_ids}
async def get_state_ids_for_events(
- self, event_ids: Iterable[str], state_filter: StateFilter = StateFilter.all()
+ self, event_ids: Iterable[str], state_filter: Optional[StateFilter] = None
) -> Dict[str, StateMap[str]]:
"""
Get the state dicts corresponding to a list of events, containing the event_ids
@@ -502,7 +502,7 @@ class StateGroupStorage:
groups = set(event_to_groups.values())
group_to_state = await self.stores.state._get_state_for_groups(
- groups, state_filter
+ groups, state_filter or StateFilter.all()
)
event_to_state = {
@@ -513,7 +513,7 @@ class StateGroupStorage:
return {event: event_to_state[event] for event in event_ids}
async def get_state_for_event(
- self, event_id: str, state_filter: StateFilter = StateFilter.all()
+ self, event_id: str, state_filter: Optional[StateFilter] = None
) -> StateMap[EventBase]:
"""
Get the state dict corresponding to a particular event
@@ -525,11 +525,13 @@ class StateGroupStorage:
Returns:
A dict from (type, state_key) -> state_event
"""
- state_map = await self.get_state_for_events([event_id], state_filter)
+ state_map = await self.get_state_for_events(
+ [event_id], state_filter or StateFilter.all()
+ )
return state_map[event_id]
async def get_state_ids_for_event(
- self, event_id: str, state_filter: StateFilter = StateFilter.all()
+ self, event_id: str, state_filter: Optional[StateFilter] = None
) -> StateMap[str]:
"""
Get the state dict corresponding to a particular event
@@ -541,11 +543,13 @@ class StateGroupStorage:
Returns:
A dict from (type, state_key) -> state_event
"""
- state_map = await self.get_state_ids_for_events([event_id], state_filter)
+ state_map = await self.get_state_ids_for_events(
+ [event_id], state_filter or StateFilter.all()
+ )
return state_map[event_id]
def _get_state_for_groups(
- self, groups: Iterable[int], state_filter: StateFilter = StateFilter.all()
+ self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
) -> Awaitable[Dict[int, MutableStateMap[str]]]:
"""Gets the state at each of a list of state groups, optionally
filtering by type/state_key
@@ -558,7 +562,9 @@ class StateGroupStorage:
Returns:
Dict of state group to state map.
"""
- return self.stores.state._get_state_for_groups(groups, state_filter)
+ return self.stores.state._get_state_for_groups(
+ groups, state_filter or StateFilter.all()
+ )
async def store_state_group(
self,
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index d4643c4fdf..32d6cc16b9 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -17,7 +17,7 @@ import logging
import threading
from collections import OrderedDict
from contextlib import contextmanager
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
import attr
@@ -91,7 +91,14 @@ class StreamIdGenerator:
# ... persist event ...
"""
- def __init__(self, db_conn, table, column, extra_tables=[], step=1):
+ def __init__(
+ self,
+ db_conn,
+ table,
+ column,
+ extra_tables: Iterable[Tuple[str, str]] = (),
+ step=1,
+ ):
assert step != 0
self._lock = threading.Lock()
self._step = step
diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py
index 60bb6ff642..20c8e2d9f5 100644
--- a/synapse/util/caches/lrucache.py
+++ b/synapse/util/caches/lrucache.py
@@ -57,12 +57,14 @@ def enumerate_leaves(node, depth):
class _Node:
__slots__ = ["prev_node", "next_node", "key", "value", "callbacks"]
- def __init__(self, prev_node, next_node, key, value, callbacks=set()):
+ def __init__(
+ self, prev_node, next_node, key, value, callbacks: Optional[set] = None
+ ):
self.prev_node = prev_node
self.next_node = next_node
self.key = key
self.value = value
- self.callbacks = callbacks
+ self.callbacks = callbacks or set()
class LruCache(Generic[KT, VT]):
@@ -176,10 +178,10 @@ class LruCache(Generic[KT, VT]):
self.len = synchronized(cache_len)
- def add_node(key, value, callbacks=set()):
+ def add_node(key, value, callbacks: Optional[set] = None):
prev_node = list_root
next_node = prev_node.next_node
- node = _Node(prev_node, next_node, key, value, callbacks)
+ node = _Node(prev_node, next_node, key, value, callbacks or set())
prev_node.next_node = node
next_node.prev_node = node
cache[key] = node
@@ -237,7 +239,7 @@ class LruCache(Generic[KT, VT]):
def cache_get(
key: KT,
default: Optional[T] = None,
- callbacks: Iterable[Callable[[], None]] = [],
+ callbacks: Iterable[Callable[[], None]] = (),
update_metrics: bool = True,
):
node = cache.get(key, None)
@@ -253,7 +255,7 @@ class LruCache(Generic[KT, VT]):
return default
@synchronized
- def cache_set(key: KT, value: VT, callbacks: Iterable[Callable[[], None]] = []):
+ def cache_set(key: KT, value: VT, callbacks: Iterable[Callable[[], None]] = ()):
node = cache.get(key, None)
if node is not None:
# We sometimes store large objects, e.g. dicts, which cause
diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py
index c306891b27..b3abc6b254 100644
--- a/synmark/suites/logging.py
+++ b/synmark/suites/logging.py
@@ -16,8 +16,7 @@
import logging
import warnings
from io import StringIO
-
-from mock import Mock
+from unittest.mock import Mock
from pyperf import perf_counter
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 34f72ae795..28d77f0ca2 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
import pymacaroons
diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py
index 467033e201..33a37fe35e 100644
--- a/tests/app/test_openid_listener.py
+++ b/tests/app/test_openid_listener.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock, patch
+from unittest.mock import Mock, patch
from parameterized import parameterized
diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py
index 0bffeb1150..03a7440eec 100644
--- a/tests/appservice/test_appservice.py
+++ b/tests/appservice/test_appservice.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import re
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py
index 97f8cad0dd..3c27d797fb 100644
--- a/tests/appservice/test_scheduler.py
+++ b/tests/appservice/test_scheduler.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 946482b7e7..a56063315b 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import time
-
-from mock import Mock
+from unittest.mock import Mock
import attr
import canonicaljson
diff --git a/tests/events/test_presence_router.py b/tests/events/test_presence_router.py
index c6e547f11c..c996ecc221 100644
--- a/tests/events/test_presence_router.py
+++ b/tests/events/test_presence_router.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
-
-from mock import Mock
+from unittest.mock import Mock
import attr
@@ -314,7 +313,8 @@ class PresenceRouterTestCase(FederatingHomeserverTestCase):
self.hs.get_federation_transport_client().send_transaction.call_args_list
)
for call in calls:
- federation_transaction = call.args[0] # type: Transaction
+ call_args = call[0]
+ federation_transaction = call_args[0] # type: Transaction
# Get the sent EDUs in this transaction
edus = federation_transaction.get_dict()["edus"]
diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py
index 8186b8ca01..701fa8379f 100644
--- a/tests/federation/test_complexity.py
+++ b/tests/federation/test_complexity.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.errors import Codes, SynapseError
from synapse.rest import admin
diff --git a/tests/federation/test_federation_catch_up.py b/tests/federation/test_federation_catch_up.py
index 95eac6a5a3..802c5ad299 100644
--- a/tests/federation/test_federation_catch_up.py
+++ b/tests/federation/test_federation_catch_up.py
@@ -1,6 +1,5 @@
from typing import List, Tuple
-
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.constants import EventTypes
from synapse.events import EventBase
diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py
index ecc3faa572..deb12433cf 100644
--- a/tests/federation/test_federation_sender.py
+++ b/tests/federation/test_federation_sender.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Optional
-
-from mock import Mock
+from unittest.mock import Mock
from signedjson import key, sign
from signedjson.types import BaseKey, SigningKey
diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py
index a01fdd0839..32669ae9ce 100644
--- a/tests/handlers/test_admin.py
+++ b/tests/handlers/test_admin.py
@@ -14,8 +14,7 @@
# limitations under the License.
from collections import Counter
-
-from mock import Mock
+from unittest.mock import Mock
import synapse.api.errors
import synapse.handlers.admin
diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py
index d5d3fdd99a..6e325b24ce 100644
--- a/tests/handlers/test_appservice.py
+++ b/tests/handlers/test_appservice.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index c9f889b511..321c5ba045 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
import pymacaroons
diff --git a/tests/handlers/test_cas.py b/tests/handlers/test_cas.py
index 7975af243c..0444b26798 100644
--- a/tests/handlers/test_cas.py
+++ b/tests/handlers/test_cas.py
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.handlers.cas_handler import CasResponse
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 863d8737b2..6ae9d4f865 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
import synapse
import synapse.api.errors
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index 5e86c5e56b..6915ac0205 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -14,7 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+from unittest import mock
from signedjson import key as key, sign as sign
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index d7498aa51a..07893302ec 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -16,8 +16,7 @@
# limitations under the License.
import copy
-
-import mock
+from unittest import mock
from synapse.api.errors import SynapseError
diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py
index c7796fb837..8702ee70e0 100644
--- a/tests/handlers/test_oidc.py
+++ b/tests/handlers/test_oidc.py
@@ -14,10 +14,9 @@
# limitations under the License.
import json
import os
+from unittest.mock import ANY, Mock, patch
from urllib.parse import parse_qs, urlparse
-from mock import ANY, Mock, patch
-
import pymacaroons
from synapse.handlers.sso import MappingException
diff --git a/tests/handlers/test_password_providers.py b/tests/handlers/test_password_providers.py
index a98a65ae67..e28e4159eb 100644
--- a/tests/handlers/test_password_providers.py
+++ b/tests/handlers/test_password_providers.py
@@ -16,8 +16,7 @@
"""Tests for the password_auth_provider interface"""
from typing import Any, Type, Union
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 77330f59a9..9f16cc65fc 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from mock import Mock, call
+from unittest.mock import Mock, call
from signedjson.key import generate_signing_key
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index 75c6a4e21c..d8b1bcac8b 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
import synapse.types
from synapse.api.errors import AuthError, SynapseError
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 94b6903594..69279a5ce9 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.auth import Auth
from synapse.api.constants import UserTypes
diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py
index 30efd43b40..8cfc184fef 100644
--- a/tests/handlers/test_saml.py
+++ b/tests/handlers/test_saml.py
@@ -13,8 +13,7 @@
# limitations under the License.
from typing import Optional
-
-from mock import Mock
+from unittest.mock import Mock
import attr
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index 24e7138196..9fa231a37a 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -16,8 +16,7 @@
import json
from typing import Dict
-
-from mock import ANY, Mock, call
+from unittest.mock import ANY, Mock, call
from twisted.internet import defer
from twisted.web.resource import Resource
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index 98b2f5b383..c68cb830af 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index 4c56253da5..ae9d4504a8 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-
-from mock import Mock
+from typing import Optional
+from unittest.mock import Mock
import treq
from netaddr import IPSet
@@ -180,7 +180,11 @@ class MatrixFederationAgentTests(unittest.TestCase):
_check_logcontext(context)
def _handle_well_known_connection(
- self, client_factory, expected_sni, content, response_headers={}
+ self,
+ client_factory,
+ expected_sni,
+ content,
+ response_headers: Optional[dict] = None,
):
"""Handle an outgoing HTTPs connection: wire it up to a server, check that the
request is for a .well-known, and send the response.
@@ -202,10 +206,12 @@ class MatrixFederationAgentTests(unittest.TestCase):
self.assertEqual(
request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
)
- self._send_well_known_response(request, content, headers=response_headers)
+ self._send_well_known_response(request, content, headers=response_headers or {})
return well_known_server
- def _send_well_known_response(self, request, content, headers={}):
+ def _send_well_known_response(
+ self, request, content, headers: Optional[dict] = None
+ ):
"""Check that an incoming request looks like a valid .well-known request, and
send back the response.
"""
@@ -213,7 +219,7 @@ class MatrixFederationAgentTests(unittest.TestCase):
self.assertEqual(request.path, b"/.well-known/matrix/server")
self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
# send back a response
- for k, v in headers.items():
+ for k, v in (headers or {}).items():
request.setHeader(k, v)
request.write(content)
request.finish()
diff --git a/tests/http/federation/test_srv_resolver.py b/tests/http/federation/test_srv_resolver.py
index fee2985d35..466ce722d9 100644
--- a/tests/http/federation/test_srv_resolver.py
+++ b/tests/http/federation/test_srv_resolver.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
from twisted.internet.defer import Deferred
diff --git a/tests/http/test_client.py b/tests/http/test_client.py
index 0ce181a51e..7e2f2a01cc 100644
--- a/tests/http/test_client.py
+++ b/tests/http/test_client.py
@@ -13,8 +13,7 @@
# limitations under the License.
from io import BytesIO
-
-from mock import Mock
+from unittest.mock import Mock
from netaddr import IPSet
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index 9c52c8fdca..21c1297171 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from netaddr import IPSet
from parameterized import parameterized
diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py
index 45089158ce..f979c96f7c 100644
--- a/tests/http/test_servlet.py
+++ b/tests/http/test_servlet.py
@@ -14,8 +14,7 @@
# limitations under the License.
import json
from io import BytesIO
-
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.errors import SynapseError
from synapse.http.servlet import (
diff --git a/tests/http/test_simple_client.py b/tests/http/test_simple_client.py
index a1cf0862d4..cc4cae320d 100644
--- a/tests/http/test_simple_client.py
+++ b/tests/http/test_simple_client.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from netaddr import IPSet
diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py
index bfe0d11c93..215fd8b0f9 100644
--- a/tests/logging/test_terse_json.py
+++ b/tests/logging/test_terse_json.py
@@ -15,8 +15,7 @@
import json
import logging
from io import BytesIO, StringIO
-
-from mock import Mock, patch
+from unittest.mock import Mock, patch
from twisted.web.server import Request
diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py
index 1d1fceeecf..349f93560e 100644
--- a/tests/module_api/test_api.py
+++ b/tests/module_api/test_api.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.constants import EduTypes
from synapse.events import EventBase
@@ -358,7 +358,8 @@ class ModuleApiTestCase(FederatingHomeserverTestCase):
self.hs.get_federation_transport_client().send_transaction.call_args_list
)
for call in calls:
- federation_transaction = call.args[0] # type: Transaction
+ call_args = call[0]
+ federation_transaction = call_args[0] # type: Transaction
# Get the sent EDUs in this transaction
edus = federation_transaction.get_dict()["edus"]
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index 60f0820cff..4074ade87a 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet.defer import Deferred
diff --git a/tests/replication/_base.py b/tests/replication/_base.py
index 1d4a592862..aff19d9fb3 100644
--- a/tests/replication/_base.py
+++ b/tests/replication/_base.py
@@ -266,7 +266,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
return resource
def make_worker_hs(
- self, worker_app: str, extra_config: dict = {}, **kwargs
+ self, worker_app: str, extra_config: Optional[dict] = None, **kwargs
) -> HomeServer:
"""Make a new worker HS instance, correctly connecting replcation
stream to the master HS.
@@ -283,7 +283,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
config = self._get_worker_hs_config()
config["worker_app"] = worker_app
- config.update(extra_config)
+ config.update(extra_config or {})
worker_hs = self.setup_test_homeserver(
homeserver_to_use=GenericWorkerServer,
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 56497b8476..83e89383f6 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from tests.replication._base import BaseStreamTestCase
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index 0ceb0f935c..db80a0bdbd 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
+from typing import Iterable, Optional
from canonicaljson import encode_canonical_json
@@ -332,15 +333,18 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
room_id=ROOM_ID,
type="m.room.message",
key=None,
- internal={},
+ internal: Optional[dict] = None,
depth=None,
- prev_events=[],
- auth_events=[],
- prev_state=[],
+ prev_events: Optional[list] = None,
+ auth_events: Optional[list] = None,
+ prev_state: Optional[list] = None,
redacts=None,
- push_actions=[],
- **content
+ push_actions: Iterable = frozenset(),
+ **content,
):
+ prev_events = prev_events or []
+ auth_events = auth_events or []
+ prev_state = prev_state or []
if depth is None:
depth = self.event_id
@@ -369,7 +373,7 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
if redacts is not None:
event_dict["redacts"] = redacts
- event = make_event_from_dict(event_dict, internal_metadata_dict=internal)
+ event = make_event_from_dict(event_dict, internal_metadata_dict=internal or {})
self.event_id += 1
state_handler = self.hs.get_state_handler()
diff --git a/tests/replication/tcp/streams/test_receipts.py b/tests/replication/tcp/streams/test_receipts.py
index 56b062ecc1..7d848e41ff 100644
--- a/tests/replication/tcp/streams/test_receipts.py
+++ b/tests/replication/tcp/streams/test_receipts.py
@@ -15,7 +15,7 @@
# type: ignore
-from mock import Mock
+from unittest.mock import Mock
from synapse.replication.tcp.streams._base import ReceiptsStream
diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py
index ca49d4dd3a..4a0b342264 100644
--- a/tests/replication/tcp/streams/test_typing.py
+++ b/tests/replication/tcp/streams/test_typing.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.handlers.typing import RoomMember
from synapse.replication.tcp.streams import TypingStream
diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py
index 0d9e3bb11d..44ad5eec57 100644
--- a/tests/replication/test_federation_ack.py
+++ b/tests/replication/test_federation_ack.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+from unittest import mock
from synapse.app.generic_worker import GenericWorkerServer
from synapse.replication.tcp.commands import FederationAckCommand
diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py
index 2f2d117858..8ca595c3ee 100644
--- a/tests/replication/test_federation_sender_shard.py
+++ b/tests/replication/test_federation_sender_shard.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.constants import EventTypes, Membership
from synapse.events.builder import EventBuilderFactory
diff --git a/tests/replication/test_pusher_shard.py b/tests/replication/test_pusher_shard.py
index ab2988a6ba..1f12bde1aa 100644
--- a/tests/replication/test_pusher_shard.py
+++ b/tests/replication/test_pusher_shard.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py
index c9b773fbd2..6c2e1674cb 100644
--- a/tests/replication/test_sharded_event_persister.py
+++ b/tests/replication/test_sharded_event_persister.py
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-
-from mock import patch
+from unittest.mock import patch
from synapse.api.room_versions import RoomVersion
from synapse.rest import admin
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
index 057e27372e..4abcbe3f55 100644
--- a/tests/rest/admin/test_admin.py
+++ b/tests/rest/admin/test_admin.py
@@ -17,8 +17,7 @@ import json
import os
import urllib.parse
from binascii import unhexlify
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet.defer import Deferred
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index b55160b70a..85f77c0a65 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -16,8 +16,7 @@
import json
import urllib.parse
from typing import List, Optional
-
-from mock import Mock
+from unittest.mock import Mock
import synapse.rest.admin
from synapse.api.constants import EventTypes, Membership
diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py
index 0c9ec133c2..5070c96984 100644
--- a/tests/rest/admin/test_user.py
+++ b/tests/rest/admin/test_user.py
@@ -19,8 +19,7 @@ import json
import urllib.parse
from binascii import unhexlify
from typing import List, Optional
-
-from mock import Mock
+from unittest.mock import Mock
import synapse.rest.admin
from synapse.api.constants import UserTypes
@@ -3012,3 +3011,287 @@ class ShadowBanRestTestCase(unittest.HomeserverTestCase):
# Ensure the user is shadow-banned (and the cache was cleared).
result = self.get_success(self.store.get_user_by_access_token(other_user_token))
self.assertTrue(result.shadow_banned)
+
+
+class RateLimitTestCase(unittest.HomeserverTestCase):
+
+ servlets = [
+ synapse.rest.admin.register_servlets,
+ login.register_servlets,
+ ]
+
+ def prepare(self, reactor, clock, hs):
+ self.store = hs.get_datastore()
+
+ self.admin_user = self.register_user("admin", "pass", admin=True)
+ self.admin_user_tok = self.login("admin", "pass")
+
+ self.other_user = self.register_user("user", "pass")
+ self.url = (
+ "/_synapse/admin/v1/users/%s/override_ratelimit"
+ % urllib.parse.quote(self.other_user)
+ )
+
+ def test_no_auth(self):
+ """
+ Try to get information of a user without authentication.
+ """
+ channel = self.make_request("GET", self.url, b"{}")
+
+ self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+ channel = self.make_request("POST", self.url, b"{}")
+
+ self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+ channel = self.make_request("DELETE", self.url, b"{}")
+
+ self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+ def test_requester_is_no_admin(self):
+ """
+ If the user is not a server admin, an error is returned.
+ """
+ other_user_token = self.login("user", "pass")
+
+ channel = self.make_request(
+ "GET",
+ self.url,
+ access_token=other_user_token,
+ )
+
+ self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=other_user_token,
+ )
+
+ self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+ channel = self.make_request(
+ "DELETE",
+ self.url,
+ access_token=other_user_token,
+ )
+
+ self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+ def test_user_does_not_exist(self):
+ """
+ Tests that a lookup for a user that does not exist returns a 404
+ """
+ url = "/_synapse/admin/v1/users/@unknown_person:test/override_ratelimit"
+
+ channel = self.make_request(
+ "GET",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(404, channel.code, msg=channel.json_body)
+ self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+ channel = self.make_request(
+ "POST",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(404, channel.code, msg=channel.json_body)
+ self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+ channel = self.make_request(
+ "DELETE",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(404, channel.code, msg=channel.json_body)
+ self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+ def test_user_is_not_local(self):
+ """
+ Tests that a lookup for a user that is not a local returns a 400
+ """
+ url = (
+ "/_synapse/admin/v1/users/@unknown_person:unknown_domain/override_ratelimit"
+ )
+
+ channel = self.make_request(
+ "GET",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(400, channel.code, msg=channel.json_body)
+ self.assertEqual("Can only lookup local users", channel.json_body["error"])
+
+ channel = self.make_request(
+ "POST",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(400, channel.code, msg=channel.json_body)
+ self.assertEqual(
+ "Only local users can be ratelimited", channel.json_body["error"]
+ )
+
+ channel = self.make_request(
+ "DELETE",
+ url,
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(400, channel.code, msg=channel.json_body)
+ self.assertEqual(
+ "Only local users can be ratelimited", channel.json_body["error"]
+ )
+
+ def test_invalid_parameter(self):
+ """
+ If parameters are invalid, an error is returned.
+ """
+ # messages_per_second is a string
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"messages_per_second": "string"},
+ )
+
+ self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+ # messages_per_second is negative
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"messages_per_second": -1},
+ )
+
+ self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+ # burst_count is a string
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"burst_count": "string"},
+ )
+
+ self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+ # burst_count is negative
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"burst_count": -1},
+ )
+
+ self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+ def test_return_zero_when_null(self):
+ """
+ If values in database are `null` API should return an int `0`
+ """
+
+ self.get_success(
+ self.store.db_pool.simple_upsert(
+ table="ratelimit_override",
+ keyvalues={"user_id": self.other_user},
+ values={
+ "messages_per_second": None,
+ "burst_count": None,
+ },
+ )
+ )
+
+ # request status
+ channel = self.make_request(
+ "GET",
+ self.url,
+ access_token=self.admin_user_tok,
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(0, channel.json_body["messages_per_second"])
+ self.assertEqual(0, channel.json_body["burst_count"])
+
+ def test_success(self):
+ """
+ Rate-limiting (set/update/delete) should succeed for an admin.
+ """
+ # request status
+ channel = self.make_request(
+ "GET",
+ self.url,
+ access_token=self.admin_user_tok,
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertNotIn("messages_per_second", channel.json_body)
+ self.assertNotIn("burst_count", channel.json_body)
+
+ # set ratelimit
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"messages_per_second": 10, "burst_count": 11},
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(10, channel.json_body["messages_per_second"])
+ self.assertEqual(11, channel.json_body["burst_count"])
+
+ # update ratelimit
+ channel = self.make_request(
+ "POST",
+ self.url,
+ access_token=self.admin_user_tok,
+ content={"messages_per_second": 20, "burst_count": 21},
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(20, channel.json_body["messages_per_second"])
+ self.assertEqual(21, channel.json_body["burst_count"])
+
+ # request status
+ channel = self.make_request(
+ "GET",
+ self.url,
+ access_token=self.admin_user_tok,
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertEqual(20, channel.json_body["messages_per_second"])
+ self.assertEqual(21, channel.json_body["burst_count"])
+
+ # delete ratelimit
+ channel = self.make_request(
+ "DELETE",
+ self.url,
+ access_token=self.admin_user_tok,
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertNotIn("messages_per_second", channel.json_body)
+ self.assertNotIn("burst_count", channel.json_body)
+
+ # request status
+ channel = self.make_request(
+ "GET",
+ self.url,
+ access_token=self.admin_user_tok,
+ )
+ self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+ self.assertNotIn("messages_per_second", channel.json_body)
+ self.assertNotIn("burst_count", channel.json_body)
diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py
index aee99bb6a0..f892a71228 100644
--- a/tests/rest/client/test_retention.py
+++ b/tests/rest/client/test_retention.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.api.constants import EventTypes
from synapse.rest import admin
diff --git a/tests/rest/client/test_shadow_banned.py b/tests/rest/client/test_shadow_banned.py
index d2cce44032..288ee12888 100644
--- a/tests/rest/client/test_shadow_banned.py
+++ b/tests/rest/client/test_shadow_banned.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock, patch
+from unittest.mock import Mock, patch
import synapse.rest.admin
from synapse.api.constants import EventTypes
diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py
index bf39014277..a7ebe0c3e9 100644
--- a/tests/rest/client/test_third_party_rules.py
+++ b/tests/rest/client/test_third_party_rules.py
@@ -14,8 +14,7 @@
# limitations under the License.
import threading
from typing import Dict
-
-from mock import Mock
+from unittest.mock import Mock
from synapse.events import EventBase
from synapse.module_api import ModuleApi
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index 171632e195..3b5747cb12 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -1,4 +1,4 @@
-from mock import Mock, call
+from unittest.mock import Mock, call
from twisted.internet import defer, reactor
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 2ae896db1e..87a18d2cb9 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -15,7 +15,7 @@
""" Tests REST events for /events paths."""
-from mock import Mock
+from unittest.mock import Mock
import synapse.rest.admin
from synapse.rest.client.v1 import events, login, room
diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py
index 988821b16f..c7b79ab8a7 100644
--- a/tests/rest/client/v1/test_login.py
+++ b/tests/rest/client/v1/test_login.py
@@ -16,10 +16,9 @@
import time
import urllib.parse
from typing import Any, Dict, List, Optional, Union
+from unittest.mock import Mock
from urllib.parse import urlencode
-from mock import Mock
-
import pymacaroons
from twisted.web.resource import Resource
diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py
index 94a5154834..c136827f79 100644
--- a/tests/rest/client/v1/test_presence.py
+++ b/tests/rest/client/v1/test_presence.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index ed65f645fc..4df20c90fd 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -19,10 +19,10 @@
"""Tests REST events for /rooms paths."""
import json
+from typing import Iterable
+from unittest.mock import Mock
from urllib import parse as urlparse
-from mock import Mock
-
import synapse.rest.admin
from synapse.api.constants import EventContentFields, EventTypes, Membership
from synapse.handlers.pagination import PurgeStatus
@@ -207,7 +207,9 @@ class RoomPermissionsTestCase(RoomBase):
)
self.assertEquals(403, channel.code, msg=channel.result["body"])
- def _test_get_membership(self, room=None, members=[], expect_code=None):
+ def _test_get_membership(
+ self, room=None, members: Iterable = frozenset(), expect_code=None
+ ):
for member in members:
path = "/rooms/%s/state/m.room.member/%s" % (room, member)
channel = self.make_request("GET", path)
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 329dbd06de..0b8f565121 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -16,7 +16,7 @@
"""Tests REST events for /rooms paths."""
-from mock import Mock
+from unittest.mock import Mock
from synapse.rest.client.v1 import room
from synapse.types import UserID
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index 946740aa5d..a6a292b20c 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -21,8 +21,7 @@ import re
import time
import urllib.parse
from typing import Any, Dict, Mapping, MutableMapping, Optional
-
-from mock import patch
+from unittest.mock import patch
import attr
@@ -132,7 +131,7 @@ class RestHelper:
src: str,
targ: str,
membership: str,
- extra_data: dict = {},
+ extra_data: Optional[dict] = None,
tok: Optional[str] = None,
expect_code: int = 200,
) -> None:
@@ -156,7 +155,7 @@ class RestHelper:
path = path + "?access_token=%s" % tok
data = {"membership": membership}
- data.update(extra_data)
+ data.update(extra_data or {})
channel = make_request(
self.hs.get_reactor(),
@@ -187,7 +186,13 @@ class RestHelper:
)
def send_event(
- self, room_id, type, content={}, txn_id=None, tok=None, expect_code=200
+ self,
+ room_id,
+ type,
+ content: Optional[dict] = None,
+ txn_id=None,
+ tok=None,
+ expect_code=200,
):
if txn_id is None:
txn_id = "m%s" % (str(time.time()))
@@ -201,7 +206,7 @@ class RestHelper:
self.site,
"PUT",
path,
- json.dumps(content).encode("utf8"),
+ json.dumps(content or {}).encode("utf8"),
)
assert (
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 27db4f551e..cd60ea7081 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -14,7 +14,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import datetime
import json
import os
@@ -22,7 +21,7 @@ import os
import pkg_resources
import synapse.rest.admin
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
from synapse.api.errors import Codes
from synapse.appservice import ApplicationService
from synapse.rest.client.v1 import login, logout
@@ -59,7 +58,9 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
)
self.hs.get_datastore().services_cache.append(appservice)
- request_data = json.dumps({"username": "as_user_kermit"})
+ request_data = json.dumps(
+ {"username": "as_user_kermit", "type": APP_SERVICE_REGISTRATION_TYPE}
+ )
channel = self.make_request(
b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
@@ -69,9 +70,31 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
det_data = {"user_id": user_id, "home_server": self.hs.hostname}
self.assertDictContainsSubset(det_data, channel.json_body)
+ def test_POST_appservice_registration_no_type(self):
+ as_token = "i_am_an_app_service"
+
+ appservice = ApplicationService(
+ as_token,
+ self.hs.config.server_name,
+ id="1234",
+ namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]},
+ sender="@as:test",
+ )
+
+ self.hs.get_datastore().services_cache.append(appservice)
+ request_data = json.dumps({"username": "as_user_kermit"})
+
+ channel = self.make_request(
+ b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
+ )
+
+ self.assertEquals(channel.result["code"], b"400", channel.result)
+
def test_POST_appservice_registration_invalid(self):
self.appservice = None # no application service exists
- request_data = json.dumps({"username": "kermit"})
+ request_data = json.dumps(
+ {"username": "kermit", "type": APP_SERVICE_REGISTRATION_TYPE}
+ )
channel = self.make_request(
b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
)
diff --git a/tests/rest/client/v2_alpha/test_relations.py b/tests/rest/client/v2_alpha/test_relations.py
index e7bb5583fc..21ee436b91 100644
--- a/tests/rest/client/v2_alpha/test_relations.py
+++ b/tests/rest/client/v2_alpha/test_relations.py
@@ -16,6 +16,7 @@
import itertools
import json
import urllib
+from typing import Optional
from synapse.api.constants import EventTypes, RelationTypes
from synapse.rest import admin
@@ -681,7 +682,7 @@ class RelationsTestCase(unittest.HomeserverTestCase):
relation_type,
event_type,
key=None,
- content={},
+ content: Optional[dict] = None,
access_token=None,
parent_id=None,
):
@@ -713,7 +714,7 @@ class RelationsTestCase(unittest.HomeserverTestCase):
"POST",
"/_matrix/client/unstable/rooms/%s/send_relation/%s/%s/%s%s"
% (self.room, original_id, relation_type, event_type, query),
- json.dumps(content).encode("utf-8"),
+ json.dumps(content or {}).encode("utf-8"),
access_token=access_token,
)
return channel
diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py
index 9d0d0ef414..eb8687ce68 100644
--- a/tests/rest/key/v2/test_remote_key_resource.py
+++ b/tests/rest/key/v2/test_remote_key_resource.py
@@ -14,8 +14,7 @@
# limitations under the License.
import urllib.parse
from io import BytesIO, StringIO
-
-from mock import Mock
+from unittest.mock import Mock
import signedjson.key
from canonicaljson import encode_canonical_json
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index 9f77125fd4..375f0b7977 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -18,10 +18,9 @@ import tempfile
from binascii import unhexlify
from io import BytesIO
from typing import Optional
+from unittest.mock import Mock
from urllib import parse
-from mock import Mock
-
import attr
from parameterized import parameterized_class
from PIL import Image as Image
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 6968502433..9067463e54 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -15,8 +15,7 @@
import json
import os
import re
-
-from mock import patch
+from unittest.mock import patch
from twisted.internet._resolver import HostResolution
from twisted.internet.address import IPv4Address, IPv6Address
diff --git a/tests/scripts/test_new_matrix_user.py b/tests/scripts/test_new_matrix_user.py
index 6f56893f5e..885b95a51f 100644
--- a/tests/scripts/test_new_matrix_user.py
+++ b/tests/scripts/test_new_matrix_user.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse._scripts.register_new_matrix_user import request_registration
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index d40d65b06a..450b4ec710 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 1ce29af5fd..e755a4db62 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -15,8 +15,7 @@
import json
import os
import tempfile
-
-from mock import Mock
+from unittest.mock import Mock
import yaml
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index 1b4fae0bb5..069db0edc4 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
from synapse.storage.background_updates import BackgroundUpdater
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index eac7e4dcd2..54e9e7f6fe 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -15,8 +15,7 @@
from collections import OrderedDict
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py
index 7791138688..b02fb32ced 100644
--- a/tests/storage/test_cleanup_extrems.py
+++ b/tests/storage/test_cleanup_extrems.py
@@ -14,9 +14,7 @@
# limitations under the License.
import os.path
-from unittest.mock import patch
-
-from mock import Mock
+from unittest.mock import Mock, patch
import synapse.rest.admin
from synapse.api.constants import EventTypes
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 34e6526097..f7f75320ba 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
import synapse.rest.admin
from synapse.http.site import XForwardedForRequest
@@ -390,7 +390,7 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
class ClientIpAuthTestCase(unittest.HomeserverTestCase):
servlets = [
- synapse.rest.admin.register_servlets_for_client_rest_resource,
+ synapse.rest.admin.register_servlets,
login.register_servlets,
]
@@ -434,7 +434,7 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase):
self.reactor,
self.site,
"GET",
- "/_synapse/admin/v1/users/" + self.user_id,
+ "/_synapse/admin/v2/users/" + self.user_id,
access_token=access_token,
custom_headers=headers1.items(),
**make_request_args,
diff --git a/tests/storage/test_database.py b/tests/storage/test_database.py
index 5a77c84962..a906d30e73 100644
--- a/tests/storage/test_database.py
+++ b/tests/storage/test_database.py
@@ -36,17 +36,6 @@ def _stub_db_engine(**kwargs) -> BaseDatabaseEngine:
class TupleComparisonClauseTestCase(unittest.TestCase):
def test_native_tuple_comparison(self):
- db_engine = _stub_db_engine(supports_tuple_comparison=True)
- clause, args = make_tuple_comparison_clause(db_engine, [("a", 1), ("b", 2)])
+ clause, args = make_tuple_comparison_clause([("a", 1), ("b", 2)])
self.assertEqual(clause, "(a,b) > (?,?)")
self.assertEqual(args, [1, 2])
-
- def test_emulated_tuple_comparison(self):
- db_engine = _stub_db_engine(supports_tuple_comparison=False)
- clause, args = make_tuple_comparison_clause(
- db_engine, [("a", 1), ("b", 2), ("c", 3)]
- )
- self.assertEqual(
- clause, "(a >= ? AND (a > ? OR (b >= ? AND (b > ? OR c > ?))))"
- )
- self.assertEqual(args, [1, 1, 2, 2, 3])
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 239f7c9faf..0289942f88 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from tests.unittest import HomeserverTestCase
diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py
index aad6bc907e..6c389fe9ac 100644
--- a/tests/storage/test_id_generators.py
+++ b/tests/storage/test_id_generators.py
@@ -12,6 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import List, Optional
+
from synapse.storage.database import DatabasePool
from synapse.storage.engines import IncorrectDatabaseSetup
from synapse.storage.util.id_generators import MultiWriterIdGenerator
@@ -43,7 +45,7 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase):
)
def _create_id_generator(
- self, instance_name="master", writers=["master"]
+ self, instance_name="master", writers: Optional[List[str]] = None
) -> MultiWriterIdGenerator:
def _create(conn):
return MultiWriterIdGenerator(
@@ -53,7 +55,7 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase):
instance_name=instance_name,
tables=[("foobar", "instance_name", "stream_id")],
sequence_name="foobar_seq",
- writers=writers,
+ writers=writers or ["master"],
)
return self.get_success_or_raise(self.db_pool.runWithConnection(_create))
@@ -476,7 +478,7 @@ class BackwardsMultiWriterIdGeneratorTestCase(HomeserverTestCase):
)
def _create_id_generator(
- self, instance_name="master", writers=["master"]
+ self, instance_name="master", writers: Optional[List[str]] = None
) -> MultiWriterIdGenerator:
def _create(conn):
return MultiWriterIdGenerator(
@@ -486,7 +488,7 @@ class BackwardsMultiWriterIdGeneratorTestCase(HomeserverTestCase):
instance_name=instance_name,
tables=[("foobar", "instance_name", "stream_id")],
sequence_name="foobar_seq",
- writers=writers,
+ writers=writers or ["master"],
positive=False,
)
@@ -612,7 +614,7 @@ class MultiTableMultiWriterIdGeneratorTestCase(HomeserverTestCase):
)
def _create_id_generator(
- self, instance_name="master", writers=["master"]
+ self, instance_name="master", writers: Optional[List[str]] = None
) -> MultiWriterIdGenerator:
def _create(conn):
return MultiWriterIdGenerator(
@@ -625,7 +627,7 @@ class MultiTableMultiWriterIdGeneratorTestCase(HomeserverTestCase):
("foobar2", "instance_name", "stream_id"),
],
sequence_name="foobar_seq",
- writers=writers,
+ writers=writers or ["master"],
)
return self.get_success_or_raise(self.db_pool.runWithConnection(_create))
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 5858c7fcc4..47556791f4 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet import defer
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 2622207639..2d2f58903c 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Optional
from canonicaljson import json
@@ -47,10 +48,15 @@ class RedactionTestCase(unittest.HomeserverTestCase):
self.depth = 1
def inject_room_member(
- self, room, user, membership, replaces_state=None, extra_content={}
+ self,
+ room,
+ user,
+ membership,
+ replaces_state=None,
+ extra_content: Optional[dict] = None,
):
content = {"membership": membership}
- content.update(extra_content)
+ content.update(extra_content or {})
builder = self.event_builder_factory.for_room_version(
RoomVersions.V1,
{
diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index b57f36e6ac..6a6cf709f6 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock, patch
+from unittest.mock import Mock, patch
from synapse.util.distributor import Distributor
diff --git a/tests/test_federation.py b/tests/test_federation.py
index fc9aab32d0..8928597d17 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from twisted.internet.defer import succeed
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 75d28a42df..7d92a16a8d 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -15,9 +15,7 @@
"""Tests REST events for /rooms paths."""
-import json
-
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
from synapse.api.errors import Codes, HttpResponseException, SynapseError
from synapse.appservice import ApplicationService
from synapse.rest.client.v2_alpha import register, sync
@@ -113,7 +111,7 @@ class TestMauLimit(unittest.HomeserverTestCase):
)
)
- self.create_user("as_kermit4", token=as_token)
+ self.create_user("as_kermit4", token=as_token, appservice=True)
def test_allowed_after_a_month_mau(self):
# Create and sync so that the MAU counts get updated
@@ -232,14 +230,15 @@ class TestMauLimit(unittest.HomeserverTestCase):
self.reactor.advance(100)
self.assertEqual(2, self.successResultOf(count))
- def create_user(self, localpart, token=None):
- request_data = json.dumps(
- {
- "username": localpart,
- "password": "monkey",
- "auth": {"type": LoginType.DUMMY},
- }
- )
+ def create_user(self, localpart, token=None, appservice=False):
+ request_data = {
+ "username": localpart,
+ "password": "monkey",
+ "auth": {"type": LoginType.DUMMY},
+ }
+
+ if appservice:
+ request_data["type"] = APP_SERVICE_REGISTRATION_TYPE
channel = self.make_request(
"POST",
diff --git a/tests/test_phone_home.py b/tests/test_phone_home.py
index e7aed092c2..0f800a075b 100644
--- a/tests/test_phone_home.py
+++ b/tests/test_phone_home.py
@@ -14,8 +14,7 @@
# limitations under the License.
import resource
-
-import mock
+from unittest import mock
from synapse.app.phone_stats_home import phone_stats_home
diff --git a/tests/test_state.py b/tests/test_state.py
index 6227a3ba95..0d626f49f6 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -12,8 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from mock import Mock
+from typing import List, Optional
+from unittest.mock import Mock
from twisted.internet import defer
@@ -37,8 +37,8 @@ def create_event(
state_key=None,
depth=2,
event_id=None,
- prev_events=[],
- **kwargs
+ prev_events: Optional[List[str]] = None,
+ **kwargs,
):
global _next_event_id
@@ -58,7 +58,7 @@ def create_event(
"sender": "@user_id:example.com",
"room_id": "!room_id:example.com",
"depth": depth,
- "prev_events": prev_events,
+ "prev_events": prev_events or [],
}
if state_key is not None:
diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py
index a743cdc3a9..0df480db9f 100644
--- a/tests/test_terms_auth.py
+++ b/tests/test_terms_auth.py
@@ -13,8 +13,7 @@
# limitations under the License.
import json
-
-from mock import Mock
+from unittest.mock import Mock
from twisted.test.proto_helpers import MemoryReactorClock
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
index 43898d8142..b557ffd692 100644
--- a/tests/test_utils/__init__.py
+++ b/tests/test_utils/__init__.py
@@ -21,8 +21,7 @@ import sys
import warnings
from asyncio import Future
from typing import Any, Awaitable, Callable, TypeVar
-
-from mock import Mock
+from unittest.mock import Mock
import attr
diff --git a/tests/test_utils/event_injection.py b/tests/test_utils/event_injection.py
index c3c4a93e1f..3dfbf8f8a9 100644
--- a/tests/test_utils/event_injection.py
+++ b/tests/test_utils/event_injection.py
@@ -33,7 +33,7 @@ async def inject_member_event(
membership: str,
target: Optional[str] = None,
extra_content: Optional[dict] = None,
- **kwargs
+ **kwargs,
) -> EventBase:
"""Inject a membership event into a room."""
if target is None:
@@ -58,7 +58,7 @@ async def inject_event(
hs: synapse.server.HomeServer,
room_version: Optional[str] = None,
prev_event_ids: Optional[List[str]] = None,
- **kwargs
+ **kwargs,
) -> EventBase:
"""Inject a generic event into a room
@@ -83,7 +83,7 @@ async def create_event(
hs: synapse.server.HomeServer,
room_version: Optional[str] = None,
prev_event_ids: Optional[List[str]] = None,
- **kwargs
+ **kwargs,
) -> Tuple[EventBase, EventContext]:
if room_version is None:
room_version = await hs.get_datastore().get_room_version_id(kwargs["room_id"])
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 510b630114..e502ac197e 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
-
-from mock import Mock
+from typing import Optional
+from unittest.mock import Mock
from twisted.internet import defer
from twisted.internet.defer import succeed
@@ -147,9 +147,11 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
return event
@defer.inlineCallbacks
- def inject_room_member(self, user_id, membership="join", extra_content={}):
+ def inject_room_member(
+ self, user_id, membership="join", extra_content: Optional[dict] = None
+ ):
content = {"membership": membership}
- content.update(extra_content)
+ content.update(extra_content or {})
builder = self.event_builder_factory.for_room_version(
RoomVersions.V1,
{
diff --git a/tests/unittest.py b/tests/unittest.py
index 57b6a395c7..92764434bd 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -21,8 +21,7 @@ import inspect
import logging
import time
from typing import Callable, Dict, Iterable, Optional, Tuple, Type, TypeVar, Union
-
-from mock import Mock, patch
+from unittest.mock import Mock, patch
from canonicaljson import json
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index e434e21aee..2d1f9360e0 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -15,8 +15,7 @@
# limitations under the License.
import logging
from typing import Set
-
-import mock
+from unittest import mock
from twisted.internet import defer, reactor
diff --git a/tests/util/caches/test_ttlcache.py b/tests/util/caches/test_ttlcache.py
index 816795c136..23018081e5 100644
--- a/tests/util/caches/test_ttlcache.py
+++ b/tests/util/caches/test_ttlcache.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.util.caches.ttlcache import TTLCache
diff --git a/tests/util/test_file_consumer.py b/tests/util/test_file_consumer.py
index 2012263184..d1372f6bc2 100644
--- a/tests/util/test_file_consumer.py
+++ b/tests/util/test_file_consumer.py
@@ -16,8 +16,7 @@
import threading
from io import StringIO
-
-from mock import NonCallableMock
+from unittest.mock import NonCallableMock
from twisted.internet import defer, reactor
diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py
index a739a6aaaf..ce4f1cc30a 100644
--- a/tests/util/test_lrucache.py
+++ b/tests/util/test_lrucache.py
@@ -14,7 +14,7 @@
# limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
from synapse.util.caches.lrucache import LruCache
from synapse.util.caches.treecache import TreeCache
diff --git a/tests/util/test_ratelimitutils.py b/tests/util/test_ratelimitutils.py
index 4d1aee91d5..3fed55090a 100644
--- a/tests/util/test_ratelimitutils.py
+++ b/tests/util/test_ratelimitutils.py
@@ -12,6 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Optional
+
from synapse.config.homeserver import HomeServerConfig
from synapse.util.ratelimitutils import FederationRateLimiter
@@ -89,9 +91,9 @@ def _await_resolution(reactor, d):
return (reactor.seconds() - start_time) * 1000
-def build_rc_config(settings={}):
+def build_rc_config(settings: Optional[dict] = None):
config_dict = default_config("test")
- config_dict.update(settings)
+ config_dict.update(settings or {})
config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
return config.rc_federation
diff --git a/tests/utils.py b/tests/utils.py
index a141ee6496..c78d3e5ba7 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -21,10 +21,9 @@ import time
import uuid
import warnings
from typing import Type
+from unittest.mock import Mock, patch
from urllib import parse as urlparse
-from mock import Mock, patch
-
from twisted.internet import defer
from synapse.api.constants import EventTypes
@@ -191,7 +190,7 @@ def setup_test_homeserver(
config=None,
reactor=None,
homeserver_to_use: Type[HomeServer] = TestHomeServer,
- **kwargs
+ **kwargs,
):
"""
Setup a homeserver suitable for running tests against. Keyword arguments
diff --git a/tox.ini b/tox.ini
index 9ff70fe312..998b04b224 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,8 @@
[tox]
-envlist = packaging, py35, py36, py37, py38, py39, check_codestyle, check_isort
+envlist = packaging, py36, py37, py38, py39, check_codestyle, check_isort
+
+# we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208
+minversion = 2.3.2
[base]
deps =
@@ -48,6 +51,7 @@ deps =
extras =
# install the optional dependendencies for tox environments without
# '-noextras' in their name
+ # (this requires tox 3)
!noextras: all
test
@@ -74,8 +78,6 @@ commands =
# we use "env" rather than putting a value in `setenv` so that it is not
# inherited by other tox environments.
#
- # keep this in sync with the copy in `testenv:py35-old`.
- #
/usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
# As of twisted 16.4, trial tries to import the tests as a package (previously
@@ -103,8 +105,9 @@ usedevelop=true
# A test suite for the oldest supported versions of Python libraries, to catch
# any uses of APIs not available in them.
-[testenv:py35-old]
-skip_install=True
+[testenv:py3-old]
+skip_install = true
+usedevelop = false
deps =
# Old automat version for Twisted
Automat == 0.3.0
@@ -120,11 +123,7 @@ commands =
# Install Synapse itself. This won't update any libraries.
pip install -e ".[test]"
- # we have to duplicate the command from `testenv` rather than refer to it
- # as `{[testenv]commands}`, because we run on ubuntu xenial, which has
- # tox 2.3.1, and https://github.com/tox-dev/tox/issues/208.
- #
- /usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+ {[testenv]commands}
[testenv:benchmark]
deps =
@@ -136,7 +135,8 @@ commands =
python -m synmark {posargs:}
[testenv:packaging]
-skip_install=True
+skip_install = true
+usedevelop = false
deps =
check-manifest
commands =
@@ -154,7 +154,8 @@ extras = lint
commands = isort -c --df --sp setup.cfg {[base]lint_targets}
[testenv:check-newsfragment]
-skip_install = True
+skip_install = true
+usedevelop = false
deps = towncrier>=18.6.0rc1
commands =
python -m towncrier.check --compare-with=origin/develop
@@ -163,7 +164,8 @@ commands =
commands = {toxinidir}/scripts-dev/generate_sample_config --check
[testenv:combine]
-skip_install = True
+skip_install = true
+usedevelop = false
deps =
coverage
pip>=10 ; python_version >= '3.6'
@@ -173,14 +175,16 @@ commands=
coverage report
[testenv:cov-erase]
-skip_install = True
+skip_install = true
+usedevelop = false
deps =
coverage
commands=
coverage erase
[testenv:cov-html]
-skip_install = True
+skip_install = true
+usedevelop = false
deps =
coverage
commands=
|
