diff --git a/.buildkite/scripts/create_postgres_db.py b/.buildkite/scripts/create_postgres_db.py
deleted file mode 100755
index cc829db216..0000000000
--- a/.buildkite/scripts/create_postgres_db.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
- # Create a PostgresEngine.
- db_engine = create_engine({"name": "psycopg2", "args": {}})
-
- # Connect to postgres to create the base database.
- # We use "postgres" as a database because it's bound to exist and the "synapse" one
- # doesn't exist yet.
- db_conn = db_engine.module.connect(
- user="postgres", host="postgres", password="postgres", dbname="postgres"
- )
- db_conn.autocommit = True
- cur = db_conn.cursor()
- cur.execute("CREATE DATABASE synapse;")
- cur.close()
- db_conn.close()
diff --git a/.buildkite/scripts/postgres_exec.py b/.buildkite/scripts/postgres_exec.py
new file mode 100755
index 0000000000..086b391724
--- /dev/null
+++ b/.buildkite/scripts/postgres_exec.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+ user="postgres", host="postgres", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+ cur.execute(c)
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
index 8914319e38..a7e2454769 100755
--- a/.buildkite/scripts/test_synapse_port_db.sh
+++ b/.buildkite/scripts/test_synapse_port_db.sh
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
#
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
+# Test script for 'synapse_port_db'.
+# - sets up synapse and deps
+# - runs the port script on a prepopulated test sqlite db
+# - also runs it against an new sqlite db
+
set -xe
cd `dirname $0`/../..
@@ -22,15 +22,32 @@ echo "--- Generate the signing key"
# Generate the server's signing key.
python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
-echo "--- Prepare the databases"
+echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
# Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
+./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against test database"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
+#####
+
+# Now do the same again, on an empty database.
+
+echo "--- Prepare empty SQLite database"
+
+# we do this by deleting the sqlite db, and then doing the same again.
+rm .buildkite/test_db.db
+
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-echo "+++ Run synapse_port_db"
+# re-create the PostgreSQL database.
+./.buildkite/scripts/postgres_exec.py \
+ "DROP DATABASE synapse" \
+ "CREATE DATABASE synapse"
-# Run the script
+echo "+++ Run synapse_port_db against empty database"
coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 12c82ac620..e7f3be1b4e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -273,7 +273,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Patch Buildkite-specific test scripts
run: |
- sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+ sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
diff --git a/CHANGES.md b/CHANGES.md
index bdeb614b9e..93efa3ce56 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,37 @@
+Synapse 1.33.2 (2021-05-11)
+===========================
+
+Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
+
+Security advisory
+-----------------
+
+This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
+
+Internal Changes
+----------------
+
+- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946))
+
+
+Synapse 1.33.1 (2021-05-06)
+===========================
+
+Bugfixes
+--------
+
+- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937))
+
+
+Synapse 1.33.0 (2021-05-05)
+===========================
+
+Features
+--------
+
+- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909))
+
+
Synapse 1.33.0rc2 (2021-04-29)
==============================
diff --git a/UPGRADE.rst b/UPGRADE.rst
index e921e0c08a..606e357b6e 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -85,6 +85,35 @@ for example:
wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
+Upgrading to v1.34.0
+====================
+
+`room_invite_state_types` configuration setting
+-----------------------------------------------
+
+The ``room_invite_state_types`` configuration setting has been deprecated and
+replaced with ``room_prejoin_state``. See the `sample configuration file <https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515>`_.
+
+If you have set ``room_invite_state_types`` to the default value you should simply
+remove it from your configuration file. The default value used to be:
+
+.. code:: yaml
+
+ room_invite_state_types:
+ - "m.room.join_rules"
+ - "m.room.canonical_alias"
+ - "m.room.avatar"
+ - "m.room.encryption"
+ - "m.room.name"
+
+If you have customised this value by adding addition state types, you should
+remove ``room_invite_state_types`` and configure ``additional_event_types`` with
+your customisations.
+
+If you have customised this value by removing state types, you should rename
+``room_invite_state_types`` to ``additional_event_types``, and set
+``disable_default_event_types`` to ``true``.
+
Upgrading to v1.33.0
====================
diff --git a/changelog.d/9881.feature b/changelog.d/9881.feature
new file mode 100644
index 0000000000..088a517e02
--- /dev/null
+++ b/changelog.d/9881.feature
@@ -0,0 +1 @@
+Add experimental option to track memory usage of the caches.
diff --git a/changelog.d/9882.misc b/changelog.d/9882.misc
new file mode 100644
index 0000000000..facfa31f38
--- /dev/null
+++ b/changelog.d/9882.misc
@@ -0,0 +1 @@
+Export jemalloc stats to Prometheus if it is being used.
diff --git a/changelog.d/9902.feature b/changelog.d/9902.feature
new file mode 100644
index 0000000000..4d9f324d4e
--- /dev/null
+++ b/changelog.d/9902.feature
@@ -0,0 +1 @@
+Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set.
diff --git a/changelog.d/9904.misc b/changelog.d/9904.misc
new file mode 100644
index 0000000000..3db1e625ae
--- /dev/null
+++ b/changelog.d/9904.misc
@@ -0,0 +1 @@
+Time response time for external cache requests.
diff --git a/changelog.d/9905.feature b/changelog.d/9905.feature
new file mode 100644
index 0000000000..96a0e7f09f
--- /dev/null
+++ b/changelog.d/9905.feature
@@ -0,0 +1 @@
+Improve performance of sending events for worker-based deployments using Redis.
diff --git a/changelog.d/9910.bugfix b/changelog.d/9910.bugfix
new file mode 100644
index 0000000000..06d523fd46
--- /dev/null
+++ b/changelog.d/9910.bugfix
@@ -0,0 +1 @@
+Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession.
diff --git a/changelog.d/9910.feature b/changelog.d/9910.feature
new file mode 100644
index 0000000000..54165cce18
--- /dev/null
+++ b/changelog.d/9910.feature
@@ -0,0 +1 @@
+Improve performance after joining a large room when presence is enabled.
diff --git a/changelog.d/9911.doc b/changelog.d/9911.doc
new file mode 100644
index 0000000000..f7fd9f1ba9
--- /dev/null
+++ b/changelog.d/9911.doc
@@ -0,0 +1 @@
+Add `port` argument to the Postgres database sample config section.
\ No newline at end of file
diff --git a/changelog.d/9913.docker b/changelog.d/9913.docker
new file mode 100644
index 0000000000..93835e14cb
--- /dev/null
+++ b/changelog.d/9913.docker
@@ -0,0 +1 @@
+Added startup_delay to docker healthcheck to reduce waiting time for coming online, updated readme for extra options, contributed by @Maquis196.
diff --git a/changelog.d/9915.feature b/changelog.d/9915.feature
new file mode 100644
index 0000000000..7b81faabea
--- /dev/null
+++ b/changelog.d/9915.feature
@@ -0,0 +1 @@
+Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration.
\ No newline at end of file
diff --git a/changelog.d/9916.feature b/changelog.d/9916.feature
new file mode 100644
index 0000000000..54165cce18
--- /dev/null
+++ b/changelog.d/9916.feature
@@ -0,0 +1 @@
+Improve performance after joining a large room when presence is enabled.
diff --git a/changelog.d/9928.bugfix b/changelog.d/9928.bugfix
new file mode 100644
index 0000000000..7b74cd9fb6
--- /dev/null
+++ b/changelog.d/9928.bugfix
@@ -0,0 +1 @@
+Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms.
diff --git a/changelog.d/9930.bugfix b/changelog.d/9930.bugfix
new file mode 100644
index 0000000000..9b22ed4458
--- /dev/null
+++ b/changelog.d/9930.bugfix
@@ -0,0 +1 @@
+Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database.
diff --git a/changelog.d/9931.misc b/changelog.d/9931.misc
new file mode 100644
index 0000000000..326adc7f3c
--- /dev/null
+++ b/changelog.d/9931.misc
@@ -0,0 +1 @@
+Minor fixes to the `make_full_schema.sh` script.
diff --git a/changelog.d/9932.misc b/changelog.d/9932.misc
new file mode 100644
index 0000000000..9e16a36173
--- /dev/null
+++ b/changelog.d/9932.misc
@@ -0,0 +1 @@
+Move database schema files into a common directory.
diff --git a/changelog.d/9935.feature b/changelog.d/9935.feature
new file mode 100644
index 0000000000..eeda5bf50e
--- /dev/null
+++ b/changelog.d/9935.feature
@@ -0,0 +1 @@
+Improve performance of backfilling in large rooms.
diff --git a/changelog.d/9945.feature b/changelog.d/9945.feature
new file mode 100644
index 0000000000..84308e8cce
--- /dev/null
+++ b/changelog.d/9945.feature
@@ -0,0 +1 @@
+Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist.
diff --git a/changelog.d/9947.feature b/changelog.d/9947.feature
new file mode 100644
index 0000000000..ce8874f810
--- /dev/null
+++ b/changelog.d/9947.feature
@@ -0,0 +1 @@
+Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary.
diff --git a/changelog.d/9950.feature b/changelog.d/9950.feature
new file mode 100644
index 0000000000..96a0e7f09f
--- /dev/null
+++ b/changelog.d/9950.feature
@@ -0,0 +1 @@
+Improve performance of sending events for worker-based deployments using Redis.
diff --git a/changelog.d/9954.feature b/changelog.d/9954.feature
new file mode 100644
index 0000000000..ce8874f810
--- /dev/null
+++ b/changelog.d/9954.feature
@@ -0,0 +1 @@
+Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary.
diff --git a/changelog.d/9959.misc b/changelog.d/9959.misc
new file mode 100644
index 0000000000..7231f29d79
--- /dev/null
+++ b/changelog.d/9959.misc
@@ -0,0 +1 @@
+Add debug logging for lost/delayed to-device messages.
diff --git a/changelog.d/9961.bugfix b/changelog.d/9961.bugfix
new file mode 100644
index 0000000000..e26d141a53
--- /dev/null
+++ b/changelog.d/9961.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped.
diff --git a/changelog.d/9965.bugfix b/changelog.d/9965.bugfix
new file mode 100644
index 0000000000..e26d141a53
--- /dev/null
+++ b/changelog.d/9965.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped.
diff --git a/changelog.d/9966.feature b/changelog.d/9966.feature
new file mode 100644
index 0000000000..7b81faabea
--- /dev/null
+++ b/changelog.d/9966.feature
@@ -0,0 +1 @@
+Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration.
\ No newline at end of file
diff --git a/debian/changelog b/debian/changelog
index fd33bfda5c..76b82c172e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,21 @@
+matrix-synapse-py3 (1.33.2) stable; urgency=medium
+
+ * New synapse release 1.33.2.
+
+ -- Synapse Packaging team <packages@matrix.org> Tue, 11 May 2021 11:17:59 +0100
+
+matrix-synapse-py3 (1.33.1) stable; urgency=medium
+
+ * New synapse release 1.33.1.
+
+ -- Synapse Packaging team <packages@matrix.org> Thu, 06 May 2021 14:06:33 +0100
+
+matrix-synapse-py3 (1.33.0) stable; urgency=medium
+
+ * New synapse release 1.33.0.
+
+ -- Synapse Packaging team <packages@matrix.org> Wed, 05 May 2021 14:15:27 +0100
+
matrix-synapse-py3 (1.32.2) stable; urgency=medium
* New synapse release 1.32.2.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4f5cd06d72..2bdc607e66 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -88,5 +88,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
ENTRYPOINT ["/start.py"]
-HEALTHCHECK --interval=1m --timeout=5s \
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
CMD curl -fSs http://localhost:8008/health || exit 1
diff --git a/docker/README.md b/docker/README.md
index a7d1e670fe..c8d3c4b3da 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -191,6 +191,16 @@ whilst running the above `docker run` commands.
```
--no-healthcheck
```
+
+## Disabling the healthcheck in docker-compose file
+
+If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
+
+```
+ healthcheck:
+ disable: true
+```
+
## Setting custom healthcheck on docker run
If you wish to point the healthcheck at a different port with docker command, add the following
@@ -202,14 +212,15 @@ If you wish to point the healthcheck at a different port with docker command, ad
## Setting the healthcheck in docker-compose file
You can add the following to set a custom healthcheck in a docker compose file.
-You will need version >2.1 for this to work.
+You will need docker-compose version >2.1 for this to work.
```
healthcheck:
test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
- interval: 1m
- timeout: 10s
+ interval: 15s
+ timeout: 5s
retries: 3
+ start_period: 5s
```
## Using jemalloc
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index e0350279ad..67ad57b1aa 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -152,6 +152,16 @@ presence:
#
#gc_thresholds: [700, 10, 10]
+# The minimum time in seconds between each GC for a generation, regardless of
+# the GC thresholds. This ensures that we don't do GC too frequently.
+#
+# A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive
+# generation 0 GCs, etc.
+#
+# Defaults to `[1s, 10s, 30s]`.
+#
+#gc_min_interval: [0.5s, 30s, 1m]
+
# Set the limit on the returned events in the timeline in the get
# and sync operations. The default value is 100. -1 means no upper limit.
#
@@ -731,6 +741,12 @@ acme:
#
#allow_profile_lookup_over_federation: false
+# Uncomment to disable device display name lookup over federation. By default, the
+# Federation API allows other homeservers to obtain device display names of any user
+# on this homeserver. Defaults to 'true'.
+#
+#allow_device_name_lookup_over_federation: false
+
## Caching ##
@@ -810,6 +826,7 @@ caches:
# password: secretpassword
# database: synapse
# host: localhost
+# port: 5432
# cp_min: 5
# cp_max: 10
#
@@ -1504,6 +1521,7 @@ room_prejoin_state:
# - m.room.avatar
# - m.room.encryption
# - m.room.name
+ # - m.room.create
#
# Uncomment the following to disable these defaults (so that only the event
# types listed in 'additional_event_types' are shared). Defaults to 'false'.
diff --git a/mypy.ini b/mypy.ini
index a40f705b76..ea655a0d4d 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -171,3 +171,6 @@ ignore_missing_imports = True
[mypy-txacme.*]
ignore_missing_imports = True
+
+[mypy-pympler.*]
+ignore_missing_imports = True
diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
index 3bb6e2c7ea..07d018db99 100755
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -21,9 +21,10 @@ DISTS = (
"debian:buster",
"debian:bullseye",
"debian:sid",
- "ubuntu:bionic",
- "ubuntu:focal",
- "ubuntu:groovy",
+ "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23)
+ "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+ "ubuntu:groovy", # 20.10 (EOL 2021-07-07)
+ "ubuntu:hirsute", # 21.04 (EOL 2022-01-05)
)
DESC = '''\
diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh
index bc8f978660..39bf30d258 100755
--- a/scripts-dev/make_full_schema.sh
+++ b/scripts-dev/make_full_schema.sh
@@ -6,7 +6,7 @@
# It does so by having Synapse generate an up-to-date SQLite DB, then running
# synapse_port_db to convert it to Postgres. It then dumps the contents of both.
-POSTGRES_HOST="localhost"
+export PGHOST="localhost"
POSTGRES_DB_NAME="synapse_full_schema.$$"
SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
@@ -32,7 +32,7 @@ usage() {
while getopts "p:co:h" opt; do
case $opt in
p)
- POSTGRES_USERNAME=$OPTARG
+ export PGUSER=$OPTARG
;;
c)
# Print all commands that are being executed
@@ -69,7 +69,7 @@ if [ ${#unsatisfied_requirements} -ne 0 ]; then
exit 1
fi
-if [ -z "$POSTGRES_USERNAME" ]; then
+if [ -z "$PGUSER" ]; then
echo "No postgres username supplied"
usage
exit 1
@@ -84,8 +84,9 @@ fi
# Create the output directory if it doesn't exist
mkdir -p "$OUTPUT_DIR"
-read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD
+read -rsp "Postgres password for '$PGUSER': " PGPASSWORD
echo ""
+export PGPASSWORD
# Exit immediately if a command fails
set -e
@@ -131,9 +132,9 @@ report_stats: false
database:
name: "psycopg2"
args:
- user: "$POSTGRES_USERNAME"
- host: "$POSTGRES_HOST"
- password: "$POSTGRES_PASSWORD"
+ user: "$PGUSER"
+ host: "$PGHOST"
+ password: "$PGPASSWORD"
database: "$POSTGRES_DB_NAME"
# Suppress the key server warning.
@@ -150,7 +151,7 @@ scripts-dev/update_database --database-config "$SQLITE_CONFIG"
# Create the PostgreSQL database.
echo "Creating postgres database..."
-createdb $POSTGRES_DB_NAME
+createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
if [ -z "$COVERAGE" ]; then
@@ -181,7 +182,7 @@ DROP TABLE user_directory_search_docsize;
DROP TABLE user_directory_search_stat;
"
sqlite3 "$SQLITE_DB" <<< "$SQL"
-psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL"
+psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index f0c93d5226..5fb5bb35f7 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -913,10 +913,11 @@ class Porter(object):
(curr_forward_id + 1,),
)
- txn.execute(
- "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
- (curr_backward_id + 1,),
- )
+ if curr_backward_id:
+ txn.execute(
+ "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
+ (curr_backward_id + 1,),
+ )
await self.postgres_store.db_pool.runInteraction(
"_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
@@ -954,10 +955,11 @@ class Porter(object):
(curr_chain_id,),
)
- await self.postgres_store.db_pool.runInteraction(
- "_setup_event_auth_chain_id", r,
- )
-
+ if curr_chain_id is not None:
+ await self.postgres_store.db_pool.runInteraction(
+ "_setup_event_auth_chain_id",
+ r,
+ )
##############################################
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 319c52be2c..ce822ccb04 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
except ImportError:
pass
-__version__ = "1.33.0rc2"
+__version__ = "1.33.2"
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
# We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 936b6534b4..3940da5c88 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -110,13 +110,18 @@ class EventTypes:
Dummy = "org.matrix.dummy_event"
+ SpaceChild = "m.space.child"
+ SpaceParent = "m.space.parent"
MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child"
MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent"
+class ToDeviceEventTypes:
+ RoomKeyRequest = "m.room_key_request"
+
+
class EduTypes:
Presence = "m.presence"
- RoomKeyRequest = "m.room_key_request"
class RejectedReason:
@@ -174,6 +179,7 @@ class EventContentFields:
SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after"
# cf https://github.com/matrix-org/matrix-doc/pull/1772
+ ROOM_TYPE = "type"
MSC1772_ROOM_TYPE = "org.matrix.msc1772.type"
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 638e01c1b2..59918d789e 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -37,6 +37,7 @@ from synapse.config.homeserver import HomeServerConfig
from synapse.crypto import context_factory
from synapse.logging.context import PreserveLoggingContext
from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.metrics.jemalloc import setup_jemalloc_stats
from synapse.util.async_helpers import Linearizer
from synapse.util.daemonize import daemonize_process
from synapse.util.rlimit import change_resource_limit
@@ -115,6 +116,7 @@ def start_reactor(
def run():
logger.info("Running")
+ setup_jemalloc_stats()
change_resource_limit(soft_file_limit)
if gc_thresholds:
gc.set_threshold(*gc_thresholds)
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index 1a15ceee81..f730cdbd78 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -454,6 +454,10 @@ def start(config_options):
config.server.update_user_directory = False
synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
+ synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
+
+ if config.server.gc_seconds:
+ synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds
hs = GenericWorkerServer(
config.server_name,
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 8e78134bbe..b2501ee4d7 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -341,6 +341,10 @@ def setup(config_options):
sys.exit(0)
events.USE_FROZEN_DICTS = config.use_frozen_dicts
+ synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
+
+ if config.server.gc_seconds:
+ synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds
hs = SynapseHomeServer(
config.server_name,
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 55c038c0c4..b18044f982 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -88,10 +88,6 @@ class ApiConfig(Config):
if not room_prejoin_state_config.get("disable_default_event_types"):
yield from _DEFAULT_PREJOIN_STATE_TYPES
- if self.spaces_enabled:
- # MSC1772 suggests adding m.room.create to the prejoin state
- yield EventTypes.Create
-
yield from room_prejoin_state_config.get("additional_event_types", [])
@@ -109,6 +105,8 @@ _DEFAULT_PREJOIN_STATE_TYPES = [
EventTypes.RoomAvatar,
EventTypes.RoomEncryption,
EventTypes.Name,
+ # Per MSC1772.
+ EventTypes.Create,
]
diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 41b9b3f51f..91165ee1ce 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -17,6 +17,8 @@ import re
import threading
from typing import Callable, Dict
+from synapse.python_dependencies import DependencyException, check_requirements
+
from ._base import Config, ConfigError
# The prefix for all cache factor-related environment variables
@@ -189,6 +191,15 @@ class CacheConfig(Config):
)
self.cache_factors[cache] = factor
+ self.track_memory_usage = cache_config.get("track_memory_usage", False)
+ if self.track_memory_usage:
+ try:
+ check_requirements("cache_memory")
+ except DependencyException as e:
+ raise ConfigError(
+ e.message # noqa: B306, DependencyException.message is a property
+ )
+
# Resize all caches (if necessary) with the new factors we've loaded
self.resize_all_caches()
diff --git a/synapse/config/database.py b/synapse/config/database.py
index 79a02706b4..c76ef1e1de 100644
--- a/synapse/config/database.py
+++ b/synapse/config/database.py
@@ -58,6 +58,7 @@ DEFAULT_CONFIG = """\
# password: secretpassword
# database: synapse
# host: localhost
+# port: 5432
# cp_min: 5
# cp_max: 10
#
diff --git a/synapse/config/federation.py b/synapse/config/federation.py
index 090ba047fa..cdd7a1ef05 100644
--- a/synapse/config/federation.py
+++ b/synapse/config/federation.py
@@ -44,6 +44,10 @@ class FederationConfig(Config):
"allow_profile_lookup_over_federation", True
)
+ self.allow_device_name_lookup_over_federation = config.get(
+ "allow_device_name_lookup_over_federation", True
+ )
+
def generate_config_section(self, config_dir_path, server_name, **kwargs):
return """\
## Federation ##
@@ -75,6 +79,12 @@ class FederationConfig(Config):
# on this homeserver. Defaults to 'true'.
#
#allow_profile_lookup_over_federation: false
+
+ # Uncomment to disable device display name lookup over federation. By default, the
+ # Federation API allows other homeservers to obtain device display names of any user
+ # on this homeserver. Defaults to 'true'.
+ #
+ #allow_device_name_lookup_over_federation: false
"""
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 21ca7b33e3..c290a35a92 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -19,7 +19,7 @@ import logging
import os.path
import re
from textwrap import indent
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
import attr
import yaml
@@ -572,6 +572,7 @@ class ServerConfig(Config):
_warn_if_webclient_configured(self.listeners)
self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
+ self.gc_seconds = self.read_gc_intervals(config.get("gc_min_interval", None))
@attr.s
class LimitRemoteRoomsConfig:
@@ -917,6 +918,16 @@ class ServerConfig(Config):
#
#gc_thresholds: [700, 10, 10]
+ # The minimum time in seconds between each GC for a generation, regardless of
+ # the GC thresholds. This ensures that we don't do GC too frequently.
+ #
+ # A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive
+ # generation 0 GCs, etc.
+ #
+ # Defaults to `[1s, 10s, 30s]`.
+ #
+ #gc_min_interval: [0.5s, 30s, 1m]
+
# Set the limit on the returned events in the timeline in the get
# and sync operations. The default value is 100. -1 means no upper limit.
#
@@ -1305,6 +1316,24 @@ class ServerConfig(Config):
help="Turn on the twisted telnet manhole service on the given port.",
)
+ def read_gc_intervals(self, durations) -> Optional[Tuple[float, float, float]]:
+ """Reads the three durations for the GC min interval option, returning seconds."""
+ if durations is None:
+ return None
+
+ try:
+ if len(durations) != 3:
+ raise ValueError()
+ return (
+ self.parse_duration(durations[0]) / 1000,
+ self.parse_duration(durations[1]) / 1000,
+ self.parse_duration(durations[2]) / 1000,
+ )
+ except Exception:
+ raise ConfigError(
+ "Value of `gc_min_interval` must be a list of three durations if set"
+ )
+
def is_threepid_reserved(reserved_threepids, threepid):
"""Check the threepid against the reserved threepid config
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index b041869758..7df4e4c3e6 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -17,7 +17,7 @@ import os
import warnings
from datetime import datetime
from hashlib import sha256
-from typing import List, Optional
+from typing import List, Optional, Pattern
from unpaddedbase64 import encode_base64
@@ -124,7 +124,7 @@ class TlsConfig(Config):
fed_whitelist_entries = []
# Support globs (*) in whitelist values
- self.federation_certificate_verification_whitelist = [] # type: List[str]
+ self.federation_certificate_verification_whitelist = [] # type: List[Pattern]
for entry in fed_whitelist_entries:
try:
entry_regex = glob_to_regex(entry.encode("ascii").decode("ascii"))
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index b729a69203..ace30aa450 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -44,7 +44,6 @@ from synapse.api.errors import (
SynapseError,
UnsupportedRoomVersionError,
)
-from synapse.api.ratelimiting import Ratelimiter
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
from synapse.events import EventBase
from synapse.federation.federation_base import FederationBase, event_from_pdu_json
@@ -865,14 +864,6 @@ class FederationHandlerRegistry:
# EDU received.
self._edu_type_to_instance = {} # type: Dict[str, List[str]]
- # A rate limiter for incoming room key requests per origin.
- self._room_key_request_rate_limiter = Ratelimiter(
- store=hs.get_datastore(),
- clock=self.clock,
- rate_hz=self.config.rc_key_requests.per_second,
- burst_count=self.config.rc_key_requests.burst_count,
- )
-
def register_edu_handler(
self, edu_type: str, handler: Callable[[str, JsonDict], Awaitable[None]]
) -> None:
@@ -926,16 +917,6 @@ class FederationHandlerRegistry:
if not self.config.use_presence and edu_type == EduTypes.Presence:
return
- # If the incoming room key requests from a particular origin are over
- # the limit, drop them.
- if (
- edu_type == EduTypes.RoomKeyRequest
- and not await self._room_key_request_rate_limiter.can_do_action(
- None, origin
- )
- ):
- return
-
# Check if we have a handler on this instance
handler = self.edu_handlers.get(edu_type)
if handler:
diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py
index 3b053ebcfb..3a2efd56ee 100644
--- a/synapse/federation/sender/per_destination_queue.py
+++ b/synapse/federation/sender/per_destination_queue.py
@@ -28,6 +28,7 @@ from synapse.api.presence import UserPresenceState
from synapse.events import EventBase
from synapse.federation.units import Edu
from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
from synapse.logging.opentracing import SynapseTags, set_tag
from synapse.metrics import sent_transactions_counter
from synapse.metrics.background_process_metrics import run_as_background_process
@@ -574,6 +575,14 @@ class PerDestinationQueue:
for content in contents
]
+ if edus:
+ issue9533_logger.debug(
+ "Sending %i to-device messages to %s, up to stream id %i",
+ len(edus),
+ self._destination,
+ stream_id,
+ )
+
return (edus, stream_id)
def _start_catching_up(self) -> None:
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ada322a81e..497848a2b7 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -995,6 +995,7 @@ class TransportLayerClient:
returned per space
exclude_rooms: a list of any rooms we can skip
"""
+ # TODO When switching to the stable endpoint, use GET instead of POST.
path = _create_path(
FEDERATION_UNSTABLE_PREFIX, "/org.matrix.msc2946/spaces/%s", room_id
)
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a3759bdda1..e1b7462474 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1376,6 +1376,32 @@ class FederationSpaceSummaryServlet(BaseFederationServlet):
PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
PATH = "/spaces/(?P<room_id>[^/]*)"
+ async def on_GET(
+ self,
+ origin: str,
+ content: JsonDict,
+ query: Mapping[bytes, Sequence[bytes]],
+ room_id: str,
+ ) -> Tuple[int, JsonDict]:
+ suggested_only = parse_boolean_from_args(query, "suggested_only", default=False)
+ max_rooms_per_space = parse_integer_from_args(query, "max_rooms_per_space")
+
+ exclude_rooms = []
+ if b"exclude_rooms" in query:
+ try:
+ exclude_rooms = [
+ room_id.decode("ascii") for room_id in query[b"exclude_rooms"]
+ ]
+ except Exception:
+ raise SynapseError(
+ 400, "Bad query parameter for exclude_rooms", Codes.INVALID_PARAM
+ )
+
+ return 200, await self.handler.federation_space_summary(
+ room_id, suggested_only, max_rooms_per_space, exclude_rooms
+ )
+
+ # TODO When switching to the stable endpoint, remove the POST handler.
async def on_POST(
self,
origin: str,
diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py
index c5d631de07..580b941595 100644
--- a/synapse/handlers/devicemessage.py
+++ b/synapse/handlers/devicemessage.py
@@ -15,7 +15,7 @@
import logging
from typing import TYPE_CHECKING, Any, Dict
-from synapse.api.constants import EduTypes
+from synapse.api.constants import ToDeviceEventTypes
from synapse.api.errors import SynapseError
from synapse.api.ratelimiting import Ratelimiter
from synapse.logging.context import run_in_background
@@ -79,6 +79,8 @@ class DeviceMessageHandler:
ReplicationUserDevicesResyncRestServlet.make_client(hs)
)
+ # a rate limiter for room key requests. The keys are
+ # (sending_user_id, sending_device_id).
self._ratelimiter = Ratelimiter(
store=self.store,
clock=hs.get_clock(),
@@ -100,12 +102,25 @@ class DeviceMessageHandler:
for user_id, by_device in content["messages"].items():
# we use UserID.from_string to catch invalid user ids
if not self.is_mine(UserID.from_string(user_id)):
- logger.warning("Request for keys for non-local user %s", user_id)
+ logger.warning("To-device message to non-local user %s", user_id)
raise SynapseError(400, "Not a user here")
if not by_device:
continue
+ # Ratelimit key requests by the sending user.
+ if message_type == ToDeviceEventTypes.RoomKeyRequest:
+ allowed, _ = await self._ratelimiter.can_do_action(
+ None, (sender_user_id, None)
+ )
+ if not allowed:
+ logger.info(
+ "Dropping room_key_request from %s to %s due to rate limit",
+ sender_user_id,
+ user_id,
+ )
+ continue
+
messages_by_device = {
device_id: {
"content": message_content,
@@ -192,13 +207,19 @@ class DeviceMessageHandler:
for user_id, by_device in messages.items():
# Ratelimit local cross-user key requests by the sending device.
if (
- message_type == EduTypes.RoomKeyRequest
+ message_type == ToDeviceEventTypes.RoomKeyRequest
and user_id != sender_user_id
- and await self._ratelimiter.can_do_action(
+ ):
+ allowed, _ = await self._ratelimiter.can_do_action(
requester, (sender_user_id, requester.device_id)
)
- ):
- continue
+ if not allowed:
+ logger.info(
+ "Dropping room_key_request from %s to %s due to rate limit",
+ sender_user_id,
+ user_id,
+ )
+ continue
# we use UserID.from_string to catch invalid user ids
if self.is_mine(UserID.from_string(user_id)):
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index de1b14cde3..4064a2b859 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -78,7 +78,7 @@ class DirectoryHandler(BaseHandler):
# TODO(erikj): Add transactions.
# TODO(erikj): Check if there is a current association.
if not servers:
- users = await self.state.get_current_users_in_room(room_id)
+ users = await self.store.get_users_in_room(room_id)
servers = {get_domain_from_id(u) for u in users}
if not servers:
@@ -270,7 +270,7 @@ class DirectoryHandler(BaseHandler):
Codes.NOT_FOUND,
)
- users = await self.state.get_current_users_in_room(room_id)
+ users = await self.store.get_users_in_room(room_id)
extra_servers = {get_domain_from_id(u) for u in users}
servers = set(extra_servers) | set(servers)
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index d82144d7fa..f134f1e234 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -103,7 +103,7 @@ class EventStreamHandler(BaseHandler):
# Send down presence.
if event.state_key == auth_user_id:
# Send down presence for everyone in the room.
- users = await self.state.get_current_users_in_room(
+ users = await self.store.get_users_in_room(
event.room_id
) # type: Iterable[str]
else:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 9d867aaf4d..798ed75b30 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -552,8 +552,12 @@ class FederationHandler(BaseHandler):
destination: str,
room_id: str,
event_id: str,
- ) -> Tuple[List[EventBase], List[EventBase]]:
- """Requests all of the room state at a given event from a remote homeserver.
+ ) -> List[EventBase]:
+ """Requests all of the room state at a given event from a remote
+ homeserver.
+
+ Will also fetch any missing events reported in the `auth_chain_ids`
+ section of `/state_ids`.
Args:
destination: The remote homeserver to query for the state.
@@ -561,8 +565,7 @@ class FederationHandler(BaseHandler):
event_id: The id of the event we want the state at.
Returns:
- A list of events in the state, not including the event itself, and
- a list of events in the auth chain for the given event.
+ A list of events in the state, not including the event itself.
"""
(
state_event_ids,
@@ -571,68 +574,53 @@ class FederationHandler(BaseHandler):
destination, room_id, event_id=event_id
)
- desired_events = set(state_event_ids + auth_event_ids)
-
- event_map = await self._get_events_from_store_or_dest(
- destination, room_id, desired_events
- )
+ # Fetch the state events from the DB, and check we have the auth events.
+ event_map = await self.store.get_events(state_event_ids, allow_rejected=True)
+ auth_events_in_store = await self.store.have_seen_events(auth_event_ids)
- failed_to_fetch = desired_events - event_map.keys()
- if failed_to_fetch:
- logger.warning(
- "Failed to fetch missing state/auth events for %s %s",
- event_id,
- failed_to_fetch,
+ # Check for missing events. We handle state and auth event seperately,
+ # as we want to pull the state from the DB, but we don't for the auth
+ # events. (Note: we likely won't use the majority of the auth chain, and
+ # it can be *huge* for large rooms, so it's worth ensuring that we don't
+ # unnecessarily pull it from the DB).
+ missing_state_events = set(state_event_ids) - set(event_map)
+ missing_auth_events = set(auth_event_ids) - set(auth_events_in_store)
+ if missing_state_events or missing_auth_events:
+ await self._get_events_and_persist(
+ destination=destination,
+ room_id=room_id,
+ events=missing_state_events | missing_auth_events,
)
- remote_state = [
- event_map[e_id] for e_id in state_event_ids if e_id in event_map
- ]
-
- auth_chain = [event_map[e_id] for e_id in auth_event_ids if e_id in event_map]
- auth_chain.sort(key=lambda e: e.depth)
-
- return remote_state, auth_chain
-
- async def _get_events_from_store_or_dest(
- self, destination: str, room_id: str, event_ids: Iterable[str]
- ) -> Dict[str, EventBase]:
- """Fetch events from a remote destination, checking if we already have them.
-
- Persists any events we don't already have as outliers.
-
- If we fail to fetch any of the events, a warning will be logged, and the event
- will be omitted from the result. Likewise, any events which turn out not to
- be in the given room.
-
- This function *does not* automatically get missing auth events of the
- newly fetched events. Callers must include the full auth chain of
- of the missing events in the `event_ids` argument, to ensure that any
- missing auth events are correctly fetched.
+ if missing_state_events:
+ new_events = await self.store.get_events(
+ missing_state_events, allow_rejected=True
+ )
+ event_map.update(new_events)
- Returns:
- map from event_id to event
- """
- fetched_events = await self.store.get_events(event_ids, allow_rejected=True)
+ missing_state_events.difference_update(new_events)
- missing_events = set(event_ids) - fetched_events.keys()
+ if missing_state_events:
+ logger.warning(
+ "Failed to fetch missing state events for %s %s",
+ event_id,
+ missing_state_events,
+ )
- if missing_events:
- logger.debug(
- "Fetching unknown state/auth events %s for room %s",
- missing_events,
- room_id,
- )
+ if missing_auth_events:
+ auth_events_in_store = await self.store.have_seen_events(
+ missing_auth_events
+ )
+ missing_auth_events.difference_update(auth_events_in_store)
- await self._get_events_and_persist(
- destination=destination, room_id=room_id, events=missing_events
- )
+ if missing_auth_events:
+ logger.warning(
+ "Failed to fetch missing auth events for %s %s",
+ event_id,
+ missing_auth_events,
+ )
- # we need to make sure we re-load from the database to get the rejected
- # state correct.
- fetched_events.update(
- (await self.store.get_events(missing_events, allow_rejected=True))
- )
+ remote_state = list(event_map.values())
# check for events which were in the wrong room.
#
@@ -640,8 +628,8 @@ class FederationHandler(BaseHandler):
# auth_events at an event in room A are actually events in room B
bad_events = [
- (event_id, event.room_id)
- for event_id, event in fetched_events.items()
+ (event.event_id, event.room_id)
+ for event in remote_state
if event.room_id != room_id
]
@@ -658,9 +646,10 @@ class FederationHandler(BaseHandler):
room_id,
)
- del fetched_events[bad_event_id]
+ if bad_events:
+ remote_state = [e for e in remote_state if e.room_id == room_id]
- return fetched_events
+ return remote_state
async def _get_state_after_missing_prev_event(
self,
@@ -963,27 +952,23 @@ class FederationHandler(BaseHandler):
# For each edge get the current state.
- auth_events = {}
state_events = {}
events_to_state = {}
for e_id in edges:
- state, auth = await self._get_state_for_room(
+ state = await self._get_state_for_room(
destination=dest,
room_id=room_id,
event_id=e_id,
)
- auth_events.update({a.event_id: a for a in auth})
- auth_events.update({s.event_id: s for s in state})
state_events.update({s.event_id: s for s in state})
events_to_state[e_id] = state
required_auth = {
a_id
- for event in events
- + list(state_events.values())
- + list(auth_events.values())
+ for event in events + list(state_events.values())
for a_id in event.auth_event_ids()
}
+ auth_events = await self.store.get_events(required_auth, allow_rejected=True)
auth_events.update(
{e_id: event_map[e_id] for e_id in required_auth if e_id in event_map}
)
@@ -2446,7 +2431,9 @@ class FederationHandler(BaseHandler):
# If we are going to send this event over federation we precaclculate
# the joined hosts.
if event.internal_metadata.get_send_on_behalf_of():
- await self.event_creation_handler.cache_joined_hosts_for_event(event)
+ await self.event_creation_handler.cache_joined_hosts_for_event(
+ event, context
+ )
return context
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 49f8aa25ea..5afb7fc261 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -51,6 +51,7 @@ from synapse.storage.state import StateFilter
from synapse.types import Requester, RoomAlias, StreamToken, UserID, create_requester
from synapse.util import json_decoder, json_encoder
from synapse.util.async_helpers import Linearizer
+from synapse.util.caches.expiringcache import ExpiringCache
from synapse.util.metrics import measure_func
from synapse.visibility import filter_events_for_client
@@ -258,7 +259,7 @@ class MessageHandler:
"Getting joined members after leaving is not implemented"
)
- users_with_profile = await self.state.get_current_users_in_room(room_id)
+ users_with_profile = await self.store.get_users_in_room_with_profiles(room_id)
# If this is an AS, double check that they are allowed to see the members.
# This can either be because the AS user is in the room or because there
@@ -457,6 +458,19 @@ class EventCreationHandler:
self._external_cache = hs.get_external_cache()
+ # Stores the state groups we've recently added to the joined hosts
+ # external cache. Note that the timeout must be significantly less than
+ # the TTL on the external cache.
+ self._external_cache_joined_hosts_updates = (
+ None
+ ) # type: Optional[ExpiringCache]
+ if self._external_cache.is_enabled():
+ self._external_cache_joined_hosts_updates = ExpiringCache(
+ "_external_cache_joined_hosts_updates",
+ self.clock,
+ expiry_ms=30 * 60 * 1000,
+ )
+
async def create_event(
self,
requester: Requester,
@@ -967,7 +981,7 @@ class EventCreationHandler:
await self.action_generator.handle_push_actions_for_event(event, context)
- await self.cache_joined_hosts_for_event(event)
+ await self.cache_joined_hosts_for_event(event, context)
try:
# If we're a worker we need to hit out to the master.
@@ -1008,7 +1022,9 @@ class EventCreationHandler:
await self.store.remove_push_actions_from_staging(event.event_id)
raise
- async def cache_joined_hosts_for_event(self, event: EventBase) -> None:
+ async def cache_joined_hosts_for_event(
+ self, event: EventBase, context: EventContext
+ ) -> None:
"""Precalculate the joined hosts at the event, when using Redis, so that
external federation senders don't have to recalculate it themselves.
"""
@@ -1016,6 +1032,9 @@ class EventCreationHandler:
if not self._external_cache.is_enabled():
return
+ # If external cache is enabled we should always have this.
+ assert self._external_cache_joined_hosts_updates is not None
+
# We actually store two mappings, event ID -> prev state group,
# state group -> joined hosts, which is much more space efficient
# than event ID -> joined hosts.
@@ -1023,22 +1042,28 @@ class EventCreationHandler:
# Note: We have to cache event ID -> prev state group, as we don't
# store that in the DB.
#
- # Note: We always set the state group -> joined hosts cache, even if
- # we already set it, so that the expiry time is reset.
+ # Note: We set the state group -> joined hosts cache if it hasn't been
+ # set for a while, so that the expiry time is reset.
state_entry = await self.state.resolve_state_groups_for_events(
event.room_id, event_ids=event.prev_event_ids()
)
if state_entry.state_group:
- joined_hosts = await self.store.get_joined_hosts(event.room_id, state_entry)
-
await self._external_cache.set(
"event_to_prev_state_group",
event.event_id,
state_entry.state_group,
expiry_ms=60 * 60 * 1000,
)
+
+ if state_entry.state_group in self._external_cache_joined_hosts_updates:
+ return
+
+ joined_hosts = await self.store.get_joined_hosts(event.room_id, state_entry)
+
+ # Note that the expiry times must be larger than the expiry time in
+ # _external_cache_joined_hosts_updates.
await self._external_cache.set(
"get_joined_hosts",
str(state_entry.state_group),
@@ -1046,6 +1071,8 @@ class EventCreationHandler:
expiry_ms=60 * 60 * 1000,
)
+ self._external_cache_joined_hosts_updates[state_entry.state_group] = None
+
async def _validate_canonical_alias(
self, directory_handler, room_alias_str: str, expected_room_id: str
) -> None:
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index b600c501a1..ba76312baa 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -1212,7 +1212,16 @@ class PresenceHandler(BasePresenceHandler):
max_pos, deltas = await self.store.get_current_state_deltas(
self._event_pos, room_max_stream_ordering
)
- await self._handle_state_delta(deltas)
+
+ # We may get multiple deltas for different rooms, but we want to
+ # handle them on a room by room basis, so we batch them up by
+ # room.
+ deltas_by_room: Dict[str, List[JsonDict]] = {}
+ for delta in deltas:
+ deltas_by_room.setdefault(delta["room_id"], []).append(delta)
+
+ for room_id, deltas_for_room in deltas_by_room.items():
+ await self._handle_state_delta(room_id, deltas_for_room)
self._event_pos = max_pos
@@ -1221,17 +1230,21 @@ class PresenceHandler(BasePresenceHandler):
max_pos
)
- async def _handle_state_delta(self, deltas: List[JsonDict]) -> None:
- """Process current state deltas to find new joins that need to be
- handled.
+ async def _handle_state_delta(self, room_id: str, deltas: List[JsonDict]) -> None:
+ """Process current state deltas for the room to find new joins that need
+ to be handled.
"""
- # A map of destination to a set of user state that they should receive
- presence_destinations = {} # type: Dict[str, Set[UserPresenceState]]
+
+ # Sets of newly joined users. Note that if the local server is
+ # joining a remote room for the first time we'll see both the joining
+ # user and all remote users as newly joined.
+ newly_joined_users = set()
for delta in deltas:
+ assert room_id == delta["room_id"]
+
typ = delta["type"]
state_key = delta["state_key"]
- room_id = delta["room_id"]
event_id = delta["event_id"]
prev_event_id = delta["prev_event_id"]
@@ -1260,72 +1273,55 @@ class PresenceHandler(BasePresenceHandler):
# Ignore changes to join events.
continue
- # Retrieve any user presence state updates that need to be sent as a result,
- # and the destinations that need to receive it
- destinations, user_presence_states = await self._on_user_joined_room(
- room_id, state_key
- )
-
- # Insert the destinations and respective updates into our destinations dict
- for destination in destinations:
- presence_destinations.setdefault(destination, set()).update(
- user_presence_states
- )
-
- # Send out user presence updates for each destination
- for destination, user_state_set in presence_destinations.items():
- self._federation_queue.send_presence_to_destinations(
- destinations=[destination], states=user_state_set
- )
-
- async def _on_user_joined_room(
- self, room_id: str, user_id: str
- ) -> Tuple[List[str], List[UserPresenceState]]:
- """Called when we detect a user joining the room via the current state
- delta stream. Returns the destinations that need to be updated and the
- presence updates to send to them.
-
- Args:
- room_id: The ID of the room that the user has joined.
- user_id: The ID of the user that has joined the room.
-
- Returns:
- A tuple of destinations and presence updates to send to them.
- """
- if self.is_mine_id(user_id):
- # If this is a local user then we need to send their presence
- # out to hosts in the room (who don't already have it)
-
- # TODO: We should be able to filter the hosts down to those that
- # haven't previously seen the user
-
- remote_hosts = await self.state.get_current_hosts_in_room(room_id)
+ newly_joined_users.add(state_key)
- # Filter out ourselves.
- filtered_remote_hosts = [
- host for host in remote_hosts if host != self.server_name
- ]
-
- state = await self.current_state_for_user(user_id)
- return filtered_remote_hosts, [state]
- else:
- # A remote user has joined the room, so we need to:
- # 1. Check if this is a new server in the room
- # 2. If so send any presence they don't already have for
- # local users in the room.
-
- # TODO: We should be able to filter the users down to those that
- # the server hasn't previously seen
-
- # TODO: Check that this is actually a new server joining the
- # room.
-
- remote_host = get_domain_from_id(user_id)
+ if not newly_joined_users:
+ # If nobody has joined then there's nothing to do.
+ return
- users = await self.state.get_current_users_in_room(room_id)
- user_ids = list(filter(self.is_mine_id, users))
+ # We want to send:
+ # 1. presence states of all local users in the room to newly joined
+ # remote servers
+ # 2. presence states of newly joined users to all remote servers in
+ # the room.
+ #
+ # TODO: Only send presence states to remote hosts that don't already
+ # have them (because they already share rooms).
+
+ # Get all the users who were already in the room, by fetching the
+ # current users in the room and removing the newly joined users.
+ users = await self.store.get_users_in_room(room_id)
+ prev_users = set(users) - newly_joined_users
+
+ # Construct sets for all the local users and remote hosts that were
+ # already in the room
+ prev_local_users = []
+ prev_remote_hosts = set()
+ for user_id in prev_users:
+ if self.is_mine_id(user_id):
+ prev_local_users.append(user_id)
+ else:
+ prev_remote_hosts.add(get_domain_from_id(user_id))
+
+ # Similarly, construct sets for all the local users and remote hosts
+ # that were *not* already in the room. Care needs to be taken with the
+ # calculating the remote hosts, as a host may have already been in the
+ # room even if there is a newly joined user from that host.
+ newly_joined_local_users = []
+ newly_joined_remote_hosts = set()
+ for user_id in newly_joined_users:
+ if self.is_mine_id(user_id):
+ newly_joined_local_users.append(user_id)
+ else:
+ host = get_domain_from_id(user_id)
+ if host not in prev_remote_hosts:
+ newly_joined_remote_hosts.add(host)
- states_d = await self.current_state_for_users(user_ids)
+ # Send presence states of all local users in the room to newly joined
+ # remote servers. (We actually only send states for local users already
+ # in the room, as we'll send states for newly joined local users below.)
+ if prev_local_users and newly_joined_remote_hosts:
+ local_states = await self.current_state_for_users(prev_local_users)
# Filter out old presence, i.e. offline presence states where
# the user hasn't been active for a week. We can change this
@@ -1335,13 +1331,27 @@ class PresenceHandler(BasePresenceHandler):
now = self.clock.time_msec()
states = [
state
- for state in states_d.values()
+ for state in local_states.values()
if state.state != PresenceState.OFFLINE
or now - state.last_active_ts < 7 * 24 * 60 * 60 * 1000
or state.status_msg is not None
]
- return [remote_host], states
+ self._federation_queue.send_presence_to_destinations(
+ destinations=newly_joined_remote_hosts,
+ states=states,
+ )
+
+ # Send presence states of newly joined users to all remote servers in
+ # the room
+ if newly_joined_local_users and (
+ prev_remote_hosts or newly_joined_remote_hosts
+ ):
+ local_states = await self.current_state_for_users(newly_joined_local_users)
+ self._federation_queue.send_presence_to_destinations(
+ destinations=prev_remote_hosts | newly_joined_remote_hosts,
+ states=list(local_states.values()),
+ )
def should_notify(old_state: UserPresenceState, new_state: UserPresenceState) -> bool:
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 5a888b7941..fb4823a5cc 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -1327,7 +1327,7 @@ class RoomShutdownHandler:
new_room_id = None
logger.info("Shutting down room %r", room_id)
- users = await self.state.get_current_users_in_room(room_id)
+ users = await self.store.get_users_in_room(room_id)
kicked_users = []
failed_to_kick_users = []
for user_id in users:
diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py
index 01e3e050f9..e35d91832b 100644
--- a/synapse/handlers/space_summary.py
+++ b/synapse/handlers/space_summary.py
@@ -14,6 +14,7 @@
import itertools
import logging
+import re
from collections import deque
from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple, cast
@@ -226,6 +227,23 @@ class SpaceSummaryHandler:
suggested_only: bool,
max_children: Optional[int],
) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]:
+ """
+ Generate a room entry and a list of event entries for a given room.
+
+ Args:
+ requester: The requesting user, or None if this is over federation.
+ room_id: The room ID to summarize.
+ suggested_only: True if only suggested children should be returned.
+ Otherwise, all children are returned.
+ max_children: The maximum number of children to return for this node.
+
+ Returns:
+ A tuple of:
+ An iterable of a single value of the room.
+
+ An iterable of the sorted children events. This may be limited
+ to a maximum size or may include all children.
+ """
if not await self._is_room_accessible(room_id, requester):
return (), ()
@@ -288,6 +306,7 @@ class SpaceSummaryHandler:
ev.data
for ev in res.events
if ev.event_type == EventTypes.MSC1772_SPACE_CHILD
+ or ev.event_type == EventTypes.SpaceChild
)
async def _is_room_accessible(self, room_id: str, requester: Optional[str]) -> bool:
@@ -331,7 +350,9 @@ class SpaceSummaryHandler:
)
# TODO: update once MSC1772 lands
- room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE)
+ room_type = create_event.content.get(EventContentFields.ROOM_TYPE)
+ if not room_type:
+ room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE)
entry = {
"room_id": stats["room_id"],
@@ -344,6 +365,7 @@ class SpaceSummaryHandler:
stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
),
"guest_can_join": stats["guest_access"] == "can_join",
+ "creation_ts": create_event.origin_server_ts,
"room_type": room_type,
}
@@ -353,6 +375,18 @@ class SpaceSummaryHandler:
return room_entry
async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
+ """
+ Get the child events for a given room.
+
+ The returned results are sorted for stability.
+
+ Args:
+ room_id: The room id to get the children of.
+
+ Returns:
+ An iterable of sorted child events.
+ """
+
# look for child rooms/spaces.
current_state_ids = await self._store.get_current_state_ids(room_id)
@@ -360,13 +394,15 @@ class SpaceSummaryHandler:
[
event_id
for key, event_id in current_state_ids.items()
- # TODO: update once MSC1772 lands
+ # TODO: update once MSC1772 has been FCP for a period of time.
if key[0] == EventTypes.MSC1772_SPACE_CHILD
+ or key[0] == EventTypes.SpaceChild
]
)
- # filter out any events without a "via" (which implies it has been redacted)
- return (e for e in events if _has_valid_via(e))
+ # filter out any events without a "via" (which implies it has been redacted),
+ # and order to ensure we return stable results.
+ return sorted(filter(_has_valid_via, events), key=_child_events_comparison_key)
@attr.s(frozen=True, slots=True)
@@ -392,3 +428,39 @@ def _is_suggested_child_event(edge_event: EventBase) -> bool:
return True
logger.debug("Ignorning not-suggested child %s", edge_event.state_key)
return False
+
+
+# Order may only contain characters in the range of \x20 (space) to \x7F (~).
+_INVALID_ORDER_CHARS_RE = re.compile(r"[^\x20-\x7F]")
+
+
+def _child_events_comparison_key(child: EventBase) -> Tuple[bool, Optional[str], str]:
+ """
+ Generate a value for comparing two child events for ordering.
+
+ The rules for ordering are supposed to be:
+
+ 1. The 'order' key, if it is valid.
+ 2. The 'origin_server_ts' of the 'm.room.create' event.
+ 3. The 'room_id'.
+
+ But we skip step 2 since we may not have any state from the room.
+
+ Args:
+ child: The event for generating a comparison key.
+
+ Returns:
+ The comparison key as a tuple of:
+ False if the ordering is valid.
+ The ordering field.
+ The room ID.
+ """
+ order = child.content.get("order")
+ # If order is not a string or doesn't meet the requirements, ignore it.
+ if not isinstance(order, str):
+ order = None
+ elif len(order) > 50 or _INVALID_ORDER_CHARS_RE.search(order):
+ order = None
+
+ # Items without an order come last.
+ return (order is None, order, child.room_id)
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index a9a3ee05c3..0fcc1532da 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1190,7 +1190,7 @@ class SyncHandler:
# Step 1b, check for newly joined rooms
for room_id in newly_joined_rooms:
- joined_users = await self.state.get_current_users_in_room(room_id)
+ joined_users = await self.store.get_users_in_room(room_id)
newly_joined_or_invited_users.update(joined_users)
# TODO: Check that these users are actually new, i.e. either they
@@ -1206,7 +1206,7 @@ class SyncHandler:
# Now find users that we no longer track
for room_id in newly_left_rooms:
- left_users = await self.state.get_current_users_in_room(room_id)
+ left_users = await self.store.get_users_in_room(room_id)
newly_left_users.update(left_users)
# Remove any users that we still share a room with.
@@ -1361,7 +1361,7 @@ class SyncHandler:
extra_users_ids = set(newly_joined_or_invited_users)
for room_id in newly_joined_rooms:
- users = await self.state.get_current_users_in_room(room_id)
+ users = await self.store.get_users_in_room(room_id)
extra_users_ids.update(users)
extra_users_ids.discard(user.to_string())
diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py
index e00969f8b1..b50a4f95eb 100644
--- a/synapse/logging/__init__.py
+++ b/synapse/logging/__init__.py
@@ -12,8 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# These are imported to allow for nicer logging configuration files.
+import logging
+
from synapse.logging._remote import RemoteHandler
from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter
+# These are imported to allow for nicer logging configuration files.
__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"]
+
+# Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc
+issue9533_logger = logging.getLogger("synapse.9533_debug")
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 31b7b3c256..fef2846669 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -535,6 +535,13 @@ class ReactorLastSeenMetric:
REGISTRY.register(ReactorLastSeenMetric())
+# The minimum time in seconds between GCs for each generation, regardless of the current GC
+# thresholds and counts.
+MIN_TIME_BETWEEN_GCS = (1.0, 10.0, 30.0)
+
+# The time (in seconds since the epoch) of the last time we did a GC for each generation.
+_last_gc = [0.0, 0.0, 0.0]
+
def runUntilCurrentTimer(reactor, func):
@functools.wraps(func)
@@ -575,11 +582,16 @@ def runUntilCurrentTimer(reactor, func):
return ret
# Check if we need to do a manual GC (since its been disabled), and do
- # one if necessary.
+ # one if necessary. Note we go in reverse order as e.g. a gen 1 GC may
+ # promote an object into gen 2, and we don't want to handle the same
+ # object multiple times.
threshold = gc.get_threshold()
counts = gc.get_count()
for i in (2, 1, 0):
- if threshold[i] < counts[i]:
+ # We check if we need to do one based on a straightforward
+ # comparison between the threshold and count. We also do an extra
+ # check to make sure that we don't a GC too often.
+ if threshold[i] < counts[i] and MIN_TIME_BETWEEN_GCS[i] < end - _last_gc[i]:
if i == 0:
logger.debug("Collecting gc %d", i)
else:
@@ -589,6 +601,8 @@ def runUntilCurrentTimer(reactor, func):
unreachable = gc.collect(i)
end = time.time()
+ _last_gc[i] = end
+
gc_time.labels(i).observe(end - start)
gc_unreachable.labels(i).set(unreachable)
@@ -615,6 +629,7 @@ try:
except AttributeError:
pass
+
__all__ = [
"MetricsResource",
"generate_latest",
diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py
new file mode 100644
index 0000000000..29ab6c0229
--- /dev/null
+++ b/synapse/metrics/jemalloc.py
@@ -0,0 +1,196 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ctypes
+import logging
+import os
+import re
+from typing import Optional
+
+from synapse.metrics import REGISTRY, GaugeMetricFamily
+
+logger = logging.getLogger(__name__)
+
+
+def _setup_jemalloc_stats():
+ """Checks to see if jemalloc is loaded, and hooks up a collector to record
+ statistics exposed by jemalloc.
+ """
+
+ # Try to find the loaded jemalloc shared library, if any. We need to
+ # introspect into what is loaded, rather than loading whatever is on the
+ # path, as if we load a *different* jemalloc version things will seg fault.
+
+ # We look in `/proc/self/maps`, which only exists on linux.
+ if not os.path.exists("/proc/self/maps"):
+ logger.debug("Not looking for jemalloc as no /proc/self/maps exist")
+ return
+
+ # We're looking for a path at the end of the line that includes
+ # "libjemalloc".
+ regex = re.compile(r"/\S+/libjemalloc.*$")
+
+ jemalloc_path = None
+ with open("/proc/self/maps") as f:
+ for line in f:
+ match = regex.search(line.strip())
+ if match:
+ jemalloc_path = match.group()
+
+ if not jemalloc_path:
+ # No loaded jemalloc was found.
+ logger.debug("jemalloc not found")
+ return
+
+ logger.debug("Found jemalloc at %s", jemalloc_path)
+
+ jemalloc = ctypes.CDLL(jemalloc_path)
+
+ def _mallctl(
+ name: str, read: bool = True, write: Optional[int] = None
+ ) -> Optional[int]:
+ """Wrapper around `mallctl` for reading and writing integers to
+ jemalloc.
+
+ Args:
+ name: The name of the option to read from/write to.
+ read: Whether to try and read the value.
+ write: The value to write, if given.
+
+ Returns:
+ The value read if `read` is True, otherwise None.
+
+ Raises:
+ An exception if `mallctl` returns a non-zero error code.
+ """
+
+ input_var = None
+ input_var_ref = None
+ input_len_ref = None
+ if read:
+ input_var = ctypes.c_size_t(0)
+ input_len = ctypes.c_size_t(ctypes.sizeof(input_var))
+
+ input_var_ref = ctypes.byref(input_var)
+ input_len_ref = ctypes.byref(input_len)
+
+ write_var_ref = None
+ write_len = ctypes.c_size_t(0)
+ if write is not None:
+ write_var = ctypes.c_size_t(write)
+ write_len = ctypes.c_size_t(ctypes.sizeof(write_var))
+
+ write_var_ref = ctypes.byref(write_var)
+
+ # The interface is:
+ #
+ # int mallctl(
+ # const char *name,
+ # void *oldp,
+ # size_t *oldlenp,
+ # void *newp,
+ # size_t newlen
+ # )
+ #
+ # Where oldp/oldlenp is a buffer where the old value will be written to
+ # (if not null), and newp/newlen is the buffer with the new value to set
+ # (if not null). Note that they're all references *except* newlen.
+ result = jemalloc.mallctl(
+ name.encode("ascii"),
+ input_var_ref,
+ input_len_ref,
+ write_var_ref,
+ write_len,
+ )
+
+ if result != 0:
+ raise Exception("Failed to call mallctl")
+
+ if input_var is None:
+ return None
+
+ return input_var.value
+
+ def _jemalloc_refresh_stats() -> None:
+ """Request that jemalloc updates its internal statistics. This needs to
+ be called before querying for stats, otherwise it will return stale
+ values.
+ """
+ try:
+ _mallctl("epoch", read=False, write=1)
+ except Exception as e:
+ logger.warning("Failed to reload jemalloc stats: %s", e)
+
+ class JemallocCollector:
+ """Metrics for internal jemalloc stats."""
+
+ def collect(self):
+ _jemalloc_refresh_stats()
+
+ g = GaugeMetricFamily(
+ "jemalloc_stats_app_memory_bytes",
+ "The stats reported by jemalloc",
+ labels=["type"],
+ )
+
+ # Read the relevant global stats from jemalloc. Note that these may
+ # not be accurate if python is configured to use its internal small
+ # object allocator (which is on by default, disable by setting the
+ # env `PYTHONMALLOC=malloc`).
+ #
+ # See the jemalloc manpage for details about what each value means,
+ # roughly:
+ # - allocated ─ Total number of bytes allocated by the app
+ # - active ─ Total number of bytes in active pages allocated by
+ # the application, this is bigger than `allocated`.
+ # - resident ─ Maximum number of bytes in physically resident data
+ # pages mapped by the allocator, comprising all pages dedicated
+ # to allocator metadata, pages backing active allocations, and
+ # unused dirty pages. This is bigger than `active`.
+ # - mapped ─ Total number of bytes in active extents mapped by the
+ # allocator.
+ # - metadata ─ Total number of bytes dedicated to jemalloc
+ # metadata.
+ for t in (
+ "allocated",
+ "active",
+ "resident",
+ "mapped",
+ "metadata",
+ ):
+ try:
+ value = _mallctl(f"stats.{t}")
+ except Exception as e:
+ # There was an error fetching the value, skip.
+ logger.warning("Failed to read jemalloc stats.%s: %s", t, e)
+ continue
+
+ g.add_metric([t], value=value)
+
+ yield g
+
+ REGISTRY.register(JemallocCollector())
+
+ logger.debug("Added jemalloc stats")
+
+
+def setup_jemalloc_stats():
+ """Try to setup jemalloc stats, if jemalloc is loaded."""
+
+ try:
+ _setup_jemalloc_stats()
+ except Exception as e:
+ # This should only happen if we find the loaded jemalloc library, but
+ # fail to load it somehow (e.g. we somehow picked the wrong version).
+ logger.info("Failed to setup collector to record jemalloc stats: %s", e)
diff --git a/synapse/notifier.py b/synapse/notifier.py
index b9531007e2..24b4e6649f 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -38,6 +38,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership
from synapse.api.errors import AuthError
from synapse.events import EventBase
from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
from synapse.logging.context import PreserveLoggingContext
from synapse.logging.opentracing import log_kv, start_active_span
from synapse.logging.utils import log_function
@@ -426,6 +427,13 @@ class Notifier:
for room in rooms:
user_streams |= self.room_to_user_streams.get(room, set())
+ if stream_key == "to_device_key":
+ issue9533_logger.debug(
+ "to-device messages stream id %s, awaking streams for %s",
+ new_token,
+ users,
+ )
+
time_now_ms = self.clock.time_msec()
for user_stream in user_streams:
try:
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 49ecb38522..98b90a4f51 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Pattern, Tuple, Union
from synapse.events import EventBase
from synapse.types import UserID
+from synapse.util import glob_to_regex, re_word_boundary
from synapse.util.caches.lrucache import LruCache
logger = logging.getLogger(__name__)
@@ -183,7 +184,7 @@ class PushRuleEvaluatorForEvent:
r = regex_cache.get((display_name, False, True), None)
if not r:
r1 = re.escape(display_name)
- r1 = _re_word_boundary(r1)
+ r1 = re_word_boundary(r1)
r = re.compile(r1, flags=re.IGNORECASE)
regex_cache[(display_name, False, True)] = r
@@ -212,7 +213,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
try:
r = regex_cache.get((glob, True, word_boundary), None)
if not r:
- r = _glob_to_re(glob, word_boundary)
+ r = glob_to_regex(glob, word_boundary)
regex_cache[(glob, True, word_boundary)] = r
return bool(r.search(value))
except re.error:
@@ -220,56 +221,6 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
return False
-def _glob_to_re(glob: str, word_boundary: bool) -> Pattern:
- """Generates regex for a given glob.
-
- Args:
- glob
- word_boundary: Whether to match against word boundaries or entire string.
- """
- if IS_GLOB.search(glob):
- r = re.escape(glob)
-
- r = r.replace(r"\*", ".*?")
- r = r.replace(r"\?", ".")
-
- # handle [abc], [a-z] and [!a-z] style ranges.
- r = GLOB_REGEX.sub(
- lambda x: (
- "[%s%s]" % (x.group(1) and "^" or "", x.group(2).replace(r"\\\-", "-"))
- ),
- r,
- )
- if word_boundary:
- r = _re_word_boundary(r)
-
- return re.compile(r, flags=re.IGNORECASE)
- else:
- r = "^" + r + "$"
-
- return re.compile(r, flags=re.IGNORECASE)
- elif word_boundary:
- r = re.escape(glob)
- r = _re_word_boundary(r)
-
- return re.compile(r, flags=re.IGNORECASE)
- else:
- r = "^" + re.escape(glob) + "$"
- return re.compile(r, flags=re.IGNORECASE)
-
-
-def _re_word_boundary(r: str) -> str:
- """
- Adds word boundary characters to the start and end of an
- expression to require that the match occur as a whole word,
- but do so respecting the fact that strings starting or ending
- with non-word characters will change word boundaries.
- """
- # we can't use \b as it chokes on unicode. however \W seems to be okay
- # as shorthand for [^0-9A-Za-z_].
- return r"(^|\W)%s(\W|$)" % (r,)
-
-
def _flatten_dict(
d: Union[EventBase, dict],
prefix: Optional[List[str]] = None,
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 2de946f464..989523c823 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -78,7 +78,8 @@ REQUIREMENTS = [
# we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note:
# Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33
# is out in November.)
- "attrs>=19.1.0",
+ # Note: 21.1.0 broke `/sync`, see #9936
+ "attrs>=19.1.0,!=21.1.0",
"netaddr>=0.7.18",
"Jinja2>=2.9",
"bleach>=1.4.3",
@@ -116,6 +117,8 @@ CONDITIONAL_REQUIREMENTS = {
# hiredis is not a *strict* dependency, but it makes things much faster.
# (if it is not installed, we fall back to slow code.)
"redis": ["txredisapi>=1.4.7", "hiredis"],
+ # Required to use experimental `caches.track_memory_usage` config option.
+ "cache_memory": ["pympler"],
}
ALL_OPTIONAL_REQUIREMENTS = set() # type: Set[str]
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index 4f3c6a18b6..62d7809175 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -51,7 +51,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-
# How long we allow callers to wait for replication updates before timing out.
_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS = 30
diff --git a/synapse/replication/tcp/external_cache.py b/synapse/replication/tcp/external_cache.py
index 1a3b051e3c..b402f82810 100644
--- a/synapse/replication/tcp/external_cache.py
+++ b/synapse/replication/tcp/external_cache.py
@@ -15,7 +15,7 @@
import logging
from typing import TYPE_CHECKING, Any, Optional
-from prometheus_client import Counter
+from prometheus_client import Counter, Histogram
from synapse.logging.context import make_deferred_yieldable
from synapse.util import json_decoder, json_encoder
@@ -35,6 +35,20 @@ get_counter = Counter(
labelnames=["cache_name", "hit"],
)
+response_timer = Histogram(
+ "synapse_external_cache_response_time_seconds",
+ "Time taken to get a response from Redis for a cache get/set request",
+ labelnames=["method"],
+ buckets=(
+ 0.001,
+ 0.002,
+ 0.005,
+ 0.01,
+ 0.02,
+ 0.05,
+ ),
+)
+
logger = logging.getLogger(__name__)
@@ -72,13 +86,14 @@ class ExternalCache:
logger.debug("Caching %s %s: %r", cache_name, key, encoded_value)
- return await make_deferred_yieldable(
- self._redis_connection.set(
- self._get_redis_key(cache_name, key),
- encoded_value,
- pexpire=expiry_ms,
+ with response_timer.labels("set").time():
+ return await make_deferred_yieldable(
+ self._redis_connection.set(
+ self._get_redis_key(cache_name, key),
+ encoded_value,
+ pexpire=expiry_ms,
+ )
)
- )
async def get(self, cache_name: str, key: str) -> Optional[Any]:
"""Look up a key/value in the named cache."""
@@ -86,9 +101,10 @@ class ExternalCache:
if self._redis_connection is None:
return None
- result = await make_deferred_yieldable(
- self._redis_connection.get(self._get_redis_key(cache_name, key))
- )
+ with response_timer.labels("get").time():
+ result = await make_deferred_yieldable(
+ self._redis_connection.get(self._get_redis_key(cache_name, key))
+ )
logger.debug("Got cache result %s %s: %r", cache_name, key, result)
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 5cab4d3c7b..51813cccbe 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -1020,6 +1020,7 @@ class RoomSpaceSummaryRestServlet(RestServlet):
max_rooms_per_space=parse_integer(request, "max_rooms_per_space"),
)
+ # TODO When switching to the stable endpoint, remove the POST handler.
async def on_POST(
self, request: SynapseRequest, room_id: str
) -> Tuple[int, JsonDict]:
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index b3bd92d37c..a1770f620e 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -213,19 +213,23 @@ class StateHandler:
return ret.state
async def get_current_users_in_room(
- self, room_id: str, latest_event_ids: Optional[List[str]] = None
+ self, room_id: str, latest_event_ids: List[str]
) -> Dict[str, ProfileInfo]:
"""
Get the users who are currently in a room.
+ Note: This is much slower than using the equivalent method
+ `DataStore.get_users_in_room` or `DataStore.get_users_in_room_with_profiles`,
+ so this should only be used when wanting the users at a particular point
+ in the room.
+
Args:
room_id: The ID of the room.
latest_event_ids: Precomputed list of latest event IDs. Will be computed if None.
Returns:
Dictionary of user IDs to their profileinfo.
"""
- if not latest_event_ids:
- latest_event_ids = await self.store.get_latest_event_ids_in_room(room_id)
+
assert latest_event_ids is not None
logger.debug("calling resolve_state_groups from get_current_users_in_room")
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 6b68d8720c..3d98d3f5f8 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -69,6 +69,7 @@ class SQLBaseStore(metaclass=ABCMeta):
self._attempt_to_invalidate_cache("is_host_joined", (room_id, host))
self._attempt_to_invalidate_cache("get_users_in_room", (room_id,))
+ self._attempt_to_invalidate_cache("get_users_in_room_with_profiles", (room_id,))
self._attempt_to_invalidate_cache("get_room_summary", (room_id,))
self._attempt_to_invalidate_cache("get_current_state_ids", (room_id,))
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 7c9d1f744e..50e7ddd735 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -15,6 +15,7 @@
import logging
from typing import List, Optional, Tuple
+from synapse.logging import issue9533_logger
from synapse.logging.opentracing import log_kv, set_tag, trace
from synapse.replication.tcp.streams import ToDeviceStream
from synapse.storage._base import SQLBaseStore, db_to_json
@@ -404,6 +405,13 @@ class DeviceInboxWorkerStore(SQLBaseStore):
],
)
+ if remote_messages_by_destination:
+ issue9533_logger.debug(
+ "Queued outgoing to-device messages with stream_id %i for %s",
+ stream_id,
+ list(remote_messages_by_destination.keys()),
+ )
+
async with self._device_inbox_id_gen.get_next() as stream_id:
now_ms = self.clock.time_msec()
await self.db_pool.runInteraction(
@@ -533,6 +541,16 @@ class DeviceInboxWorkerStore(SQLBaseStore):
],
)
+ issue9533_logger.debug(
+ "Stored to-device messages with stream_id %i for %s",
+ stream_id,
+ [
+ (user_id, device_id)
+ for (user_id, messages_by_device) in local_by_user_then_device.items()
+ for device_id in messages_by_device.keys()
+ ],
+ )
+
class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py
index 88afe97c41..398d6b6acb 100644
--- a/synapse/storage/databases/main/end_to_end_keys.py
+++ b/synapse/storage/databases/main/end_to_end_keys.py
@@ -84,7 +84,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore):
if keys:
result["keys"] = keys
- device_display_name = device.display_name
+ device_display_name = None
+ if self.hs.config.allow_device_name_lookup_over_federation:
+ device_display_name = device.display_name
if device_display_name:
result["device_display_name"] = device_display_name
diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py
index 2a8532f8c1..5fc3bb5a7d 100644
--- a/synapse/storage/databases/main/roommember.py
+++ b/synapse/storage/databases/main/roommember.py
@@ -205,8 +205,12 @@ class RoomMemberWorkerStore(EventsWorkerStore):
def _get_users_in_room_with_profiles(txn) -> Dict[str, ProfileInfo]:
sql = """
- SELECT user_id, display_name, avatar_url FROM room_memberships
- WHERE room_id = ? AND membership = ?
+ SELECT state_key, display_name, avatar_url FROM room_memberships as m
+ INNER JOIN current_state_events as c
+ ON m.event_id = c.event_id
+ AND m.room_id = c.room_id
+ AND m.user_id = c.state_key
+ WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?
"""
txn.execute(sql, (room_id, Membership.JOIN))
diff --git a/synapse/storage/databases/main/schema/full_schemas/README.md b/synapse/storage/databases/main/schema/full_schemas/README.md
deleted file mode 100644
index c00f287190..0000000000
--- a/synapse/storage/databases/main/schema/full_schemas/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Synapse Database Schemas
-
-These schemas are used as a basis to create brand new Synapse databases, on both
-SQLite3 and Postgres.
-
-## Building full schema dumps
-
-If you want to recreate these schemas, they need to be made from a database that
-has had all background updates run.
-
-To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
-`full.sql.postgres ` and `full.sql.sqlite` files.
-
-Ensure postgres is installed and your user has the ability to run bash commands
-such as `createdb`, then call
-
- ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
-
-There are currently two folders with full-schema snapshots. `16` is a snapshot
-from 2015, for historical reference. The other contains the most recent full
-schema snapshot.
diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py
index 7a082fdd21..a6bfb4902a 100644
--- a/synapse/storage/databases/main/user_directory.py
+++ b/synapse/storage/databases/main/user_directory.py
@@ -142,8 +142,6 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
batch_size (int): Maximum number of state events to process
per cycle.
"""
- state = self.hs.get_state_handler()
-
# If we don't have progress filed, delete everything.
if not progress:
await self.delete_all_from_user_dir()
@@ -197,7 +195,7 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
room_id
)
- users_with_profile = await state.get_current_users_in_room(room_id)
+ users_with_profile = await self.get_users_in_room_with_profiles(room_id)
user_ids = set(users_with_profile)
# Update each user in the user directory.
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 7a2cbee426..3799d46734 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -26,16 +26,13 @@ from synapse.config.homeserver import HomeServerConfig
from synapse.storage.database import LoggingDatabaseConnection
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.schema import SCHEMA_VERSION
from synapse.storage.types import Cursor
logger = logging.getLogger(__name__)
-# Remember to update this number every time a change is made to database
-# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 59
-
-dir_path = os.path.abspath(os.path.dirname(__file__))
+schema_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "schema")
class PrepareDatabaseException(Exception):
@@ -167,7 +164,14 @@ def _setup_new_database(
Example directory structure:
- schema/
+ schema/
+ common/
+ delta/
+ ...
+ full_schemas/
+ 11/
+ foo.sql
+ main/
delta/
...
full_schemas/
@@ -175,15 +179,14 @@ def _setup_new_database(
test.sql
...
11/
- foo.sql
bar.sql
...
In the example foo.sql and bar.sql would be run, and then any delta files
for versions strictly greater than 11.
- Note: we apply the full schemas and deltas from the top level `schema/`
- folder as well those in the data stores specified.
+ Note: we apply the full schemas and deltas from the `schema/common`
+ folder as well those in the databases specified.
Args:
cur: a database cursor
@@ -195,12 +198,12 @@ def _setup_new_database(
# configured to our liking.
database_engine.check_new_database(cur)
- current_dir = os.path.join(dir_path, "schema", "full_schemas")
+ full_schemas_dir = os.path.join(schema_path, "common", "full_schemas")
# First we find the highest full schema version we have
valid_versions = []
- for filename in os.listdir(current_dir):
+ for filename in os.listdir(full_schemas_dir):
try:
ver = int(filename)
except ValueError:
@@ -218,15 +221,13 @@ def _setup_new_database(
logger.debug("Initialising schema v%d", max_current_ver)
- # Now lets find all the full schema files, both in the global schema and
- # in data store schemas.
- directories = [os.path.join(current_dir, str(max_current_ver))]
+ # Now let's find all the full schema files, both in the common schema and
+ # in database schemas.
+ directories = [os.path.join(full_schemas_dir, str(max_current_ver))]
directories.extend(
os.path.join(
- dir_path,
- "databases",
+ schema_path,
database,
- "schema",
"full_schemas",
str(max_current_ver),
)
@@ -357,6 +358,9 @@ def _upgrade_existing_database(
check_database_before_upgrade(cur, database_engine, config)
start_ver = current_version
+
+ # if we got to this schema version by running a full_schema rather than a series
+ # of deltas, we should not run the deltas for this version.
if not upgraded:
start_ver += 1
@@ -385,12 +389,10 @@ def _upgrade_existing_database(
# directories for schema updates.
# First we find the directories to search in
- delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
+ delta_dir = os.path.join(schema_path, "common", "delta", str(v))
directories = [delta_dir]
for database in databases:
- directories.append(
- os.path.join(dir_path, "databases", database, "schema", "delta", str(v))
- )
+ directories.append(os.path.join(schema_path, database, "delta", str(v)))
# Used to check if we have any duplicate file names
file_name_counter = Counter() # type: CounterType[str]
@@ -621,8 +623,8 @@ def _get_or_create_schema_state(
txn: Cursor, database_engine: BaseDatabaseEngine
) -> Optional[Tuple[int, List[str], bool]]:
# Bluntly try creating the schema_version tables.
- schema_path = os.path.join(dir_path, "schema", "schema_version.sql")
- executescript(txn, schema_path)
+ sql_path = os.path.join(schema_path, "common", "schema_version.sql")
+ executescript(txn, sql_path)
txn.execute("SELECT version, upgraded FROM schema_version")
row = txn.fetchone()
diff --git a/synapse/storage/schema/README.md b/synapse/storage/schema/README.md
new file mode 100644
index 0000000000..030153db64
--- /dev/null
+++ b/synapse/storage/schema/README.md
@@ -0,0 +1,37 @@
+# Synapse Database Schemas
+
+This directory contains the schema files used to build Synapse databases.
+
+Synapse supports splitting its datastore across multiple physical databases (which can
+be useful for large installations), and the schema files are therefore split according
+to the logical database they are apply to.
+
+At the time of writing, the following "logical" databases are supported:
+
+* `state` - used to store Matrix room state (more specifically, `state_groups`,
+ their relationships and contents.)
+* `main` - stores everything else.
+
+Addionally, the `common` directory contains schema files for tables which must be
+present on *all* physical databases.
+
+## Full schema dumps
+
+In the `full_schemas` directories, only the most recently-numbered snapshot is useful
+(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical
+reference only.
+
+## Building full schema dumps
+
+If you want to recreate these schemas, they need to be made from a database that
+has had all background updates run.
+
+To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
+`full.sql.postgres` and `full.sql.sqlite` files.
+
+Ensure postgres is installed, then run:
+
+ ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+
+NB at the time of writing, this script predates the split into separate `state`/`main`
+databases so will require updates to handle that correctly.
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
new file mode 100644
index 0000000000..f0d9f23167
--- /dev/null
+++ b/synapse/storage/schema/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remember to update this number every time a change is made to database
+# schema files, so the users will be informed on server restarts.
+SCHEMA_VERSION = 59
diff --git a/synapse/storage/schema/delta/25/00background_updates.sql b/synapse/storage/schema/common/delta/25/00background_updates.sql
index 2ad9e8fa56..2ad9e8fa56 100644
--- a/synapse/storage/schema/delta/25/00background_updates.sql
+++ b/synapse/storage/schema/common/delta/25/00background_updates.sql
diff --git a/synapse/storage/schema/delta/35/00background_updates_add_col.sql b/synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
index c2d2a4f836..c2d2a4f836 100644
--- a/synapse/storage/schema/delta/35/00background_updates_add_col.sql
+++ b/synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
diff --git a/synapse/storage/schema/delta/58/00background_update_ordering.sql b/synapse/storage/schema/common/delta/58/00background_update_ordering.sql
index 02dae587cc..02dae587cc 100644
--- a/synapse/storage/schema/delta/58/00background_update_ordering.sql
+++ b/synapse/storage/schema/common/delta/58/00background_update_ordering.sql
diff --git a/synapse/storage/schema/full_schemas/54/full.sql b/synapse/storage/schema/common/full_schemas/54/full.sql
index 1005880466..1005880466 100644
--- a/synapse/storage/schema/full_schemas/54/full.sql
+++ b/synapse/storage/schema/common/full_schemas/54/full.sql
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/common/schema_version.sql
index 42e5cb6df5..42e5cb6df5 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/common/schema_version.sql
diff --git a/synapse/storage/databases/main/schema/delta/12/v12.sql b/synapse/storage/schema/main/delta/12/v12.sql
index 5964c5aaac..5964c5aaac 100644
--- a/synapse/storage/databases/main/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/main/delta/12/v12.sql
diff --git a/synapse/storage/databases/main/schema/delta/13/v13.sql b/synapse/storage/schema/main/delta/13/v13.sql
index f8649e5d99..f8649e5d99 100644
--- a/synapse/storage/databases/main/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/main/delta/13/v13.sql
diff --git a/synapse/storage/databases/main/schema/delta/14/v14.sql b/synapse/storage/schema/main/delta/14/v14.sql
index a831920da6..a831920da6 100644
--- a/synapse/storage/databases/main/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/main/delta/14/v14.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/main/delta/15/appservice_txns.sql
index e4f5e76aec..e4f5e76aec 100644
--- a/synapse/storage/databases/main/schema/delta/15/appservice_txns.sql
+++ b/synapse/storage/schema/main/delta/15/appservice_txns.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/presence_indices.sql b/synapse/storage/schema/main/delta/15/presence_indices.sql
index 6b8d0f1ca7..6b8d0f1ca7 100644
--- a/synapse/storage/databases/main/schema/delta/15/presence_indices.sql
+++ b/synapse/storage/schema/main/delta/15/presence_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/v15.sql b/synapse/storage/schema/main/delta/15/v15.sql
index 9523d2bcc3..9523d2bcc3 100644
--- a/synapse/storage/databases/main/schema/delta/15/v15.sql
+++ b/synapse/storage/schema/main/delta/15/v15.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/events_order_index.sql b/synapse/storage/schema/main/delta/16/events_order_index.sql
index a48f215170..a48f215170 100644
--- a/synapse/storage/databases/main/schema/delta/16/events_order_index.sql
+++ b/synapse/storage/schema/main/delta/16/events_order_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
index 7a15265cb1..7a15265cb1 100644
--- a/synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql
+++ b/synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/main/delta/16/remove_duplicates.sql
index 65c97b5e2f..65c97b5e2f 100644
--- a/synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql
+++ b/synapse/storage/schema/main/delta/16/remove_duplicates.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/main/delta/16/room_alias_index.sql
index f82486132b..f82486132b 100644
--- a/synapse/storage/databases/main/schema/delta/16/room_alias_index.sql
+++ b/synapse/storage/schema/main/delta/16/room_alias_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/main/delta/16/unique_constraints.sql
index 5b8de52c33..5b8de52c33 100644
--- a/synapse/storage/databases/main/schema/delta/16/unique_constraints.sql
+++ b/synapse/storage/schema/main/delta/16/unique_constraints.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/users.sql b/synapse/storage/schema/main/delta/16/users.sql
index cd0709250d..cd0709250d 100644
--- a/synapse/storage/databases/main/schema/delta/16/users.sql
+++ b/synapse/storage/schema/main/delta/16/users.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/drop_indexes.sql b/synapse/storage/schema/main/delta/17/drop_indexes.sql
index 7c9a90e27f..7c9a90e27f 100644
--- a/synapse/storage/databases/main/schema/delta/17/drop_indexes.sql
+++ b/synapse/storage/schema/main/delta/17/drop_indexes.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/server_keys.sql b/synapse/storage/schema/main/delta/17/server_keys.sql
index 70b247a06b..70b247a06b 100644
--- a/synapse/storage/databases/main/schema/delta/17/server_keys.sql
+++ b/synapse/storage/schema/main/delta/17/server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/user_threepids.sql b/synapse/storage/schema/main/delta/17/user_threepids.sql
index c17715ac80..c17715ac80 100644
--- a/synapse/storage/databases/main/schema/delta/17/user_threepids.sql
+++ b/synapse/storage/schema/main/delta/17/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
index 6e0871c92b..6e0871c92b 100644
--- a/synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql
+++ b/synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
diff --git a/synapse/storage/databases/main/schema/delta/19/event_index.sql b/synapse/storage/schema/main/delta/19/event_index.sql
index 18b97b4332..18b97b4332 100644
--- a/synapse/storage/databases/main/schema/delta/19/event_index.sql
+++ b/synapse/storage/schema/main/delta/19/event_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/dummy.sql b/synapse/storage/schema/main/delta/20/dummy.sql
index e0ac49d1ec..e0ac49d1ec 100644
--- a/synapse/storage/databases/main/schema/delta/20/dummy.sql
+++ b/synapse/storage/schema/main/delta/20/dummy.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/pushers.py b/synapse/storage/schema/main/delta/20/pushers.py
index 45b846e6a7..45b846e6a7 100644
--- a/synapse/storage/databases/main/schema/delta/20/pushers.py
+++ b/synapse/storage/schema/main/delta/20/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/main/delta/21/end_to_end_keys.sql
index 4c2fb20b77..4c2fb20b77 100644
--- a/synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql
+++ b/synapse/storage/schema/main/delta/21/end_to_end_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/21/receipts.sql b/synapse/storage/schema/main/delta/21/receipts.sql
index d070845477..d070845477 100644
--- a/synapse/storage/databases/main/schema/delta/21/receipts.sql
+++ b/synapse/storage/schema/main/delta/21/receipts.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/receipts_index.sql b/synapse/storage/schema/main/delta/22/receipts_index.sql
index bfc0b3bcaa..bfc0b3bcaa 100644
--- a/synapse/storage/databases/main/schema/delta/22/receipts_index.sql
+++ b/synapse/storage/schema/main/delta/22/receipts_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/main/delta/22/user_threepids_unique.sql
index 87edfa454c..87edfa454c 100644
--- a/synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql
+++ b/synapse/storage/schema/main/delta/22/user_threepids_unique.sql
diff --git a/synapse/storage/databases/main/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/main/delta/24/stats_reporting.sql
index acea7483bd..acea7483bd 100644
--- a/synapse/storage/databases/main/schema/delta/24/stats_reporting.sql
+++ b/synapse/storage/schema/main/delta/24/stats_reporting.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py
index 21f57825d4..21f57825d4 100644
--- a/synapse/storage/databases/main/schema/delta/25/fts.py
+++ b/synapse/storage/schema/main/delta/25/fts.py
diff --git a/synapse/storage/databases/main/schema/delta/25/guest_access.sql b/synapse/storage/schema/main/delta/25/guest_access.sql
index 1ea389b471..1ea389b471 100644
--- a/synapse/storage/databases/main/schema/delta/25/guest_access.sql
+++ b/synapse/storage/schema/main/delta/25/guest_access.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/history_visibility.sql b/synapse/storage/schema/main/delta/25/history_visibility.sql
index f468fc1897..f468fc1897 100644
--- a/synapse/storage/databases/main/schema/delta/25/history_visibility.sql
+++ b/synapse/storage/schema/main/delta/25/history_visibility.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/tags.sql b/synapse/storage/schema/main/delta/25/tags.sql
index 7a32ce68e4..7a32ce68e4 100644
--- a/synapse/storage/databases/main/schema/delta/25/tags.sql
+++ b/synapse/storage/schema/main/delta/25/tags.sql
diff --git a/synapse/storage/databases/main/schema/delta/26/account_data.sql b/synapse/storage/schema/main/delta/26/account_data.sql
index e395de2b5e..e395de2b5e 100644
--- a/synapse/storage/databases/main/schema/delta/26/account_data.sql
+++ b/synapse/storage/schema/main/delta/26/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/account_data.sql b/synapse/storage/schema/main/delta/27/account_data.sql
index bf0558b5b3..bf0558b5b3 100644
--- a/synapse/storage/databases/main/schema/delta/27/account_data.sql
+++ b/synapse/storage/schema/main/delta/27/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql b/synapse/storage/schema/main/delta/27/forgotten_memberships.sql
index e2094f37fe..e2094f37fe 100644
--- a/synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql
+++ b/synapse/storage/schema/main/delta/27/forgotten_memberships.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py
index 1c6058063f..1c6058063f 100644
--- a/synapse/storage/databases/main/schema/delta/27/ts.py
+++ b/synapse/storage/schema/main/delta/27/ts.py
diff --git a/synapse/storage/databases/main/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/main/delta/28/event_push_actions.sql
index 4d519849df..4d519849df 100644
--- a/synapse/storage/databases/main/schema/delta/28/event_push_actions.sql
+++ b/synapse/storage/schema/main/delta/28/event_push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/events_room_stream.sql b/synapse/storage/schema/main/delta/28/events_room_stream.sql
index 36609475f1..36609475f1 100644
--- a/synapse/storage/databases/main/schema/delta/28/events_room_stream.sql
+++ b/synapse/storage/schema/main/delta/28/events_room_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/public_roms_index.sql b/synapse/storage/schema/main/delta/28/public_roms_index.sql
index 6c1fd68c5b..6c1fd68c5b 100644
--- a/synapse/storage/databases/main/schema/delta/28/public_roms_index.sql
+++ b/synapse/storage/schema/main/delta/28/public_roms_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql b/synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
index cb84c69baa..cb84c69baa 100644
--- a/synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql
+++ b/synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/upgrade_times.sql b/synapse/storage/schema/main/delta/28/upgrade_times.sql
index 3e4a9ab455..3e4a9ab455 100644
--- a/synapse/storage/databases/main/schema/delta/28/upgrade_times.sql
+++ b/synapse/storage/schema/main/delta/28/upgrade_times.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/main/delta/28/users_is_guest.sql
index 21d2b420bf..21d2b420bf 100644
--- a/synapse/storage/databases/main/schema/delta/28/users_is_guest.sql
+++ b/synapse/storage/schema/main/delta/28/users_is_guest.sql
diff --git a/synapse/storage/databases/main/schema/delta/29/push_actions.sql b/synapse/storage/schema/main/delta/29/push_actions.sql
index 84b21cf813..84b21cf813 100644
--- a/synapse/storage/databases/main/schema/delta/29/push_actions.sql
+++ b/synapse/storage/schema/main/delta/29/push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/alias_creator.sql b/synapse/storage/schema/main/delta/30/alias_creator.sql
index c9d0dde638..c9d0dde638 100644
--- a/synapse/storage/databases/main/schema/delta/30/alias_creator.sql
+++ b/synapse/storage/schema/main/delta/30/alias_creator.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py
index 7f08fabe9f..7f08fabe9f 100644
--- a/synapse/storage/databases/main/schema/delta/30/as_users.py
+++ b/synapse/storage/schema/main/delta/30/as_users.py
diff --git a/synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql b/synapse/storage/schema/main/delta/30/deleted_pushers.sql
index 712c454aa1..712c454aa1 100644
--- a/synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql
+++ b/synapse/storage/schema/main/delta/30/deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/presence_stream.sql b/synapse/storage/schema/main/delta/30/presence_stream.sql
index 606bbb037d..606bbb037d 100644
--- a/synapse/storage/databases/main/schema/delta/30/presence_stream.sql
+++ b/synapse/storage/schema/main/delta/30/presence_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/public_rooms.sql b/synapse/storage/schema/main/delta/30/public_rooms.sql
index f09db4faa6..f09db4faa6 100644
--- a/synapse/storage/databases/main/schema/delta/30/public_rooms.sql
+++ b/synapse/storage/schema/main/delta/30/public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql b/synapse/storage/schema/main/delta/30/push_rule_stream.sql
index 735aa8d5f6..735aa8d5f6 100644
--- a/synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql
+++ b/synapse/storage/schema/main/delta/30/push_rule_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql b/synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
index 0dd2f1360c..0dd2f1360c 100644
--- a/synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql
+++ b/synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/invites.sql b/synapse/storage/schema/main/delta/31/invites.sql
index 2c57846d5a..2c57846d5a 100644
--- a/synapse/storage/databases/main/schema/delta/31/invites.sql
+++ b/synapse/storage/schema/main/delta/31/invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql b/synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
index 9efb4280eb..9efb4280eb 100644
--- a/synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql
+++ b/synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers.py b/synapse/storage/schema/main/delta/31/pushers.py
index 5be81c806a..5be81c806a 100644
--- a/synapse/storage/databases/main/schema/delta/31/pushers.py
+++ b/synapse/storage/schema/main/delta/31/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers_index.sql b/synapse/storage/schema/main/delta/31/pushers_index.sql
index a82add88fd..a82add88fd 100644
--- a/synapse/storage/databases/main/schema/delta/31/pushers_index.sql
+++ b/synapse/storage/schema/main/delta/31/pushers_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py
index b84c844e3a..b84c844e3a 100644
--- a/synapse/storage/databases/main/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/main/delta/31/search_update.py
diff --git a/synapse/storage/databases/main/schema/delta/32/events.sql b/synapse/storage/schema/main/delta/32/events.sql
index 1dd0f9e170..1dd0f9e170 100644
--- a/synapse/storage/databases/main/schema/delta/32/events.sql
+++ b/synapse/storage/schema/main/delta/32/events.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/openid.sql b/synapse/storage/schema/main/delta/32/openid.sql
index 36f37b11c8..36f37b11c8 100644
--- a/synapse/storage/databases/main/schema/delta/32/openid.sql
+++ b/synapse/storage/schema/main/delta/32/openid.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql b/synapse/storage/schema/main/delta/32/pusher_throttle.sql
index d86d30c13c..d86d30c13c 100644
--- a/synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql
+++ b/synapse/storage/schema/main/delta/32/pusher_throttle.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/remove_indices.sql b/synapse/storage/schema/main/delta/32/remove_indices.sql
index 2de50d408c..2de50d408c 100644
--- a/synapse/storage/databases/main/schema/delta/32/remove_indices.sql
+++ b/synapse/storage/schema/main/delta/32/remove_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/reports.sql b/synapse/storage/schema/main/delta/32/reports.sql
index d13609776f..d13609776f 100644
--- a/synapse/storage/databases/main/schema/delta/32/reports.sql
+++ b/synapse/storage/schema/main/delta/32/reports.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql b/synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
index 61ad3fe3e8..61ad3fe3e8 100644
--- a/synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql
+++ b/synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices.sql b/synapse/storage/schema/main/delta/33/devices.sql
index eca7268d82..eca7268d82 100644
--- a/synapse/storage/databases/main/schema/delta/33/devices.sql
+++ b/synapse/storage/schema/main/delta/33/devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
index aa4a3b9f2f..aa4a3b9f2f 100644
--- a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql
+++ b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
index 6671573398..6671573398 100644
--- a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
+++ b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py
index e928c66a8f..e928c66a8f 100644
--- a/synapse/storage/databases/main/schema/delta/33/event_fields.py
+++ b/synapse/storage/schema/main/delta/33/event_fields.py
diff --git a/synapse/storage/databases/main/schema/delta/33/remote_media_ts.py b/synapse/storage/schema/main/delta/33/remote_media_ts.py
index 3907189e29..3907189e29 100644
--- a/synapse/storage/databases/main/schema/delta/33/remote_media_ts.py
+++ b/synapse/storage/schema/main/delta/33/remote_media_ts.py
diff --git a/synapse/storage/databases/main/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/main/delta/33/user_ips_index.sql
index 473f75a78e..473f75a78e 100644
--- a/synapse/storage/databases/main/schema/delta/33/user_ips_index.sql
+++ b/synapse/storage/schema/main/delta/33/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/appservice_stream.sql b/synapse/storage/schema/main/delta/34/appservice_stream.sql
index 69e16eda0f..69e16eda0f 100644
--- a/synapse/storage/databases/main/schema/delta/34/appservice_stream.sql
+++ b/synapse/storage/schema/main/delta/34/appservice_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/cache_stream.py b/synapse/storage/schema/main/delta/34/cache_stream.py
index cf09e43e2b..cf09e43e2b 100644
--- a/synapse/storage/databases/main/schema/delta/34/cache_stream.py
+++ b/synapse/storage/schema/main/delta/34/cache_stream.py
diff --git a/synapse/storage/databases/main/schema/delta/34/device_inbox.sql b/synapse/storage/schema/main/delta/34/device_inbox.sql
index e68844c74a..e68844c74a 100644
--- a/synapse/storage/databases/main/schema/delta/34/device_inbox.sql
+++ b/synapse/storage/schema/main/delta/34/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql b/synapse/storage/schema/main/delta/34/push_display_name_rename.sql
index 0d9fe1a99a..0d9fe1a99a 100644
--- a/synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql
+++ b/synapse/storage/schema/main/delta/34/push_display_name_rename.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/received_txn_purge.py b/synapse/storage/schema/main/delta/34/received_txn_purge.py
index 67d505e68b..67d505e68b 100644
--- a/synapse/storage/databases/main/schema/delta/34/received_txn_purge.py
+++ b/synapse/storage/schema/main/delta/34/received_txn_purge.py
diff --git a/synapse/storage/databases/main/schema/delta/35/contains_url.sql b/synapse/storage/schema/main/delta/35/contains_url.sql
index 6cd123027b..6cd123027b 100644
--- a/synapse/storage/databases/main/schema/delta/35/contains_url.sql
+++ b/synapse/storage/schema/main/delta/35/contains_url.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_outbox.sql b/synapse/storage/schema/main/delta/35/device_outbox.sql
index 17e6c43105..17e6c43105 100644
--- a/synapse/storage/databases/main/schema/delta/35/device_outbox.sql
+++ b/synapse/storage/schema/main/delta/35/device_outbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_stream_id.sql b/synapse/storage/schema/main/delta/35/device_stream_id.sql
index 7ab7d942e2..7ab7d942e2 100644
--- a/synapse/storage/databases/main/schema/delta/35/device_stream_id.sql
+++ b/synapse/storage/schema/main/delta/35/device_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/main/delta/35/event_push_actions_index.sql
index 2e836d8e9c..2e836d8e9c 100644
--- a/synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql
+++ b/synapse/storage/schema/main/delta/35/event_push_actions_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql b/synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
index dd2bf2e28a..dd2bf2e28a 100644
--- a/synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql
+++ b/synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql b/synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
index 2b945d8a57..2b945d8a57 100644
--- a/synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql
+++ b/synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
diff --git a/synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql b/synapse/storage/schema/main/delta/36/readd_public_rooms.sql
index 90d8fd18f9..90d8fd18f9 100644
--- a/synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql
+++ b/synapse/storage/schema/main/delta/36/readd_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py b/synapse/storage/schema/main/delta/37/remove_auth_idx.py
index a377884169..a377884169 100644
--- a/synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py
+++ b/synapse/storage/schema/main/delta/37/remove_auth_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/37/user_threepids.sql b/synapse/storage/schema/main/delta/37/user_threepids.sql
index cf7a90dd10..cf7a90dd10 100644
--- a/synapse/storage/databases/main/schema/delta/37/user_threepids.sql
+++ b/synapse/storage/schema/main/delta/37/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql b/synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
index 515e6b8e84..515e6b8e84 100644
--- a/synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql
+++ b/synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql b/synapse/storage/schema/main/delta/39/appservice_room_list.sql
index 74bdc49073..74bdc49073 100644
--- a/synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql
+++ b/synapse/storage/schema/main/delta/39/appservice_room_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql b/synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
index 00be801e90..00be801e90 100644
--- a/synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql
+++ b/synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/event_push_index.sql b/synapse/storage/schema/main/delta/39/event_push_index.sql
index de2ad93e5c..de2ad93e5c 100644
--- a/synapse/storage/databases/main/schema/delta/39/event_push_index.sql
+++ b/synapse/storage/schema/main/delta/39/event_push_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/federation_out_position.sql b/synapse/storage/schema/main/delta/39/federation_out_position.sql
index 5af814290b..5af814290b 100644
--- a/synapse/storage/databases/main/schema/delta/39/federation_out_position.sql
+++ b/synapse/storage/schema/main/delta/39/federation_out_position.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/membership_profile.sql b/synapse/storage/schema/main/delta/39/membership_profile.sql
index 1bf911c8ab..1bf911c8ab 100644
--- a/synapse/storage/databases/main/schema/delta/39/membership_profile.sql
+++ b/synapse/storage/schema/main/delta/39/membership_profile.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/current_state_idx.sql b/synapse/storage/schema/main/delta/40/current_state_idx.sql
index 7ffa189f39..7ffa189f39 100644
--- a/synapse/storage/databases/main/schema/delta/40/current_state_idx.sql
+++ b/synapse/storage/schema/main/delta/40/current_state_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_inbox.sql b/synapse/storage/schema/main/delta/40/device_inbox.sql
index b9fe1f0480..b9fe1f0480 100644
--- a/synapse/storage/databases/main/schema/delta/40/device_inbox.sql
+++ b/synapse/storage/schema/main/delta/40/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/main/delta/40/device_list_streams.sql
index dd6dcb65f1..dd6dcb65f1 100644
--- a/synapse/storage/databases/main/schema/delta/40/device_list_streams.sql
+++ b/synapse/storage/schema/main/delta/40/device_list_streams.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/event_push_summary.sql b/synapse/storage/schema/main/delta/40/event_push_summary.sql
index 3918f0b794..3918f0b794 100644
--- a/synapse/storage/databases/main/schema/delta/40/event_push_summary.sql
+++ b/synapse/storage/schema/main/delta/40/event_push_summary.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/pushers.sql b/synapse/storage/schema/main/delta/40/pushers.sql
index 054a223f14..054a223f14 100644
--- a/synapse/storage/databases/main/schema/delta/40/pushers.sql
+++ b/synapse/storage/schema/main/delta/40/pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql b/synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
index b7bee8b692..b7bee8b692 100644
--- a/synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql
+++ b/synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql b/synapse/storage/schema/main/delta/41/device_outbound_index.sql
index 62f0b9892b..62f0b9892b 100644
--- a/synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql
+++ b/synapse/storage/schema/main/delta/41/device_outbound_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql b/synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
index 5d9cfecf36..5d9cfecf36 100644
--- a/synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql
+++ b/synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/ratelimit.sql b/synapse/storage/schema/main/delta/41/ratelimit.sql
index a194bf0238..a194bf0238 100644
--- a/synapse/storage/databases/main/schema/delta/41/ratelimit.sql
+++ b/synapse/storage/schema/main/delta/41/ratelimit.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/current_state_delta.sql b/synapse/storage/schema/main/delta/42/current_state_delta.sql
index d28851aff8..d28851aff8 100644
--- a/synapse/storage/databases/main/schema/delta/42/current_state_delta.sql
+++ b/synapse/storage/schema/main/delta/42/current_state_delta.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql b/synapse/storage/schema/main/delta/42/device_list_last_id.sql
index 9ab8c14fa3..9ab8c14fa3 100644
--- a/synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql
+++ b/synapse/storage/schema/main/delta/42/device_list_last_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql b/synapse/storage/schema/main/delta/42/event_auth_state_only.sql
index b8821ac759..b8821ac759 100644
--- a/synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql
+++ b/synapse/storage/schema/main/delta/42/event_auth_state_only.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/user_dir.py b/synapse/storage/schema/main/delta/42/user_dir.py
index 506f326f4d..506f326f4d 100644
--- a/synapse/storage/databases/main/schema/delta/42/user_dir.py
+++ b/synapse/storage/schema/main/delta/42/user_dir.py
diff --git a/synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql b/synapse/storage/schema/main/delta/43/blocked_rooms.sql
index 0e3cd143ff..0e3cd143ff 100644
--- a/synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql
+++ b/synapse/storage/schema/main/delta/43/blocked_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/quarantine_media.sql b/synapse/storage/schema/main/delta/43/quarantine_media.sql
index 630907ec4f..630907ec4f 100644
--- a/synapse/storage/databases/main/schema/delta/43/quarantine_media.sql
+++ b/synapse/storage/schema/main/delta/43/quarantine_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/url_cache.sql b/synapse/storage/schema/main/delta/43/url_cache.sql
index 45ebe020da..45ebe020da 100644
--- a/synapse/storage/databases/main/schema/delta/43/url_cache.sql
+++ b/synapse/storage/schema/main/delta/43/url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/user_share.sql b/synapse/storage/schema/main/delta/43/user_share.sql
index ee7062abe4..ee7062abe4 100644
--- a/synapse/storage/databases/main/schema/delta/43/user_share.sql
+++ b/synapse/storage/schema/main/delta/43/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql b/synapse/storage/schema/main/delta/44/expire_url_cache.sql
index b12f9b2ebf..b12f9b2ebf 100644
--- a/synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql
+++ b/synapse/storage/schema/main/delta/44/expire_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/group_server.sql b/synapse/storage/schema/main/delta/45/group_server.sql
index b2333848a0..b2333848a0 100644
--- a/synapse/storage/databases/main/schema/delta/45/group_server.sql
+++ b/synapse/storage/schema/main/delta/45/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/profile_cache.sql b/synapse/storage/schema/main/delta/45/profile_cache.sql
index e5ddc84df0..e5ddc84df0 100644
--- a/synapse/storage/databases/main/schema/delta/45/profile_cache.sql
+++ b/synapse/storage/schema/main/delta/45/profile_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql b/synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
index 68c48a89a9..68c48a89a9 100644
--- a/synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql
+++ b/synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql b/synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
index bb307889c1..bb307889c1 100644
--- a/synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql
+++ b/synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/group_server.sql b/synapse/storage/schema/main/delta/46/group_server.sql
index 097679bc9a..097679bc9a 100644
--- a/synapse/storage/databases/main/schema/delta/46/group_server.sql
+++ b/synapse/storage/schema/main/delta/46/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql b/synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
index bbfc7f5d1a..bbfc7f5d1a 100644
--- a/synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql
+++ b/synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql b/synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
index cb0d5a2576..cb0d5a2576 100644
--- a/synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql
+++ b/synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql b/synapse/storage/schema/main/delta/46/user_dir_typos.sql
index d9505f8da1..d9505f8da1 100644
--- a/synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql
+++ b/synapse/storage/schema/main/delta/46/user_dir_typos.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/last_access_media.sql b/synapse/storage/schema/main/delta/47/last_access_media.sql
index f505fb22b5..f505fb22b5 100644
--- a/synapse/storage/databases/main/schema/delta/47/last_access_media.sql
+++ b/synapse/storage/schema/main/delta/47/last_access_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql b/synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
index 31d7a817eb..31d7a817eb 100644
--- a/synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql
+++ b/synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql b/synapse/storage/schema/main/delta/47/push_actions_staging.sql
index edccf4a96f..edccf4a96f 100644
--- a/synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql
+++ b/synapse/storage/schema/main/delta/47/push_actions_staging.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/main/delta/48/add_user_consent.sql
index 5237491506..5237491506 100644
--- a/synapse/storage/databases/main/schema/delta/48/add_user_consent.sql
+++ b/synapse/storage/schema/main/delta/48/add_user_consent.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
index 9248b0b24a..9248b0b24a 100644
--- a/synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql
+++ b/synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/main/delta/48/deactivated_users.sql
index e9013a6969..e9013a6969 100644
--- a/synapse/storage/databases/main/schema/delta/48/deactivated_users.sql
+++ b/synapse/storage/schema/main/delta/48/deactivated_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/main/delta/48/group_unique_indexes.py
index 49f5f2c003..49f5f2c003 100644
--- a/synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py
+++ b/synapse/storage/schema/main/delta/48/group_unique_indexes.py
diff --git a/synapse/storage/databases/main/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/main/delta/48/groups_joinable.sql
index ce26eaf0c9..ce26eaf0c9 100644
--- a/synapse/storage/databases/main/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/main/delta/48/groups_joinable.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
index 14dcf18d73..14dcf18d73 100644
--- a/synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql
+++ b/synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
index 3dd478196f..3dd478196f 100644
--- a/synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql
+++ b/synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
index 3a4ed59b5b..3a4ed59b5b 100644
--- a/synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql
+++ b/synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
index c93ae47532..c93ae47532 100644
--- a/synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql
+++ b/synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/erasure_store.sql b/synapse/storage/schema/main/delta/50/erasure_store.sql
index 5d8641a9ab..5d8641a9ab 100644
--- a/synapse/storage/databases/main/schema/delta/50/erasure_store.sql
+++ b/synapse/storage/schema/main/delta/50/erasure_store.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py
index acd6ad1e1f..acd6ad1e1f 100644
--- a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py
+++ b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py
diff --git a/synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/main/delta/51/e2e_room_keys.sql
index c0e66a697d..c0e66a697d 100644
--- a/synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql
+++ b/synapse/storage/schema/main/delta/51/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/main/delta/51/monthly_active_users.sql
index c9d537d5a3..c9d537d5a3 100644
--- a/synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/main/delta/51/monthly_active_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql b/synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
index 91e03d13e1..91e03d13e1 100644
--- a/synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql
+++ b/synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql b/synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
index bfa49e6f92..bfa49e6f92 100644
--- a/synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql
+++ b/synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/main/delta/52/e2e_room_keys.sql
index db687cccae..db687cccae 100644
--- a/synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql
+++ b/synapse/storage/schema/main/delta/52/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql b/synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
index 88ec2f83e5..88ec2f83e5 100644
--- a/synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql
+++ b/synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql b/synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
index e372f5a44a..e372f5a44a 100644
--- a/synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql
+++ b/synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/event_format_version.sql b/synapse/storage/schema/main/delta/53/event_format_version.sql
index 1d977c2834..1d977c2834 100644
--- a/synapse/storage/databases/main/schema/delta/53/event_format_version.sql
+++ b/synapse/storage/schema/main/delta/53/event_format_version.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql b/synapse/storage/schema/main/delta/53/user_dir_populate.sql
index ffcc896b58..ffcc896b58 100644
--- a/synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql
+++ b/synapse/storage/schema/main/delta/53/user_dir_populate.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/main/delta/53/user_ips_index.sql
index b812c5794f..b812c5794f 100644
--- a/synapse/storage/databases/main/schema/delta/53/user_ips_index.sql
+++ b/synapse/storage/schema/main/delta/53/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_share.sql b/synapse/storage/schema/main/delta/53/user_share.sql
index 5831b1a6f8..5831b1a6f8 100644
--- a/synapse/storage/databases/main/schema/delta/53/user_share.sql
+++ b/synapse/storage/schema/main/delta/53/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql b/synapse/storage/schema/main/delta/53/user_threepid_id.sql
index 80c2c573b6..80c2c573b6 100644
--- a/synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql
+++ b/synapse/storage/schema/main/delta/53/user_threepid_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql b/synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
index f7827ca6d2..f7827ca6d2 100644
--- a/synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql
+++ b/synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql b/synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
index 0adb2ad55e..0adb2ad55e 100644
--- a/synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql
+++ b/synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql b/synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
index c01aa9d2d9..c01aa9d2d9 100644
--- a/synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql
+++ b/synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
index b062ec840c..b062ec840c 100644
--- a/synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql
+++ b/synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
index dbbe682697..dbbe682697 100644
--- a/synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql
+++ b/synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql b/synapse/storage/schema/main/delta/54/drop_presence_list.sql
index e6ee70c623..e6ee70c623 100644
--- a/synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql
+++ b/synapse/storage/schema/main/delta/54/drop_presence_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/relations.sql b/synapse/storage/schema/main/delta/54/relations.sql
index 134862b870..134862b870 100644
--- a/synapse/storage/databases/main/schema/delta/54/relations.sql
+++ b/synapse/storage/schema/main/delta/54/relations.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats.sql b/synapse/storage/schema/main/delta/54/stats.sql
index 652e58308e..652e58308e 100644
--- a/synapse/storage/databases/main/schema/delta/54/stats.sql
+++ b/synapse/storage/schema/main/delta/54/stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats2.sql b/synapse/storage/schema/main/delta/54/stats2.sql
index 3b2d48447f..3b2d48447f 100644
--- a/synapse/storage/databases/main/schema/delta/54/stats2.sql
+++ b/synapse/storage/schema/main/delta/54/stats2.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql b/synapse/storage/schema/main/delta/55/access_token_expiry.sql
index 4590604bfd..4590604bfd 100644
--- a/synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql
+++ b/synapse/storage/schema/main/delta/55/access_token_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql b/synapse/storage/schema/main/delta/55/track_threepid_validations.sql
index a8eced2e0a..a8eced2e0a 100644
--- a/synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql
+++ b/synapse/storage/schema/main/delta/55/track_threepid_validations.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql b/synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
index dabdde489b..dabdde489b 100644
--- a/synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql
+++ b/synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql b/synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
index 41807eb1e7..41807eb1e7 100644
--- a/synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql
+++ b/synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership.sql
index 473018676f..473018676f 100644
--- a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql
+++ b/synapse/storage/schema/main/delta/56/current_state_events_membership.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
index 3133d42d4a..3133d42d4a 100644
--- a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql
+++ b/synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql b/synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
index 1d2ddb1b1a..1d2ddb1b1a 100644
--- a/synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql
+++ b/synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql b/synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
index f00889290b..f00889290b 100644
--- a/synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql
+++ b/synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres b/synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
index b9bbb18a91..b9bbb18a91 100644
--- a/synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres
+++ b/synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql b/synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
index c2f557fde9..c2f557fde9 100644
--- a/synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql
+++ b/synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/main/delta/56/devices_last_seen.sql
index dfa902d0ba..dfa902d0ba 100644
--- a/synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql
+++ b/synapse/storage/schema/main/delta/56/devices_last_seen.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql b/synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
index 9f09922c67..9f09922c67 100644
--- a/synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql
+++ b/synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_expiry.sql b/synapse/storage/schema/main/delta/56/event_expiry.sql
index 81a36a8b1d..81a36a8b1d 100644
--- a/synapse/storage/databases/main/schema/delta/56/event_expiry.sql
+++ b/synapse/storage/schema/main/delta/56/event_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels.sql b/synapse/storage/schema/main/delta/56/event_labels.sql
index ccf287971c..ccf287971c 100644
--- a/synapse/storage/databases/main/schema/delta/56/event_labels.sql
+++ b/synapse/storage/schema/main/delta/56/event_labels.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql b/synapse/storage/schema/main/delta/56/event_labels_background_update.sql
index 5f5e0499ae..5f5e0499ae 100644
--- a/synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql
+++ b/synapse/storage/schema/main/delta/56/event_labels_background_update.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql b/synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
index 014cb3b538..014cb3b538 100644
--- a/synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql
+++ b/synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices.sql b/synapse/storage/schema/main/delta/56/hidden_devices.sql
index 67f8b20297..67f8b20297 100644
--- a/synapse/storage/databases/main/schema/delta/56/hidden_devices.sql
+++ b/synapse/storage/schema/main/delta/56/hidden_devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite b/synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
index e8b1fd35d8..e8b1fd35d8 100644
--- a/synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite
+++ b/synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql b/synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
index 4f24c1405d..4f24c1405d 100644
--- a/synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql
+++ b/synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql b/synapse/storage/schema/main/delta/56/public_room_list_idx.sql
index 7be31ffebb..7be31ffebb 100644
--- a/synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql
+++ b/synapse/storage/schema/main/delta/56/public_room_list_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor.sql b/synapse/storage/schema/main/delta/56/redaction_censor.sql
index ea95db0ed7..ea95db0ed7 100644
--- a/synapse/storage/databases/main/schema/delta/56/redaction_censor.sql
+++ b/synapse/storage/schema/main/delta/56/redaction_censor.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql b/synapse/storage/schema/main/delta/56/redaction_censor2.sql
index 49ce35d794..49ce35d794 100644
--- a/synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql
+++ b/synapse/storage/schema/main/delta/56/redaction_censor2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres b/synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
index 67471f3ef5..67471f3ef5 100644
--- a/synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres
+++ b/synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql b/synapse/storage/schema/main/delta/56/redaction_censor4.sql
index b7550f6f4e..b7550f6f4e 100644
--- a/synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql
+++ b/synapse/storage/schema/main/delta/56/redaction_censor4.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql b/synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
index aeb17813d3..aeb17813d3 100644
--- a/synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql
+++ b/synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_key_etag.sql b/synapse/storage/schema/main/delta/56/room_key_etag.sql
index 7d70dd071e..7d70dd071e 100644
--- a/synapse/storage/databases/main/schema/delta/56/room_key_etag.sql
+++ b/synapse/storage/schema/main/delta/56/room_key_etag.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql b/synapse/storage/schema/main/delta/56/room_membership_idx.sql
index 92ab1f5e65..92ab1f5e65 100644
--- a/synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql
+++ b/synapse/storage/schema/main/delta/56/room_membership_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_retention.sql b/synapse/storage/schema/main/delta/56/room_retention.sql
index ee6cdf7a14..ee6cdf7a14 100644
--- a/synapse/storage/databases/main/schema/delta/56/room_retention.sql
+++ b/synapse/storage/schema/main/delta/56/room_retention.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys.sql b/synapse/storage/schema/main/delta/56/signing_keys.sql
index 5c5fffcafb..5c5fffcafb 100644
--- a/synapse/storage/databases/main/schema/delta/56/signing_keys.sql
+++ b/synapse/storage/schema/main/delta/56/signing_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql b/synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
index 0aa90ebf0c..0aa90ebf0c 100644
--- a/synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql
+++ b/synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/stats_separated.sql b/synapse/storage/schema/main/delta/56/stats_separated.sql
index bbdde121e8..bbdde121e8 100644
--- a/synapse/storage/databases/main/schema/delta/56/stats_separated.sql
+++ b/synapse/storage/schema/main/delta/56/stats_separated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/main/delta/56/unique_user_filter_index.py
index bb7296852a..bb7296852a 100644
--- a/synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py
+++ b/synapse/storage/schema/main/delta/56/unique_user_filter_index.py
diff --git a/synapse/storage/databases/main/schema/delta/56/user_external_ids.sql b/synapse/storage/schema/main/delta/56/user_external_ids.sql
index 91390c4527..91390c4527 100644
--- a/synapse/storage/databases/main/schema/delta/56/user_external_ids.sql
+++ b/synapse/storage/schema/main/delta/56/user_external_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql b/synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
index 149f8be8b6..149f8be8b6 100644
--- a/synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql
+++ b/synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql b/synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
index aec06c8261..aec06c8261 100644
--- a/synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql
+++ b/synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql b/synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
index c3b6de2099..c3b6de2099 100644
--- a/synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql
+++ b/synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py b/synapse/storage/schema/main/delta/57/local_current_membership.py
index 66989222e6..66989222e6 100644
--- a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py
+++ b/synapse/storage/schema/main/delta/57/local_current_membership.py
diff --git a/synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql b/synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
index 133d80af35..133d80af35 100644
--- a/synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql
+++ b/synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql b/synapse/storage/schema/main/delta/57/rooms_version_column.sql
index 352a66f5b0..352a66f5b0 100644
--- a/synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql
+++ b/synapse/storage/schema/main/delta/57/rooms_version_column.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
index c601cff6de..c601cff6de 100644
--- a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres
+++ b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
index 335c6f2074..335c6f2074 100644
--- a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite
+++ b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
index 92aaadde0d..92aaadde0d 100644
--- a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres
+++ b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
index e19dab97cb..e19dab97cb 100644
--- a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite
+++ b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql b/synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
index fdc39e9ba5..fdc39e9ba5 100644
--- a/synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql
+++ b/synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql b/synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
index dcb593fc2d..dcb593fc2d 100644
--- a/synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql
+++ b/synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres b/synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
index aa46eb0e10..aa46eb0e10 100644
--- a/synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py b/synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
index d353f2bcb3..d353f2bcb3 100644
--- a/synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py
+++ b/synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
index 3275ae2b20..3275ae2b20 100644
--- a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
index 1d0c04b53a..1d0c04b53a 100644
--- a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
+++ b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql b/synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
index 4cc96a5341..4cc96a5341 100644
--- a/synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql
+++ b/synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
index 597f2ffd3d..597f2ffd3d 100644
--- a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
index 69db89ac0e..69db89ac0e 100644
--- a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite
+++ b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql b/synapse/storage/schema/main/delta/58/09shadow_ban.sql
index 260b009b48..260b009b48 100644
--- a/synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql
+++ b/synapse/storage/schema/main/delta/58/09shadow_ban.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql b/synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
index 847aebd85e..847aebd85e 100644
--- a/synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql
+++ b/synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql b/synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
index eb57203e46..eb57203e46 100644
--- a/synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql
+++ b/synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql b/synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
index 1cc2633aad..1cc2633aad 100644
--- a/synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql
+++ b/synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11dehydration.sql b/synapse/storage/schema/main/delta/58/11dehydration.sql
index 7851a0a825..7851a0a825 100644
--- a/synapse/storage/databases/main/schema/delta/58/11dehydration.sql
+++ b/synapse/storage/schema/main/delta/58/11dehydration.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11fallback.sql b/synapse/storage/schema/main/delta/58/11fallback.sql
index 4ed981dbf8..4ed981dbf8 100644
--- a/synapse/storage/databases/main/schema/delta/58/11fallback.sql
+++ b/synapse/storage/schema/main/delta/58/11fallback.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11user_id_seq.py b/synapse/storage/schema/main/delta/58/11user_id_seq.py
index 4310ec12ce..4310ec12ce 100644
--- a/synapse/storage/databases/main/schema/delta/58/11user_id_seq.py
+++ b/synapse/storage/schema/main/delta/58/11user_id_seq.py
diff --git a/synapse/storage/databases/main/schema/delta/58/12room_stats.sql b/synapse/storage/schema/main/delta/58/12room_stats.sql
index fd733adf13..fd733adf13 100644
--- a/synapse/storage/databases/main/schema/delta/58/12room_stats.sql
+++ b/synapse/storage/schema/main/delta/58/12room_stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql b/synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
index 15421b99ac..15421b99ac 100644
--- a/synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql
+++ b/synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql b/synapse/storage/schema/main/delta/58/14events_instance_name.sql
index 98ff76d709..98ff76d709 100644
--- a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql
+++ b/synapse/storage/schema/main/delta/58/14events_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres b/synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
index c31f9af82a..c31f9af82a 100644
--- a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql b/synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
index ebfbed7925..ebfbed7925 100644
--- a/synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql
+++ b/synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql b/synapse/storage/schema/main/delta/58/15unread_count.sql
index 317fba8a5d..317fba8a5d 100644
--- a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql
+++ b/synapse/storage/schema/main/delta/58/15unread_count.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql b/synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
index 55f5d0f732..55f5d0f732 100644
--- a/synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql
+++ b/synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql b/synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
index a67aa5e500..a67aa5e500 100644
--- a/synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql
+++ b/synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/18stream_positions.sql b/synapse/storage/schema/main/delta/58/18stream_positions.sql
index 985fd949a2..985fd949a2 100644
--- a/synapse/storage/databases/main/schema/delta/58/18stream_positions.sql
+++ b/synapse/storage/schema/main/delta/58/18stream_positions.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres b/synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
index 841186b826..841186b826 100644
--- a/synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/19txn_id.sql b/synapse/storage/schema/main/delta/58/19txn_id.sql
index b2454121a8..b2454121a8 100644
--- a/synapse/storage/databases/main/schema/delta/58/19txn_id.sql
+++ b/synapse/storage/schema/main/delta/58/19txn_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql b/synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
index ad1f481428..ad1f481428 100644
--- a/synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql
+++ b/synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql b/synapse/storage/schema/main/delta/58/20user_daily_visits.sql
index b0b5dcddce..b0b5dcddce 100644
--- a/synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql
+++ b/synapse/storage/schema/main/delta/58/20user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql b/synapse/storage/schema/main/delta/58/21as_device_stream.sql
index 7b84a207fd..7b84a207fd 100644
--- a/synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql
+++ b/synapse/storage/schema/main/delta/58/21as_device_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql b/synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
index 01ea6eddcf..01ea6eddcf 100644
--- a/synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql
+++ b/synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22puppet_token.sql b/synapse/storage/schema/main/delta/58/22puppet_token.sql
index 00a9431a97..00a9431a97 100644
--- a/synapse/storage/databases/main/schema/delta/58/22puppet_token.sql
+++ b/synapse/storage/schema/main/delta/58/22puppet_token.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql b/synapse/storage/schema/main/delta/58/22users_have_local_media.sql
index e1a35be831..e1a35be831 100644
--- a/synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql
+++ b/synapse/storage/schema/main/delta/58/22users_have_local_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql b/synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
index 75c3915a94..75c3915a94 100644
--- a/synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql
+++ b/synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql b/synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
index 8a39d54aed..8a39d54aed 100644
--- a/synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql
+++ b/synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql b/synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
index 8f5e65aa71..8f5e65aa71 100644
--- a/synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql
+++ b/synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql b/synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
index 1a101cd5eb..1a101cd5eb 100644
--- a/synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql
+++ b/synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/27local_invites.sql b/synapse/storage/schema/main/delta/58/27local_invites.sql
index 44b2a0572f..44b2a0572f 100644
--- a/synapse/storage/databases/main/schema/delta/58/27local_invites.sql
+++ b/synapse/storage/schema/main/delta/58/27local_invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
index de57645019..de57645019 100644
--- a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres
+++ b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
index ee0e3521bf..ee0e3521bf 100644
--- a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite
+++ b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/59/01ignored_user.py b/synapse/storage/schema/main/delta/59/01ignored_user.py
index 9e8f35c1d2..9e8f35c1d2 100644
--- a/synapse/storage/databases/main/schema/delta/59/01ignored_user.py
+++ b/synapse/storage/schema/main/delta/59/01ignored_user.py
diff --git a/synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql b/synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
index d781a92fec..d781a92fec 100644
--- a/synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql
+++ b/synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres b/synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
index 45a845a3a5..45a845a3a5 100644
--- a/synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres
+++ b/synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
index 729196cfd5..729196cfd5 100644
--- a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql
+++ b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
index e8a035bbeb..e8a035bbeb 100644
--- a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres
+++ b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql b/synapse/storage/schema/main/delta/59/04drop_account_data.sql
index 64ab696cfe..64ab696cfe 100644
--- a/synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql
+++ b/synapse/storage/schema/main/delta/59/04drop_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql b/synapse/storage/schema/main/delta/59/05cache_invalidation.sql
index fb71b360a0..fb71b360a0 100644
--- a/synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql
+++ b/synapse/storage/schema/main/delta/59/05cache_invalidation.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql b/synapse/storage/schema/main/delta/59/06chain_cover_index.sql
index fe3dca71dd..fe3dca71dd 100644
--- a/synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql
+++ b/synapse/storage/schema/main/delta/59/06chain_cover_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql b/synapse/storage/schema/main/delta/59/06shard_account_data.sql
index 46abf8d562..46abf8d562 100644
--- a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql
+++ b/synapse/storage/schema/main/delta/59/06shard_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres b/synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
index 4a6e6c74f5..4a6e6c74f5 100644
--- a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres
+++ b/synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql b/synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
index 9f2b5ebc5a..9f2b5ebc5a 100644
--- a/synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql
+++ b/synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql b/synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
index 0ec6764150..0ec6764150 100644
--- a/synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql
+++ b/synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql b/synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
index 85196db288..85196db288 100644
--- a/synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql
+++ b/synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql b/synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
index cc9b267c7d..cc9b267c7d 100644
--- a/synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql
+++ b/synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql b/synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
index 87cb1f3cfd..87cb1f3cfd 100644
--- a/synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql
+++ b/synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
index 54c1bca3b1..54c1bca3b1 100644
--- a/synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres
+++ b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql b/synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql
index 4836dac16e..4836dac16e 100644
--- a/synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql
+++ b/synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
index b6ba0bda1a..b6ba0bda1a 100644
--- a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql
+++ b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
index 02b182adf9..02b182adf9 100644
--- a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres
+++ b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/12users_to_send_full_presence_to.sql b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql
index ab37a48ed9..ab37a48ed9 100644
--- a/synapse/storage/databases/main/schema/delta/59/12users_to_send_full_presence_to.sql
+++ b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/main/full_schemas/16/application_services.sql
index 883fcd10b2..883fcd10b2 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/main/full_schemas/16/application_services.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/main/full_schemas/16/event_edges.sql
index 10ce2aa7a0..10ce2aa7a0 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/main/full_schemas/16/event_edges.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/main/full_schemas/16/event_signatures.sql
index 95826da431..95826da431 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql
+++ b/synapse/storage/schema/main/full_schemas/16/event_signatures.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/im.sql b/synapse/storage/schema/main/full_schemas/16/im.sql
index a1a2aa8e5b..a1a2aa8e5b 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/main/full_schemas/16/im.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/keys.sql b/synapse/storage/schema/main/full_schemas/16/keys.sql
index 11cdffdbb3..11cdffdbb3 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/main/full_schemas/16/keys.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/main/full_schemas/16/media_repository.sql
index 8f3759bb2a..8f3759bb2a 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql
+++ b/synapse/storage/schema/main/full_schemas/16/media_repository.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/presence.sql b/synapse/storage/schema/main/full_schemas/16/presence.sql
index 01d2d8f833..01d2d8f833 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/main/full_schemas/16/presence.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/main/full_schemas/16/profiles.sql
index c04f4747d9..c04f4747d9 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/profiles.sql
+++ b/synapse/storage/schema/main/full_schemas/16/profiles.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/push.sql b/synapse/storage/schema/main/full_schemas/16/push.sql
index e44465cf45..e44465cf45 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/push.sql
+++ b/synapse/storage/schema/main/full_schemas/16/push.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/main/full_schemas/16/redactions.sql
index 318f0d9aa5..318f0d9aa5 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/redactions.sql
+++ b/synapse/storage/schema/main/full_schemas/16/redactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/main/full_schemas/16/room_aliases.sql
index d47da3b12f..d47da3b12f 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql
+++ b/synapse/storage/schema/main/full_schemas/16/room_aliases.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/state.sql b/synapse/storage/schema/main/full_schemas/16/state.sql
index 96391a8f0e..96391a8f0e 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/main/full_schemas/16/state.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/main/full_schemas/16/transactions.sql
index 17e67bedac..17e67bedac 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/main/full_schemas/16/transactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/users.sql b/synapse/storage/schema/main/full_schemas/16/users.sql
index f013aa8b18..f013aa8b18 100644
--- a/synapse/storage/databases/main/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/main/full_schemas/16/users.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/main/full_schemas/54/full.sql.postgres
index 889a9a0ce4..889a9a0ce4 100644
--- a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres
+++ b/synapse/storage/schema/main/full_schemas/54/full.sql.postgres
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
index 308124e531..308124e531 100644
--- a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite
+++ b/synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql b/synapse/storage/schema/main/full_schemas/54/stream_positions.sql
index 91d21b2921..91d21b2921 100644
--- a/synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql
+++ b/synapse/storage/schema/main/full_schemas/54/stream_positions.sql
diff --git a/synapse/storage/databases/state/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/state/delta/23/drop_state_index.sql
index ae09fa0065..ae09fa0065 100644
--- a/synapse/storage/databases/state/schema/delta/23/drop_state_index.sql
+++ b/synapse/storage/schema/state/delta/23/drop_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/30/state_stream.sql b/synapse/storage/schema/state/delta/30/state_stream.sql
index e85699e82e..e85699e82e 100644
--- a/synapse/storage/databases/state/schema/delta/30/state_stream.sql
+++ b/synapse/storage/schema/state/delta/30/state_stream.sql
diff --git a/synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql b/synapse/storage/schema/state/delta/32/remove_state_indices.sql
index 1450313bfa..1450313bfa 100644
--- a/synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql
+++ b/synapse/storage/schema/state/delta/32/remove_state_indices.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/add_state_index.sql b/synapse/storage/schema/state/delta/35/add_state_index.sql
index 33980d02f0..33980d02f0 100644
--- a/synapse/storage/databases/state/schema/delta/35/add_state_index.sql
+++ b/synapse/storage/schema/state/delta/35/add_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state.sql b/synapse/storage/schema/state/delta/35/state.sql
index 0f1fa68a89..0f1fa68a89 100644
--- a/synapse/storage/databases/state/schema/delta/35/state.sql
+++ b/synapse/storage/schema/state/delta/35/state.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state_dedupe.sql b/synapse/storage/schema/state/delta/35/state_dedupe.sql
index 97e5067ef4..97e5067ef4 100644
--- a/synapse/storage/databases/state/schema/delta/35/state_dedupe.sql
+++ b/synapse/storage/schema/state/delta/35/state_dedupe.sql
diff --git a/synapse/storage/databases/state/schema/delta/47/state_group_seq.py b/synapse/storage/schema/state/delta/47/state_group_seq.py
index 9fd1ccf6f7..9fd1ccf6f7 100644
--- a/synapse/storage/databases/state/schema/delta/47/state_group_seq.py
+++ b/synapse/storage/schema/state/delta/47/state_group_seq.py
diff --git a/synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql b/synapse/storage/schema/state/delta/56/state_group_room_idx.sql
index 7916ef18b2..7916ef18b2 100644
--- a/synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql
+++ b/synapse/storage/schema/state/delta/56/state_group_room_idx.sql
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/full.sql b/synapse/storage/schema/state/full_schemas/54/full.sql
index 35f97d6b3d..35f97d6b3d 100644
--- a/synapse/storage/databases/state/schema/full_schemas/54/full.sql
+++ b/synapse/storage/schema/state/full_schemas/54/full.sql
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres b/synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
index fcd926c9fb..fcd926c9fb 100644
--- a/synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres
+++ b/synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 0f84fa3f4e..b69f562ca5 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -15,6 +15,7 @@
import json
import logging
import re
+from typing import Pattern
import attr
from frozendict import frozendict
@@ -26,6 +27,9 @@ from synapse.logging import context
logger = logging.getLogger(__name__)
+_WILDCARD_RUN = re.compile(r"([\?\*]+)")
+
+
def _reject_invalid_json(val):
"""Do not allow Infinity, -Infinity, or NaN values in JSON."""
raise ValueError("Invalid JSON value: '%s'" % val)
@@ -158,25 +162,54 @@ def log_failure(failure, msg, consumeErrors=True):
return failure
-def glob_to_regex(glob):
+def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern:
"""Converts a glob to a compiled regex object.
- The regex is anchored at the beginning and end of the string.
-
Args:
- glob (str)
+ glob: pattern to match
+ word_boundary: If True, the pattern will be allowed to match at word boundaries
+ anywhere in the string. Otherwise, the pattern is anchored at the start and
+ end of the string.
Returns:
- re.RegexObject
+ compiled regex pattern
"""
- res = ""
- for c in glob:
- if c == "*":
- res = res + ".*"
- elif c == "?":
- res = res + "."
+
+ # Patterns with wildcards must be simplified to avoid performance cliffs
+ # - The glob `?**?**?` is equivalent to the glob `???*`
+ # - The glob `???*` is equivalent to the regex `.{3,}`
+ chunks = []
+ for chunk in _WILDCARD_RUN.split(glob):
+ # No wildcards? re.escape()
+ if not _WILDCARD_RUN.match(chunk):
+ chunks.append(re.escape(chunk))
+ continue
+
+ # Wildcards? Simplify.
+ qmarks = chunk.count("?")
+ if "*" in chunk:
+ chunks.append(".{%d,}" % qmarks)
else:
- res = res + re.escape(c)
+ chunks.append(".{%d}" % qmarks)
+
+ res = "".join(chunks)
- # \A anchors at start of string, \Z at end of string
- return re.compile(r"\A" + res + r"\Z", re.IGNORECASE)
+ if word_boundary:
+ res = re_word_boundary(res)
+ else:
+ # \A anchors at start of string, \Z at end of string
+ res = r"\A" + res + r"\Z"
+
+ return re.compile(res, re.IGNORECASE)
+
+
+def re_word_boundary(r: str) -> str:
+ """
+ Adds word boundary characters to the start and end of an
+ expression to require that the match occur as a whole word,
+ but do so respecting the fact that strings starting or ending
+ with non-word characters will change word boundaries.
+ """
+ # we can't use \b as it chokes on unicode. however \W seems to be okay
+ # as shorthand for [^0-9A-Za-z_].
+ return r"(^|\W)%s(\W|$)" % (r,)
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 46af7fa473..ca36f07c20 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -24,6 +24,11 @@ from synapse.config.cache import add_resizable_cache
logger = logging.getLogger(__name__)
+
+# Whether to track estimated memory usage of the LruCaches.
+TRACK_MEMORY_USAGE = False
+
+
caches_by_name = {} # type: Dict[str, Sized]
collectors_by_name = {} # type: Dict[str, CacheMetric]
@@ -32,6 +37,11 @@ cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"])
cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"])
cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"])
cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"])
+cache_memory_usage = Gauge(
+ "synapse_util_caches_cache_size_bytes",
+ "Estimated memory usage of the caches",
+ ["name"],
+)
response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"])
response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"])
@@ -52,6 +62,7 @@ class CacheMetric:
hits = attr.ib(default=0)
misses = attr.ib(default=0)
evicted_size = attr.ib(default=0)
+ memory_usage = attr.ib(default=None)
def inc_hits(self):
self.hits += 1
@@ -62,6 +73,19 @@ class CacheMetric:
def inc_evictions(self, size=1):
self.evicted_size += size
+ def inc_memory_usage(self, memory: int):
+ if self.memory_usage is None:
+ self.memory_usage = 0
+
+ self.memory_usage += memory
+
+ def dec_memory_usage(self, memory: int):
+ self.memory_usage -= memory
+
+ def clear_memory_usage(self):
+ if self.memory_usage is not None:
+ self.memory_usage = 0
+
def describe(self):
return []
@@ -81,6 +105,13 @@ class CacheMetric:
cache_total.labels(self._cache_name).set(self.hits + self.misses)
if getattr(self._cache, "max_size", None):
cache_max_size.labels(self._cache_name).set(self._cache.max_size)
+
+ if TRACK_MEMORY_USAGE:
+ # self.memory_usage can be None if nothing has been inserted
+ # into the cache yet.
+ cache_memory_usage.labels(self._cache_name).set(
+ self.memory_usage or 0
+ )
if self._collect_callback:
self._collect_callback()
except Exception as e:
diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py
index 10b0ec6b75..1be675e014 100644
--- a/synapse/util/caches/lrucache.py
+++ b/synapse/util/caches/lrucache.py
@@ -32,9 +32,36 @@ from typing import (
from typing_extensions import Literal
from synapse.config import cache as cache_config
+from synapse.util import caches
from synapse.util.caches import CacheMetric, register_cache
from synapse.util.caches.treecache import TreeCache
+try:
+ from pympler.asizeof import Asizer
+
+ def _get_size_of(val: Any, *, recurse=True) -> int:
+ """Get an estimate of the size in bytes of the object.
+
+ Args:
+ val: The object to size.
+ recurse: If true will include referenced values in the size,
+ otherwise only sizes the given object.
+ """
+ # Ignore singleton values when calculating memory usage.
+ if val in ((), None, ""):
+ return 0
+
+ sizer = Asizer()
+ sizer.exclude_refs((), None, "")
+ return sizer.asizeof(val, limit=100 if recurse else 0)
+
+
+except ImportError:
+
+ def _get_size_of(val: Any, *, recurse=True) -> int:
+ return 0
+
+
# Function type: the type used for invalidation callbacks
FT = TypeVar("FT", bound=Callable[..., Any])
@@ -56,7 +83,7 @@ def enumerate_leaves(node, depth):
class _Node:
- __slots__ = ["prev_node", "next_node", "key", "value", "callbacks"]
+ __slots__ = ["prev_node", "next_node", "key", "value", "callbacks", "memory"]
def __init__(
self,
@@ -84,6 +111,16 @@ class _Node:
self.add_callbacks(callbacks)
+ self.memory = 0
+ if caches.TRACK_MEMORY_USAGE:
+ self.memory = (
+ _get_size_of(key)
+ + _get_size_of(value)
+ + _get_size_of(self.callbacks, recurse=False)
+ + _get_size_of(self, recurse=False)
+ )
+ self.memory += _get_size_of(self.memory, recurse=False)
+
def add_callbacks(self, callbacks: Collection[Callable[[], None]]) -> None:
"""Add to stored list of callbacks, removing duplicates."""
@@ -233,6 +270,9 @@ class LruCache(Generic[KT, VT]):
if size_callback:
cached_cache_len[0] += size_callback(node.value)
+ if caches.TRACK_MEMORY_USAGE and metrics:
+ metrics.inc_memory_usage(node.memory)
+
def move_node_to_front(node):
prev_node = node.prev_node
next_node = node.next_node
@@ -258,6 +298,9 @@ class LruCache(Generic[KT, VT]):
node.run_and_clear_callbacks()
+ if caches.TRACK_MEMORY_USAGE and metrics:
+ metrics.dec_memory_usage(node.memory)
+
return deleted_len
@overload
@@ -373,6 +416,9 @@ class LruCache(Generic[KT, VT]):
if size_callback:
cached_cache_len[0] = 0
+ if caches.TRACK_MEMORY_USAGE and metrics:
+ metrics.clear_memory_usage()
+
@synchronized
def cache_contains(key: KT) -> bool:
return key in cache
diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py
index 8508b6bd3b..1737891564 100644
--- a/tests/federation/test_federation_server.py
+++ b/tests/federation/test_federation_server.py
@@ -74,6 +74,25 @@ class ServerACLsTestCase(unittest.TestCase):
self.assertFalse(server_matches_acl_event("[1:2::]", e))
self.assertTrue(server_matches_acl_event("1:2:3:4", e))
+ def test_wildcard_matching(self):
+ e = _create_acl_event({"allow": ["good*.com"]})
+ self.assertTrue(
+ server_matches_acl_event("good.com", e),
+ "* matches 0 characters",
+ )
+ self.assertTrue(
+ server_matches_acl_event("GOOD.COM", e),
+ "pattern is case-insensitive",
+ )
+ self.assertTrue(
+ server_matches_acl_event("good.aa.com", e),
+ "* matches several characters, including '.'",
+ )
+ self.assertFalse(
+ server_matches_acl_event("ishgood.com", e),
+ "pattern does not allow prefixes",
+ )
+
class StateQueryTests(unittest.FederatingHomeserverTestCase):
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index ce330e79cc..1ffab709fc 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -729,7 +729,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
)
self.assertEqual(expected_state.state, PresenceState.ONLINE)
self.federation_sender.send_presence_to_destinations.assert_called_once_with(
- destinations=["server2"], states={expected_state}
+ destinations={"server2"}, states=[expected_state]
)
#
@@ -740,7 +740,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
self._add_new_user(room_id, "@bob:server3")
self.federation_sender.send_presence_to_destinations.assert_called_once_with(
- destinations=["server3"], states={expected_state}
+ destinations={"server3"}, states=[expected_state]
)
def test_remote_gets_presence_when_local_user_joins(self):
@@ -788,14 +788,8 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
self.presence_handler.current_state_for_user("@test2:server")
)
self.assertEqual(expected_state.state, PresenceState.ONLINE)
- self.assertEqual(
- self.federation_sender.send_presence_to_destinations.call_count, 2
- )
- self.federation_sender.send_presence_to_destinations.assert_any_call(
- destinations=["server3"], states={expected_state}
- )
- self.federation_sender.send_presence_to_destinations.assert_any_call(
- destinations=["server2"], states={expected_state}
+ self.federation_sender.send_presence_to_destinations.assert_called_once_with(
+ destinations={"server2", "server3"}, states=[expected_state]
)
def _add_new_user(self, room_id, user_id):
diff --git a/tests/handlers/test_space_summary.py b/tests/handlers/test_space_summary.py
new file mode 100644
index 0000000000..2c5e81531b
--- /dev/null
+++ b/tests/handlers/test_space_summary.py
@@ -0,0 +1,81 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Optional
+from unittest import mock
+
+from synapse.handlers.space_summary import _child_events_comparison_key
+
+from tests import unittest
+
+
+def _create_event(room_id: str, order: Optional[Any] = None):
+ result = mock.Mock()
+ result.room_id = room_id
+ result.content = {}
+ if order is not None:
+ result.content["order"] = order
+ return result
+
+
+def _order(*events):
+ return sorted(events, key=_child_events_comparison_key)
+
+
+class TestSpaceSummarySort(unittest.TestCase):
+ def test_no_order_last(self):
+ """An event with no ordering is placed behind those with an ordering."""
+ ev1 = _create_event("!abc:test")
+ ev2 = _create_event("!xyz:test", "xyz")
+
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ def test_order(self):
+ """The ordering should be used."""
+ ev1 = _create_event("!abc:test", "xyz")
+ ev2 = _create_event("!xyz:test", "abc")
+
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ def test_order_room_id(self):
+ """Room ID is a tie-breaker for ordering."""
+ ev1 = _create_event("!abc:test", "abc")
+ ev2 = _create_event("!xyz:test", "abc")
+
+ self.assertEqual([ev1, ev2], _order(ev1, ev2))
+
+ def test_invalid_ordering_type(self):
+ """Invalid orderings are considered the same as missing."""
+ ev1 = _create_event("!abc:test", 1)
+ ev2 = _create_event("!xyz:test", "xyz")
+
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ ev1 = _create_event("!abc:test", {})
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ ev1 = _create_event("!abc:test", [])
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ ev1 = _create_event("!abc:test", True)
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ def test_invalid_ordering_value(self):
+ """Invalid orderings are considered the same as missing."""
+ ev1 = _create_event("!abc:test", "foo\n")
+ ev2 = _create_event("!xyz:test", "xyz")
+
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+ ev1 = _create_event("!abc:test", "a" * 51)
+ self.assertEqual([ev2, ev1], _order(ev1, ev2))
diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py
index 45906ce720..a52e89e407 100644
--- a/tests/push/test_push_rule_evaluator.py
+++ b/tests/push/test_push_rule_evaluator.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Any, Dict
+
from synapse.api.room_versions import RoomVersions
from synapse.events import FrozenEvent
from synapse.push import push_rule_evaluator
@@ -66,6 +68,170 @@ class PushRuleEvaluatorTestCase(unittest.TestCase):
# A display name with spaces should work fine.
self.assertTrue(evaluator.matches(condition, "@user:test", "foo bar"))
+ def _assert_matches(
+ self, condition: Dict[str, Any], content: Dict[str, Any], msg=None
+ ) -> None:
+ evaluator = self._get_evaluator(content)
+ self.assertTrue(evaluator.matches(condition, "@user:test", "display_name"), msg)
+
+ def _assert_not_matches(
+ self, condition: Dict[str, Any], content: Dict[str, Any], msg=None
+ ) -> None:
+ evaluator = self._get_evaluator(content)
+ self.assertFalse(
+ evaluator.matches(condition, "@user:test", "display_name"), msg
+ )
+
+ def test_event_match_body(self):
+ """Check that event_match conditions on content.body work as expected"""
+
+ # if the key is `content.body`, the pattern matches substrings.
+
+ # non-wildcards should match
+ condition = {
+ "kind": "event_match",
+ "key": "content.body",
+ "pattern": "foobaz",
+ }
+ self._assert_matches(
+ condition,
+ {"body": "aaa FoobaZ zzz"},
+ "patterns should match and be case-insensitive",
+ )
+ self._assert_not_matches(
+ condition,
+ {"body": "aa xFoobaZ yy"},
+ "pattern should only match at word boundaries",
+ )
+ self._assert_not_matches(
+ condition,
+ {"body": "aa foobazx yy"},
+ "pattern should only match at word boundaries",
+ )
+
+ # wildcards should match
+ condition = {
+ "kind": "event_match",
+ "key": "content.body",
+ "pattern": "f?o*baz",
+ }
+
+ self._assert_matches(
+ condition,
+ {"body": "aaa FoobarbaZ zzz"},
+ "* should match string and pattern should be case-insensitive",
+ )
+ self._assert_matches(
+ condition, {"body": "aa foobaz yy"}, "* should match 0 characters"
+ )
+ self._assert_not_matches(
+ condition, {"body": "aa fobbaz yy"}, "? should not match 0 characters"
+ )
+ self._assert_not_matches(
+ condition, {"body": "aa fiiobaz yy"}, "? should not match 2 characters"
+ )
+ self._assert_not_matches(
+ condition,
+ {"body": "aa xfooxbaz yy"},
+ "pattern should only match at word boundaries",
+ )
+ self._assert_not_matches(
+ condition,
+ {"body": "aa fooxbazx yy"},
+ "pattern should only match at word boundaries",
+ )
+
+ # test backslashes
+ condition = {
+ "kind": "event_match",
+ "key": "content.body",
+ "pattern": r"f\oobaz",
+ }
+ self._assert_matches(
+ condition,
+ {"body": r"F\oobaz"},
+ "backslash should match itself",
+ )
+ condition = {
+ "kind": "event_match",
+ "key": "content.body",
+ "pattern": r"f\?obaz",
+ }
+ self._assert_matches(
+ condition,
+ {"body": r"F\oobaz"},
+ r"? after \ should match any character",
+ )
+
+ def test_event_match_non_body(self):
+ """Check that event_match conditions on other keys work as expected"""
+
+ # if the key is anything other than 'content.body', the pattern must match the
+ # whole value.
+
+ # non-wildcards should match
+ condition = {
+ "kind": "event_match",
+ "key": "content.value",
+ "pattern": "foobaz",
+ }
+ self._assert_matches(
+ condition,
+ {"value": "FoobaZ"},
+ "patterns should match and be case-insensitive",
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "xFoobaZ"},
+ "pattern should only match at the start/end of the value",
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "FoobaZz"},
+ "pattern should only match at the start/end of the value",
+ )
+
+ # wildcards should match
+ condition = {
+ "kind": "event_match",
+ "key": "content.value",
+ "pattern": "f?o*baz",
+ }
+ self._assert_matches(
+ condition,
+ {"value": "FoobarbaZ"},
+ "* should match string and pattern should be case-insensitive",
+ )
+ self._assert_matches(
+ condition, {"value": "foobaz"}, "* should match 0 characters"
+ )
+ self._assert_not_matches(
+ condition, {"value": "fobbaz"}, "? should not match 0 characters"
+ )
+ self._assert_not_matches(
+ condition, {"value": "fiiobaz"}, "? should not match 2 characters"
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "xfooxbaz"},
+ "pattern should only match at the start/end of the value",
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "fooxbazx"},
+ "pattern should only match at the start/end of the value",
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "x\nfooxbaz"},
+ "pattern should not match after a newline",
+ )
+ self._assert_not_matches(
+ condition,
+ {"value": "fooxbaz\nx"},
+ "pattern should not match before a newline",
+ )
+
def test_no_body(self):
"""Not having a body shouldn't break the evaluator."""
evaluator = self._get_evaluator({})
diff --git a/tests/rest/client/v2_alpha/test_sendtodevice.py b/tests/rest/client/v2_alpha/test_sendtodevice.py
new file mode 100644
index 0000000000..c9c99cc5d7
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_sendtodevice.py
@@ -0,0 +1,201 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest import admin
+from synapse.rest.client.v1 import login
+from synapse.rest.client.v2_alpha import sendtodevice, sync
+
+from tests.unittest import HomeserverTestCase, override_config
+
+
+class SendToDeviceTestCase(HomeserverTestCase):
+ servlets = [
+ admin.register_servlets,
+ login.register_servlets,
+ sendtodevice.register_servlets,
+ sync.register_servlets,
+ ]
+
+ def test_user_to_user(self):
+ """A to-device message from one user to another should get delivered"""
+
+ user1 = self.register_user("u1", "pass")
+ user1_tok = self.login("u1", "pass", "d1")
+
+ user2 = self.register_user("u2", "pass")
+ user2_tok = self.login("u2", "pass", "d2")
+
+ # send the message
+ test_msg = {"foo": "bar"}
+ chan = self.make_request(
+ "PUT",
+ "/_matrix/client/r0/sendToDevice/m.test/1234",
+ content={"messages": {user2: {"d2": test_msg}}},
+ access_token=user1_tok,
+ )
+ self.assertEqual(chan.code, 200, chan.result)
+
+ # check it appears
+ channel = self.make_request("GET", "/sync", access_token=user2_tok)
+ self.assertEqual(channel.code, 200, channel.result)
+ expected_result = {
+ "events": [
+ {
+ "sender": user1,
+ "type": "m.test",
+ "content": test_msg,
+ }
+ ]
+ }
+ self.assertEqual(channel.json_body["to_device"], expected_result)
+
+ # it should re-appear if we do another sync
+ channel = self.make_request("GET", "/sync", access_token=user2_tok)
+ self.assertEqual(channel.code, 200, channel.result)
+ self.assertEqual(channel.json_body["to_device"], expected_result)
+
+ # it should *not* appear if we do an incremental sync
+ sync_token = channel.json_body["next_batch"]
+ channel = self.make_request(
+ "GET", f"/sync?since={sync_token}", access_token=user2_tok
+ )
+ self.assertEqual(channel.code, 200, channel.result)
+ self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), [])
+
+ @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+ def test_local_room_key_request(self):
+ """m.room_key_request has special-casing; test from local user"""
+ user1 = self.register_user("u1", "pass")
+ user1_tok = self.login("u1", "pass", "d1")
+
+ user2 = self.register_user("u2", "pass")
+ user2_tok = self.login("u2", "pass", "d2")
+
+ # send three messages
+ for i in range(3):
+ chan = self.make_request(
+ "PUT",
+ f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}",
+ content={"messages": {user2: {"d2": {"idx": i}}}},
+ access_token=user1_tok,
+ )
+ self.assertEqual(chan.code, 200, chan.result)
+
+ # now sync: we should get two of the three
+ channel = self.make_request("GET", "/sync", access_token=user2_tok)
+ self.assertEqual(channel.code, 200, channel.result)
+ msgs = channel.json_body["to_device"]["events"]
+ self.assertEqual(len(msgs), 2)
+ for i in range(2):
+ self.assertEqual(
+ msgs[i],
+ {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}},
+ )
+ sync_token = channel.json_body["next_batch"]
+
+ # ... time passes
+ self.reactor.advance(1)
+
+ # and we can send more messages
+ chan = self.make_request(
+ "PUT",
+ "/_matrix/client/r0/sendToDevice/m.room_key_request/3",
+ content={"messages": {user2: {"d2": {"idx": 3}}}},
+ access_token=user1_tok,
+ )
+ self.assertEqual(chan.code, 200, chan.result)
+
+ # ... which should arrive
+ channel = self.make_request(
+ "GET", f"/sync?since={sync_token}", access_token=user2_tok
+ )
+ self.assertEqual(channel.code, 200, channel.result)
+ msgs = channel.json_body["to_device"]["events"]
+ self.assertEqual(len(msgs), 1)
+ self.assertEqual(
+ msgs[0],
+ {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}},
+ )
+
+ @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+ def test_remote_room_key_request(self):
+ """m.room_key_request has special-casing; test from remote user"""
+ user2 = self.register_user("u2", "pass")
+ user2_tok = self.login("u2", "pass", "d2")
+
+ federation_registry = self.hs.get_federation_registry()
+
+ # send three messages
+ for i in range(3):
+ self.get_success(
+ federation_registry.on_edu(
+ "m.direct_to_device",
+ "remote_server",
+ {
+ "sender": "@user:remote_server",
+ "type": "m.room_key_request",
+ "messages": {user2: {"d2": {"idx": i}}},
+ "message_id": f"{i}",
+ },
+ )
+ )
+
+ # now sync: we should get two of the three
+ channel = self.make_request("GET", "/sync", access_token=user2_tok)
+ self.assertEqual(channel.code, 200, channel.result)
+ msgs = channel.json_body["to_device"]["events"]
+ self.assertEqual(len(msgs), 2)
+ for i in range(2):
+ self.assertEqual(
+ msgs[i],
+ {
+ "sender": "@user:remote_server",
+ "type": "m.room_key_request",
+ "content": {"idx": i},
+ },
+ )
+ sync_token = channel.json_body["next_batch"]
+
+ # ... time passes
+ self.reactor.advance(1)
+
+ # and we can send more messages
+ self.get_success(
+ federation_registry.on_edu(
+ "m.direct_to_device",
+ "remote_server",
+ {
+ "sender": "@user:remote_server",
+ "type": "m.room_key_request",
+ "messages": {user2: {"d2": {"idx": 3}}},
+ "message_id": "3",
+ },
+ )
+ )
+
+ # ... which should arrive
+ channel = self.make_request(
+ "GET", f"/sync?since={sync_token}", access_token=user2_tok
+ )
+ self.assertEqual(channel.code, 200, channel.result)
+ msgs = channel.json_body["to_device"]["events"]
+ self.assertEqual(len(msgs), 1)
+ self.assertEqual(
+ msgs[0],
+ {
+ "sender": "@user:remote_server",
+ "type": "m.room_key_request",
+ "content": {"idx": 3},
+ },
+ )
diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py
index aa20588bbe..77c4fe721c 100644
--- a/tests/storage/test_cleanup_extrems.py
+++ b/tests/storage/test_cleanup_extrems.py
@@ -47,10 +47,8 @@ class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase):
)
schema_path = os.path.join(
- prepare_database.dir_path,
- "databases",
+ prepare_database.schema_path,
"main",
- "schema",
"delta",
"54",
"delete_forward_extremities.sql",
diff --git a/tests/util/test_glob_to_regex.py b/tests/util/test_glob_to_regex.py
new file mode 100644
index 0000000000..220accb92b
--- /dev/null
+++ b/tests/util/test_glob_to_regex.py
@@ -0,0 +1,59 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.util import glob_to_regex
+
+from tests.unittest import TestCase
+
+
+class GlobToRegexTestCase(TestCase):
+ def test_literal_match(self):
+ """patterns without wildcards should match"""
+ pat = glob_to_regex("foobaz")
+ self.assertTrue(
+ pat.match("FoobaZ"), "patterns should match and be case-insensitive"
+ )
+ self.assertFalse(
+ pat.match("x foobaz"), "pattern should not match at word boundaries"
+ )
+
+ def test_wildcard_match(self):
+ pat = glob_to_regex("f?o*baz")
+
+ self.assertTrue(
+ pat.match("FoobarbaZ"),
+ "* should match string and pattern should be case-insensitive",
+ )
+ self.assertTrue(pat.match("foobaz"), "* should match 0 characters")
+ self.assertFalse(pat.match("fooxaz"), "the character after * must match")
+ self.assertFalse(pat.match("fobbaz"), "? should not match 0 characters")
+ self.assertFalse(pat.match("fiiobaz"), "? should not match 2 characters")
+
+ def test_multi_wildcard(self):
+ """patterns with multiple wildcards in a row should match"""
+ pat = glob_to_regex("**baz")
+ self.assertTrue(pat.match("agsgsbaz"), "** should match any string")
+ self.assertTrue(pat.match("baz"), "** should match the empty string")
+ self.assertEqual(pat.pattern, r"\A.{0,}baz\Z")
+
+ pat = glob_to_regex("*?baz")
+ self.assertTrue(pat.match("agsgsbaz"), "*? should match any string")
+ self.assertTrue(pat.match("abaz"), "*? should match a single char")
+ self.assertFalse(pat.match("baz"), "*? should not match the empty string")
+ self.assertEqual(pat.pattern, r"\A.{1,}baz\Z")
+
+ pat = glob_to_regex("a?*?*?baz")
+ self.assertTrue(pat.match("a g baz"), "?*?*? should match 3 chars")
+ self.assertFalse(pat.match("a..baz"), "?*?*? should not match 2 chars")
+ self.assertTrue(pat.match("a.gg.baz"), "?*?*? should match 4 chars")
+ self.assertEqual(pat.pattern, r"\Aa.{3,}baz\Z")
|
