summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore1
-rw-r--r--changelog.d/15853.misc1
-rw-r--r--changelog.d/15854.misc1
-rw-r--r--changelog.d/15861.misc1
-rw-r--r--changelog.d/15862.bugfix3
-rw-r--r--flake.nix2
-rw-r--r--poetry.lock26
-rw-r--r--synapse/storage/databases/main/devices.py14
-rw-r--r--synapse/storage/engines/postgres.py13
9 files changed, 43 insertions, 19 deletions
diff --git a/.gitignore b/.gitignore
index 8cf504324b..a89f149ec1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,6 +34,7 @@ __pycache__/
 /logs
 /media_store/
 /uploads
+/homeserver-config-overrides.d
 
 # For direnv users
 /.envrc
diff --git a/changelog.d/15853.misc b/changelog.d/15853.misc
new file mode 100644
index 0000000000..3e9516b1ad
--- /dev/null
+++ b/changelog.d/15853.misc
@@ -0,0 +1 @@
+Add a timeout that aborts any Postgres statement taking more than 1 hour.
\ No newline at end of file
diff --git a/changelog.d/15854.misc b/changelog.d/15854.misc
new file mode 100644
index 0000000000..8c940dd9c5
--- /dev/null
+++ b/changelog.d/15854.misc
@@ -0,0 +1 @@
+Fix the `devenv up` configuration which was ignoring the config overrides.
\ No newline at end of file
diff --git a/changelog.d/15861.misc b/changelog.d/15861.misc
new file mode 100644
index 0000000000..6f320eab81
--- /dev/null
+++ b/changelog.d/15861.misc
@@ -0,0 +1 @@
+Optimised cleanup of old entries in device_lists_stream.
diff --git a/changelog.d/15862.bugfix b/changelog.d/15862.bugfix
new file mode 100644
index 0000000000..8eb6aa9a7f
--- /dev/null
+++ b/changelog.d/15862.bugfix
@@ -0,0 +1,3 @@
+Pin `pydantic` to ^=1.7.4 to avoid backwards-incompatible API changes from the 2.0.0 release.
+Resolves https://github.com/matrix-org/synapse/issues/15858.
+Contributed by @PaarthShah.
diff --git a/flake.nix b/flake.nix
index 8b7c0e6a5b..bb42c9ff9b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -178,7 +178,7 @@
                   EOF
                 '';
                 # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
 
                 # Define the perl modules we require to run SyTest.
                 #
diff --git a/poetry.lock b/poetry.lock
index 5e0d192c96..9aaf5c7de7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -53,13 +53,13 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte
 
 [[package]]
 name = "authlib"
-version = "1.2.0"
+version = "1.2.1"
 description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients."
 optional = true
 python-versions = "*"
 files = [
-    {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"},
-    {file = "Authlib-1.2.0.tar.gz", hash = "sha256:4fa3e80883a5915ef9f5bc28630564bc4ed5b5af39812a3ff130ec76bd631e9d"},
+    {file = "Authlib-1.2.1-py2.py3-none-any.whl", hash = "sha256:c88984ea00149a90e3537c964327da930779afa4564e354edfd98410bea01911"},
+    {file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"},
 ]
 
 [package.dependencies]
@@ -837,13 +837,13 @@ files = [
 
 [[package]]
 name = "importlib-metadata"
-version = "6.6.0"
+version = "6.7.0"
 description = "Read metadata from Python packages"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"},
-    {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"},
+    {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"},
+    {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"},
 ]
 
 [package.dependencies]
@@ -853,7 +853,7 @@ zipp = ">=0.5"
 [package.extras]
 docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
 perf = ["ipython"]
-testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"]
+testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"]
 
 [[package]]
 name = "importlib-resources"
@@ -2301,13 +2301,13 @@ doc = ["Sphinx", "sphinx-rtd-theme"]
 
 [[package]]
 name = "sentry-sdk"
-version = "1.25.1"
+version = "1.26.0"
 description = "Python client for Sentry (https://sentry.io)"
 optional = true
 python-versions = "*"
 files = [
-    {file = "sentry-sdk-1.25.1.tar.gz", hash = "sha256:aa796423eb6a2f4a8cd7a5b02ba6558cb10aab4ccdc0537f63a47b038c520c38"},
-    {file = "sentry_sdk-1.25.1-py2.py3-none-any.whl", hash = "sha256:79afb7c896014038e358401ad1d36889f97a129dfa8031c49b3f238cd1aa3935"},
+    {file = "sentry-sdk-1.26.0.tar.gz", hash = "sha256:760e4fb6d01c994110507133e08ecd4bdf4d75ee4be77f296a3579796cf73134"},
+    {file = "sentry_sdk-1.26.0-py2.py3-none-any.whl", hash = "sha256:0c9f858337ec3781cf4851972ef42bba8c9828aea116b0dbed8f38c5f9a1896c"},
 ]
 
 [package.dependencies]
@@ -2964,13 +2964,13 @@ files = [
 
 [[package]]
 name = "types-pyopenssl"
-version = "23.2.0.0"
+version = "23.2.0.1"
 description = "Typing stubs for pyOpenSSL"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-pyOpenSSL-23.2.0.0.tar.gz", hash = "sha256:43e307e8dfb3a7a8208a19874ca060305f460c529d4eaca8a2669ea89499f244"},
-    {file = "types_pyOpenSSL-23.2.0.0-py3-none-any.whl", hash = "sha256:ba803a99440b0c2e9ab4e197084aeefc55bdfe8a580d367b2aa4210810a21240"},
+    {file = "types-pyOpenSSL-23.2.0.1.tar.gz", hash = "sha256:beeb5d22704c625a1e4b6dc756355c5b4af0b980138b702a9d9f932acf020903"},
+    {file = "types_pyOpenSSL-23.2.0.1-py3-none-any.whl", hash = "sha256:0568553f104466f1b8e0db3360fbe6770137d02e21a1a45c209bf2b1b03d90d4"},
 ]
 
 [package.dependencies]
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index f677d048aa..d9df437e51 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -1950,12 +1950,16 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
 
         # Delete older entries in the table, as we really only care about
         # when the latest change happened.
-        txn.execute_batch(
-            """
+        cleanup_obsolete_stmt = """
             DELETE FROM device_lists_stream
-            WHERE user_id = ? AND device_id = ? AND stream_id < ?
-            """,
-            [(user_id, device_id, min_stream_id) for device_id in device_ids],
+            WHERE user_id = ? AND stream_id < ? AND %s
+        """
+        device_ids_clause, device_ids_args = make_in_list_sql_clause(
+            txn.database_engine, "device_id", device_ids
+        )
+        txn.execute(
+            cleanup_obsolete_stmt % (device_ids_clause,),
+            [user_id, min_stream_id] + device_ids_args,
         )
 
         self.db_pool.simple_insert_many_txn(
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index b350f57ccb..05a72dc554 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -45,6 +45,15 @@ class PostgresEngine(
 
         psycopg2.extensions.register_adapter(bytes, _disable_bytes_adapter)
         self.synchronous_commit: bool = database_config.get("synchronous_commit", True)
+        # Set the statement timeout to 1 hour by default.
+        # Any query taking more than 1 hour should probably be considered a bug;
+        # most of the time this is a sign that work needs to be split up or that
+        # some degenerate query plan has been created and the client has probably
+        # timed out/walked off anyway.
+        # This is in milliseconds.
+        self.statement_timeout: Optional[int] = database_config.get(
+            "statement_timeout", 60 * 60 * 1000
+        )
         self._version: Optional[int] = None  # unknown as yet
 
         self.isolation_level_map: Mapping[int, int] = {
@@ -157,6 +166,10 @@ class PostgresEngine(
         if not self.synchronous_commit:
             cursor.execute("SET synchronous_commit TO OFF")
 
+        # Abort really long-running statements and turn them into errors.
+        if self.statement_timeout is not None:
+            cursor.execute("SET statement_timeout TO ?", (self.statement_timeout,))
+
         cursor.close()
         db_conn.commit()
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-01-29 02:55:19 +0000