From 76aa5537ad4f41cad130862a230c1f4cc4bfcbcf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 Dec 2021 16:33:35 +0000 Subject: Disable aggregation bundling on `/sync` responses (#11583) * Disable aggregation bundling on `/sync` responses A partial revert of #11478. This turns out to have had a significant CPU impact on initial-sync handling. For now, let's disable it, until we find a more efficient way of achieving this. * Fix tests. Co-authored-by: Patrick Cloke --- changelog.d/11583.bugfix | 1 + synapse/rest/client/sync.py | 10 +++++++++- tests/rest/client/test_relations.py | 10 +++++----- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 changelog.d/11583.bugfix diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix new file mode 100644 index 0000000000..d2ed113e21 --- /dev/null +++ b/changelog.d/11583.bugfix @@ -0,0 +1 @@ +Fix a performance regression in `/sync` handling, introduced in 1.49.0. diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 88e4f5e063..7f5846d389 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -522,7 +522,15 @@ class SyncRestServlet(RestServlet): time_now=time_now, # Don't bother to bundle aggregations if the timeline is unlimited, # as clients will have all the necessary information. - bundle_aggregations=room.timeline.limited, + # bundle_aggregations=room.timeline.limited, + # + # richvdh 2021-12-15: disable this temporarily as it has too high an + # overhead for initialsyncs. We need to figure out a way that the + # bundling can be done *before* the events are stored in the + # SyncResponseCache so that this part can be synchronous. + # + # Ensure to re-enable the test at tests/rest/client/test_relations.py::RelationsTestCase.test_bundled_aggregations. + bundle_aggregations=False, token_id=token_id, event_format=event_formatter, only_event_fields=only_fields, diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 397c12c2a6..1b58b73136 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -574,11 +574,11 @@ class RelationsTestCase(unittest.HomeserverTestCase): assert_bundle(channel.json_body["event"]["unsigned"].get("m.relations")) # Request sync. - channel = self.make_request("GET", "/sync", access_token=self.user_token) - self.assertEquals(200, channel.code, channel.json_body) - room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] - self.assertTrue(room_timeline["limited"]) - _find_and_assert_event(room_timeline["events"]) + # channel = self.make_request("GET", "/sync", access_token=self.user_token) + # self.assertEquals(200, channel.code, channel.json_body) + # room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] + # self.assertTrue(room_timeline["limited"]) + # _find_and_assert_event(room_timeline["events"]) # Note that /relations is tested separately in test_aggregation_get_event_for_thread # since it needs different data configured. -- cgit 1.4.1 From 8c36d332d598f55f471196fb9415bf27db3e6fcd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:07:41 +0000 Subject: 1.49.1 --- CHANGES.md | 9 +++++++++ changelog.d/11583.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/11583.bugfix diff --git a/CHANGES.md b/CHANGES.md index 58217e655e..bdfa40686f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.49.1 (2021-12-21) +=========================== + +Bugfixes +-------- + +- Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) + + Synapse 1.49.0 (2021-12-14) =========================== diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix deleted file mode 100644 index d2ed113e21..0000000000 --- a/changelog.d/11583.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a performance regression in `/sync` handling, introduced in 1.49.0. diff --git a/debian/changelog b/debian/changelog index 794d8efa40..61a1d003b2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.49.1) stable; urgency=medium + + * New synapse release 1.49.1. + + -- Synapse Packaging team Tue, 21 Dec 2021 11:07:30 +0000 + matrix-synapse-py3 (1.49.0) stable; urgency=medium * New synapse release 1.49.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index d44a03a2cb..d45e0f1883 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.49.0" +__version__ = "1.49.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when -- cgit 1.4.1 From b9f2f6d3c414a5b0d2bbce15ef12d669dbcea82a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:23:35 +0000 Subject: more words --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index bdfa40686f..0fcfb3c782 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.49.1 (2021-12-21) =========================== +This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client. + Bugfixes -------- -- cgit 1.4.1 From aa58e8a28a05c5a71003570f8bdcd1256276f243 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:24:24 +0000 Subject: typopo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0fcfb3c782..6932045848 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.49.1 (2021-12-21) =========================== -This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client. +This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. Bugfixes -------- -- cgit 1.4.1 From 57ca8ab10f74e9f36b3a93ff31438cabb146fb5f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 12:06:31 +0000 Subject: Add notes about dropping support for Python 3.6 and Postgres 9.6. --- CHANGES.md | 4 ++++ docs/deprecation_policy.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6932045848..dc6fa17794 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,10 @@ Synapse 1.49.1 (2021-12-21) This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. +**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10. + +**Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. + Bugfixes -------- diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 06ea340559..359dac07c3 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -14,8 +14,8 @@ i.e. when a version reaches End of Life Synapse will withdraw support for that version in future releases. Details on the upstream support life cycles for Python and PostgreSQL are -documented at https://endoflife.date/python and -https://endoflife.date/postgresql. +documented at [https://endoflife.date/python](https://endoflife.date/python) and +[https://endoflife.date/postgresql](https://endoflife.date/postgresql). Context -- cgit 1.4.1 From 2bf31f7807c7a0c229170803c97090d612dc16f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 Dec 2021 17:26:04 +0000 Subject: Pin to frozendict<2.1.2 (#11625) ... to work around breakage on buster (https://github.com/Marco-Sulla/python-frozendict/issues/41) --- changelog.d/11625.misc | 1 + synapse/python_dependencies.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11625.misc diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc new file mode 100644 index 0000000000..82ee78e724 --- /dev/null +++ b/changelog.d/11625.misc @@ -0,0 +1 @@ +Work around a build problem on Debian Buster. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 7d26954244..13fb69460e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -50,7 +50,8 @@ logger = logging.getLogger(__name__) REQUIREMENTS = [ # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 "jsonschema>=3.0.0", - "frozendict>=1", + # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 + "frozendict>=1,<2.1.2", "unpaddedbase64>=1.1.0", "canonicaljson>=1.4.0", # we use the type definitions added in signedjson 1.1. -- cgit 1.4.1 From aa874a13902fcf7f3e4024cd95a8491f308367bb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 17:32:16 +0000 Subject: 1.49.2 --- CHANGES.md | 13 ++++++++++++- changelog.d/11625.misc | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/11625.misc diff --git a/CHANGES.md b/CHANGES.md index dc6fa17794..9301ec8804 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -Synapse 1.49.1 (2021-12-21) +Synapse 1.49.2 (2021-12-21) =========================== This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. @@ -12,6 +12,17 @@ Bugfixes - Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) +Internal Changes +---------------- + +- Work around a build problem on Debian Buster. ([\#11625](https://github.com/matrix-org/synapse/issues/11625)) + + +Synapse 1.49.1 (2021-12-21) +=========================== + +Not released due to problems building the debian packages. + Synapse 1.49.0 (2021-12-14) =========================== diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc deleted file mode 100644 index 82ee78e724..0000000000 --- a/changelog.d/11625.misc +++ /dev/null @@ -1 +0,0 @@ -Work around a build problem on Debian Buster. diff --git a/debian/changelog b/debian/changelog index 61a1d003b2..ebe3e0cbf9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.49.2) stable; urgency=medium + + * New synapse release 1.49.2. + + -- Synapse Packaging team Tue, 21 Dec 2021 17:31:03 +0000 + matrix-synapse-py3 (1.49.1) stable; urgency=medium * New synapse release 1.49.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index d45e0f1883..95a49c20be 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.49.1" +__version__ = "1.49.2" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when -- cgit 1.4.1 From 6b6dcdc338a3fcfa1da6c3e2a38f62c268df1cbc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 17:41:34 +0000 Subject: update changelog postgres 10 _+_ --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9301ec8804..9f6e29631d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ Synapse 1.49.2 (2021-12-21) This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. -**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10. +**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10+. **Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. -- cgit 1.4.1