From 72626b78ef4aa9ab0bd11e332495f34bd43bbc26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Oct 2021 12:33:21 +0100 Subject: Fix thread BG update to not seq scan event_json (#11192) For some reason the query optimiser decided to seq scan both tables, rather than index scanning `event_json`. --- changelog.d/11192.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11192.feature (limited to 'changelog.d') diff --git a/changelog.d/11192.feature b/changelog.d/11192.feature new file mode 100644 index 0000000000..76b0d28084 --- /dev/null +++ b/changelog.d/11192.feature @@ -0,0 +1 @@ +Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). -- cgit 1.5.1 From f3a4be870092e28531957702931c5d74b9d1f18f Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 27 Oct 2021 13:04:56 +0100 Subject: Shut down the DNS threadpool (#11190) The DNS threadpool must be explicitly stopped, otherwise Synapse will hang indefinitely when asked to shut down. --- changelog.d/11190.bugfix | 1 + synapse/app/_base.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/11190.bugfix (limited to 'changelog.d') diff --git a/changelog.d/11190.bugfix b/changelog.d/11190.bugfix new file mode 100644 index 0000000000..0d913805ac --- /dev/null +++ b/changelog.d/11190.bugfix @@ -0,0 +1 @@ +Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 03627cdcba..f4c3f867a8 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -346,6 +346,7 @@ async def start(hs: "HomeServer"): # numbers of DNS requests don't starve out other users of the threadpool. resolver_threadpool = ThreadPool(name="gai_resolver") resolver_threadpool.start() + reactor.addSystemEventTrigger("during", "shutdown", resolver_threadpool.stop) reactor.installNameResolver( GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool) ) -- cgit 1.5.1 From b615fc35d6a425f0241f1e9cf92886028ec36a72 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Oct 2021 13:22:48 +0100 Subject: Update changelog with new changes --- CHANGES.md | 4 ++-- changelog.d/11190.bugfix | 1 - changelog.d/11192.feature | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/11190.bugfix delete mode 100644 changelog.d/11192.feature (limited to 'changelog.d') diff --git a/CHANGES.md b/CHANGES.md index 56cd04756e..a58ab7c250 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Features - Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024)) - Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180)) -- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181)) +- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181), [\#11192](https://github.com/matrix-org/synapse/issues/11192)) - Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174)) @@ -27,7 +27,7 @@ Bugfixes - Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112)) - Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120)) - Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145)) -- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177)) +- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177), [\#11190](https://github.com/matrix-org/synapse/issues/11190)) - Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975)) diff --git a/changelog.d/11190.bugfix b/changelog.d/11190.bugfix deleted file mode 100644 index 0d913805ac..0000000000 --- a/changelog.d/11190.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. diff --git a/changelog.d/11192.feature b/changelog.d/11192.feature deleted file mode 100644 index 76b0d28084..0000000000 --- a/changelog.d/11192.feature +++ /dev/null @@ -1 +0,0 @@ -Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). -- cgit 1.5.1 From b3e843be88d67633d11711ecc80d4e0390b1e723 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 27 Oct 2021 10:48:02 -0400 Subject: Fix URL preview errors when previewing XML documents. (#11196) --- changelog.d/11196.bugfix | 1 + synapse/rest/media/v1/preview_url_resource.py | 9 ++++++--- tests/test_preview.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 changelog.d/11196.bugfix (limited to 'changelog.d') diff --git a/changelog.d/11196.bugfix b/changelog.d/11196.bugfix new file mode 100644 index 0000000000..3861eeb908 --- /dev/null +++ b/changelog.d/11196.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail. diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 278fd901e2..8ca97b5b18 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -718,9 +718,12 @@ def decode_body( if not body: return None + # The idea here is that multiple encodings are tried until one works. + # Unfortunately the result is never used and then LXML will decode the string + # again with the found encoding. for encoding in get_html_media_encodings(body, content_type): try: - body_str = body.decode(encoding) + body.decode(encoding) except Exception: pass else: @@ -732,11 +735,11 @@ def decode_body( from lxml import etree # Create an HTML parser. - parser = etree.HTMLParser(recover=True, encoding="utf-8") + parser = etree.HTMLParser(recover=True, encoding=encoding) # Attempt to parse the body. Returns None if the body was successfully # parsed, but no tree was found. - return etree.fromstring(body_str, parser) + return etree.fromstring(body, parser) def _calc_og(tree: "etree.Element", media_uri: str) -> Dict[str, Optional[str]]: diff --git a/tests/test_preview.py b/tests/test_preview.py index 9a576f9a4e..40b89fb2ef 100644 --- a/tests/test_preview.py +++ b/tests/test_preview.py @@ -277,6 +277,21 @@ class CalcOgTestCase(unittest.TestCase): tree = decode_body(html, "http://example.com/test.html") self.assertIsNone(tree) + def test_xml(self): + """Test decoding XML and ensure it works properly.""" + # Note that the strip() call is important to ensure the xml tag starts + # at the initial byte. + html = b""" + + + + + FooSome text. + """.strip() + tree = decode_body(html, "http://example.com/test.html") + og = _calc_og(tree, "http://example.com/test.html") + self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."}) + def test_invalid_encoding(self): """An invalid character encoding should be ignored and treated as UTF-8, if possible.""" html = b""" -- cgit 1.5.1