From d2ef1a79cf942b2f8c1a4724eb0c4bca0b60edbe Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Fri, 4 Mar 2022 22:40:24 +0000
Subject: Relax version guard for packaging (#12166)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It’s just occurred to me that #12088 pulled in the “packaging” package (~=21.3). I pulled in the newest version I had at the time.

I only use it for packaging.requirements.Requirements. Which was added in packaging 16.1: https://github.com/pypa/packaging/releases/tag/16.1

https://pkgs.org/download/python3-packaging suggests that the oldest version we care about is 17.1 in Ubuntu Bionic. So I think with this bound we're hunky dory.
---
 synapse/python_dependencies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/python_dependencies.py')

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 8f48a33936..b40a7bbb76 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -83,8 +83,8 @@ REQUIREMENTS = [
     # ijson 3.1.4 fixes a bug with "." in property names
     "ijson>=3.1.4",
     "matrix-common~=1.1.0",
-    # For runtime introspection of our dependencies
-    "packaging~=21.3",
+    # We need packaging.requirements.Requirement, added in 16.1.
+    "packaging>=16.1",
 ]
 
 CONDITIONAL_REQUIREMENTS = {
-- 
cgit 1.5.1


From 562718278847375636ead2ed3afcc9d9d482ef96 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 8 Mar 2022 15:58:14 +0000
Subject: Use `ParamSpec` in type hints for `synapse.logging.context` (#12150)

Signed-off-by: Sean Quah <seanq@element.io>
---
 changelog.d/12150.misc                    |  1 +
 synapse/handlers/initial_sync.py          |  5 ++--
 synapse/logging/context.py                | 44 +++++++++++++++++--------------
 synapse/python_dependencies.py            |  3 ++-
 synapse/rest/media/v1/storage_provider.py |  9 +++++--
 5 files changed, 37 insertions(+), 25 deletions(-)
 create mode 100644 changelog.d/12150.misc

(limited to 'synapse/python_dependencies.py')

diff --git a/changelog.d/12150.misc b/changelog.d/12150.misc
new file mode 100644
index 0000000000..2d2706dac7
--- /dev/null
+++ b/changelog.d/12150.misc
@@ -0,0 +1 @@
+Use `ParamSpec` in type hints for `synapse.logging.context`.
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 316cfae24f..a7db8feb57 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -153,8 +153,9 @@ class InitialSyncHandler:
 
         public_room_ids = await self.store.get_public_room_ids()
 
-        limit = pagin_config.limit
-        if limit is None:
+        if pagin_config.limit is not None:
+            limit = pagin_config.limit
+        else:
             limit = 10
 
         serializer_options = SerializeEventConfig(as_client_event=as_client_event)
diff --git a/synapse/logging/context.py b/synapse/logging/context.py
index c31c2960ad..88cd8a9e1c 100644
--- a/synapse/logging/context.py
+++ b/synapse/logging/context.py
@@ -29,7 +29,6 @@ import warnings
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
-    Any,
     Awaitable,
     Callable,
     Optional,
@@ -41,7 +40,7 @@ from typing import (
 )
 
 import attr
-from typing_extensions import Literal
+from typing_extensions import Literal, ParamSpec
 
 from twisted.internet import defer, threads
 from twisted.python.threadpool import ThreadPool
@@ -719,32 +718,33 @@ def nested_logging_context(suffix: str) -> LoggingContext:
     )
 
 
+P = ParamSpec("P")
 R = TypeVar("R")
 
 
 @overload
 def preserve_fn(  # type: ignore[misc]
-    f: Callable[..., Awaitable[R]],
-) -> Callable[..., "defer.Deferred[R]"]:
+    f: Callable[P, Awaitable[R]],
+) -> Callable[P, "defer.Deferred[R]"]:
     # The `type: ignore[misc]` above suppresses
     # "Overloaded function signatures 1 and 2 overlap with incompatible return types"
     ...
 
 
 @overload
-def preserve_fn(f: Callable[..., R]) -> Callable[..., "defer.Deferred[R]"]:
+def preserve_fn(f: Callable[P, R]) -> Callable[P, "defer.Deferred[R]"]:
     ...
 
 
 def preserve_fn(
     f: Union[
-        Callable[..., R],
-        Callable[..., Awaitable[R]],
+        Callable[P, R],
+        Callable[P, Awaitable[R]],
     ]
-) -> Callable[..., "defer.Deferred[R]"]:
+) -> Callable[P, "defer.Deferred[R]"]:
     """Function decorator which wraps the function with run_in_background"""
 
-    def g(*args: Any, **kwargs: Any) -> "defer.Deferred[R]":
+    def g(*args: P.args, **kwargs: P.kwargs) -> "defer.Deferred[R]":
         return run_in_background(f, *args, **kwargs)
 
     return g
@@ -752,7 +752,7 @@ def preserve_fn(
 
 @overload
 def run_in_background(  # type: ignore[misc]
-    f: Callable[..., Awaitable[R]], *args: Any, **kwargs: Any
+    f: Callable[P, Awaitable[R]], *args: P.args, **kwargs: P.kwargs
 ) -> "defer.Deferred[R]":
     # The `type: ignore[misc]` above suppresses
     # "Overloaded function signatures 1 and 2 overlap with incompatible return types"
@@ -761,18 +761,22 @@ def run_in_background(  # type: ignore[misc]
 
 @overload
 def run_in_background(
-    f: Callable[..., R], *args: Any, **kwargs: Any
+    f: Callable[P, R], *args: P.args, **kwargs: P.kwargs
 ) -> "defer.Deferred[R]":
     ...
 
 
-def run_in_background(
+def run_in_background(  # type: ignore[misc]
+    # The `type: ignore[misc]` above suppresses
+    # "Overloaded function implementation does not accept all possible arguments of signature 1"
+    # "Overloaded function implementation does not accept all possible arguments of signature 2"
+    # which seems like a bug in mypy.
     f: Union[
-        Callable[..., R],
-        Callable[..., Awaitable[R]],
+        Callable[P, R],
+        Callable[P, Awaitable[R]],
     ],
-    *args: Any,
-    **kwargs: Any,
+    *args: P.args,
+    **kwargs: P.kwargs,
 ) -> "defer.Deferred[R]":
     """Calls a function, ensuring that the current context is restored after
     return from the function, and that the sentinel context is set once the
@@ -872,7 +876,7 @@ def _set_context_cb(result: ResultT, context: LoggingContext) -> ResultT:
 
 
 def defer_to_thread(
-    reactor: "ISynapseReactor", f: Callable[..., R], *args: Any, **kwargs: Any
+    reactor: "ISynapseReactor", f: Callable[P, R], *args: P.args, **kwargs: P.kwargs
 ) -> "defer.Deferred[R]":
     """
     Calls the function `f` using a thread from the reactor's default threadpool and
@@ -908,9 +912,9 @@ def defer_to_thread(
 def defer_to_threadpool(
     reactor: "ISynapseReactor",
     threadpool: ThreadPool,
-    f: Callable[..., R],
-    *args: Any,
-    **kwargs: Any,
+    f: Callable[P, R],
+    *args: P.args,
+    **kwargs: P.kwargs,
 ) -> "defer.Deferred[R]":
     """
     A wrapper for twisted.internet.threads.deferToThreadpool, which handles
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index b40a7bbb76..1dd39f06cf 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -76,7 +76,8 @@ REQUIREMENTS = [
     "netaddr>=0.7.18",
     "Jinja2>=2.9",
     "bleach>=1.4.3",
-    "typing-extensions>=3.7.4",
+    # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
+    "typing-extensions>=3.10.0",
     # We enforce that we have a `cryptography` version that bundles an `openssl`
     # with the latest security patches.
     "cryptography>=3.4.7",
diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py
index 18bf977d3d..1c9b71d69c 100644
--- a/synapse/rest/media/v1/storage_provider.py
+++ b/synapse/rest/media/v1/storage_provider.py
@@ -16,7 +16,7 @@ import abc
 import logging
 import os
 import shutil
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Callable, Optional
 
 from synapse.config._base import Config
 from synapse.logging.context import defer_to_thread, run_in_background
@@ -150,8 +150,13 @@ class FileStorageProviderBackend(StorageProvider):
         dirname = os.path.dirname(backup_fname)
         os.makedirs(dirname, exist_ok=True)
 
+        # mypy needs help inferring the type of the second parameter, which is generic
+        shutil_copyfile: Callable[[str, str], str] = shutil.copyfile
         await defer_to_thread(
-            self.hs.get_reactor(), shutil.copyfile, primary_fname, backup_fname
+            self.hs.get_reactor(),
+            shutil_copyfile,
+            primary_fname,
+            backup_fname,
         )
 
     async def fetch(self, path: str, file_info: FileInfo) -> Optional[Responder]:
-- 
cgit 1.5.1


From 8810c93e828a9ed1dcb008d08be8aa9fcb4d28c3 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 24 Mar 2022 17:16:13 +0000
Subject: Replace instances of deprecated `Jinja2.Markup` with
 `markupsafe.Markup` (#12289)

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
---
 changelog.d/12289.misc         | 1 +
 synapse/push/mailer.py         | 9 +++++----
 synapse/python_dependencies.py | 1 +
 3 files changed, 7 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/12289.misc

(limited to 'synapse/python_dependencies.py')

diff --git a/changelog.d/12289.misc b/changelog.d/12289.misc
new file mode 100644
index 0000000000..f80ccf3284
--- /dev/null
+++ b/changelog.d/12289.misc
@@ -0,0 +1 @@
+Remove uses of the long-deprecated `Jinja2.Markup` which would prevent Synapse from starting with Jinja2 3.1.0 or above installed. This does not affect deployments of Synapse using our Docker images or Debian packages.
\ No newline at end of file
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 649a4f49d0..5ccdd88364 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, TypeVar
 
 import bleach
 import jinja2
+from markupsafe import Markup
 
 from synapse.api.constants import EventTypes, Membership, RoomTypes
 from synapse.api.errors import StoreError
@@ -867,7 +868,7 @@ class Mailer:
         )
 
 
-def safe_markup(raw_html: str) -> jinja2.Markup:
+def safe_markup(raw_html: str) -> Markup:
     """
     Sanitise a raw HTML string to a set of allowed tags and attributes, and linkify any bare URLs.
 
@@ -877,7 +878,7 @@ def safe_markup(raw_html: str) -> jinja2.Markup:
     Returns:
         A Markup object ready to safely use in a Jinja template.
     """
-    return jinja2.Markup(
+    return Markup(
         bleach.linkify(
             bleach.clean(
                 raw_html,
@@ -891,7 +892,7 @@ def safe_markup(raw_html: str) -> jinja2.Markup:
     )
 
 
-def safe_text(raw_text: str) -> jinja2.Markup:
+def safe_text(raw_text: str) -> Markup:
     """
     Sanitise text (escape any HTML tags), and then linkify any bare URLs.
 
@@ -901,7 +902,7 @@ def safe_text(raw_text: str) -> jinja2.Markup:
     Returns:
         A Markup object ready to safely use in a Jinja template.
     """
-    return jinja2.Markup(
+    return Markup(
         bleach.linkify(bleach.clean(raw_text, tags=[], attributes=[], strip=False))
     )
 
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 1dd39f06cf..232f9a9595 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -75,6 +75,7 @@ REQUIREMENTS = [
     "attrs>=19.2.0,!=21.1.0",
     "netaddr>=0.7.18",
     "Jinja2>=2.9",
+    "MarkupSafe>=2.0",
     "bleach>=1.4.3",
     # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
     "typing-extensions>=3.10.0",
-- 
cgit 1.5.1


From 000426095224401649827d6cdc5a7d5e0ef4e17c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 24 Mar 2022 18:42:04 +0000
Subject: Revert "Replace instances of deprecated `Jinja2.Markup` with
 `markupsafe.Markup`" (#12296)

This reverts commit 8810c93e828a9ed1dcb008d08be8aa9fcb4d28c3.
---
 synapse/push/mailer.py         | 9 ++++-----
 synapse/python_dependencies.py | 1 -
 2 files changed, 4 insertions(+), 6 deletions(-)

(limited to 'synapse/python_dependencies.py')

diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 5ccdd88364..649a4f49d0 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -18,7 +18,6 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, TypeVar
 
 import bleach
 import jinja2
-from markupsafe import Markup
 
 from synapse.api.constants import EventTypes, Membership, RoomTypes
 from synapse.api.errors import StoreError
@@ -868,7 +867,7 @@ class Mailer:
         )
 
 
-def safe_markup(raw_html: str) -> Markup:
+def safe_markup(raw_html: str) -> jinja2.Markup:
     """
     Sanitise a raw HTML string to a set of allowed tags and attributes, and linkify any bare URLs.
 
@@ -878,7 +877,7 @@ def safe_markup(raw_html: str) -> Markup:
     Returns:
         A Markup object ready to safely use in a Jinja template.
     """
-    return Markup(
+    return jinja2.Markup(
         bleach.linkify(
             bleach.clean(
                 raw_html,
@@ -892,7 +891,7 @@ def safe_markup(raw_html: str) -> Markup:
     )
 
 
-def safe_text(raw_text: str) -> Markup:
+def safe_text(raw_text: str) -> jinja2.Markup:
     """
     Sanitise text (escape any HTML tags), and then linkify any bare URLs.
 
@@ -902,7 +901,7 @@ def safe_text(raw_text: str) -> Markup:
     Returns:
         A Markup object ready to safely use in a Jinja template.
     """
-    return Markup(
+    return jinja2.Markup(
         bleach.linkify(bleach.clean(raw_text, tags=[], attributes=[], strip=False))
     )
 
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 232f9a9595..1dd39f06cf 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -75,7 +75,6 @@ REQUIREMENTS = [
     "attrs>=19.2.0,!=21.1.0",
     "netaddr>=0.7.18",
     "Jinja2>=2.9",
-    "MarkupSafe>=2.0",
     "bleach>=1.4.3",
     # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
     "typing-extensions>=3.10.0",
-- 
cgit 1.5.1


From 84eb14c4d21315798048aed123c8222cfa96c566 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 24 Mar 2022 18:59:31 +0000
Subject: Pin Jinja to <3.1.0 (#12297)

as 3.1.0 removed the deprecated jinja2.Markup class which we still rely on.
---
 changelog.d/12297.misc         | 1 +
 synapse/python_dependencies.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/12297.misc

(limited to 'synapse/python_dependencies.py')

diff --git a/changelog.d/12297.misc b/changelog.d/12297.misc
new file mode 100644
index 0000000000..7bf05305f6
--- /dev/null
+++ b/changelog.d/12297.misc
@@ -0,0 +1 @@
+Pin Jinja to <3.1.0, as Synapse fails to start with Jinja 3.1.0.
\ No newline at end of file
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 1dd39f06cf..79ae06ce5d 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -74,7 +74,8 @@ REQUIREMENTS = [
     # Note: 21.1.0 broke `/sync`, see #9936
     "attrs>=19.2.0,!=21.1.0",
     "netaddr>=0.7.18",
-    "Jinja2>=2.9",
+    # Jinja2 3.1.0 removes the deprecated jinja2.Markup class, which we rely on.
+    "Jinja2<3.1.0",
     "bleach>=1.4.3",
     # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
     "typing-extensions>=3.10.0",
-- 
cgit 1.5.1