Merge tag 'v1.42.0rc2' into develop

Synapse 1.42.0rc2 (2021-09-06)
==============================

This version of Synapse removes deprecated room-management admin APIs, removes out-of-date
email pushers, and improves error handling for fallback templates for user-interactive
authentication. For more information on these points, server administrators are
encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420).

Features
--------

- Support room version 9 from [MSC3375](https://github.com/matrix-org/matrix-doc/pull/3375). ([\#10747](https://github.com/matrix-org/synapse/issues/10747))

Internal Changes
----------------

- Print a warning when using one of the deprecated `template_dir` settings. ([\#10768](https://github.com/matrix-org/synapse/issues/10768))

9 files changed, 145 insertions, 4 deletions

diff --git a/CHANGES.md b/CHANGES.md
index 986efbba0d..f9659d596d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,8 +1,25 @@
-Synapse 1.42.0rc1 (2021-09-01)
+Synapse 1.42.0rc2 (2021-09-06)
 ==============================
 
-Server administrators are reminded to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420).
+This version of Synapse removes deprecated room-management admin APIs, removes out-of-date
+email pushers, and improves error handling for fallback templates for user-interactive
+authentication. For more information on these points, server administrators are
+encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420).
+
+Features
+--------
 
+- Support room version 9 from [MSC3375](https://github.com/matrix-org/matrix-doc/pull/3375). ([\#10747](https://github.com/matrix-org/synapse/issues/10747))
+
+
+Internal Changes
+----------------
+
+- Print a warning when using one of the deprecated `template_dir` settings. ([\#10768](https://github.com/matrix-org/synapse/issues/10768))
+
+
+Synapse 1.42.0rc1 (2021-09-01)
+==============================
 
 Features
 --------
diff --git a/debian/changelog b/debian/changelog
index 0f7dbdf71e..e865e0d2f6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.42.0~rc2) stable; urgency=medium
+
+  * New synapse release 1.42.0~rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 06 Sep 2021 15:25:13 +0100
+
 matrix-synapse-py3 (1.42.0~rc1) stable; urgency=medium
 
   * New synapse release 1.42.0rc1.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index e5b075c53b..e4302d81a8 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.42.0rc1"
+__version__ = "1.42.0rc2"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py
index 8abcdfd4fd..a19be6707a 100644
--- a/synapse/api/room_versions.py
+++ b/synapse/api/room_versions.py
@@ -70,6 +70,9 @@ class RoomVersion:
     msc2176_redaction_rules = attr.ib(type=bool)
     # MSC3083: Support the 'restricted' join_rule.
     msc3083_join_rules = attr.ib(type=bool)
+    # MSC3375: Support for the proper redaction rules for MSC3083. This mustn't
+    #          be enabled if MSC3083 is not.
+    msc3375_redaction_rules = attr.ib(type=bool)
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
     msc2403_knocking = attr.ib(type=bool)
@@ -92,6 +95,7 @@ class RoomVersions:
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -107,6 +111,7 @@ class RoomVersions:
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -122,6 +127,7 @@ class RoomVersions:
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -137,6 +143,7 @@ class RoomVersions:
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -152,6 +159,7 @@ class RoomVersions:
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -167,6 +175,7 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -182,6 +191,7 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=True,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -197,6 +207,7 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -212,6 +223,23 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
         msc3083_join_rules=True,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+    )
+    V9 = RoomVersion(
+        "9",
+        RoomDisposition.STABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
@@ -227,6 +255,7 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=True,
         msc2716_historical=True,
         msc2716_redactions=False,
@@ -242,6 +271,7 @@ class RoomVersions:
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
         msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
         msc2403_knocking=True,
         msc2716_historical=True,
         msc2716_redactions=True,
@@ -261,6 +291,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V7,
         RoomVersions.MSC2716,
         RoomVersions.V8,
+        RoomVersions.V9,
     )
 }
 
diff --git a/synapse/config/account_validity.py b/synapse/config/account_validity.py
index 52e63ab1f6..ffaffc4931 100644
--- a/synapse/config/account_validity.py
+++ b/synapse/config/account_validity.py
@@ -11,8 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
+
 from synapse.config._base import Config, ConfigError
 
+logger = logging.getLogger(__name__)
+
+LEGACY_TEMPLATE_DIR_WARNING = """
+This server's configuration file is using the deprecated 'template_dir' setting in the
+'account_validity' section. Support for this setting has been deprecated and will be
+removed in a future version of Synapse. Server admins should instead use the new
+'custom_templates_directory' setting documented here:
+https://matrix-org.github.io/synapse/latest/templates.html
+---------------------------------------------------------------------------------------"""
+
 
 class AccountValidityConfig(Config):
     section = "account_validity"
@@ -69,6 +81,8 @@ class AccountValidityConfig(Config):
 
         # Load account validity templates.
         account_validity_template_dir = account_validity_config.get("template_dir")
+        if account_validity_template_dir is not None:
+            logger.warning(LEGACY_TEMPLATE_DIR_WARNING)
 
         account_renewed_template_filename = account_validity_config.get(
             "account_renewed_html_path", "account_renewed.html"
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 4477419196..936abe6178 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -16,6 +16,7 @@
 
 # This file can't be called email.py because if it is, we cannot:
 import email.utils
+import logging
 import os
 from enum import Enum
 from typing import Optional
@@ -24,6 +25,8 @@ import attr
 
 from ._base import Config, ConfigError
 
+logger = logging.getLogger(__name__)
+
 MISSING_PASSWORD_RESET_CONFIG_ERROR = """\
 Password reset emails are enabled on this homeserver due to a partial
 'email' block. However, the following required keys are missing:
@@ -44,6 +47,14 @@ DEFAULT_SUBJECTS = {
     "email_validation": "[%(server_name)s] Validate your email",
 }
 
+LEGACY_TEMPLATE_DIR_WARNING = """
+This server's configuration file is using the deprecated 'template_dir' setting in the
+'email' section. Support for this setting has been deprecated and will be removed in a
+future version of Synapse. Server admins should instead use the new
+'custom_templates_directory' setting documented here:
+https://matrix-org.github.io/synapse/latest/templates.html
+---------------------------------------------------------------------------------------"""
+
 
 @attr.s(slots=True, frozen=True)
 class EmailSubjectConfig:
@@ -105,6 +116,9 @@ class EmailConfig(Config):
 
         # A user-configurable template directory
         template_dir = email_config.get("template_dir")
+        if template_dir is not None:
+            logger.warning(LEGACY_TEMPLATE_DIR_WARNING)
+
         if isinstance(template_dir, str):
             # We need an absolute path, because we change directory after starting (and
             # we don't yet know what auxiliary templates like mail.css we will need).
diff --git a/synapse/config/sso.py b/synapse/config/sso.py
index fe1177ab81..524a7ff3aa 100644
--- a/synapse/config/sso.py
+++ b/synapse/config/sso.py
@@ -11,12 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 from typing import Any, Dict, Optional
 
 import attr
 
 from ._base import Config
 
+logger = logging.getLogger(__name__)
+
+LEGACY_TEMPLATE_DIR_WARNING = """
+This server's configuration file is using the deprecated 'template_dir' setting in the
+'sso' section. Support for this setting has been deprecated and will be removed in a
+future version of Synapse. Server admins should instead use the new
+'custom_templates_directory' setting documented here:
+https://matrix-org.github.io/synapse/latest/templates.html
+---------------------------------------------------------------------------------------"""
+
 
 @attr.s(frozen=True)
 class SsoAttributeRequirement:
@@ -43,6 +54,8 @@ class SSOConfig(Config):
 
         # The sso-specific template_dir
         self.sso_template_dir = sso_config.get("template_dir")
+        if self.sso_template_dir is not None:
+            logger.warning(LEGACY_TEMPLATE_DIR_WARNING)
 
         # Read templates from disk
         custom_template_directories = (
diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 738a151cef..fb22337e27 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -104,6 +104,8 @@ def prune_event_dict(room_version: RoomVersion, event_dict: dict) -> dict:
 
     if event_type == EventTypes.Member:
         add_fields("membership")
+        if room_version.msc3375_redaction_rules:
+            add_fields("join_authorised_via_users_server")
     elif event_type == EventTypes.Create:
         # MSC2176 rules state that create events cannot be redacted.
         if room_version.msc2176_redaction_rules:
diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py
index 7a826c086e..5446fda5e7 100644
--- a/tests/events/test_utils.py
+++ b/tests/events/test_utils.py
@@ -322,7 +322,7 @@ class PruneEventTestCase(unittest.TestCase):
             },
         )
 
-        # After MSC3083, alias events have no special behavior.
+        # After MSC3083, the allow key is protected from redaction.
         self.run_test(
             {
                 "type": "m.room.join_rules",
@@ -344,6 +344,50 @@ class PruneEventTestCase(unittest.TestCase):
             room_version=RoomVersions.V8,
         )
 
+    def test_member(self):
+        """Member events have changed behavior starting with MSC3375."""
+        self.run_test(
+            {
+                "type": "m.room.member",
+                "event_id": "$test:domain",
+                "content": {
+                    "membership": "join",
+                    "join_authorised_via_users_server": "@user:domain",
+                    "other_key": "stripped",
+                },
+            },
+            {
+                "type": "m.room.member",
+                "event_id": "$test:domain",
+                "content": {"membership": "join"},
+                "signatures": {},
+                "unsigned": {},
+            },
+        )
+
+        # After MSC3375, the join_authorised_via_users_server key is protected
+        # from redaction.
+        self.run_test(
+            {
+                "type": "m.room.member",
+                "content": {
+                    "membership": "join",
+                    "join_authorised_via_users_server": "@user:domain",
+                    "other_key": "stripped",
+                },
+            },
+            {
+                "type": "m.room.member",
+                "content": {
+                    "membership": "join",
+                    "join_authorised_via_users_server": "@user:domain",
+                },
+                "signatures": {},
+                "unsigned": {},
+            },
+            room_version=RoomVersions.V9,
+        )
+
 
 class SerializeEventTestCase(unittest.TestCase):
     def serialize(self, ev, fields):