Support MSC1767's `content.body` behaviour; Add base rules from MSC3933 (#14524)

* Support MSC1767's `content.body` behaviour in push rules

* Add the base rules from MSC3933

* Changelog entry

* Flip condition around for finding `m.markup`

* Remove forgotten import

7 files changed, 316 insertions, 4 deletions

diff --git a/changelog.d/14524.feature b/changelog.d/14524.feature
new file mode 100644
index 0000000000..210acaa8ee
--- /dev/null
+++ b/changelog.d/14524.feature
@@ -0,0 +1 @@
+Add unstable support for an Extensible Events room version (`org.matrix.msc1767.10`) via [MSC1767](https://github.com/matrix-org/matrix-spec-proposals/pull/1767), [MSC3931](https://github.com/matrix-org/matrix-spec-proposals/pull/3931), [MSC3932](https://github.com/matrix-org/matrix-spec-proposals/pull/3932), and [MSC3933](https://github.com/matrix-org/matrix-spec-proposals/pull/3933).
\ No newline at end of file
diff --git a/rust/src/push/base_rules.rs b/rust/src/push/base_rules.rs
index 49802fa4eb..35129691ca 100644
--- a/rust/src/push/base_rules.rs
+++ b/rust/src/push/base_rules.rs
@@ -275,6 +275,156 @@ pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.encrypted_room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.encrypted")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.message.room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.message")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.file.room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.file")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.image.room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.image")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.video.room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.video")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed(
+            "global/underride/.org.matrix.msc3933.rule.extensible.audio.room_one_to_one",
+        ),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("org.matrix.msc1767.audio")),
+                pattern_type: None,
+            })),
+            Condition::Known(KnownCondition::RoomMemberCount {
+                is: Some(Cow::Borrowed("2")),
+            }),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, SOUND_ACTION, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
         rule_id: Cow::Borrowed("global/underride/.m.rule.message"),
         priority_class: 1,
         conditions: Cow::Borrowed(&[Condition::Known(KnownCondition::EventMatch(
@@ -303,6 +453,126 @@ pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.encrypted"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.encrypted")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.message"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.message")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.file"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.file")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.image"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.image")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.video"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.video")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
+        rule_id: Cow::Borrowed("global/underride/.org.matrix.msc1767.rule.extensible.audio"),
+        priority_class: 1,
+        conditions: Cow::Borrowed(&[
+            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
+                key: Cow::Borrowed("type"),
+                // MSC3933: Type changed from template rule - see MSC.
+                pattern: Some(Cow::Borrowed("m.audio")),
+                pattern_type: None,
+            })),
+            // MSC3933: Add condition on top of template rule - see MSC.
+            Condition::Known(KnownCondition::RoomVersionSupports {
+                // RoomVersionFeatures::ExtensibleEvents.as_str(), ideally
+                feature: Cow::Borrowed("org.matrix.msc3932.extensible_events"),
+            }),
+        ]),
+        actions: Cow::Borrowed(&[Action::Notify, HIGHLIGHT_FALSE_ACTION]),
+        default: true,
+        default_enabled: true,
+    },
+    PushRule {
         rule_id: Cow::Borrowed("global/underride/.im.vector.jitsi"),
         priority_class: 1,
         conditions: Cow::Borrowed(&[
diff --git a/rust/src/push/evaluator.rs b/rust/src/push/evaluator.rs
index b4c3039aba..1cd54f7e2c 100644
--- a/rust/src/push/evaluator.rs
+++ b/rust/src/push/evaluator.rs
@@ -483,7 +483,7 @@ fn test_requires_room_version_supports_condition() {
     };
     let rules = PushRules::new(vec![custom_rule]);
     result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, true),
         None,
         None,
     );
diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs
index eef39f6472..2e9d3e38a1 100644
--- a/rust/src/push/mod.rs
+++ b/rust/src/push/mod.rs
@@ -412,6 +412,7 @@ pub struct FilteredPushRules {
     push_rules: PushRules,
     enabled_map: BTreeMap<String, bool>,
     msc3664_enabled: bool,
+    msc1767_enabled: bool,
 }
 
 #[pymethods]
@@ -421,11 +422,13 @@ impl FilteredPushRules {
         push_rules: PushRules,
         enabled_map: BTreeMap<String, bool>,
         msc3664_enabled: bool,
+        msc1767_enabled: bool,
     ) -> Self {
         Self {
             push_rules,
             enabled_map,
             msc3664_enabled,
+            msc1767_enabled,
         }
     }
 
@@ -450,6 +453,10 @@ impl FilteredPushRules {
                     return false;
                 }
 
+                if !self.msc1767_enabled && rule.rule_id.contains("org.matrix.msc1767") {
+                    return false;
+                }
+
                 true
             })
             .map(|r| {
diff --git a/stubs/synapse/synapse_rust/push.pyi b/stubs/synapse/synapse_rust/push.pyi
index cbeb49663c..a6a586a0b5 100644
--- a/stubs/synapse/synapse_rust/push.pyi
+++ b/stubs/synapse/synapse_rust/push.pyi
@@ -26,7 +26,11 @@ class PushRules:
 
 class FilteredPushRules:
     def __init__(
-        self, push_rules: PushRules, enabled_map: Dict[str, bool], msc3664_enabled: bool
+        self,
+        push_rules: PushRules,
+        enabled_map: Dict[str, bool],
+        msc3664_enabled: bool,
+        msc1767_enabled: bool,
     ): ...
     def rules(self) -> Collection[Tuple[PushRule, bool]]: ...
 
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 9cc3da6d91..d6b377860f 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -29,6 +29,7 @@ from typing import (
 from prometheus_client import Counter
 
 from synapse.api.constants import MAIN_TIMELINE, EventTypes, Membership, RelationTypes
+from synapse.api.room_versions import PushRuleRoomFlag, RoomVersion
 from synapse.event_auth import auth_types_for_event, get_user_power_level
 from synapse.events import EventBase, relation_from_event
 from synapse.events.snapshot import EventContext
@@ -343,7 +344,7 @@ class BulkPushRuleEvaluator:
             room_version_features = []
 
         evaluator = PushRuleEvaluator(
-            _flatten_dict(event),
+            _flatten_dict(event, room_version=event.room_version),
             room_member_count,
             sender_power_level,
             notification_levels,
@@ -426,6 +427,7 @@ StateGroup = Union[object, int]
 
 def _flatten_dict(
     d: Union[EventBase, Mapping[str, Any]],
+    room_version: Optional[RoomVersion] = None,
     prefix: Optional[List[str]] = None,
     result: Optional[Dict[str, str]] = None,
 ) -> Dict[str, str]:
@@ -437,6 +439,31 @@ def _flatten_dict(
         if isinstance(value, str):
             result[".".join(prefix + [key])] = value.lower()
         elif isinstance(value, Mapping):
+            # do not set `room_version` due to recursion considerations below
             _flatten_dict(value, prefix=(prefix + [key]), result=result)
 
+    # `room_version` should only ever be set when looking at the top level of an event
+    if (
+        room_version is not None
+        and PushRuleRoomFlag.EXTENSIBLE_EVENTS in room_version.msc3931_push_features
+        and isinstance(d, EventBase)
+    ):
+        # Room supports extensible events: replace `content.body` with the plain text
+        # representation from `m.markup`, as per MSC1767.
+        markup = d.get("content").get("m.markup")
+        if room_version.identifier.startswith("org.matrix.msc1767."):
+            markup = d.get("content").get("org.matrix.msc1767.markup")
+        if markup is not None and isinstance(markup, list):
+            text = ""
+            for rep in markup:
+                if not isinstance(rep, dict):
+                    # invalid markup - skip all processing
+                    break
+                if rep.get("mimetype", "text/plain") == "text/plain":
+                    rep_text = rep.get("body")
+                    if rep_text is not None and isinstance(rep_text, str):
+                        text = rep_text.lower()
+                        break
+            result["content.body"] = text
+
     return result
diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py
index 12ad44dbb3..d4c64c46ad 100644
--- a/synapse/storage/databases/main/push_rule.py
+++ b/synapse/storage/databases/main/push_rule.py
@@ -84,7 +84,10 @@ def _load_rules(
     push_rules = PushRules(ruleslist)
 
     filtered_rules = FilteredPushRules(
-        push_rules, enabled_map, msc3664_enabled=experimental_config.msc3664_enabled
+        push_rules,
+        enabled_map,
+        msc3664_enabled=experimental_config.msc3664_enabled,
+        msc1767_enabled=experimental_config.msc1767_enabled,
     )
 
     return filtered_rules