Port `EventInternalMetadata` class to Rust (#16782)

There are a couple of things we need to be careful of here:

1. The current python code does no validation when loading from the DB,
so we need to be careful to ignore such errors (at least on jki.re there
are some old events with internal metadata fields of the wrong type).
2. We want to be memory efficient, as we often have many hundreds of
thousands of events in the cache at a time.

---------

Co-authored-by: Quentin Gliech <quenting@element.io>

5 files changed, 119 insertions, 119 deletions

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index c52e726661..92b406e336 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -42,6 +42,7 @@ from unpaddedbase64 import encode_base64
 
 from synapse.api.constants import RelationTypes
 from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions
+from synapse.synapse_rust.events import EventInternalMetadata
 from synapse.types import JsonDict, StrCollection
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
@@ -74,7 +75,7 @@ T = TypeVar("T")
 #
 # Note that DictProperty/DefaultDictProperty cannot actually be used with
 # EventBuilder as it lacks a _dict property.
-_DictPropertyInstance = Union["_EventInternalMetadata", "EventBase", "EventBuilder"]
+_DictPropertyInstance = Union["EventBase", "EventBuilder"]
 
 
 class DictProperty(Generic[T]):
@@ -111,7 +112,7 @@ class DictProperty(Generic[T]):
         if instance is None:
             return self
         try:
-            assert isinstance(instance, (EventBase, _EventInternalMetadata))
+            assert isinstance(instance, EventBase)
             return instance._dict[self.key]
         except KeyError as e1:
             # We want this to look like a regular attribute error (mostly so that
@@ -127,11 +128,11 @@ class DictProperty(Generic[T]):
             ) from e1.__context__
 
     def __set__(self, instance: _DictPropertyInstance, v: T) -> None:
-        assert isinstance(instance, (EventBase, _EventInternalMetadata))
+        assert isinstance(instance, EventBase)
         instance._dict[self.key] = v
 
     def __delete__(self, instance: _DictPropertyInstance) -> None:
-        assert isinstance(instance, (EventBase, _EventInternalMetadata))
+        assert isinstance(instance, EventBase)
         try:
             del instance._dict[self.key]
         except KeyError as e1:
@@ -176,118 +177,10 @@ class DefaultDictProperty(DictProperty, Generic[T]):
     ) -> Union[T, "DefaultDictProperty"]:
         if instance is None:
             return self
-        assert isinstance(instance, (EventBase, _EventInternalMetadata))
+        assert isinstance(instance, EventBase)
         return instance._dict.get(self.key, self.default)
 
 
-class _EventInternalMetadata:
-    __slots__ = ["_dict", "stream_ordering", "outlier"]
-
-    def __init__(self, internal_metadata_dict: JsonDict):
-        # we have to copy the dict, because it turns out that the same dict is
-        # reused. TODO: fix that
-        self._dict = dict(internal_metadata_dict)
-
-        # the stream ordering of this event. None, until it has been persisted.
-        self.stream_ordering: Optional[int] = None
-
-        # whether this event is an outlier (ie, whether we have the state at that point
-        # in the DAG)
-        self.outlier = False
-
-    out_of_band_membership: DictProperty[bool] = DictProperty("out_of_band_membership")
-    send_on_behalf_of: DictProperty[str] = DictProperty("send_on_behalf_of")
-    recheck_redaction: DictProperty[bool] = DictProperty("recheck_redaction")
-    soft_failed: DictProperty[bool] = DictProperty("soft_failed")
-    proactively_send: DictProperty[bool] = DictProperty("proactively_send")
-    redacted: DictProperty[bool] = DictProperty("redacted")
-
-    txn_id: DictProperty[str] = DictProperty("txn_id")
-    """The transaction ID, if it was set when the event was created."""
-
-    token_id: DictProperty[int] = DictProperty("token_id")
-    """The access token ID of the user who sent this event, if any."""
-
-    device_id: DictProperty[str] = DictProperty("device_id")
-    """The device ID of the user who sent this event, if any."""
-
-    def get_dict(self) -> JsonDict:
-        return dict(self._dict)
-
-    def is_outlier(self) -> bool:
-        return self.outlier
-
-    def is_out_of_band_membership(self) -> bool:
-        """Whether this event is an out-of-band membership.
-
-        OOB memberships are a special case of outlier events: they are membership events
-        for federated rooms that we aren't full members of. Examples include invites
-        received over federation, and rejections for such invites.
-
-        The concept of an OOB membership is needed because these events need to be
-        processed as if they're new regular events (e.g. updating membership state in
-        the database, relaying to clients via /sync, etc) despite being outliers.
-
-        See also https://element-hq.github.io/synapse/develop/development/room-dag-concepts.html#out-of-band-membership-events.
-
-        (Added in synapse 0.99.0, so may be unreliable for events received before that)
-        """
-        return self._dict.get("out_of_band_membership", False)
-
-    def get_send_on_behalf_of(self) -> Optional[str]:
-        """Whether this server should send the event on behalf of another server.
-        This is used by the federation "send_join" API to forward the initial join
-        event for a server in the room.
-
-        returns a str with the name of the server this event is sent on behalf of.
-        """
-        return self._dict.get("send_on_behalf_of")
-
-    def need_to_check_redaction(self) -> bool:
-        """Whether the redaction event needs to be rechecked when fetching
-        from the database.
-
-        Starting in room v3 redaction events are accepted up front, and later
-        checked to see if the redacter and redactee's domains match.
-
-        If the sender of the redaction event is allowed to redact any event
-        due to auth rules, then this will always return false.
-        """
-        return self._dict.get("recheck_redaction", False)
-
-    def is_soft_failed(self) -> bool:
-        """Whether the event has been soft failed.
-
-        Soft failed events should be handled as usual, except:
-            1. They should not go down sync or event streams, or generally
-               sent to clients.
-            2. They should not be added to the forward extremities (and
-               therefore not to current state).
-        """
-        return self._dict.get("soft_failed", False)
-
-    def should_proactively_send(self) -> bool:
-        """Whether the event, if ours, should be sent to other clients and
-        servers.
-
-        This is used for sending dummy events internally. Servers and clients
-        can still explicitly fetch the event.
-        """
-        return self._dict.get("proactively_send", True)
-
-    def is_redacted(self) -> bool:
-        """Whether the event has been redacted.
-
-        This is used for efficiently checking whether an event has been
-        marked as redacted without needing to make another database call.
-        """
-        return self._dict.get("redacted", False)
-
-    def is_notifiable(self) -> bool:
-        """Whether this event can trigger a push notification"""
-        return not self.is_outlier() or self.is_out_of_band_membership()
-
-
 class EventBase(metaclass=abc.ABCMeta):
     @property
     @abc.abstractmethod
@@ -313,7 +206,7 @@ class EventBase(metaclass=abc.ABCMeta):
 
         self._dict = event_dict
 
-        self.internal_metadata = _EventInternalMetadata(internal_metadata_dict)
+        self.internal_metadata = EventInternalMetadata(internal_metadata_dict)
 
     depth: DictProperty[int] = DictProperty("depth")
     content: DictProperty[JsonDict] = DictProperty("content")
diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index ae7092daaa..f32449c7da 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -31,9 +31,10 @@ from synapse.api.room_versions import (
 )
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.event_auth import auth_types_for_event
-from synapse.events import EventBase, _EventInternalMetadata, make_event_from_dict
+from synapse.events import EventBase, make_event_from_dict
 from synapse.state import StateHandler
 from synapse.storage.databases.main import DataStore
+from synapse.synapse_rust.events import EventInternalMetadata
 from synapse.types import EventID, JsonDict, StrCollection
 from synapse.types.state import StateFilter
 from synapse.util import Clock
@@ -93,8 +94,8 @@ class EventBuilder:
     _redacts: Optional[str] = None
     _origin_server_ts: Optional[int] = None
 
-    internal_metadata: _EventInternalMetadata = attr.Factory(
-        lambda: _EventInternalMetadata({})
+    internal_metadata: EventInternalMetadata = attr.Factory(
+        lambda: EventInternalMetadata({})
     )
 
     @property
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index c412063091..e3679d8f37 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -1155,7 +1155,7 @@ class FederationClient(FederationBase):
             # NB: We *need* to copy to ensure that we don't have multiple
             # references being passed on, as that causes... issues.
             for s in signed_state:
-                s.internal_metadata = copy.deepcopy(s.internal_metadata)
+                s.internal_metadata = s.internal_metadata.copy()
 
             # double-check that the auth chain doesn't include a different create event
             auth_chain_create_events = [
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
index 78ffeeaa46..1fd458b510 100644
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -1496,7 +1496,7 @@ class EventsWorkerStore(SQLBaseStore):
                     room_version_id=row[5],
                     rejected_reason=row[6],
                     redactions=[],
-                    outlier=row[7],
+                    outlier=bool(row[7]),  # This is an int in SQLite3
                 )
 
             # check for redactions
diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi
new file mode 100644
index 0000000000..423ede5969
--- /dev/null
+++ b/synapse/synapse_rust/events.pyi
@@ -0,0 +1,106 @@
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+from typing import Optional
+
+from synapse.types import JsonDict
+
+class EventInternalMetadata:
+    def __init__(self, internal_metadata_dict: JsonDict): ...
+
+    stream_ordering: Optional[int]
+    """the stream ordering of this event. None, until it has been persisted."""
+
+    outlier: bool
+    """whether this event is an outlier (ie, whether we have the state at that
+    point in the DAG)"""
+
+    out_of_band_membership: bool
+    send_on_behalf_of: str
+    recheck_redaction: bool
+    soft_failed: bool
+    proactively_send: bool
+    redacted: bool
+
+    txn_id: str
+    """The transaction ID, if it was set when the event was created."""
+    token_id: int
+    """The access token ID of the user who sent this event, if any."""
+    device_id: str
+    """The device ID of the user who sent this event, if any."""
+
+    def get_dict(self) -> JsonDict: ...
+    def is_outlier(self) -> bool: ...
+    def copy(self) -> "EventInternalMetadata": ...
+    def is_out_of_band_membership(self) -> bool:
+        """Whether this event is an out-of-band membership.
+
+        OOB memberships are a special case of outlier events: they are membership events
+        for federated rooms that we aren't full members of. Examples include invites
+        received over federation, and rejections for such invites.
+
+        The concept of an OOB membership is needed because these events need to be
+        processed as if they're new regular events (e.g. updating membership state in
+        the database, relaying to clients via /sync, etc) despite being outliers.
+
+        See also https://element-hq.github.io/synapse/develop/development/room-dag-concepts.html#out-of-band-membership-events.
+
+        (Added in synapse 0.99.0, so may be unreliable for events received before that)
+        """
+        ...
+    def get_send_on_behalf_of(self) -> Optional[str]:
+        """Whether this server should send the event on behalf of another server.
+        This is used by the federation "send_join" API to forward the initial join
+        event for a server in the room.
+
+        returns a str with the name of the server this event is sent on behalf of.
+        """
+        ...
+    def need_to_check_redaction(self) -> bool:
+        """Whether the redaction event needs to be rechecked when fetching
+        from the database.
+
+        Starting in room v3 redaction events are accepted up front, and later
+        checked to see if the redacter and redactee's domains match.
+
+        If the sender of the redaction event is allowed to redact any event
+        due to auth rules, then this will always return false.
+        """
+        ...
+    def is_soft_failed(self) -> bool:
+        """Whether the event has been soft failed.
+
+        Soft failed events should be handled as usual, except:
+            1. They should not go down sync or event streams, or generally
+               sent to clients.
+            2. They should not be added to the forward extremities (and
+               therefore not to current state).
+        """
+        ...
+    def should_proactively_send(self) -> bool:
+        """Whether the event, if ours, should be sent to other clients and
+        servers.
+
+        This is used for sending dummy events internally. Servers and clients
+        can still explicitly fetch the event.
+        """
+        ...
+    def is_redacted(self) -> bool:
+        """Whether the event has been redacted.
+
+        This is used for efficiently checking whether an event has been
+        marked as redacted without needing to make another database call.
+        """
+        ...
+    def is_notifiable(self) -> bool:
+        """Whether this event can trigger a push notification"""
+        ...