From df3a661e4adb7676682a5e3c298a2dfda18b08a1 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 10:04:47 +0000
Subject: Search for messages across predecessor rooms

Signed-off-by: Andrew Morgan <andrew@amorgan.xyz>
---
 synapse/handlers/search.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index ec936bbb4e..77e7e4e0fb 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -37,6 +37,54 @@ class SearchHandler(BaseHandler):
     def __init__(self, hs):
         super(SearchHandler, self).__init__(hs)
 
+    @defer.inlineCallbacks
+    def get_old_rooms_from_upgraded_room(self, room_id):
+        """Retrieves room IDs of old rooms in the history of an upgraded room.
+
+        We do so by checking the m.room.create event of the room for a
+        `predecessor` key. If it exists, we add the room ID to our return
+        list and then check that room for a m.room.create event and so on
+        until we can no longer find any more previous rooms.
+
+        The full list of all found rooms in then returned.
+
+        Args:
+            room_id (str): The ID of the room to search through.
+
+        Returns:
+            dict of past room IDs as strings
+        """
+
+        historical_room_ids = []
+
+        while True:
+            state_ids = yield self.store.get_current_state_ids(room_id)
+            create_id = state_ids.get((EventTypes.Create, ""))
+
+            # If we can't find the create event, assume we've hit a dead end
+            if not create_id:
+                break
+
+            # Retrieve the room's create event
+            create_event = yield self.store.get_event(create_id)
+
+            if not create_event:
+                break
+
+            # Check if a predecessor room is present
+            predecessor = create_event.content.get("predecessor", None)
+            if not predecessor:
+                break
+
+            # Add predecessor's room ID
+            historical_room_id = predecessor["room_id"]
+            historical_room_ids.append(historical_room_id)
+
+            # Scan through the old room for further predecessors
+            room_id = historical_room_id
+
+        defer.returnValue(historical_room_ids)
+
     @defer.inlineCallbacks
     def search(self, user, content, batch=None):
         """Performs a full text search for a user.
@@ -139,6 +187,27 @@ class SearchHandler(BaseHandler):
 
         room_ids = search_filter.filter_rooms(room_ids)
 
+        # If doing a subset of all rooms seearch, check if any of the rooms
+        # are from an upgraded room, and search their contents as well
+        # XXX: There is the possibility that we don't have a create event for
+        # the room in question, in which case we can't return all the results
+        # we want to.
+        # Ideally we would just return the results we can get now, and
+        # try to get more results from other servers in the background.
+        if search_filter.rooms:
+            historical_room_ids = []
+            for room_id in room_ids:
+                # Add any previous rooms to the search if they exist
+                ids = yield self.get_old_rooms_from_upgraded_room(room_id)
+                historical_room_ids += ids
+
+            # Add any found rooms to the list to search
+            for historical_room_id in historical_room_ids:
+                room_ids.add(historical_room_id)
+
+            # Prevent any historical events from being filtered
+            search_filter.add_room_ids(historical_room_ids)
+
         if batch_group == "room_id":
             room_ids.intersection_update({batch_group_key})
 
-- 
cgit 1.5.1


From 75942af1dbe1a4aa65fc986e5662e12c1337b7de Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 17 Jan 2019 14:11:24 +0000
Subject: Fix typo

---
 synapse/handlers/room.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index cb8c5f77dd..8898602eeb 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -125,7 +125,7 @@ class RoomCreationHandler(BaseHandler):
             )
             yield self.auth.check_from_context(tombstone_event, tombstone_context)
 
-            yield self.clone_exiting_room(
+            yield self.clone_existing_room(
                 requester,
                 old_room_id=old_room_id,
                 new_room_id=new_room_id,
@@ -230,7 +230,7 @@ class RoomCreationHandler(BaseHandler):
         )
 
     @defer.inlineCallbacks
-    def clone_exiting_room(
+    def clone_existing_room(
             self, requester, old_room_id, new_room_id, new_room_version,
             tombstone_event_id,
     ):
-- 
cgit 1.5.1


From 4ff6d2224543146723e3109681636de73a738f25 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 17 Jan 2019 15:22:03 +0000
Subject: Preserve DM status of a room on upgrade

Signed-off-by: Andrew Morgan <andrew@amorgan.xyz>
---
 synapse/handlers/room.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 8898602eeb..3bfba6c744 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -260,8 +260,27 @@ class RoomCreationHandler(BaseHandler):
             }
         }
 
+        # Copy over whether this room is considered a direct message by this
+        # user or not
+        user_account_data = yield self.store.get_account_data_for_user(
+            user_id,
+        )
+
+        if user_account_data:
+            direct_rooms = user_account_data[0]["m.direct"]
+            # Check if this room was a DM
+            if old_room_id in direct_rooms[user_id]:
+                # Add this room ID to the list of direct rooms
+                direct_rooms[user_id].append(new_room_id)
+
+                # Add this room ID to the list of direct rooms for this user
+                yield self.store.add_account_data_for_user(
+                    user_id, "m.direct", direct_rooms,
+                )
+
         initial_state = dict()
 
+        # Replicate relevant room events
         types_to_copy = (
             (EventTypes.JoinRules, ""),
             (EventTypes.Name, ""),
-- 
cgit 1.5.1


From 887ca93a1b3b01d1da2c80549d5d2b1166e0608b Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 17 Jan 2019 15:46:39 +0000
Subject: Prevent crash on user who doesn't have any direct rooms

---
 synapse/handlers/room.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 3bfba6c744..626c9f9166 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -266,7 +266,7 @@ class RoomCreationHandler(BaseHandler):
             user_id,
         )
 
-        if user_account_data:
+        if user_account_data and "m.direct" in user_account_data[0]:
             direct_rooms = user_account_data[0]["m.direct"]
             # Check if this room was a DM
             if old_room_id in direct_rooms[user_id]:
-- 
cgit 1.5.1


From ea8903fcc904c88027bcec62dc6646377941db41 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 15:12:47 +0000
Subject: Migrating dm and room tags work for migrator

---
 synapse/handlers/room.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 626c9f9166..07cf5f3f18 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -260,12 +260,16 @@ class RoomCreationHandler(BaseHandler):
             }
         }
 
-        # Copy over whether this room is considered a direct message by this
-        # user or not
+        # Copy over room account data for this user
         user_account_data = yield self.store.get_account_data_for_user(
             user_id,
         )
 
+        room_tags = yield self.store.get_tags_for_room(
+            user_id, old_room_id,
+        )
+
+        # Copy direct message state if applicable
         if user_account_data and "m.direct" in user_account_data[0]:
             direct_rooms = user_account_data[0]["m.direct"]
             # Check if this room was a DM
@@ -278,6 +282,13 @@ class RoomCreationHandler(BaseHandler):
                     user_id, "m.direct", direct_rooms,
                 )
 
+        # Copy room tags if applicable
+        if room_tags:
+            # Copy each room tag to the new room
+            for tag in room_tags.keys():
+                tag_content = room_tags[tag]
+                yield self.store.add_tag_to_room(user_id, new_room_id, tag, tag_content)
+
         initial_state = dict()
 
         # Replicate relevant room events
-- 
cgit 1.5.1


From 25d64a846ab4974da0d8a51b3a1ff014a10d319d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 15:27:11 +0000
Subject: Fix typos

---
 changelog.d/4412.bugfix               |  2 +-
 synapse/handlers/room_member.py       | 16 ++++++++--------
 synapse/rest/client/v1/room.py        |  8 ++++----
 synapse/storage/background_updates.py |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'synapse/handlers')

diff --git a/changelog.d/4412.bugfix b/changelog.d/4412.bugfix
index 2388ed5b59..007be1b7db 100644
--- a/changelog.d/4412.bugfix
+++ b/changelog.d/4412.bugfix
@@ -1 +1 @@
-Copy over whether a room is a direct message or not on room upgrade.
\ No newline at end of file
+Copy over whether a room is a direct message and any associated room tags on room upgrade.
\ No newline at end of file
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 07fd3e82fc..1d337ad95c 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -63,7 +63,7 @@ class RoomMemberHandler(object):
         self.directory_handler = hs.get_handlers().directory_handler
         self.registration_handler = hs.get_handlers().registration_handler
         self.profile_handler = hs.get_profile_handler()
-        self.event_creation_hander = hs.get_event_creation_handler()
+        self.event_creation_handler = hs.get_event_creation_handler()
 
         self.member_linearizer = Linearizer(name="member")
 
@@ -168,7 +168,7 @@ class RoomMemberHandler(object):
         if requester.is_guest:
             content["kind"] = "guest"
 
-        event, context = yield self.event_creation_hander.create_event(
+        event, context = yield self.event_creation_handler.create_event(
             requester,
             {
                 "type": EventTypes.Member,
@@ -186,14 +186,14 @@ class RoomMemberHandler(object):
         )
 
         # Check if this event matches the previous membership event for the user.
-        duplicate = yield self.event_creation_hander.deduplicate_state_event(
+        duplicate = yield self.event_creation_handler.deduplicate_state_event(
             event, context,
         )
         if duplicate is not None:
             # Discard the new event since this membership change is a no-op.
             defer.returnValue(duplicate)
 
-        yield self.event_creation_hander.handle_new_client_event(
+        yield self.event_creation_handler.handle_new_client_event(
             requester,
             event,
             context,
@@ -493,7 +493,7 @@ class RoomMemberHandler(object):
         else:
             requester = synapse.types.create_requester(target_user)
 
-        prev_event = yield self.event_creation_hander.deduplicate_state_event(
+        prev_event = yield self.event_creation_handler.deduplicate_state_event(
             event, context,
         )
         if prev_event is not None:
@@ -513,7 +513,7 @@ class RoomMemberHandler(object):
             if is_blocked:
                 raise SynapseError(403, "This room has been blocked on this server")
 
-        yield self.event_creation_hander.handle_new_client_event(
+        yield self.event_creation_handler.handle_new_client_event(
             requester,
             event,
             context,
@@ -527,7 +527,7 @@ class RoomMemberHandler(object):
         )
 
         if event.membership == Membership.JOIN:
-            # Only fire user_joined_room if the user has acutally joined the
+            # Only fire user_joined_room if the user has actually joined the
             # room. Don't bother if the user is just changing their profile
             # info.
             newly_joined = True
@@ -755,7 +755,7 @@ class RoomMemberHandler(object):
             )
         )
 
-        yield self.event_creation_hander.create_and_send_nonmember_event(
+        yield self.event_creation_handler.create_and_send_nonmember_event(
             requester,
             {
                 "type": EventTypes.ThirdPartyInvite,
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index fcfe7857f6..48da4d557f 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -89,7 +89,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(RoomStateEventRestServlet, self).__init__(hs)
         self.handlers = hs.get_handlers()
-        self.event_creation_hander = hs.get_event_creation_handler()
+        self.event_creation_handler = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
         self.message_handler = hs.get_message_handler()
 
@@ -172,7 +172,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
                 content=content,
             )
         else:
-            event = yield self.event_creation_hander.create_and_send_nonmember_event(
+            event = yield self.event_creation_handler.create_and_send_nonmember_event(
                 requester,
                 event_dict,
                 txn_id=txn_id,
@@ -189,7 +189,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomSendEventRestServlet, self).__init__(hs)
-        self.event_creation_hander = hs.get_event_creation_handler()
+        self.event_creation_handler = hs.get_event_creation_handler()
 
     def register(self, http_server):
         # /rooms/$roomid/send/$event_type[/$txn_id]
@@ -211,7 +211,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
         if b'ts' in request.args and requester.app_service:
             event_dict['origin_server_ts'] = parse_integer(request, "ts", 0)
 
-        event = yield self.event_creation_hander.create_and_send_nonmember_event(
+        event = yield self.event_creation_handler.create_and_send_nonmember_event(
             requester,
             event_dict,
             txn_id=txn_id,
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index 5fe1ca2de7..60cdc884e6 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -240,7 +240,7 @@ class BackgroundUpdateStore(SQLBaseStore):
         * An integer count of the number of items to update in this batch.
 
         The handler should return a deferred integer count of items updated.
-        The hander is responsible for updating the progress of the update.
+        The handler is responsible for updating the progress of the update.
 
         Args:
             update_name(str): The name of the update that this code handles.
-- 
cgit 1.5.1


From 8c85f0833d568f90d397dad30bc0ba28cf5d538b Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 17:03:09 +0000
Subject: tags, m.direct copying over correctly

---
 synapse/handlers/room.py        | 22 ++++++++++--------
 synapse/handlers/room_member.py | 51 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 62 insertions(+), 11 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 07cf5f3f18..9ac04eda50 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -272,15 +272,19 @@ class RoomCreationHandler(BaseHandler):
         # Copy direct message state if applicable
         if user_account_data and "m.direct" in user_account_data[0]:
             direct_rooms = user_account_data[0]["m.direct"]
-            # Check if this room was a DM
-            if old_room_id in direct_rooms[user_id]:
-                # Add this room ID to the list of direct rooms
-                direct_rooms[user_id].append(new_room_id)
-
-                # Add this room ID to the list of direct rooms for this user
-                yield self.store.add_account_data_for_user(
-                    user_id, "m.direct", direct_rooms,
-                )
+
+            # Check which key this room is under
+            for key, room_id_list in direct_rooms.items():
+                for room_id in room_id_list:
+                    if room_id == old_room_id:
+                        # Add new room_id to this key
+                        direct_rooms[key].append(new_room_id)
+
+                        # Save back to user's m.direct account data
+                        yield self.store.add_account_data_for_user(
+                            user_id, "m.direct", direct_rooms,
+                        )
+                        break
 
         # Copy room tags if applicable
         if room_tags:
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 1d337ad95c..35dd448c02 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -161,6 +161,8 @@ class RoomMemberHandler(object):
         ratelimit=True,
         content=None,
     ):
+        user_id = target.to_string()
+
         if content is None:
             content = {}
 
@@ -175,7 +177,7 @@ class RoomMemberHandler(object):
                 "content": content,
                 "room_id": room_id,
                 "sender": requester.user.to_string(),
-                "state_key": target.to_string(),
+                "state_key": user_id,
 
                 # For backwards compatibility:
                 "membership": membership,
@@ -204,7 +206,7 @@ class RoomMemberHandler(object):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
         prev_member_event_id = prev_state_ids.get(
-            (EventTypes.Member, target.to_string()),
+            (EventTypes.Member, user_id),
             None
         )
 
@@ -218,6 +220,51 @@ class RoomMemberHandler(object):
                 newly_joined = prev_member_event.membership != Membership.JOIN
             if newly_joined:
                 yield self._user_joined_room(target, room_id)
+
+            # Copy over direct message status and room tags if this is a join
+            # on an upgraded room
+
+            # Check if this is an upgraded room
+            state_ids = yield self.store.get_current_state_ids(room_id)
+            create_id = state_ids.get((EventTypes.Create, ""))
+            if not create_id:
+                return
+            create_event = yield self.store.get_event(create_id)
+
+            if "predecessor" in create_event["content"]:
+                old_room_id = create_event["content"]["predecessor"]["room_id"]
+
+                # Copy over room account data from predecessor room to upgraded room
+                user_account_data = yield self.store.get_account_data_for_user(
+                    user_id,
+                )
+                room_tags = yield self.store.get_tags_for_room(
+                    user_id, old_room_id,
+                )
+
+                # Copy direct message state if applicable
+                if user_account_data and "m.direct" in user_account_data[0]:
+                    direct_rooms = user_account_data[0]["m.direct"]
+
+                    # Check which key this room is under
+                    for key, room_id_list in direct_rooms.items():
+                        for rid in room_id_list:
+                            if rid == old_room_id:
+                                # Add new room_id to this key
+                                direct_rooms[key].append(room_id)
+
+                                # Save back to user's m.direct account data
+                                yield self.store.add_account_data_for_user(
+                                    user_id, "m.direct", direct_rooms,
+                                )
+                                break
+
+                # Copy room tags if applicable
+                if room_tags:
+                    # Copy each room tag to the new room
+                    for tag in room_tags.keys():
+                        tag_content = room_tags[tag]
+                        yield self.store.add_tag_to_room(user_id, room_id, tag, tag_content)
         elif event.membership == Membership.LEAVE:
             if prev_member_event_id:
                 prev_member_event = yield self.store.get_event(prev_member_event_id)
-- 
cgit 1.5.1


From 48951f437fb792d5d4da70c95fa8d20467428604 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 10:52:56 +0000
Subject: Join logic covers both room creator and arbitrary users

---
 synapse/handlers/room.py | 33 ---------------------------------
 1 file changed, 33 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 9ac04eda50..388302de09 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -260,39 +260,6 @@ class RoomCreationHandler(BaseHandler):
             }
         }
 
-        # Copy over room account data for this user
-        user_account_data = yield self.store.get_account_data_for_user(
-            user_id,
-        )
-
-        room_tags = yield self.store.get_tags_for_room(
-            user_id, old_room_id,
-        )
-
-        # Copy direct message state if applicable
-        if user_account_data and "m.direct" in user_account_data[0]:
-            direct_rooms = user_account_data[0]["m.direct"]
-
-            # Check which key this room is under
-            for key, room_id_list in direct_rooms.items():
-                for room_id in room_id_list:
-                    if room_id == old_room_id:
-                        # Add new room_id to this key
-                        direct_rooms[key].append(new_room_id)
-
-                        # Save back to user's m.direct account data
-                        yield self.store.add_account_data_for_user(
-                            user_id, "m.direct", direct_rooms,
-                        )
-                        break
-
-        # Copy room tags if applicable
-        if room_tags:
-            # Copy each room tag to the new room
-            for tag in room_tags.keys():
-                tag_content = room_tags[tag]
-                yield self.store.add_tag_to_room(user_id, new_room_id, tag, tag_content)
-
         initial_state = dict()
 
         # Replicate relevant room events
-- 
cgit 1.5.1


From 8086a5c05e5ddf7a24662882cbb090ec826110b4 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 10:56:35 +0000
Subject: Fix comments

---
 synapse/handlers/room_member.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 35dd448c02..d14da4bc7f 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -211,7 +211,7 @@ class RoomMemberHandler(object):
         )
 
         if event.membership == Membership.JOIN:
-            # Only fire user_joined_room if the user has acutally joined the
+            # Only fire user_joined_room if the user has actually joined the
             # room. Don't bother if the user is just changing their profile
             # info.
             newly_joined = True
@@ -234,7 +234,7 @@ class RoomMemberHandler(object):
             if "predecessor" in create_event["content"]:
                 old_room_id = create_event["content"]["predecessor"]["room_id"]
 
-                # Copy over room account data from predecessor room to upgraded room
+                # Retrieve room account data for predecessor room
                 user_account_data = yield self.store.get_account_data_for_user(
                     user_id,
                 )
-- 
cgit 1.5.1


From c9bfb058d85f6205fada062c78a4d1eca119417c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 11:12:48 +0000
Subject: Fix a bug with single-room search searching all rooms

* Create a new method for getting predecessor rooms
* Remove formatting change
---
 synapse/api/filtering.py   | 15 +++++++++++++--
 synapse/handlers/search.py | 42 ++++++++++--------------------------------
 synapse/storage/state.py   | 29 ++++++++++++++++++++++++++++-
 3 files changed, 51 insertions(+), 35 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 84000e6422..0d8957175d 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -444,8 +444,19 @@ class Filter(object):
     def include_redundant_members(self):
         return self.filter_json.get("include_redundant_members", False)
 
-    def add_room_ids(self, room_ids):
-        self.rooms += room_ids
+    def with_room_ids(self, room_ids):
+        """Returns a new filter with the given room IDs appended.
+
+        Args:
+            room_ids (list): A list of room_ids.
+
+        Returns:
+            filter: A new filter including the given rooms and the old
+                    filter's rooms.
+        """
+        newFilter = self
+        newFilter.rooms += room_ids
+        return newFilter
 
 
 def _matches_wildcard(actual_value, filter_value):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 77e7e4e0fb..75c26fe065 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -49,39 +49,26 @@ class SearchHandler(BaseHandler):
         The full list of all found rooms in then returned.
 
         Args:
-            room_id (str): The ID of the room to search through.
+            room_id (str): id of the room to search through.
 
         Returns:
-            dict of past room IDs as strings
+            Deferred[iterable[str]]: predecessor room ids
         """
 
         historical_room_ids = []
 
         while True:
-            state_ids = yield self.store.get_current_state_ids(room_id)
-            create_id = state_ids.get((EventTypes.Create, ""))
+            predecessor = yield self.store.get_room_predecessor(room_id)
 
-            # If we can't find the create event, assume we've hit a dead end
-            if not create_id:
-                break
-
-            # Retrieve the room's create event
-            create_event = yield self.store.get_event(create_id)
-
-            if not create_event:
-                break
-
-            # Check if a predecessor room is present
-            predecessor = create_event.content.get("predecessor", None)
+            # If no predecessor, assume we've hit a dead end
             if not predecessor:
                 break
 
             # Add predecessor's room ID
-            historical_room_id = predecessor["room_id"]
-            historical_room_ids.append(historical_room_id)
+            historical_room_ids.append(predecessor["room_id"])
 
             # Scan through the old room for further predecessors
-            room_id = historical_room_id
+            room_id = predecessor["room_id"]
 
         defer.returnValue(historical_room_ids)
 
@@ -185,28 +172,19 @@ class SearchHandler(BaseHandler):
         )
         room_ids = set(r.room_id for r in rooms)
 
-        room_ids = search_filter.filter_rooms(room_ids)
-
         # If doing a subset of all rooms seearch, check if any of the rooms
         # are from an upgraded room, and search their contents as well
-        # XXX: There is the possibility that we don't have a create event for
-        # the room in question, in which case we can't return all the results
-        # we want to.
-        # Ideally we would just return the results we can get now, and
-        # try to get more results from other servers in the background.
         if search_filter.rooms:
             historical_room_ids = []
-            for room_id in room_ids:
+            for room_id in search_filter.rooms:
                 # Add any previous rooms to the search if they exist
                 ids = yield self.get_old_rooms_from_upgraded_room(room_id)
                 historical_room_ids += ids
 
-            # Add any found rooms to the list to search
-            for historical_room_id in historical_room_ids:
-                room_ids.add(historical_room_id)
-
             # Prevent any historical events from being filtered
-            search_filter.add_room_ids(historical_room_ids)
+            search_filter = search_filter.with_room_ids(historical_room_ids)
+
+        room_ids = search_filter.filter_rooms(room_ids)
 
         if batch_group == "room_id":
             room_ids.intersection_update({batch_group_key})
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 49b3ff4a71..b064671851 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -437,6 +437,34 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         create_event = yield self.get_event(create_id)
         defer.returnValue(create_event.content.get("room_version", "1"))
 
+    @defer.inlineCallbacks
+    def get_room_predecessor(self, room_id):
+        """Get the predecessor room of an upgraded room if one exists.
+        Otherwise return None.
+
+        Args:
+            room_id (str)
+
+        Returns:
+            Deferred[str]: predecessor room id
+        """
+
+        state_ids = yield self.get_current_state_ids(room_id)
+        create_id = state_ids.get((EventTypes.Create, ""))
+
+        # If we can't find the create event, assume we've hit a dead end
+        if not create_id:
+            return None
+
+        # Retrieve the room's create event
+        create_event = yield self.get_event(create_id)
+
+        if not create_event:
+            return None
+
+        # Return predecessor if present
+        return create_event.content.get("predecessor", None)
+
     @cached(max_entries=100000, iterable=True)
     def get_current_state_ids(self, room_id):
         """Get the current state event ids for a room based on the
@@ -448,7 +476,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             deferred: dict of (type, state_key) -> event_id
         """
-
         def _get_current_state_ids_txn(txn):
             txn.execute(
                 """SELECT type, state_key, event_id FROM current_state_events
-- 
cgit 1.5.1


From c4875d8c767035036fa1e5c1ee58e9c01ac1e08d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 12:13:46 +0000
Subject: Prevent duplicate room IDs in m.direct

---
 synapse/handlers/room_member.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index d14da4bc7f..cdaf65cdef 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -248,16 +248,15 @@ class RoomMemberHandler(object):
 
                     # Check which key this room is under
                     for key, room_id_list in direct_rooms.items():
-                        for rid in room_id_list:
-                            if rid == old_room_id:
-                                # Add new room_id to this key
-                                direct_rooms[key].append(room_id)
-
-                                # Save back to user's m.direct account data
-                                yield self.store.add_account_data_for_user(
-                                    user_id, "m.direct", direct_rooms,
-                                )
-                                break
+                        if old_room_id in room_id_list and room_id not in room_id_list:
+                            # Add new room_id to this key
+                            direct_rooms[key].append(room_id)
+
+                            # Save back to user's m.direct account data
+                            yield self.store.add_account_data_for_user(
+                                user_id, "m.direct", direct_rooms,
+                            )
+                            break
 
                 # Copy room tags if applicable
                 if room_tags:
-- 
cgit 1.5.1


From 766a172b9911ff04ae3b314155462dfd40c76fba Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 13:23:26 +0000
Subject: lint

---
 synapse/handlers/room_member.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index cdaf65cdef..eb46f5f4fa 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -263,7 +263,9 @@ class RoomMemberHandler(object):
                     # Copy each room tag to the new room
                     for tag in room_tags.keys():
                         tag_content = room_tags[tag]
-                        yield self.store.add_tag_to_room(user_id, room_id, tag, tag_content)
+                        yield self.store.add_tag_to_room(
+                            user_id, room_id, tag, tag_content
+                        )
         elif event.membership == Membership.LEAVE:
             if prev_member_event_id:
                 prev_member_event = yield self.store.get_event(prev_member_event_id)
-- 
cgit 1.5.1


From 67cd4dad81ed2932009472da2d13648ca11eab73 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 16:50:06 +0000
Subject: Implement MSC 1813 - Add room version to make APIs

We also implement `make_membership_event` converting the returned
room version to an event format version.
---
 synapse/events/__init__.py              | 17 ++++++++++++++++-
 synapse/federation/federation_client.py | 21 ++++++++++++++++-----
 synapse/federation/federation_server.py |  8 +++++++-
 synapse/handlers/federation.py          |  2 +-
 4 files changed, 40 insertions(+), 8 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 9dd6940385..e6f94e68af 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,7 +18,7 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import EventFormatVersions
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -235,3 +235,18 @@ class FrozenEvent(EventBase):
             self.get("type", None),
             self.get("state_key", None),
         )
+
+
+def room_version_to_event_format(room_version):
+    """Converts a room version string to the event format
+
+    Args:
+        room_version (str)
+
+    Returns:
+        int
+    """
+    if room_version not in KNOWN_ROOM_VERSIONS:
+        raise
+
+    return EventFormatVersions.V1
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..0757ad12f4 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,14 +25,19 @@ from prometheus_client import Counter
 
 from twisted.internet import defer
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    Membership,
+    RoomVersions,
+)
 from synapse.api.errors import (
     CodeMessageException,
     FederationDeniedError,
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import builder
+from synapse.events import builder, room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -536,8 +541,9 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred: resolves to a tuple of (origin (str), event (object))
-            where origin is the remote homeserver which generated the event.
+            Deferred[tuple[str, dict, int]]: resolves to a tuple of
+            `(origin, event, event_format)` where origin is the remote
+            homeserver which generated the event.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
@@ -557,6 +563,11 @@ class FederationClient(FederationBase):
                 destination, room_id, user_id, membership, params,
             )
 
+            # Note: If not supplied, the room version may be either v1 or v2,
+            # however either way the event format version will be v1.
+            room_version = ret.get("room_version", RoomVersions.V1)
+            event_format = room_version_to_event_format(room_version)
+
             pdu_dict = ret.get("event", None)
             if not isinstance(pdu_dict, dict):
                 raise InvalidResponseError("Bad 'event' field in response")
@@ -574,7 +585,7 @@ class FederationClient(FederationBase):
             ev = builder.EventBuilder(pdu_dict)
 
             defer.returnValue(
-                (destination, ev)
+                (destination, ev, event_format)
             )
 
         return self._try_destination_list(
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 37d29e7027..17eccaaea0 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -400,8 +400,14 @@ class FederationServer(FederationBase):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
         pdu = yield self.handler.on_make_leave_request(room_id, user_id)
+
+        room_version = yield self.store.get_room_version(room_id)
+
         time_now = self._clock.time_msec()
-        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
+        defer.returnValue({
+            "event": pdu.get_pdu_json(time_now),
+            "room_version": room_version,
+        })
 
     @defer.inlineCallbacks
     def on_send_leave_request(self, origin, content):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..d1ba1450e9 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1336,7 +1336,7 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={}, params=None):
-        origin, pdu = yield self.federation_client.make_membership_event(
+        origin, pdu, _ = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
-- 
cgit 1.5.1


From 6a41d2a187ecef484a3aa67518ec9b4b0638c614 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 17:19:58 +0000
Subject: Add room_version param to get_pdu

When we add new event format we'll need to know the event format or room
version when parsing events.
---
 synapse/federation/federation_base.py   | 11 +++++---
 synapse/federation/federation_client.py | 46 ++++++++++++++++++++++++++++-----
 synapse/federation/federation_server.py |  4 ++-
 synapse/handlers/federation.py          | 12 +++++++--
 4 files changed, 60 insertions(+), 13 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index b7ad729c63..d749bfdd3a 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -43,8 +43,8 @@ class FederationBase(object):
         self._clock = hs.get_clock()
 
     @defer.inlineCallbacks
-    def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
-                                       include_none=False):
+    def _check_sigs_and_hash_and_fetch(self, origin, pdus, room_version,
+                                       outlier=False, include_none=False):
         """Takes a list of PDUs and checks the signatures and hashs of each
         one. If a PDU fails its signature check then we check if we have it in
         the database and if not then request if from the originating server of
@@ -56,8 +56,12 @@ class FederationBase(object):
         a new list.
 
         Args:
+            origin (str)
             pdu (list)
-            outlier (bool)
+            room_version (str)
+            outlier (bool): Whether the events are outliers or not
+            include_none (str): Whether to include None in the returned list
+                for events that have failed their checks
 
         Returns:
             Deferred : A list of PDUs that have valid signatures and hashes.
@@ -84,6 +88,7 @@ class FederationBase(object):
                     res = yield self.get_pdu(
                         destinations=[pdu.origin],
                         event_id=pdu.event_id,
+                        room_version=room_version,
                         outlier=outlier,
                         timeout=10000,
                     )
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..4e171f9b56 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,7 +25,12 @@ from prometheus_client import Counter
 
 from twisted.internet import defer
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    Membership,
+    RoomVersions,
+)
 from synapse.api.errors import (
     CodeMessageException,
     FederationDeniedError,
@@ -202,7 +207,8 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
+    def get_pdu(self, destinations, event_id, room_version, outlier=False,
+                timeout=None):
         """Requests the PDU with given origin and ID from the remote home
         servers.
 
@@ -212,6 +218,7 @@ class FederationClient(FederationBase):
         Args:
             destinations (list): Which home servers to query
             event_id (str): event to fetch
+            room_version (str): version of the room
             outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if
                 it's from an arbitary point in the context as opposed to part
                 of the current block of PDUs. Defaults to `False`
@@ -352,10 +359,13 @@ class FederationClient(FederationBase):
             ev.event_id for ev in itertools.chain(pdus, auth_chain)
         ])
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
-            outlier=True
+            outlier=True,
+            room_version=room_version,
         )
         signed_pdus.extend(
             seen_events[p.event_id] for p in pdus if p.event_id in seen_events
@@ -364,7 +374,8 @@ class FederationClient(FederationBase):
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in auth_chain if p.event_id not in seen_events],
-            outlier=True
+            outlier=True,
+            room_version=room_version,
         )
         signed_auth.extend(
             seen_events[p.event_id] for p in auth_chain if p.event_id in seen_events
@@ -411,6 +422,8 @@ class FederationClient(FederationBase):
             random.shuffle(srvs)
             return srvs
 
+        room_version = yield self.store.get_room_version(room_id)
+
         batch_size = 20
         missing_events = list(missing_events)
         for i in range(0, len(missing_events), batch_size):
@@ -421,6 +434,7 @@ class FederationClient(FederationBase):
                     self.get_pdu,
                     destinations=random_server_list(),
                     event_id=e_id,
+                    room_version=room_version,
                 )
                 for e_id in batch
             ]
@@ -450,8 +464,11 @@ class FederationClient(FederationBase):
             for p in res["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain, outlier=True
+            destination, auth_chain,
+            outlier=True, room_version=room_version,
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -650,9 +667,20 @@ class FederationClient(FederationBase):
                 for p in itertools.chain(state, auth_chain)
             }
 
+            room_version = None
+            for e in state:
+                if (e.type, e.state_key) == (EventTypes.Create, ""):
+                    room_version = e.content.get("room_version", RoomVersions.V1)
+                    break
+
+            if room_version is None:
+                # We use this error has that is what
+                raise SynapseError(400, "No create event in state")
+
             valid_pdus = yield self._check_sigs_and_hash_and_fetch(
                 destination, list(pdus.values()),
                 outlier=True,
+                room_version=room_version,
             )
 
             valid_pdus_map = {
@@ -790,8 +818,10 @@ class FederationClient(FederationBase):
             for e in content["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain, outlier=True
+            destination, auth_chain, outlier=True, room_version=room_version,
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -838,8 +868,10 @@ class FederationClient(FederationBase):
                 for e in content.get("events", [])
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_events = yield self._check_sigs_and_hash_and_fetch(
-                destination, events, outlier=False
+                destination, events, outlier=False, room_version=room_version,
             )
         except HttpResponseException as e:
             if not e.code == 400:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 37d29e7027..cb729c69ea 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -457,8 +457,10 @@ class FederationServer(FederationBase):
                 for e in content["auth_chain"]
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
-                origin, auth_chain, outlier=True
+                origin, auth_chain, outlier=True, room_version=room_version,
             )
 
             ret = yield self.handler.on_query_auth(
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..a9dc4a4e4e 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -34,6 +34,7 @@ from synapse.api.constants import (
     EventTypes,
     Membership,
     RejectedReason,
+    RoomVersions,
 )
 from synapse.api.errors import (
     AuthError,
@@ -342,6 +343,8 @@ class FederationHandler(BaseHandler):
                             room_id, event_id, p,
                         )
 
+                        room_version = yield self.store.get_room_version(room_id)
+
                         with logcontext.nested_logging_context(p):
                             # note that if any of the missing prevs share missing state or
                             # auth events, the requests to fetch those events are deduped
@@ -355,7 +358,7 @@ class FederationHandler(BaseHandler):
                             # we want the state *after* p; get_state_for_room returns the
                             # state *before* p.
                             remote_event = yield self.federation_client.get_pdu(
-                                [origin], p, outlier=True,
+                                [origin], p, room_version, outlier=True,
                             )
 
                             if remote_event is None:
@@ -379,7 +382,6 @@ class FederationHandler(BaseHandler):
                             for x in remote_state:
                                 event_map[x.event_id] = x
 
-                    room_version = yield self.store.get_room_version(room_id)
                     state_map = yield resolve_events_with_store(
                         room_version, state_maps, event_map,
                         state_res_store=StateResolutionStore(self.store),
@@ -655,6 +657,8 @@ class FederationHandler(BaseHandler):
         if dest == self.server_name:
             raise SynapseError(400, "Can't backfill from self.")
 
+        room_version = yield self.store.get_room_version(room_id)
+
         events = yield self.federation_client.backfill(
             dest,
             room_id,
@@ -748,6 +752,7 @@ class FederationHandler(BaseHandler):
                             self.federation_client.get_pdu,
                             [dest],
                             event_id,
+                            room_version=room_version,
                             outlier=True,
                             timeout=10000,
                         )
@@ -1659,6 +1664,8 @@ class FederationHandler(BaseHandler):
                 create_event = e
                 break
 
+        room_version = create_event.content.get("room_version", RoomVersions.V1)
+
         missing_auth_events = set()
         for e in itertools.chain(auth_events, state, [event]):
             for e_id in e.auth_event_ids():
@@ -1669,6 +1676,7 @@ class FederationHandler(BaseHandler):
             m_ev = yield self.federation_client.get_pdu(
                 [origin],
                 e_id,
+                room_version=room_version,
                 outlier=True,
                 timeout=10000,
             )
-- 
cgit 1.5.1


From 886e5acc762b879b606773b511ff92345aef14c6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 Jan 2019 15:13:07 +0000
Subject: Store rejected remote invite events as outliers

Currently they're stored as non-outliers even though the server isn't in
the room, which can be problematic in places where the code assumes it
has the state for all non outlier events.

In particular, there is an edge case where persisting the leave event
triggers a state resolution, which requires looking up the room version
from state. Since the server doesn't have the state, this causes an
exception to be thrown.
---
 synapse/federation/federation_client.py | 10 ++++++--
 synapse/handlers/federation.py          | 44 +++++++++------------------------
 synapse/storage/roommember.py           |  5 +---
 3 files changed, 21 insertions(+), 38 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..8fa726759e 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -32,7 +32,6 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -66,6 +65,8 @@ class FederationClient(FederationBase):
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
 
+        self.event_builder_factory = hs.get_event_builder_factory()
+
         self._get_pdu_cache = ExpiringCache(
             cache_name="get_pdu_cache",
             clock=self._clock,
@@ -571,7 +572,12 @@ class FederationClient(FederationBase):
             if "prev_state" not in pdu_dict:
                 pdu_dict["prev_state"] = []
 
-            ev = builder.EventBuilder(pdu_dict)
+            # Strip off the fields that we want to clobber.
+            pdu_dict.pop("origin", None)
+            pdu_dict.pop("origin_server_ts", None)
+            pdu_dict.pop("unsigned", None)
+
+            ev = self.event_builder_factory.new(pdu_dict)
 
             defer.returnValue(
                 (destination, ev)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..70be87cd3d 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,10 +43,7 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
-from synapse.crypto.event_signing import (
-    add_hashes_and_signatures,
-    compute_event_signature,
-)
+from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -58,7 +55,6 @@ from synapse.types import UserID, get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room
-from synapse.util.frozenutils import unfreeze
 from synapse.util.logutils import log_function
 from synapse.util.retryutils import NotRetryingDestination
 from synapse.visibility import filter_events_for_server
@@ -1083,7 +1079,9 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            event = self._sign_event(event)
+            self._sign_event(event)
+            event.internal_metadata.outlier = False
+
             # Try the host we successfully got a response to /make_join/
             # request first.
             try:
@@ -1289,13 +1287,7 @@ class FederationHandler(BaseHandler):
         event.internal_metadata.outlier = True
         event.internal_metadata.invite_from_remote = True
 
-        event.signatures.update(
-            compute_event_signature(
-                event,
-                self.hs.hostname,
-                self.hs.config.signing_key[0]
-            )
-        )
+        self._sign_event(event)
 
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
@@ -1313,7 +1305,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        event = self._sign_event(event)
+        self._sign_event(event)
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
@@ -1358,26 +1350,14 @@ class FederationHandler(BaseHandler):
         defer.returnValue((origin, event))
 
     def _sign_event(self, event):
-        event.internal_metadata.outlier = False
-
-        builder = self.event_builder_factory.new(
-            unfreeze(event.get_pdu_json())
-        )
-
-        builder.event_id = self.event_builder_factory.create_event_id()
-        builder.origin = self.hs.hostname
-
-        if not hasattr(event, "signatures"):
-            builder.signatures = {}
-
-        add_hashes_and_signatures(
-            builder,
-            self.hs.hostname,
-            self.hs.config.signing_key[0],
+        event.signatures.update(
+            compute_event_signature(
+                event,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
         )
 
-        return builder.build()
-
     @defer.inlineCallbacks
     @log_function
     def on_make_leave_request(self, room_id, user_id):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 0707f9a86a..c7488f4259 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -591,10 +591,7 @@ class RoomMemberStore(RoomMemberWorkerStore):
             # i.e., its something that has just happened.
             # The only current event that can also be an outlier is if its an
             # invite that has come in across federation.
-            is_new_state = not backfilled and (
-                not event.internal_metadata.is_outlier()
-                or event.internal_metadata.is_invite_from_remote()
-            )
+            is_new_state = not backfilled
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
                 if event.membership == Membership.INVITE:
-- 
cgit 1.5.1


From 07f62da55ac8903f7ea224255b8defd122724ec4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 19:44:37 +0000
Subject: Remove unnecessary '_sign_event'

---
 synapse/federation/federation_client.py |  9 ++++++++-
 synapse/handlers/federation.py          | 14 --------------
 2 files changed, 8 insertions(+), 15 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 8fa726759e..f4adcb556d 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -32,6 +32,7 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
+from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -577,7 +578,13 @@ class FederationClient(FederationBase):
             pdu_dict.pop("origin_server_ts", None)
             pdu_dict.pop("unsigned", None)
 
-            ev = self.event_builder_factory.new(pdu_dict)
+            builder = self.event_builder_factory.new(pdu_dict)
+            add_hashes_and_signatures(
+                builder,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
+            ev = builder.build()
 
             defer.returnValue(
                 (destination, ev)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 70be87cd3d..9a14ba4517 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,7 +43,6 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
-from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -1079,7 +1078,6 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            self._sign_event(event)
             event.internal_metadata.outlier = False
 
             # Try the host we successfully got a response to /make_join/
@@ -1287,8 +1285,6 @@ class FederationHandler(BaseHandler):
         event.internal_metadata.outlier = True
         event.internal_metadata.invite_from_remote = True
 
-        self._sign_event(event)
-
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
@@ -1305,7 +1301,6 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        self._sign_event(event)
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
@@ -1349,15 +1344,6 @@ class FederationHandler(BaseHandler):
         assert(event.room_id == room_id)
         defer.returnValue((origin, event))
 
-    def _sign_event(self, event):
-        event.signatures.update(
-            compute_event_signature(
-                event,
-                self.hs.hostname,
-                self.hs.config.signing_key[0]
-            )
-        )
-
     @defer.inlineCallbacks
     @log_function
     def on_make_leave_request(self, room_id, user_id):
-- 
cgit 1.5.1


From 7c288c22500e2045d36a29c38d2671fad6484e30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 20:05:44 +0000
Subject: Clarify the invite flows

---
 synapse/events/__init__.py     |  8 ++++++--
 synapse/handlers/federation.py | 12 +++++++++++-
 synapse/storage/roommember.py  | 11 +++++++----
 3 files changed, 24 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 84c75495d5..5030636c7e 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -41,8 +41,12 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
-    def is_invite_from_remote(self):
-        return getattr(self, "invite_from_remote", False)
+    def is_new_remote_event(self):
+        """Whether this is a new remote event, like an invite or an invite
+        rejection. This is needed as those events are marked as outliers, but
+        they still need to be processed.
+        """
+        return getattr(self, "new_remote_event", False)
 
     def get_send_on_behalf_of(self):
         """Whether this server should send the event on behalf of another server.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 9a14ba4517..e017cab777 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,6 +43,7 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
+from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -1283,7 +1284,15 @@ class FederationHandler(BaseHandler):
             )
 
         event.internal_metadata.outlier = True
-        event.internal_metadata.invite_from_remote = True
+        event.internal_metadata.new_remote_event = True
+
+        event.signatures.update(
+            compute_event_signature(
+                event,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
+        )
 
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
@@ -1301,6 +1310,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
+        event.internal_metadata.new_remote_event = True
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index c7488f4259..40b13de80b 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -588,10 +588,13 @@ class RoomMemberStore(RoomMemberWorkerStore):
             )
 
             # We update the local_invites table only if the event is "current",
-            # i.e., its something that has just happened.
-            # The only current event that can also be an outlier is if its an
-            # invite that has come in across federation.
-            is_new_state = not backfilled
+            # i.e., its something that has just happened. If the event is an
+            # outlier it is only current if its a "new remote event", like a
+            # remote invite or a rejection of a remote invite.
+            is_new_state = not backfilled and (
+                not event.internal_metadata.is_outlier()
+                or event.internal_metadata.is_new_remote_event()
+            )
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
                 if event.membership == Membership.INVITE:
-- 
cgit 1.5.1


From 03c85335d1d386c0523af3b6bf992f83bfb905d7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 24 Jan 2019 17:22:09 +0000
Subject: Apply suggestions from code review

Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com>
---
 synapse/handlers/search.py | 2 +-
 synapse/storage/state.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 75c26fe065..49c439313e 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -52,7 +52,7 @@ class SearchHandler(BaseHandler):
             room_id (str): id of the room to search through.
 
         Returns:
-            Deferred[iterable[str]]: predecessor room ids
+            Deferred[iterable[unicode]]: predecessor room ids
         """
 
         historical_room_ids = []
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index fceb9744aa..0a0691cd00 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -446,7 +446,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
             room_id (str)
 
         Returns:
-            Deferred[str]: predecessor room id
+            Deferred[unicode|None]: predecessor room id
         """
         state_ids = yield self.get_current_state_ids(room_id)
         create_id = state_ids.get((EventTypes.Create, ""))
-- 
cgit 1.5.1


From b8082a54451bb4db30e3b2a4d19dc8cb23330eb7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 17:33:19 +0000
Subject: Use term 'out of band membership' instead

---
 synapse/events/__init__.py     | 9 +++++----
 synapse/handlers/federation.py | 4 ++--
 synapse/storage/roommember.py  | 6 +++---
 3 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 5030636c7e..48289cad06 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -41,12 +41,13 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
-    def is_new_remote_event(self):
-        """Whether this is a new remote event, like an invite or an invite
+    def is_out_of_band_membership(self):
+        """Whether this is an out of band membership, like an invite or an invite
         rejection. This is needed as those events are marked as outliers, but
-        they still need to be processed.
+        they still need to be processed as if they're new events (e.g. updating
+        invite state in the database, relaying to clients, etc).
         """
-        return getattr(self, "new_remote_event", False)
+        return getattr(self, "out_of_band_membership", False)
 
     def get_send_on_behalf_of(self):
         """Whether this server should send the event on behalf of another server.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e017cab777..242719b7ce 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1284,7 +1284,7 @@ class FederationHandler(BaseHandler):
             )
 
         event.internal_metadata.outlier = True
-        event.internal_metadata.new_remote_event = True
+        event.internal_metadata.out_of_band_membership = True
 
         event.signatures.update(
             compute_event_signature(
@@ -1310,7 +1310,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        event.internal_metadata.new_remote_event = True
+        event.internal_metadata.out_of_band_membership = True
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 40b13de80b..592c1bcd33 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -589,11 +589,11 @@ class RoomMemberStore(RoomMemberWorkerStore):
 
             # We update the local_invites table only if the event is "current",
             # i.e., its something that has just happened. If the event is an
-            # outlier it is only current if its a "new remote event", like a
-            # remote invite or a rejection of a remote invite.
+            # outlier it is only current if its an "out of band membership",
+            # like a remote invite or a rejection of a remote invite.
             is_new_state = not backfilled and (
                 not event.internal_metadata.is_outlier()
-                or event.internal_metadata.is_new_remote_event()
+                or event.internal_metadata.is_out_of_band_membership()
             )
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
-- 
cgit 1.5.1


From 9139b87be420fcfce22e70a7c35ba52b2ea32f3a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:04:02 +0000
Subject: Remove unecessary setting of outlier bit

---
 synapse/handlers/federation.py | 2 --
 1 file changed, 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 242719b7ce..d53b716ffb 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1079,8 +1079,6 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            event.internal_metadata.outlier = False
-
             # Try the host we successfully got a response to /make_join/
             # request first.
             try:
-- 
cgit 1.5.1


From d148c43050f7a85523a743ff6069683c644a517d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:31:23 +0000
Subject: Review comments

---
 synapse/federation/federation_client.py | 3 ++-
 synapse/handlers/federation.py          | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 4e171f9b56..7fb5736142 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -674,7 +674,8 @@ class FederationClient(FederationBase):
                     break
 
             if room_version is None:
-                # We use this error has that is what
+                # If the state doesn't have a create event then the room is
+                # invalid, and it would fail auth checks anyway.
                 raise SynapseError(400, "No create event in state")
 
             valid_pdus = yield self._check_sigs_and_hash_and_fetch(
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a9dc4a4e4e..5280d88a50 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1664,6 +1664,11 @@ class FederationHandler(BaseHandler):
                 create_event = e
                 break
 
+        if create_event is None:
+            # If the state doesn't have a create event then the room is
+            # invalid, and it would fail auth checks anyway.
+            raise SynapseError(400, "No create event in state")
+
         room_version = create_event.content.get("room_version", RoomVersions.V1)
 
         missing_auth_events = set()
-- 
cgit 1.5.1


From a50cf929c13044f25c3776802287458fe5695c37 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 20:21:33 +0000
Subject: Require event format version to parse or create events

---
 synapse/events/__init__.py              | 24 ++++++++++-
 synapse/events/builder.py               | 51 ++++++++++++++++++++++-
 synapse/federation/federation_base.py   |  9 +++--
 synapse/federation/federation_client.py | 60 ++++++++++++++++-----------
 synapse/federation/federation_server.py | 33 +++++++++------
 synapse/federation/transport/server.py  |  4 +-
 synapse/handlers/federation.py          | 72 ++++++++++++++++++++-------------
 synapse/handlers/message.py             | 10 ++++-
 8 files changed, 191 insertions(+), 72 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 38470ad176..3fe52aaa45 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,7 +18,11 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions
+from synapse.api.constants import (
+    KNOWN_EVENT_FORMAT_VERSIONS,
+    KNOWN_ROOM_VERSIONS,
+    EventFormatVersions,
+)
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -256,3 +260,21 @@ def room_version_to_event_format(room_version):
         raise RuntimeError("Unrecognized room version %s" % (room_version,))
 
     return EventFormatVersions.V1
+
+
+def event_type_from_format_version(format_version):
+    """Returns the python type to use to construct an Event object for the
+    given event format version.
+
+    Args:
+        format_version (int): The event format version
+
+    Returns:
+        type: A type that can be initialized as per the initializer of
+        `FrozenEvent`
+    """
+    if format_version not in KNOWN_EVENT_FORMAT_VERSIONS:
+        raise Exception(
+            "No event format %r" % (format_version,)
+        )
+    return FrozenEvent
diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index e662eaef10..7e63371095 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -15,12 +15,39 @@
 
 import copy
 
+from synapse.api.constants import RoomVersions
 from synapse.types import EventID
 from synapse.util.stringutils import random_string
 
 from . import EventBase, FrozenEvent, _event_dict_property
 
 
+def get_event_builder(room_version, key_values={}, internal_metadata_dict={}):
+    """Generate an event builder appropriate for the given room version
+
+    Args:
+        room_version (str): Version of the room that we're creating an
+            event builder for
+        key_values (dict): Fields used as the basis of the new event
+        internal_metadata_dict (dict): Used to create the `_EventInternalMetadata`
+            object.
+
+    Returns:
+        EventBuilder
+    """
+    if room_version in {
+        RoomVersions.V1,
+        RoomVersions.V2,
+        RoomVersions.VDH_TEST,
+        RoomVersions.STATE_V2_TEST,
+    }:
+        return EventBuilder(key_values, internal_metadata_dict)
+    else:
+        raise Exception(
+            "No event format defined for version %r" % (room_version,)
+        )
+
+
 class EventBuilder(EventBase):
     def __init__(self, key_values={}, internal_metadata_dict={}):
         signatures = copy.deepcopy(key_values.pop("signatures", {}))
@@ -58,7 +85,29 @@ class EventBuilderFactory(object):
 
         return e_id.to_string()
 
-    def new(self, key_values={}):
+    def new(self, room_version, key_values={}):
+        """Generate an event builder appropriate for the given room version
+
+        Args:
+            room_version (str): Version of the room that we're creating an
+                event builder for
+            key_values (dict): Fields used as the basis of the new event
+
+        Returns:
+            EventBuilder
+        """
+
+        # There's currently only the one event version defined
+        if room_version not in {
+            RoomVersions.V1,
+            RoomVersions.V2,
+            RoomVersions.VDH_TEST,
+            RoomVersions.STATE_V2_TEST,
+        }:
+            raise Exception(
+                "No event format defined for version %r" % (room_version,)
+            )
+
         key_values["event_id"] = self.create_event_id()
 
         time_now = int(self.clock.time_msec())
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index d749bfdd3a..5c31e5f85f 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -23,7 +23,7 @@ from twisted.internet.defer import DeferredList
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
-from synapse.events import FrozenEvent
+from synapse.events import event_type_from_format_version
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
 from synapse.types import get_domain_from_id
@@ -302,11 +302,12 @@ def _is_invite_via_3pid(event):
     )
 
 
-def event_from_pdu_json(pdu_json, outlier=False):
+def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
     """Construct a FrozenEvent from an event json received over federation
 
     Args:
         pdu_json (object): pdu as received over federation
+        event_format_version (int): The event format version
         outlier (bool): True to mark this event as an outlier
 
     Returns:
@@ -330,8 +331,8 @@ def event_from_pdu_json(pdu_json, outlier=False):
     elif depth > MAX_DEPTH:
         raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
 
-    event = FrozenEvent(
-        pdu_json
+    event = event_type_from_format_version(event_format_version)(
+        pdu_json,
     )
 
     event.internal_metadata.outlier = outlier
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 33ecabca29..71809893c5 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -170,13 +170,13 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def backfill(self, dest, context, limit, extremities):
+    def backfill(self, dest, room_id, limit, extremities):
         """Requests some more historic PDUs for the given context from the
         given destination server.
 
         Args:
             dest (str): The remote home server to ask.
-            context (str): The context to backfill.
+            room_id (str): The room_id to backfill.
             limit (int): The maximum number of PDUs to return.
             extremities (list): List of PDU id and origins of the first pdus
                 we have seen from the context
@@ -191,12 +191,15 @@ class FederationClient(FederationBase):
             return
 
         transaction_data = yield self.transport_layer.backfill(
-            dest, context, extremities, limit)
+            dest, room_id, extremities, limit)
 
         logger.debug("backfill transaction_data=%s", repr(transaction_data))
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         pdus = [
-            event_from_pdu_json(p, outlier=False)
+            event_from_pdu_json(p, format_ver, outlier=False)
             for p in transaction_data["pdus"]
         ]
 
@@ -240,6 +243,8 @@ class FederationClient(FederationBase):
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
+        format_ver = room_version_to_event_format(room_version)
+
         signed_pdu = None
         for destination in destinations:
             now = self._clock.time_msec()
@@ -255,7 +260,7 @@ class FederationClient(FederationBase):
                 logger.debug("transaction_data %r", transaction_data)
 
                 pdu_list = [
-                    event_from_pdu_json(p, outlier=outlier)
+                    event_from_pdu_json(p, format_ver, outlier=outlier)
                     for p in transaction_data["pdus"]
                 ]
 
@@ -349,12 +354,16 @@ class FederationClient(FederationBase):
             destination, room_id, event_id=event_id,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         pdus = [
-            event_from_pdu_json(p, outlier=True) for p in result["pdus"]
+            event_from_pdu_json(p, format_ver, outlier=True)
+            for p in result["pdus"]
         ]
 
         auth_chain = [
-            event_from_pdu_json(p, outlier=True)
+            event_from_pdu_json(p, format_ver, outlier=True)
             for p in result.get("auth_chain", [])
         ]
 
@@ -362,8 +371,6 @@ class FederationClient(FederationBase):
             ev.event_id for ev in itertools.chain(pdus, auth_chain)
         ])
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
@@ -462,13 +469,14 @@ class FederationClient(FederationBase):
             destination, room_id, event_id,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         auth_chain = [
-            event_from_pdu_json(p, outlier=True)
+            event_from_pdu_json(p, format_ver, outlier=True)
             for p in res["auth_chain"]
         ]
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain,
             outlier=True, room_version=room_version,
@@ -605,7 +613,7 @@ class FederationClient(FederationBase):
             pdu_dict.pop("origin_server_ts", None)
             pdu_dict.pop("unsigned", None)
 
-            builder = self.event_builder_factory.new(pdu_dict)
+            builder = self.event_builder_factory.new(room_version, pdu_dict)
             add_hashes_and_signatures(
                 builder,
                 self.hs.hostname,
@@ -621,7 +629,7 @@ class FederationClient(FederationBase):
             "make_" + membership, destinations, send_request,
         )
 
-    def send_join(self, destinations, pdu):
+    def send_join(self, destinations, pdu, event_format_version):
         """Sends a join event to one of a list of homeservers.
 
         Doing so will cause the remote server to add the event to the graph,
@@ -631,6 +639,7 @@ class FederationClient(FederationBase):
             destinations (str): Candidate homeservers which are probably
                 participating in the room.
             pdu (BaseEvent): event to be sent
+            event_format_version (int): The event format version
 
         Return:
             Deferred: resolves to a dict with members ``origin`` (a string
@@ -676,12 +685,12 @@ class FederationClient(FederationBase):
             logger.debug("Got content: %s", content)
 
             state = [
-                event_from_pdu_json(p, outlier=True)
+                event_from_pdu_json(p, event_format_version, outlier=True)
                 for p in content.get("state", [])
             ]
 
             auth_chain = [
-                event_from_pdu_json(p, outlier=True)
+                event_from_pdu_json(p, event_format_version, outlier=True)
                 for p in content.get("auth_chain", [])
             ]
 
@@ -759,7 +768,10 @@ class FederationClient(FederationBase):
 
         logger.debug("Got response to send_invite: %s", pdu_dict)
 
-        pdu = event_from_pdu_json(pdu_dict)
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
+        pdu = event_from_pdu_json(pdu_dict, format_ver)
 
         # Check signatures are correct.
         pdu = yield self._check_sigs_and_hash(pdu)
@@ -837,13 +849,14 @@ class FederationClient(FederationBase):
             content=send_content,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         auth_chain = [
-            event_from_pdu_json(e)
+            event_from_pdu_json(e, format_ver)
             for e in content["auth_chain"]
         ]
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain, outlier=True, room_version=room_version,
         )
@@ -887,13 +900,14 @@ class FederationClient(FederationBase):
                 timeout=timeout,
             )
 
+            room_version = yield self.store.get_room_version(room_id)
+            format_ver = room_version_to_event_format(room_version)
+
             events = [
-                event_from_pdu_json(e)
+                event_from_pdu_json(e, format_ver)
                 for e in content.get("events", [])
             ]
 
-            room_version = yield self.store.get_room_version(room_id)
-
             signed_events = yield self._check_sigs_and_hash_and_fetch(
                 destination, events, outlier=False, room_version=room_version,
             )
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index dde166e295..4aa04b9588 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -34,6 +34,7 @@ from synapse.api.errors import (
     SynapseError,
 )
 from synapse.crypto.event_signing import compute_event_signature
+from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
@@ -178,14 +179,13 @@ class FederationServer(FederationBase):
                 continue
 
             try:
-                # In future we will actually use the room version to parse the
-                # PDU into an event.
-                yield self.store.get_room_version(room_id)
+                room_version = yield self.store.get_room_version(room_id)
+                format_ver = room_version_to_event_format(room_version)
             except NotFoundError:
                 logger.info("Ignoring PDU for unknown room_id: %s", room_id)
                 continue
 
-            event = event_from_pdu_json(p)
+            event = event_from_pdu_json(p, format_ver)
             pdus_by_room.setdefault(room_id, []).append(event)
 
         pdu_results = {}
@@ -370,7 +370,9 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
-        pdu = event_from_pdu_json(content)
+        format_ver = room_version_to_event_format(room_version)
+
+        pdu = event_from_pdu_json(content, format_ver)
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
@@ -378,9 +380,12 @@ class FederationServer(FederationBase):
         defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    def on_send_join_request(self, origin, content):
+    def on_send_join_request(self, origin, content, room_id):
         logger.debug("on_send_join_request: content: %s", content)
-        pdu = event_from_pdu_json(content)
+
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+        pdu = event_from_pdu_json(content, format_ver)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -410,9 +415,12 @@ class FederationServer(FederationBase):
         })
 
     @defer.inlineCallbacks
-    def on_send_leave_request(self, origin, content):
+    def on_send_leave_request(self, origin, content, room_id):
         logger.debug("on_send_leave_request: content: %s", content)
-        pdu = event_from_pdu_json(content)
+
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+        pdu = event_from_pdu_json(content, format_ver)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -458,13 +466,14 @@ class FederationServer(FederationBase):
             origin_host, _ = parse_server_name(origin)
             yield self.check_server_matches_acl(origin_host, room_id)
 
+            room_version = yield self.store.get_room_version(room_id)
+            format_ver = room_version_to_event_format(room_version)
+
             auth_chain = [
-                event_from_pdu_json(e)
+                event_from_pdu_json(e, format_ver)
                 for e in content["auth_chain"]
             ]
 
-            room_version = yield self.store.get_room_version(room_id)
-
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
                 origin, auth_chain, outlier=True, room_version=room_version,
             )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 4557a9e66e..67ae0212c3 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -469,7 +469,7 @@ class FederationSendLeaveServlet(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, origin, content, query, room_id, event_id):
-        content = yield self.handler.on_send_leave_request(origin, content)
+        content = yield self.handler.on_send_leave_request(origin, content, room_id)
         defer.returnValue((200, content))
 
 
@@ -487,7 +487,7 @@ class FederationSendJoinServlet(BaseFederationServlet):
     def on_PUT(self, origin, content, query, context, event_id):
         # TODO(paul): assert that context/event_id parsed from path actually
         #   match those given in content
-        content = yield self.handler.on_send_join_request(origin, content)
+        content = yield self.handler.on_send_join_request(origin, content, context)
         defer.returnValue((200, content))
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index c52dca1b81..a4b771049c 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1061,7 +1061,7 @@ class FederationHandler(BaseHandler):
         """
         logger.debug("Joining %s to %s", joinee, room_id)
 
-        origin, event = yield self._make_and_verify_event(
+        origin, event, event_format_version = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             joinee,
@@ -1091,7 +1091,9 @@ class FederationHandler(BaseHandler):
                 target_hosts.insert(0, origin)
             except ValueError:
                 pass
-            ret = yield self.federation_client.send_join(target_hosts, event)
+            ret = yield self.federation_client.send_join(
+                target_hosts, event, event_format_version,
+            )
 
             origin = ret["origin"]
             state = ret["state"]
@@ -1164,13 +1166,18 @@ class FederationHandler(BaseHandler):
         """
         event_content = {"membership": Membership.JOIN}
 
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "content": event_content,
-            "room_id": room_id,
-            "sender": user_id,
-            "state_key": user_id,
-        })
+        room_version = yield self.store.get_room_version(room_id)
+
+        builder = self.event_builder_factory.new(
+            room_version,
+            {
+                "type": EventTypes.Member,
+                "content": event_content,
+                "room_id": room_id,
+                "sender": user_id,
+                "state_key": user_id,
+            }
+        )
 
         try:
             event, context = yield self.event_creation_handler.create_new_client_event(
@@ -1304,7 +1311,7 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
-        origin, event = yield self._make_and_verify_event(
+        origin, event, event_format_version = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             user_id,
@@ -1336,7 +1343,7 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={}, params=None):
-        origin, pdu, _ = yield self.federation_client.make_membership_event(
+        origin, event, format_ver = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
@@ -1345,9 +1352,7 @@ class FederationHandler(BaseHandler):
             params=params,
         )
 
-        logger.debug("Got response to make_%s: %s", membership, pdu)
-
-        event = pdu
+        logger.debug("Got response to make_%s: %s", membership, event)
 
         # We should assert some things.
         # FIXME: Do this in a nicer way
@@ -1355,7 +1360,7 @@ class FederationHandler(BaseHandler):
         assert(event.user_id == user_id)
         assert(event.state_key == user_id)
         assert(event.room_id == room_id)
-        defer.returnValue((origin, event))
+        defer.returnValue((origin, event, format_ver))
 
     @defer.inlineCallbacks
     @log_function
@@ -1364,13 +1369,17 @@ class FederationHandler(BaseHandler):
         leave event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
         """
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "content": {"membership": Membership.LEAVE},
-            "room_id": room_id,
-            "sender": user_id,
-            "state_key": user_id,
-        })
+        room_version = yield self.store.get_room_version(room_id)
+        builder = self.event_builder_factory.new(
+            room_version,
+            {
+                "type": EventTypes.Member,
+                "content": {"membership": Membership.LEAVE},
+                "room_id": room_id,
+                "sender": user_id,
+                "state_key": user_id,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
@@ -2266,14 +2275,16 @@ class FederationHandler(BaseHandler):
         }
 
         if (yield self.auth.check_host_in_room(room_id, self.hs.hostname)):
-            builder = self.event_builder_factory.new(event_dict)
+            room_version = yield self.store.get_room_version(room_id)
+            builder = self.event_builder_factory.new(room_version, event_dict)
+
             EventValidator().validate_new(builder)
             event, context = yield self.event_creation_handler.create_new_client_event(
                 builder=builder
             )
 
             event, context = yield self.add_display_name_to_third_party_invite(
-                event_dict, event, context
+                room_version, event_dict, event, context
             )
 
             try:
@@ -2304,14 +2315,18 @@ class FederationHandler(BaseHandler):
         Returns:
             Deferred: resolves (to None)
         """
-        builder = self.event_builder_factory.new(event_dict)
+        room_version = yield self.store.get_room_version(room_id)
+
+        # NB: event_dict has a particular specced format we might need to fudge
+        # if we change event formats too much.
+        builder = self.event_builder_factory.new(room_version, event_dict)
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
         )
 
         event, context = yield self.add_display_name_to_third_party_invite(
-            event_dict, event, context
+            room_version, event_dict, event, context
         )
 
         try:
@@ -2331,7 +2346,8 @@ class FederationHandler(BaseHandler):
         yield member_handler.send_membership_event(None, event, context)
 
     @defer.inlineCallbacks
-    def add_display_name_to_third_party_invite(self, event_dict, event, context):
+    def add_display_name_to_third_party_invite(self, room_version, event_dict,
+                                               event, context):
         key = (
             EventTypes.ThirdPartyInvite,
             event.content["third_party_invite"]["signed"]["token"]
@@ -2355,7 +2371,7 @@ class FederationHandler(BaseHandler):
             # auth checks. If we need the invite and don't have it then the
             # auth check code will explode appropriately.
 
-        builder = self.event_builder_factory.new(event_dict)
+        builder = self.event_builder_factory.new(room_version, event_dict)
         EventValidator().validate_new(builder)
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index a7cd779b02..7aaa4fba33 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -278,7 +278,15 @@ class EventCreationHandler(object):
         """
         yield self.auth.check_auth_blocking(requester.user.to_string())
 
-        builder = self.event_builder_factory.new(event_dict)
+        if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "":
+            room_version = event_dict["content"]["room_version"]
+        else:
+            try:
+                room_version = yield self.store.get_room_version(event_dict["room_id"])
+            except NotFoundError:
+                raise AuthError(403, "Unknown room")
+
+        builder = self.event_builder_factory.new(room_version, event_dict)
 
         self.validator.validate_new(builder)
 
-- 
cgit 1.5.1


From 0862d35b8e23151174dac3740c37ddb18e78a74a Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:09:34 +0000
Subject: Move tag and direct state copying into separate function

---
 synapse/handlers/room_member.py | 106 ++++++++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 43 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index eb46f5f4fa..a340566bb0 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -223,49 +223,7 @@ class RoomMemberHandler(object):
 
             # Copy over direct message status and room tags if this is a join
             # on an upgraded room
-
-            # Check if this is an upgraded room
-            state_ids = yield self.store.get_current_state_ids(room_id)
-            create_id = state_ids.get((EventTypes.Create, ""))
-            if not create_id:
-                return
-            create_event = yield self.store.get_event(create_id)
-
-            if "predecessor" in create_event["content"]:
-                old_room_id = create_event["content"]["predecessor"]["room_id"]
-
-                # Retrieve room account data for predecessor room
-                user_account_data = yield self.store.get_account_data_for_user(
-                    user_id,
-                )
-                room_tags = yield self.store.get_tags_for_room(
-                    user_id, old_room_id,
-                )
-
-                # Copy direct message state if applicable
-                if user_account_data and "m.direct" in user_account_data[0]:
-                    direct_rooms = user_account_data[0]["m.direct"]
-
-                    # Check which key this room is under
-                    for key, room_id_list in direct_rooms.items():
-                        if old_room_id in room_id_list and room_id not in room_id_list:
-                            # Add new room_id to this key
-                            direct_rooms[key].append(room_id)
-
-                            # Save back to user's m.direct account data
-                            yield self.store.add_account_data_for_user(
-                                user_id, "m.direct", direct_rooms,
-                            )
-                            break
-
-                # Copy room tags if applicable
-                if room_tags:
-                    # Copy each room tag to the new room
-                    for tag in room_tags.keys():
-                        tag_content = room_tags[tag]
-                        yield self.store.add_tag_to_room(
-                            user_id, room_id, tag, tag_content
-                        )
+            self.copy_room_tags_and_direct_to_room(old_room_id, room_id, user_id)
         elif event.membership == Membership.LEAVE:
             if prev_member_event_id:
                 prev_member_event = yield self.store.get_event(prev_member_event_id)
@@ -274,6 +232,68 @@ class RoomMemberHandler(object):
 
         defer.returnValue(event)
 
+    @defer.inlineCallbacks
+    def copy_room_tags_and_direct_to_room(
+        self,
+        old_room_id,
+        new_room_id,
+        user_id,
+    ):
+        """Copies the tags and direct room state from one room to another.
+
+        Args:
+            old_room_id (str)
+            new_room_id (str)
+            user_id (str)
+
+        Returns:
+            Deferred|None
+        """
+        # Check if this is an upgraded room
+        state_ids = yield self.store.get_filtered_current_state_ids(
+            new_room_id, StateFilter.from_types(((EventTypes.Create, ""))),
+        )
+        create_id = state_ids.get((EventTypes.Create, ""))
+        if not create_id:
+            return
+        create_event = yield self.store.get_event(create_id)
+
+        if "predecessor" in create_event["content"]:
+            old_room_id = create_event["content"]["predecessor"]["room_id"]
+
+            # Retrieve room account data for predecessor room
+            user_account_data = yield self.store.get_account_data_for_user(
+                user_id,
+            )
+            room_tags = yield self.store.get_tags_for_room(
+                user_id, old_room_id,
+            )
+
+            # Copy direct message state if applicable
+            if user_account_data and "m.direct" in user_account_data[0]:
+                direct_rooms = user_account_data[0]["m.direct"]
+
+                # Check which key this room is under
+                for key, room_id_list in direct_rooms.items():
+                    if old_room_id in room_id_list and new_room_id not in room_id_list:
+                        # Add new room_id to this key
+                        direct_rooms[key].append(new_room_id)
+
+                        # Save back to user's m.direct account data
+                        yield self.store.add_account_data_for_user(
+                            user_id, "m.direct", direct_rooms,
+                        )
+                        break
+
+            # Copy room tags if applicable
+            if room_tags:
+                # Copy each room tag to the new room
+                for tag in room_tags.keys():
+                    tag_content = room_tags[tag]
+                    yield self.store.add_tag_to_room(
+                        user_id, room_id, tag, tag_content
+                    )
+
     @defer.inlineCallbacks
     def update_membership(
             self,
-- 
cgit 1.5.1


From 6f3fda79ce50059c314b9e1566007062045624b4 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:21:25 +0000
Subject: Move room_tag declaration to be closer to its use

---
 synapse/handlers/room_member.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index a340566bb0..67c601b696 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -261,13 +261,10 @@ class RoomMemberHandler(object):
         if "predecessor" in create_event["content"]:
             old_room_id = create_event["content"]["predecessor"]["room_id"]
 
-            # Retrieve room account data for predecessor room
+            # Retrieve user account data for predecessor room
             user_account_data = yield self.store.get_account_data_for_user(
                 user_id,
             )
-            room_tags = yield self.store.get_tags_for_room(
-                user_id, old_room_id,
-            )
 
             # Copy direct message state if applicable
             if user_account_data and "m.direct" in user_account_data[0]:
@@ -286,6 +283,10 @@ class RoomMemberHandler(object):
                         break
 
             # Copy room tags if applicable
+            room_tags = yield self.store.get_tags_for_room(
+                user_id, old_room_id,
+            )
+
             if room_tags:
                 # Copy each room tag to the new room
                 for tag in room_tags.keys():
-- 
cgit 1.5.1


From 516456b763c7652b1401ad502bb6b621f822b15a Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:22:14 +0000
Subject: Remove unnecessary null check

---
 synapse/handlers/room_member.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 67c601b696..3d4500fb18 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -287,13 +287,12 @@ class RoomMemberHandler(object):
                 user_id, old_room_id,
             )
 
-            if room_tags:
-                # Copy each room tag to the new room
-                for tag in room_tags.keys():
-                    tag_content = room_tags[tag]
-                    yield self.store.add_tag_to_room(
-                        user_id, room_id, tag, tag_content
-                    )
+            # Copy each room tag to the new room
+            for tag in room_tags.keys():
+                tag_content = room_tags[tag]
+                yield self.store.add_tag_to_room(
+                    user_id, room_id, tag, tag_content
+                )
 
     @defer.inlineCallbacks
     def update_membership(
-- 
cgit 1.5.1


From c4cdafa81fdf167961257f16118735b304535f62 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:24:28 +0000
Subject: Destructure account data tuple before use

---
 synapse/handlers/room_member.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 3d4500fb18..bde4798d0c 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -262,13 +262,13 @@ class RoomMemberHandler(object):
             old_room_id = create_event["content"]["predecessor"]["room_id"]
 
             # Retrieve user account data for predecessor room
-            user_account_data = yield self.store.get_account_data_for_user(
+            user_account_data, _ = yield self.store.get_account_data_for_user(
                 user_id,
             )
 
             # Copy direct message state if applicable
-            if user_account_data and "m.direct" in user_account_data[0]:
-                direct_rooms = user_account_data[0]["m.direct"]
+            if user_account_data and "m.direct" in user_account_data:
+                direct_rooms = user_account_data["m.direct"]
 
                 # Check which key this room is under
                 for key, room_id_list in direct_rooms.items():
-- 
cgit 1.5.1


From 8265995498e375cefd728b77930ea3e65306c69a Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:26:06 +0000
Subject: Use python magic

---
 synapse/handlers/room_member.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index bde4798d0c..ee12428fe7 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -288,8 +288,7 @@ class RoomMemberHandler(object):
             )
 
             # Copy each room tag to the new room
-            for tag in room_tags.keys():
-                tag_content = room_tags[tag]
+            for tag, tag_content in room_tags.items():
                 yield self.store.add_tag_to_room(
                     user_id, room_id, tag, tag_content
                 )
-- 
cgit 1.5.1


From da0d2219d2bdde68165b16d51483357dd234ad1d Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:37:12 +0000
Subject: Clean up direct_rooms access

---
 synapse/handlers/room_member.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index ee12428fe7..b9de146913 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -267,10 +267,10 @@ class RoomMemberHandler(object):
             )
 
             # Copy direct message state if applicable
-            if user_account_data and "m.direct" in user_account_data:
-                direct_rooms = user_account_data["m.direct"]
+            direct_rooms = user_account_data.get("m.direct", {})
 
-                # Check which key this room is under
+            # Check which key this room is under
+            if isinstance(direct_rooms, dict):
                 for key, room_id_list in direct_rooms.items():
                     if old_room_id in room_id_list and new_room_id not in room_id_list:
                         # Add new room_id to this key
-- 
cgit 1.5.1


From 9244a3089eae772c0fca69372a8230d1b5ab9947 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 25 Jan 2019 11:48:38 +0000
Subject: Fixes

---
 synapse/handlers/room_member.py | 78 ++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index b9de146913..aabc488753 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -29,6 +29,7 @@ import synapse.server
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.storage.state import StateFilter
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -223,7 +224,18 @@ class RoomMemberHandler(object):
 
             # Copy over direct message status and room tags if this is a join
             # on an upgraded room
-            self.copy_room_tags_and_direct_to_room(old_room_id, room_id, user_id)
+            # Check if this is an upgraded room
+            state_ids = yield self.store.get_filtered_current_state_ids(
+                room_id, StateFilter.from_types([(EventTypes.Create, "")]),
+            )
+            create_id = state_ids.get((EventTypes.Create, ""))
+            if not create_id:
+                return
+            create_event = yield self.store.get_event(create_id)
+
+            if "predecessor" in create_event["content"]:
+                old_room_id = create_event["content"]["predecessor"]["room_id"]
+                self.copy_room_tags_and_direct_to_room(old_room_id, room_id, user_id)
         elif event.membership == Membership.LEAVE:
             if prev_member_event_id:
                 prev_member_event = yield self.store.get_event(prev_member_event_id)
@@ -249,49 +261,37 @@ class RoomMemberHandler(object):
         Returns:
             Deferred|None
         """
-        # Check if this is an upgraded room
-        state_ids = yield self.store.get_filtered_current_state_ids(
-            new_room_id, StateFilter.from_types(((EventTypes.Create, ""))),
+        # Retrieve user account data for predecessor room
+        user_account_data, _ = yield self.store.get_account_data_for_user(
+            user_id,
         )
-        create_id = state_ids.get((EventTypes.Create, ""))
-        if not create_id:
-            return
-        create_event = yield self.store.get_event(create_id)
 
-        if "predecessor" in create_event["content"]:
-            old_room_id = create_event["content"]["predecessor"]["room_id"]
+        # Copy direct message state if applicable
+        direct_rooms = user_account_data.get("m.direct", {})
 
-            # Retrieve user account data for predecessor room
-            user_account_data, _ = yield self.store.get_account_data_for_user(
-                user_id,
-            )
+        # Check which key this room is under
+        if isinstance(direct_rooms, dict):
+            for key, room_id_list in direct_rooms.items():
+                if old_room_id in room_id_list and new_room_id not in room_id_list:
+                    # Add new room_id to this key
+                    direct_rooms[key].append(new_room_id)
 
-            # Copy direct message state if applicable
-            direct_rooms = user_account_data.get("m.direct", {})
-
-            # Check which key this room is under
-            if isinstance(direct_rooms, dict):
-                for key, room_id_list in direct_rooms.items():
-                    if old_room_id in room_id_list and new_room_id not in room_id_list:
-                        # Add new room_id to this key
-                        direct_rooms[key].append(new_room_id)
-
-                        # Save back to user's m.direct account data
-                        yield self.store.add_account_data_for_user(
-                            user_id, "m.direct", direct_rooms,
-                        )
-                        break
-
-            # Copy room tags if applicable
-            room_tags = yield self.store.get_tags_for_room(
-                user_id, old_room_id,
-            )
+                    # Save back to user's m.direct account data
+                    yield self.store.add_account_data_for_user(
+                        user_id, "m.direct", direct_rooms,
+                    )
+                    break
 
-            # Copy each room tag to the new room
-            for tag, tag_content in room_tags.items():
-                yield self.store.add_tag_to_room(
-                    user_id, room_id, tag, tag_content
-                )
+        # Copy room tags if applicable
+        room_tags = yield self.store.get_tags_for_room(
+            user_id, old_room_id,
+        )
+
+        # Copy each room tag to the new room
+        for tag, tag_content in room_tags.items():
+            yield self.store.add_tag_to_room(
+                user_id, new_room_id, tag, tag_content
+            )
 
     @defer.inlineCallbacks
     def update_membership(
-- 
cgit 1.5.1


From ae2a957dbacc38f1126e2eca160f17322c710d26 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Jan 2019 18:31:41 +0000
Subject: Pass through room version to event auth

---
 synapse/api/auth.py            | 14 ++++++++++----
 synapse/event_auth.py          |  3 ++-
 synapse/handlers/federation.py | 20 ++++++++++++--------
 synapse/handlers/message.py    |  7 ++++++-
 synapse/handlers/room.py       |  5 ++++-
 synapse/state/__init__.py      |  2 +-
 synapse/state/v1.py            | 14 +++++++++++---
 synapse/state/v2.py            | 14 +++++++++-----
 tests/state/test_v2.py         |  4 +++-
 tests/test_event_auth.py       | 13 +++++++++++--
 10 files changed, 69 insertions(+), 27 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index e37b807c94..7b213e54c8 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -65,7 +65,7 @@ class Auth(object):
         register_cache("cache", "token_cache", self.token_cache)
 
     @defer.inlineCallbacks
-    def check_from_context(self, event, context, do_sig_check=True):
+    def check_from_context(self, room_version, event, context, do_sig_check=True):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         auth_events_ids = yield self.compute_auth_events(
             event, prev_state_ids, for_verification=True,
@@ -74,12 +74,16 @@ class Auth(object):
         auth_events = {
             (e.type, e.state_key): e for e in itervalues(auth_events)
         }
-        self.check(event, auth_events=auth_events, do_sig_check=do_sig_check)
+        self.check(
+            room_version, event,
+            auth_events=auth_events, do_sig_check=do_sig_check,
+        )
 
-    def check(self, event, auth_events, do_sig_check=True):
+    def check(self, room_version, event, auth_events, do_sig_check=True):
         """ Checks if this event is correctly authed.
 
         Args:
+            room_version (str): version of the room
             event: the event being checked.
             auth_events (dict: event-key -> event): the existing room state.
 
@@ -88,7 +92,9 @@ class Auth(object):
             True if the auth checks pass.
         """
         with Measure(self.clock, "auth.check"):
-            event_auth.check(event, auth_events, do_sig_check=do_sig_check)
+            event_auth.check(
+                room_version, event, auth_events, do_sig_check=do_sig_check
+            )
 
     @defer.inlineCallbacks
     def check_joined_room(self, room_id, user_id, current_state=None):
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index c81d8e6729..9adedbbb02 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -27,10 +27,11 @@ from synapse.types import UserID, get_domain_from_id
 logger = logging.getLogger(__name__)
 
 
-def check(event, auth_events, do_sig_check=True, do_size_check=True):
+def check(room_version, event, auth_events, do_sig_check=True, do_size_check=True):
     """ Checks if this event is correctly authed.
 
     Args:
+        room_version (str): the version of the room
         event: the event being checked.
         auth_events (dict: event-key -> event): the existing room state.
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a4b771049c..5adbe7b538 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1189,7 +1189,9 @@ class FederationHandler(BaseHandler):
 
         # The remote hasn't signed it yet, obviously. We'll do the full checks
         # when we get the event back in `on_send_join_request`
-        yield self.auth.check_from_context(event, context, do_sig_check=False)
+        yield self.auth.check_from_context(
+            room_version, event, context, do_sig_check=False,
+        )
 
         defer.returnValue(event)
 
@@ -1388,7 +1390,9 @@ class FederationHandler(BaseHandler):
         try:
             # The remote hasn't signed it yet, obviously. We'll do the full checks
             # when we get the event back in `on_send_leave_request`
-            yield self.auth.check_from_context(event, context, do_sig_check=False)
+            yield self.auth.check_from_context(
+                room_version, event, context, do_sig_check=False,
+            )
         except AuthError as e:
             logger.warn("Failed to create new leave %r because %s", event, e)
             raise e
@@ -1683,7 +1687,7 @@ class FederationHandler(BaseHandler):
                 auth_for_e[(EventTypes.Create, "")] = create_event
 
             try:
-                self.auth.check(e, auth_events=auth_for_e)
+                self.auth.check(room_version, e, auth_events=auth_for_e)
             except SynapseError as err:
                 # we may get SynapseErrors here as well as AuthErrors. For
                 # instance, there are a couple of (ancient) events in some
@@ -1927,6 +1931,8 @@ class FederationHandler(BaseHandler):
         current_state = set(e.event_id for e in auth_events.values())
         different_auth = event_auth_events - current_state
 
+        room_version = yield self.store.get_room_version(event.room_id)
+
         if different_auth and not event.internal_metadata.is_outlier():
             # Do auth conflict res.
             logger.info("Different auth: %s", different_auth)
@@ -1951,8 +1957,6 @@ class FederationHandler(BaseHandler):
                     (d.type, d.state_key): d for d in different_events if d
                 })
 
-                room_version = yield self.store.get_room_version(event.room_id)
-
                 new_state = yield self.state_handler.resolve_events(
                     room_version,
                     [list(local_view.values()), list(remote_view.values())],
@@ -2052,7 +2056,7 @@ class FederationHandler(BaseHandler):
                 )
 
         try:
-            self.auth.check(event, auth_events=auth_events)
+            self.auth.check(room_version, event, auth_events=auth_events)
         except AuthError as e:
             logger.warn("Failed auth resolution for %r because %s", event, e)
             raise e
@@ -2288,7 +2292,7 @@ class FederationHandler(BaseHandler):
             )
 
             try:
-                yield self.auth.check_from_context(event, context)
+                yield self.auth.check_from_context(room_version, event, context)
             except AuthError as e:
                 logger.warn("Denying new third party invite %r because %s", event, e)
                 raise e
@@ -2330,7 +2334,7 @@ class FederationHandler(BaseHandler):
         )
 
         try:
-            self.auth.check_from_context(event, context)
+            self.auth.check_from_context(room_version, event, context)
         except AuthError as e:
             logger.warn("Denying third party invite %r because %s", event, e)
             raise e
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 7aaa4fba33..10a7ed4c5e 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -611,8 +611,13 @@ class EventCreationHandler(object):
             extra_users (list(UserID)): Any extra users to notify about event
         """
 
+        if event.is_state() and (event.type, event.state_key) == (EventTypes.Create, ""):
+            room_version = event.content["room_version"]
+        else:
+            room_version = yield self.store.get_room_version(event.room_id)
+
         try:
-            yield self.auth.check_from_context(event, context)
+            yield self.auth.check_from_context(room_version, event, context)
         except AuthError as err:
             logger.warn("Denying new event %r because %s", event, err)
             raise err
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index cb8c5f77dd..19b4ee35d2 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -123,7 +123,10 @@ class RoomCreationHandler(BaseHandler):
                     token_id=requester.access_token_id,
                 )
             )
-            yield self.auth.check_from_context(tombstone_event, tombstone_context)
+            old_room_version = yield self.store.get_room_version(old_room_id)
+            yield self.auth.check_from_context(
+                old_room_version, tombstone_event, tombstone_context,
+            )
 
             yield self.clone_exiting_room(
                 requester,
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index e9ecb00277..2fca51d0b2 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -611,7 +611,7 @@ def resolve_events_with_store(room_version, state_sets, event_map, state_res_sto
         RoomVersions.VDH_TEST, RoomVersions.STATE_V2_TEST, RoomVersions.V2,
     ):
         return v2.resolve_events_with_store(
-            state_sets, event_map, state_res_store,
+            room_version, state_sets, event_map, state_res_store,
         )
     else:
         # This should only happen if we added a version but forgot to add it to
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index 19e091ce3b..6d3afcae7c 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -21,7 +21,7 @@ from six import iteritems, iterkeys, itervalues
 from twisted.internet import defer
 
 from synapse import event_auth
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, RoomVersions
 from synapse.api.errors import AuthError
 
 logger = logging.getLogger(__name__)
@@ -274,7 +274,11 @@ def _resolve_auth_events(events, auth_events):
         auth_events[(prev_event.type, prev_event.state_key)] = prev_event
         try:
             # The signatures have already been checked at this point
-            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
+            event_auth.check(
+                RoomVersions.V1, event, auth_events,
+                do_sig_check=False,
+                do_size_check=False,
+            )
             prev_event = event
         except AuthError:
             return prev_event
@@ -286,7 +290,11 @@ def _resolve_normal_events(events, auth_events):
     for event in _ordered_events(events):
         try:
             # The signatures have already been checked at this point
-            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
+            event_auth.check(
+                RoomVersions.V1, event, auth_events,
+                do_sig_check=False,
+                do_size_check=False,
+            )
             return event
         except AuthError:
             pass
diff --git a/synapse/state/v2.py b/synapse/state/v2.py
index 3573bb0028..650995c92c 100644
--- a/synapse/state/v2.py
+++ b/synapse/state/v2.py
@@ -29,10 +29,12 @@ logger = logging.getLogger(__name__)
 
 
 @defer.inlineCallbacks
-def resolve_events_with_store(state_sets, event_map, state_res_store):
+def resolve_events_with_store(room_version, state_sets, event_map, state_res_store):
     """Resolves the state using the v2 state resolution algorithm
 
     Args:
+        room_version (str): The room version
+
         state_sets(list): List of dicts of (type, state_key) -> event_id,
             which are the different state groups to resolve.
 
@@ -104,7 +106,7 @@ def resolve_events_with_store(state_sets, event_map, state_res_store):
 
     # Now sequentially auth each one
     resolved_state = yield _iterative_auth_checks(
-        sorted_power_events, unconflicted_state, event_map,
+        room_version, sorted_power_events, unconflicted_state, event_map,
         state_res_store,
     )
 
@@ -129,7 +131,7 @@ def resolve_events_with_store(state_sets, event_map, state_res_store):
     logger.debug("resolving remaining events")
 
     resolved_state = yield _iterative_auth_checks(
-        leftover_events, resolved_state, event_map,
+        room_version, leftover_events, resolved_state, event_map,
         state_res_store,
     )
 
@@ -350,11 +352,13 @@ def _reverse_topological_power_sort(event_ids, event_map, state_res_store, auth_
 
 
 @defer.inlineCallbacks
-def _iterative_auth_checks(event_ids, base_state, event_map, state_res_store):
+def _iterative_auth_checks(room_version, event_ids, base_state, event_map,
+                           state_res_store):
     """Sequentially apply auth checks to each event in given list, updating the
     state as it goes along.
 
     Args:
+        room_version (str)
         event_ids (list[str]): Ordered list of events to apply auth checks to
         base_state (dict[tuple[str, str], str]): The set of state to start with
         event_map (dict[str,FrozenEvent])
@@ -385,7 +389,7 @@ def _iterative_auth_checks(event_ids, base_state, event_map, state_res_store):
 
         try:
             event_auth.check(
-                event, auth_events,
+                room_version, event, auth_events,
                 do_sig_check=False,
                 do_size_check=False
             )
diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py
index 2e073a3afc..9a5c816927 100644
--- a/tests/state/test_v2.py
+++ b/tests/state/test_v2.py
@@ -19,7 +19,7 @@ from six.moves import zip
 
 import attr
 
-from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.api.constants import EventTypes, JoinRules, Membership, RoomVersions
 from synapse.event_auth import auth_types_for_event
 from synapse.events import FrozenEvent
 from synapse.state.v2 import lexicographical_topological_sort, resolve_events_with_store
@@ -539,6 +539,7 @@ class StateTestCase(unittest.TestCase):
                 state_before = dict(state_at_event[prev_events[0]])
             else:
                 state_d = resolve_events_with_store(
+                    RoomVersions.V2,
                     [state_at_event[n] for n in prev_events],
                     event_map=event_map,
                     state_res_store=TestStateResolutionStore(event_map),
@@ -685,6 +686,7 @@ class SimpleParamStateTestCase(unittest.TestCase):
         # Test that we correctly handle passing `None` as the event_map
 
         state_d = resolve_events_with_store(
+            RoomVersions.V2,
             [self.state_at_bob, self.state_at_charlie],
             event_map=None,
             state_res_store=TestStateResolutionStore(self.event_map),
diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py
index 411b4a9f86..7ee318e4e8 100644
--- a/tests/test_event_auth.py
+++ b/tests/test_event_auth.py
@@ -16,6 +16,7 @@
 import unittest
 
 from synapse import event_auth
+from synapse.api.constants import RoomVersions
 from synapse.api.errors import AuthError
 from synapse.events import FrozenEvent
 
@@ -35,12 +36,16 @@ class EventAuthTestCase(unittest.TestCase):
         }
 
         # creator should be able to send state
-        event_auth.check(_random_state_event(creator), auth_events, do_sig_check=False)
+        event_auth.check(
+            RoomVersions.V1, _random_state_event(creator), auth_events,
+            do_sig_check=False,
+        )
 
         # joiner should not be able to send state
         self.assertRaises(
             AuthError,
             event_auth.check,
+            RoomVersions.V1,
             _random_state_event(joiner),
             auth_events,
             do_sig_check=False,
@@ -69,13 +74,17 @@ class EventAuthTestCase(unittest.TestCase):
         self.assertRaises(
             AuthError,
             event_auth.check,
+            RoomVersions.V1,
             _random_state_event(pleb),
             auth_events,
             do_sig_check=False,
         ),
 
         # king should be able to send state
-        event_auth.check(_random_state_event(king), auth_events, do_sig_check=False)
+        event_auth.check(
+            RoomVersions.V1, _random_state_event(king), auth_events,
+            do_sig_check=False,
+        )
 
 
 # helpers for making events
-- 
cgit 1.5.1


From 1ce463963db06158d8e740b1986392ef968df0c0 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 28 Jan 2019 14:08:18 +0000
Subject: Reuse predecessor method

---
 synapse/handlers/room_member.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index aabc488753..6163c6bdee 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -29,7 +29,6 @@ import synapse.server
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, SynapseError
-from synapse.storage.state import StateFilter
 from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -224,18 +223,15 @@ class RoomMemberHandler(object):
 
             # Copy over direct message status and room tags if this is a join
             # on an upgraded room
+
             # Check if this is an upgraded room
-            state_ids = yield self.store.get_filtered_current_state_ids(
-                room_id, StateFilter.from_types([(EventTypes.Create, "")]),
-            )
-            create_id = state_ids.get((EventTypes.Create, ""))
-            if not create_id:
-                return
-            create_event = yield self.store.get_event(create_id)
-
-            if "predecessor" in create_event["content"]:
-                old_room_id = create_event["content"]["predecessor"]["room_id"]
-                self.copy_room_tags_and_direct_to_room(old_room_id, room_id, user_id)
+            predecessor = yield self.store.get_room_predecessor(room_id)
+
+            if predecessor:
+                # It is an upgraded room. Copy over old tags
+                self.copy_room_tags_and_direct_to_room(
+                    predecessor["room_id"], room_id, user_id,
+                )
         elif event.membership == Membership.LEAVE:
             if prev_member_event_id:
                 prev_member_event = yield self.store.get_event(prev_member_event_id)
-- 
cgit 1.5.1


From f0e96ab66a09db8ffec34fddf67d088fe03f8ea6 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 28 Jan 2019 14:09:45 +0000
Subject: Change return syntax in doc string

---
 synapse/handlers/room_member.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 6163c6bdee..9ed5a05cca 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -255,7 +255,7 @@ class RoomMemberHandler(object):
             user_id (str)
 
         Returns:
-            Deferred|None
+            Deferred[None]
         """
         # Retrieve user account data for predecessor room
         user_account_data, _ = yield self.store.get_account_data_for_user(
-- 
cgit 1.5.1


From b872c7b1b43431b8933e2afd2f226aa34ad81a0f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 28 Jan 2019 17:00:14 +0000
Subject: Split up event validation between event and builder

The validator was being run on the EventBuilder objects, and so the
validator only checked a subset of fields. With the upcoming
EventBuilder refactor even fewer fields will be there to validate.

To get around this we split the validation into those that can be run
against an EventBuilder and those run against a fully fledged event.
---
 synapse/events/validator.py    | 73 ++++++++++++++++++++++++++++++------------
 synapse/handlers/federation.py |  7 ++--
 synapse/handlers/message.py    |  4 ++-
 3 files changed, 61 insertions(+), 23 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/validator.py b/synapse/events/validator.py
index cf184748a1..55d44d093d 100644
--- a/synapse/events/validator.py
+++ b/synapse/events/validator.py
@@ -24,14 +24,13 @@ class EventValidator(object):
 
     def validate(self, event):
         EventID.from_string(event.event_id)
-        RoomID.from_string(event.room_id)
 
         required = [
-            # "auth_events",
+            "auth_events",
             "content",
-            # "hashes",
+            "hashes",
             "origin",
-            # "prev_events",
+            "prev_events",
             "sender",
             "type",
         ]
@@ -43,33 +42,60 @@ class EventValidator(object):
         # Check that the following keys have string values
         strings = [
             "origin",
-            "sender",
-            "type",
         ]
 
-        if hasattr(event, "state_key"):
-            strings.append("state_key")
-
         for s in strings:
             if not isinstance(getattr(event, s), string_types):
                 raise SynapseError(400, "Not '%s' a string type" % (s,))
 
-        if event.type == EventTypes.Member:
-            if "membership" not in event.content:
-                raise SynapseError(400, "Content has not membership key")
+    def validate_new(self, event):
+        """Validates the event has roughly the right format
 
-            if event.content["membership"] not in Membership.LIST:
-                raise SynapseError(400, "Invalid membership key")
+        Args:
+            event (FrozenEvent)
+        """
+        self.validate_builder(event)
+        self.validate(event)
 
-        # Check that the following keys have dictionary values
-        # TODO
+        UserID.from_string(event.sender)
+
+        if event.type == EventTypes.Message:
+            strings = [
+                "body",
+                "msgtype",
+            ]
 
-        # Check that the following keys have the correct format for DAGs
-        # TODO
+            self._ensure_strings(event.content, strings)
 
-    def validate_new(self, event):
-        self.validate(event)
+        elif event.type == EventTypes.Topic:
+            self._ensure_strings(event.content, ["topic"])
 
+        elif event.type == EventTypes.Name:
+            self._ensure_strings(event.content, ["name"])
+
+    def validate_builder(self, event):
+        """Validates that the builder/event has roughly the right format. Only
+        checks values that we expect a proto event to have, rather than all the
+        fields an event would have
+
+        Args:
+            event (EventBuilder|FrozenEvent)
+        """
+
+        strings = [
+            "room_id",
+            "sender",
+            "type",
+        ]
+
+        if hasattr(event, "state_key"):
+            strings.append("state_key")
+
+        for s in strings:
+            if not isinstance(getattr(event, s), string_types):
+                raise SynapseError(400, "Not '%s' a string type" % (s,))
+
+        RoomID.from_string(event.room_id)
         UserID.from_string(event.sender)
 
         if event.type == EventTypes.Message:
@@ -86,6 +112,13 @@ class EventValidator(object):
         elif event.type == EventTypes.Name:
             self._ensure_strings(event.content, ["name"])
 
+        elif event.type == EventTypes.Member:
+            if "membership" not in event.content:
+                raise SynapseError(400, "Content has not membership key")
+
+            if event.content["membership"] not in Membership.LIST:
+                raise SynapseError(400, "Invalid membership key")
+
     def _ensure_strings(self, d, keys):
         for s in keys:
             if s not in d:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a4b771049c..13333818ae 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -2278,7 +2278,7 @@ class FederationHandler(BaseHandler):
             room_version = yield self.store.get_room_version(room_id)
             builder = self.event_builder_factory.new(room_version, event_dict)
 
-            EventValidator().validate_new(builder)
+            EventValidator().validate_builder(builder)
             event, context = yield self.event_creation_handler.create_new_client_event(
                 builder=builder
             )
@@ -2287,6 +2287,8 @@ class FederationHandler(BaseHandler):
                 room_version, event_dict, event, context
             )
 
+            EventValidator().validate_new(event)
+
             try:
                 yield self.auth.check_from_context(event, context)
             except AuthError as e:
@@ -2372,10 +2374,11 @@ class FederationHandler(BaseHandler):
             # auth check code will explode appropriately.
 
         builder = self.event_builder_factory.new(room_version, event_dict)
-        EventValidator().validate_new(builder)
+        EventValidator().validate_builder(builder)
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
         )
+        EventValidator().validate_new(event)
         defer.returnValue((event, context))
 
     @defer.inlineCallbacks
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 7aaa4fba33..d2aab25111 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -288,7 +288,7 @@ class EventCreationHandler(object):
 
         builder = self.event_builder_factory.new(room_version, event_dict)
 
-        self.validator.validate_new(builder)
+        self.validator.validate_builder(builder)
 
         if builder.type == EventTypes.Member:
             membership = builder.content.get("membership", None)
@@ -326,6 +326,8 @@ class EventCreationHandler(object):
             prev_events_and_hashes=prev_events_and_hashes,
         )
 
+        self.validator.validate_new(event)
+
         defer.returnValue((event, context))
 
     def _is_exempt_from_privacy_policy(self, builder, requester):
-- 
cgit 1.5.1


From d758d5310e5a5753f181d0eef447acb1a3dd29cc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 28 Jan 2019 17:26:39 +0000
Subject: Correctly use default room version if none is set

---
 synapse/handlers/message.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 10a7ed4c5e..67b364a6c3 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -22,7 +22,7 @@ from canonicaljson import encode_canonical_json, json
 from twisted.internet import defer
 from twisted.internet.defer import succeed
 
-from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
+from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -612,7 +612,7 @@ class EventCreationHandler(object):
         """
 
         if event.is_state() and (event.type, event.state_key) == (EventTypes.Create, ""):
-            room_version = event.content["room_version"]
+            room_version = event.content.get("room_version", RoomVersions.V1)
         else:
             room_version = yield self.store.get_room_version(event.room_id)
 
-- 
cgit 1.5.1


From be47cfa9c97b4acfd884440f1953ed000225eb37 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Jan 2019 17:19:31 +0000
Subject: Refactor event building into EventBuilder

This is so that everything is done in one place, making it easier to
change the event format based on room version
---
 synapse/events/builder.py               | 284 ++++++++++++++++++++++++--------
 synapse/federation/federation_client.py |  20 +--
 synapse/handlers/message.py             |  34 +---
 synapse/server.py                       |   5 +-
 synapse/storage/event_federation.py     |  23 +++
 5 files changed, 254 insertions(+), 112 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index 7e63371095..225b5fd670 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -13,79 +13,156 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
+import attr
 
-from synapse.api.constants import RoomVersions
+from twisted.internet import defer
+
+from synapse.api.constants import (
+    KNOWN_EVENT_FORMAT_VERSIONS,
+    KNOWN_ROOM_VERSIONS,
+    MAX_DEPTH,
+)
+from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.types import EventID
 from synapse.util.stringutils import random_string
 
-from . import EventBase, FrozenEvent, _event_dict_property
+from . import (
+    _EventInternalMetadata,
+    event_type_from_format_version,
+    room_version_to_event_format,
+)
 
 
-def get_event_builder(room_version, key_values={}, internal_metadata_dict={}):
-    """Generate an event builder appropriate for the given room version
+@attr.s(slots=True, cmp=False, frozen=True)
+class EventBuilder(object):
+    """A format independent event builder used to build up the event content
+    before signing the event.
 
-    Args:
-        room_version (str): Version of the room that we're creating an
-            event builder for
-        key_values (dict): Fields used as the basis of the new event
-        internal_metadata_dict (dict): Used to create the `_EventInternalMetadata`
-            object.
+    (Note that while objects of this class are frozen, the
+    content/unsigned/internal_metadata fields are still mutable)
 
-    Returns:
-        EventBuilder
+    Attributes:
+        format_version (int): Event format version
+        room_id (str)
+        type (str)
+        sender (str)
+        content (dict)
+        unsigned (dict)
+        internal_metadata (_EventInternalMetadata)
+
+        _state (StateHandler)
+        _auth (synapse.api.Auth)
+        _store (DataStore)
+        _clock (Clock)
+        _hostname (str): The hostname of the server creating the event
+        _signing_key: The signing key to use to sign the event as the server
     """
-    if room_version in {
-        RoomVersions.V1,
-        RoomVersions.V2,
-        RoomVersions.VDH_TEST,
-        RoomVersions.STATE_V2_TEST,
-    }:
-        return EventBuilder(key_values, internal_metadata_dict)
-    else:
-        raise Exception(
-            "No event format defined for version %r" % (room_version,)
-        )
 
+    _state = attr.ib()
+    _auth = attr.ib()
+    _store = attr.ib()
+    _clock = attr.ib()
+    _hostname = attr.ib()
+    _signing_key = attr.ib()
+
+    format_version = attr.ib()
+
+    room_id = attr.ib()
+    type = attr.ib()
+    sender = attr.ib()
+
+    content = attr.ib(default=attr.Factory(dict))
+    unsigned = attr.ib(default=attr.Factory(dict))
+
+    # These only exist on a subset of events, so they raise AttributeError if
+    # someone tries to get them when they don't exist.
+    _state_key = attr.ib(default=None)
+    _redacts = attr.ib(default=None)
+
+    internal_metadata = attr.ib(default=attr.Factory(lambda: _EventInternalMetadata({})))
+
+    @property
+    def state_key(self):
+        if self._state_key is not None:
+            return self._state_key
+
+        raise AttributeError("state_key")
+
+    def is_state(self):
+        return self._state_key is not None
 
-class EventBuilder(EventBase):
-    def __init__(self, key_values={}, internal_metadata_dict={}):
-        signatures = copy.deepcopy(key_values.pop("signatures", {}))
-        unsigned = copy.deepcopy(key_values.pop("unsigned", {}))
+    @defer.inlineCallbacks
+    def build(self, prev_event_ids):
+        """Transform into a fully signed and hashed event
 
-        super(EventBuilder, self).__init__(
-            key_values,
-            signatures=signatures,
-            unsigned=unsigned,
-            internal_metadata_dict=internal_metadata_dict,
+        Args:
+            prev_event_ids (list[str]): The event IDs to use as the prev events
+
+        Returns:
+            Deferred[FrozenEvent]
+        """
+
+        state_ids = yield self._state.get_current_state_ids(
+            self.room_id, prev_event_ids,
+        )
+        auth_ids = yield self._auth.compute_auth_events(
+            self, state_ids,
         )
 
-    event_id = _event_dict_property("event_id")
-    state_key = _event_dict_property("state_key")
-    type = _event_dict_property("type")
+        auth_events = yield self._store.add_event_hashes(auth_ids)
+        prev_events = yield self._store.add_event_hashes(prev_event_ids)
 
-    def build(self):
-        return FrozenEvent.from_event(self)
+        old_depth = yield self._store.get_max_depth_of(
+            prev_event_ids,
+        )
+        depth = old_depth + 1
 
+        # we cap depth of generated events, to ensure that they are not
+        # rejected by other servers (and so that they can be persisted in
+        # the db)
+        depth = min(depth, MAX_DEPTH)
 
-class EventBuilderFactory(object):
-    def __init__(self, clock, hostname):
-        self.clock = clock
-        self.hostname = hostname
+        event_dict = {
+            "auth_events": auth_events,
+            "prev_events": prev_events,
+            "type": self.type,
+            "room_id": self.room_id,
+            "sender": self.sender,
+            "content": self.content,
+            "unsigned": self.unsigned,
+            "depth": depth,
+            "prev_state": [],
+        }
+
+        if self.is_state():
+            event_dict["state_key"] = self._state_key
 
-        self.event_id_count = 0
+        if self._redacts is not None:
+            event_dict["redacts"] = self._redacts
 
-    def create_event_id(self):
-        i = str(self.event_id_count)
-        self.event_id_count += 1
+        defer.returnValue(
+            create_local_event_from_event_dict(
+                clock=self._clock,
+                hostname=self._hostname,
+                signing_key=self._signing_key,
+                format_version=self.format_version,
+                event_dict=event_dict,
+                internal_metadata_dict=self.internal_metadata.get_dict(),
+            )
+        )
 
-        local_part = str(int(self.clock.time())) + i + random_string(5)
 
-        e_id = EventID(local_part, self.hostname)
+class EventBuilderFactory(object):
+    def __init__(self, hs):
+        self.clock = hs.get_clock()
+        self.hostname = hs.hostname
+        self.signing_key = hs.config.signing_key[0]
 
-        return e_id.to_string()
+        self.store = hs.get_datastore()
+        self.state = hs.get_state_handler()
+        self.auth = hs.get_auth()
 
-    def new(self, room_version, key_values={}):
+    def new(self, room_version, key_values):
         """Generate an event builder appropriate for the given room version
 
         Args:
@@ -98,27 +175,104 @@ class EventBuilderFactory(object):
         """
 
         # There's currently only the one event version defined
-        if room_version not in {
-            RoomVersions.V1,
-            RoomVersions.V2,
-            RoomVersions.VDH_TEST,
-            RoomVersions.STATE_V2_TEST,
-        }:
+        if room_version not in KNOWN_ROOM_VERSIONS:
             raise Exception(
                 "No event format defined for version %r" % (room_version,)
             )
 
-        key_values["event_id"] = self.create_event_id()
+        key_values["event_id"] = _create_event_id(self.clock, self.hostname)
+
+        return EventBuilder(
+            store=self.store,
+            state=self.state,
+            auth=self.auth,
+            clock=self.clock,
+            hostname=self.hostname,
+            signing_key=self.signing_key,
+            format_version=room_version_to_event_format(room_version),
+            type=key_values["type"],
+            state_key=key_values.get("state_key"),
+            room_id=key_values["room_id"],
+            sender=key_values["sender"],
+            content=key_values.get("content", {}),
+            unsigned=key_values.get("unsigned", {}),
+            redacts=key_values.get("redacts", None),
+        )
+
+
+def create_local_event_from_event_dict(clock, hostname, signing_key,
+                                       format_version, event_dict,
+                                       internal_metadata_dict=None):
+    """Takes a fully formed event dict, ensuring that fields like `origin`
+    and `origin_server_ts` have correct values for a locally produced event,
+    then signs and hashes it.
+
+    Args:
+        clock (Clock)
+        hostname (str)
+        signing_key
+        format_version (int)
+        event_dict (dict)
+        internal_metadata_dict (dict|None)
+
+    Returns:
+        FrozenEvent
+    """
+
+    # There's currently only the one event version defined
+    if format_version not in KNOWN_EVENT_FORMAT_VERSIONS:
+        raise Exception(
+            "No event format defined for version %r" % (format_version,)
+        )
+
+    if internal_metadata_dict is None:
+        internal_metadata_dict = {}
+
+    time_now = int(clock.time_msec())
+
+    event_dict["event_id"] = _create_event_id(clock, hostname)
+
+    event_dict["origin"] = hostname
+    event_dict["origin_server_ts"] = time_now
+
+    event_dict.setdefault("unsigned", {})
+    age = event_dict["unsigned"].pop("age", 0)
+    event_dict["unsigned"].setdefault("age_ts", time_now - age)
+
+    event_dict.setdefault("signatures", {})
+
+    add_hashes_and_signatures(
+        event_dict,
+        hostname,
+        signing_key,
+    )
+    return event_type_from_format_version(format_version)(
+        event_dict, internal_metadata_dict=internal_metadata_dict,
+    )
+
+
+# A counter used when generating new event IDs
+_event_id_counter = 0
+
+
+def _create_event_id(clock, hostname):
+    """Create a new event ID
+
+    Args:
+        clock (Clock)
+        hostname (str): The server name for the event ID
+
+    Returns:
+        str
+    """
 
-        time_now = int(self.clock.time_msec())
+    global _event_id_counter
 
-        key_values.setdefault("origin", self.hostname)
-        key_values.setdefault("origin_server_ts", time_now)
+    i = str(_event_id_counter)
+    _event_id_counter += 1
 
-        key_values.setdefault("unsigned", {})
-        age = key_values["unsigned"].pop("age", 0)
-        key_values["unsigned"].setdefault("age_ts", time_now - age)
+    local_part = str(int(clock.time())) + i + random_string(5)
 
-        key_values["signatures"] = {}
+    e_id = EventID(local_part, hostname)
 
-        return EventBuilder(key_values=key_values,)
+    return e_id.to_string()
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 71809893c5..be3bb59431 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -37,8 +37,7 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
-from synapse.crypto.event_signing import add_hashes_and_signatures
-from synapse.events import room_version_to_event_format
+from synapse.events import builder, room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -72,7 +71,8 @@ class FederationClient(FederationBase):
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
 
-        self.event_builder_factory = hs.get_event_builder_factory()
+        self.hostname = hs.hostname
+        self.signing_key = hs.config.signing_key[0]
 
         self._get_pdu_cache = ExpiringCache(
             cache_name="get_pdu_cache",
@@ -608,18 +608,10 @@ class FederationClient(FederationBase):
             if "prev_state" not in pdu_dict:
                 pdu_dict["prev_state"] = []
 
-            # Strip off the fields that we want to clobber.
-            pdu_dict.pop("origin", None)
-            pdu_dict.pop("origin_server_ts", None)
-            pdu_dict.pop("unsigned", None)
-
-            builder = self.event_builder_factory.new(room_version, pdu_dict)
-            add_hashes_and_signatures(
-                builder,
-                self.hs.hostname,
-                self.hs.config.signing_key[0]
+            ev = builder.create_local_event_from_event_dict(
+                self._clock, self.hostname, self.signing_key,
+                format_version=event_format, event_dict=pdu_dict,
             )
-            ev = builder.build()
 
             defer.returnValue(
                 (destination, ev, event_format)
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 05d1370c18..ac6f4fd985 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -22,7 +22,7 @@ from canonicaljson import encode_canonical_json, json
 from twisted.internet import defer
 from twisted.internet.defer import succeed
 
-from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions
+from synapse.api.constants import EventTypes, Membership, RoomVersions
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -31,7 +31,6 @@ from synapse.api.errors import (
     SynapseError,
 )
 from synapse.api.urls import ConsentURIBuilder
-from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
 from synapse.events.validator import EventValidator
 from synapse.replication.http.send_event import ReplicationSendEventRestServlet
@@ -545,40 +544,17 @@ class EventCreationHandler(object):
             prev_events_and_hashes = \
                 yield self.store.get_prev_events_for_room(builder.room_id)
 
-        if prev_events_and_hashes:
-            depth = max([d for _, _, d in prev_events_and_hashes]) + 1
-            # we cap depth of generated events, to ensure that they are not
-            # rejected by other servers (and so that they can be persisted in
-            # the db)
-            depth = min(depth, MAX_DEPTH)
-        else:
-            depth = 1
-
         prev_events = [
             (event_id, prev_hashes)
             for event_id, prev_hashes, _ in prev_events_and_hashes
         ]
 
-        builder.prev_events = prev_events
-        builder.depth = depth
-
-        context = yield self.state.compute_event_context(builder)
-        if requester:
-            context.app_service = requester.app_service
-
-        if builder.is_state():
-            builder.prev_state = yield self.store.add_event_hashes(
-                context.prev_state_events
-            )
-
-        yield self.auth.add_auth_events(builder, context)
-
-        signing_key = self.hs.config.signing_key[0]
-        add_hashes_and_signatures(
-            builder, self.server_name, signing_key
+        event = yield builder.build(
+            prev_event_ids=[p for p, _ in prev_events],
         )
+        context = yield self.state.compute_event_context(event)
 
-        event = builder.build()
+        self.validator.validate_new(event)
 
         logger.debug(
             "Created event %s",
diff --git a/synapse/server.py b/synapse/server.py
index c8914302cf..6c52101616 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -355,10 +355,7 @@ class HomeServer(object):
         return Keyring(self)
 
     def build_event_builder_factory(self):
-        return EventBuilderFactory(
-            clock=self.get_clock(),
-            hostname=self.hostname,
-        )
+        return EventBuilderFactory(self)
 
     def build_filtering(self):
         return Filtering(self)
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index d3b9dea1d6..38809ed0fc 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -125,6 +125,29 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
 
         return dict(txn)
 
+    @defer.inlineCallbacks
+    def get_max_depth_of(self, event_ids):
+        """Returns the max depth of a set of event IDs
+
+        Args:
+            event_ids (list[str])
+
+        Returns
+            Deferred[int]
+        """
+        rows = yield self._simple_select_many_batch(
+            table="events",
+            column="event_id",
+            iterable=event_ids,
+            retcols=("depth",),
+            desc="get_max_depth_of",
+        )
+
+        if not rows:
+            defer.returnValue(0)
+        else:
+            defer.returnValue(max(row["depth"] for row in rows))
+
     def _get_oldest_events_in_room_txn(self, txn, room_id):
         return self._simple_select_onecol_txn(
             txn,
-- 
cgit 1.5.1


From 6598992b01629062791cd4d98f062aeabd4f7baf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 29 Jan 2019 11:28:02 +0000
Subject: Fixup calls to `comput_event_signature`

We currently pass FrozenEvent instead of `dict` to
`compute_event_signature`, which works by accident due to `dict(event)`
producing the correct result.

This fixes PR #4493 commit 855a151
---
 synapse/federation/federation_server.py | 2 +-
 synapse/handlers/federation.py          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 4aa04b9588..6681614232 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -322,7 +322,7 @@ class FederationServer(FederationBase):
             if self.hs.is_mine_id(event.event_id):
                 event.signatures.update(
                     compute_event_signature(
-                        event,
+                        event.get_pdu_json(),
                         self.hs.hostname,
                         self.hs.config.signing_key[0]
                     )
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index adf59db7a8..fcaf7530b0 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1300,7 +1300,7 @@ class FederationHandler(BaseHandler):
 
         event.signatures.update(
             compute_event_signature(
-                event,
+                event.get_pdu_json(),
                 self.hs.hostname,
                 self.hs.config.signing_key[0]
             )
-- 
cgit 1.5.1


From 7709d2bd167e27493b134e938410c307f8c10396 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 28 Jan 2019 21:09:45 +0000
Subject: Implement rechecking of redactions

---
 synapse/api/auth.py              |  4 ++--
 synapse/event_auth.py            | 24 ++++++++++++++++++------
 synapse/events/__init__.py       |  3 +++
 synapse/handlers/message.py      |  6 +++++-
 synapse/storage/events_worker.py | 26 +++++++++++++++++++++++++-
 5 files changed, 53 insertions(+), 10 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 7b213e54c8..963e0e7d60 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -627,7 +627,7 @@ class Auth(object):
 
         defer.returnValue(auth_ids)
 
-    def check_redaction(self, event, auth_events):
+    def check_redaction(self, room_version, event, auth_events):
         """Check whether the event sender is allowed to redact the target event.
 
         Returns:
@@ -640,7 +640,7 @@ class Auth(object):
             AuthError if the event sender is definitely not allowed to redact
             the target event.
         """
-        return event_auth.check_redaction(event, auth_events)
+        return event_auth.check_redaction(room_version, event, auth_events)
 
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 9adedbbb02..a95d142f0c 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -20,7 +20,13 @@ from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 from unpaddedbase64 import decode_base64
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, JoinRules, Membership
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    JoinRules,
+    Membership,
+    RoomVersions,
+)
 from synapse.api.errors import AuthError, EventSizeError, SynapseError
 from synapse.types import UserID, get_domain_from_id
 
@@ -168,7 +174,7 @@ def check(room_version, event, auth_events, do_sig_check=True, do_size_check=Tru
         _check_power_levels(event, auth_events)
 
     if event.type == EventTypes.Redaction:
-        check_redaction(event, auth_events)
+        check_redaction(room_version, event, auth_events)
 
     logger.debug("Allowing! %s", event)
 
@@ -422,7 +428,7 @@ def _can_send_event(event, auth_events):
     return True
 
 
-def check_redaction(event, auth_events):
+def check_redaction(room_version, event, auth_events):
     """Check whether the event sender is allowed to redact the target event.
 
     Returns:
@@ -442,10 +448,16 @@ def check_redaction(event, auth_events):
     if user_level >= redact_level:
         return False
 
-    redacter_domain = get_domain_from_id(event.event_id)
-    redactee_domain = get_domain_from_id(event.redacts)
-    if redacter_domain == redactee_domain:
+    if room_version in (RoomVersions.V1, RoomVersions.V2, RoomVersions.VDH_TEST):
+        redacter_domain = get_domain_from_id(event.event_id)
+        redactee_domain = get_domain_from_id(event.redacts)
+        if redacter_domain == redactee_domain:
+            return True
+    elif room_version == RoomVersions.V3:
+        event.internal_metadata.recheck_redaction = True
         return True
+    else:
+        raise RuntimeError("Unrecognized room version %r" % (room_version,))
 
     raise AuthError(
         403,
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 3fe52aaa45..70d3c0fbd9 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -63,6 +63,9 @@ class _EventInternalMetadata(object):
         """
         return getattr(self, "send_on_behalf_of", None)
 
+    def need_to_check_redaction(self):
+        return getattr(self, "recheck_redaction", False)
+
 
 def _event_dict_property(key):
     # We want to be able to use hasattr with the event dict properties.
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 05d1370c18..0cfced43d5 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -767,7 +767,8 @@ class EventCreationHandler(object):
             auth_events = {
                 (e.type, e.state_key): e for e in auth_events.values()
             }
-            if self.auth.check_redaction(event, auth_events=auth_events):
+            room_version = yield self.store.get_room_version(event.room_id)
+            if self.auth.check_redaction(room_version, event, auth_events=auth_events):
                 original_event = yield self.store.get_event(
                     event.redacts,
                     check_redacted=False,
@@ -781,6 +782,9 @@ class EventCreationHandler(object):
                         "You don't have permission to redact events"
                     )
 
+                # We've already checked.
+                event.internal_metadata.recheck_redaction = False
+
         if event.type == EventTypes.Create:
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             if prev_state_ids:
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 0a0ca58fc4..9ce19430e8 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -21,13 +21,14 @@ from canonicaljson import json
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventFormatVersions
+from synapse.api.constants import EventFormatVersions, EventTypes
 from synapse.api.errors import NotFoundError
 from synapse.events import FrozenEvent, event_type_from_format_version  # noqa: F401
 # these are only included to make the type annotations work
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.events.utils import prune_event
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.types import get_domain_from_id
 from synapse.util.logcontext import (
     LoggingContext,
     PreserveLoggingContext,
@@ -174,6 +175,29 @@ class EventsWorkerStore(SQLBaseStore):
             if not entry:
                 continue
 
+            # Some redactions in room version v3 need to be rechecked if we
+            # didn't have the redacted event at the time, so we recheck on read
+            # instead.
+            if not allow_rejected and entry.event.type == EventTypes.Redaction:
+                if entry.event.internal_metadata.need_to_check_redaction():
+                    orig = yield self.get_event(
+                        entry.event.redacts,
+                        allow_none=True,
+                        allow_rejected=True,
+                        get_prev_content=False,
+                    )
+                    expected_domain = get_domain_from_id(entry.event.sender)
+                    if orig and get_domain_from_id(orig.sender) == expected_domain:
+                        # This redaction event is allowed. Mark as not needing a
+                        # recheck.
+                        entry.event.recheck_redaction = False
+                    else:
+                        # We don't have the event that is being redacted, so we
+                        # assume that the event isn't authorized for now. (If we
+                        # later receive the event, then we will always redact
+                        # it anyway, since we have this redaction)
+                        continue
+
             if allow_rejected or not entry.event.rejected_reason:
                 if check_redacted and entry.redacted_event:
                     event = entry.redacted_event
-- 
cgit 1.5.1


From 5891a6edc89039cf4fa76217c3f34384a2aa14eb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 29 Jan 2019 12:09:10 +0000
Subject: Correctly set context.app_service

---
 synapse/handlers/message.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index ac6f4fd985..37a7dca794 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -553,6 +553,8 @@ class EventCreationHandler(object):
             prev_event_ids=[p for p, _ in prev_events],
         )
         context = yield self.state.compute_event_context(event)
+        if requester:
+            context.app_service = requester.app_service
 
         self.validator.validate_new(event)
 
-- 
cgit 1.5.1


From b1fffca3458c6ba26da1e61e4e70ac6f7419d839 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 29 Jan 2019 16:11:19 +0000
Subject: Remove event ID usage when checking if new room

The event ID is changing, so we can no longer get the domain from it. On
the other hand, the check is unnecessary.
---
 synapse/handlers/room_member.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 9ed5a05cca..2beffdf41e 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -940,7 +940,8 @@ class RoomMemberHandler(object):
         # first member event?
         create_event_id = current_state_ids.get(("m.room.create", ""))
         if len(current_state_ids) == 1 and create_event_id:
-            defer.returnValue(self.hs.is_mine_id(create_event_id))
+            # We can only get here if we're in the process of creating the room
+            defer.returnValue(True)
 
         for etype, state_key in current_state_ids:
             if etype != EventTypes.Member or not self.hs.is_mine_id(state_key):
-- 
cgit 1.5.1


From 55d90248356b0068b201c5be7298e0f3ae1c8ace Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 29 Jan 2019 16:15:16 +0000
Subject: Use snder and not event ID domain to check if ours

The transaction queue only sends out events that we generate. This was
done by checking domain of event ID, but that can no longer be used.
Instead, we may as well use the sender field.
---
 synapse/federation/transaction_queue.py | 2 +-
 synapse/handlers/federation.py          | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index fe787abaeb..1f0b67f5f8 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -175,7 +175,7 @@ class TransactionQueue(object):
                 def handle_event(event):
                     # Only send events for this server.
                     send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of()
-                    is_mine = self.is_mine_id(event.event_id)
+                    is_mine = self.is_mine_id(event.sender)
                     if not is_mine and send_on_behalf_of is None:
                         return
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index fcaf7530b0..f89dabb9eb 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -2293,6 +2293,10 @@ class FederationHandler(BaseHandler):
 
             EventValidator().validate_new(event)
 
+            # We need to tell the transaction queue to send this out, even
+            # though the sender isn't a local user.
+            event.internal_metadata.send_on_behalf_of = self.hs.hostname
+
             try:
                 yield self.auth.check_from_context(room_version, event, context)
             except AuthError as e:
@@ -2342,6 +2346,10 @@ class FederationHandler(BaseHandler):
             raise e
         yield self._check_signature(event, context)
 
+        # We need to tell the transaction queue to send this out, even
+        # though the sender isn't a local user.
+        event.internal_metadata.send_on_behalf_of = get_domain_from_id(event.sender)
+
         # XXX we send the invite here, but send_membership_event also sends it,
         # so we end up making two requests. I think this is redundant.
         returned_invite = yield self.send_invite(origin, event)
-- 
cgit 1.5.1