Pull out event ids rather than full events for state

4 files changed, 119 insertions, 55 deletions

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index 8a475417a6..cf11b4aa2e 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -15,6 +15,14 @@
 
 
 class EventContext(object):
+    def _set_current_state(self, current_state):
+        if current_state is not None:
+            self.current_state_ids = {k: e.event_id for k, e in current_state.items()}
+        else:
+            self.current_state_ids = None
+        self._current_state = current_state
+
+    current_state = property(lambda self: self._current_state, _set_current_state)
 
     def __init__(self, current_state=None):
         self.current_state = current_state
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 01a761715b..92679532b9 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -560,6 +560,18 @@ class FederationHandler(BaseHandler):
         ]))
         states = dict(zip(event_ids, [s[1] for s in states]))
 
+        state_map = yield self.store.get_events(
+            [e_id for ids in states.values() for e_id in ids],
+            get_prev_content=False
+        )
+        states = {
+            key: {
+                k: state_map[e_id]
+                for k, e_id in state_dict.items()
+                if e_id in state_map
+            } for key, state_dict in states.items()
+        }
+
         for e_id, _ in sorted_extremeties_tuple:
             likely_domains = get_domains_from_state(states[e_id])
 
diff --git a/synapse/state.py b/synapse/state.py
index ef1bc470be..2249b7fffb 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -93,8 +93,12 @@ class StateHandler(object):
         if not latest_event_ids:
             latest_event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
 
-        res = yield self.resolve_state_groups(room_id, latest_event_ids)
-        state = res[1]
+        _, state = yield self.resolve_state_groups(room_id, latest_event_ids)
+
+        state_map = yield self.store.get_events(state.values(), get_prev_content=False)
+        state = {
+            key: state_map[e_id] for key, e_id in state.items() if e_id in state_map
+        }
 
         if event_type:
             defer.returnValue(state.get((event_type, state_key)))
@@ -159,7 +163,15 @@ class StateHandler(object):
                 event.room_id, [e for e, _ in event.prev_events],
             )
 
-        group, curr_state, prev_state = ret
+        group, curr_state = ret
+
+        state_map = yield self.store.get_events(
+            curr_state.values(),
+            get_prev_content=False
+        )
+        curr_state = {
+            key: state_map[e_id] for key, e_id in curr_state.items() if e_id in state_map
+        }
 
         context.current_state = curr_state
         context.state_group = group if not event.is_state() else None
@@ -170,7 +182,7 @@ class StateHandler(object):
                 replaces = context.current_state[key]
                 event.unsigned["replaces_state"] = replaces.event_id
 
-        context.prev_state_events = prev_state
+        context.prev_state_events = []
         defer.returnValue(context)
 
     @defer.inlineCallbacks
@@ -187,72 +199,83 @@ class StateHandler(object):
         """
         logger.debug("resolve_state_groups event_ids %s", event_ids)
 
-        state_groups = yield self.store.get_state_groups(
+        state_groups_ids = yield self.store.get_state_groups_ids(
             room_id, event_ids
         )
 
         logger.debug(
             "resolve_state_groups state_groups %s",
-            state_groups.keys()
+            state_groups_ids.keys()
         )
 
-        group_names = frozenset(state_groups.keys())
+        group_names = frozenset(state_groups_ids.keys())
         if len(group_names) == 1:
-            name, state_list = state_groups.items().pop()
-            state = {
-                (e.type, e.state_key): e
-                for e in state_list
-            }
-            prev_state = state.get((event_type, state_key), None)
-            if prev_state:
-                prev_state = prev_state.event_id
-                prev_states = [prev_state]
-            else:
-                prev_states = []
+            name, state_list = state_groups_ids.items().pop()
 
-            defer.returnValue((name, state, prev_states))
+            defer.returnValue((name, state_list,))
 
         if self._state_cache is not None:
             cache = self._state_cache.get(group_names, None)
             if cache:
                 cache.ts = self.clock.time_msec()
 
-                event_dict = yield self.store.get_events(cache.state.values())
-                state = {(e.type, e.state_key): e for e in event_dict.values()}
-
-                prev_state = state.get((event_type, state_key), None)
-                if prev_state:
-                    prev_state = prev_state.event_id
-                    prev_states = [prev_state]
-                else:
-                    prev_states = []
                 defer.returnValue(
-                    (cache.state_group, state, prev_states)
+                    (cache.state_group, cache.state,)
                 )
 
-        logger.info("Resolving state for %s with %d groups", room_id, len(state_groups))
-
-        new_state, prev_states = self._resolve_events(
-            state_groups.values(), event_type, state_key
+        logger.info(
+            "Resolving state for %s with %d groups", room_id, len(state_groups_ids)
         )
 
+        state = {}
+        for st in state_groups_ids.values():
+            for key, e_id in st.items():
+                state.setdefault(key, set()).add(e_id)
+
+        conflicted_state = {
+            k: list(v)
+            for k, v in state.items()
+            if len(v) > 1
+        }
+
+        if conflicted_state:
+            logger.info("Resolving conflicted state for %r", room_id)
+            state_map = yield self.store.get_events(
+                [e_id for st in state_groups_ids.values() for e_id in st.values()],
+                get_prev_content=False
+            )
+            state_sets = [
+                [state_map[e_id] for key, e_id in st.items() if e_id in state_map]
+                for st in state_groups_ids.values()
+            ]
+            new_state, _ = self._resolve_events(
+                state_sets, event_type, state_key
+            )
+            new_state = {
+                key: e.event_id for key, e in new_state.items()
+            }
+        else:
+            new_state = {
+                key: e_ids.pop() for key, e_ids in state.items()
+            }
+
         state_group = None
-        new_state_event_ids = frozenset(e.event_id for e in new_state.values())
-        for sg, events in state_groups.items():
-            if new_state_event_ids == frozenset(e.event_id for e in events):
+        new_state_event_ids = frozenset(new_state.values())
+        for sg, events in state_groups_ids.items():
+            if new_state_event_ids == frozenset(e_id for e_id in events):
                 state_group = sg
                 break
 
         if self._state_cache is not None:
             cache = _StateCacheEntry(
-                state={key: event.event_id for key, event in new_state.items()},
+                state=new_state,
                 state_group=state_group,
                 ts=self.clock.time_msec()
             )
 
             self._state_cache[group_names] = cache
 
-        defer.returnValue((state_group, new_state, prev_states))
+        defer.returnValue((state_group, new_state,))
 
     def resolve_events(self, state_sets, event):
         logger.info(
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 0e8fa93e1f..fa40af6933 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -44,11 +44,7 @@ class StateStore(SQLBaseStore):
     """
 
     @defer.inlineCallbacks
-    def get_state_groups(self, room_id, event_ids):
-        """ Get the state groups for the given list of event_ids
-
-        The return value is a dict mapping group names to lists of events.
-        """
+    def get_state_groups_ids(self, room_id, event_ids):
         if not event_ids:
             defer.returnValue({})
 
@@ -59,9 +55,32 @@ class StateStore(SQLBaseStore):
         groups = set(event_to_groups.values())
         group_to_state = yield self._get_state_for_groups(groups)
 
+        defer.returnValue(group_to_state)
+
+    @defer.inlineCallbacks
+    def get_state_groups(self, room_id, event_ids):
+        """ Get the state groups for the given list of event_ids
+
+        The return value is a dict mapping group names to lists of events.
+        """
+        if not event_ids:
+            defer.returnValue({})
+
+        group_to_ids = yield self.get_state_groups_ids(room_id, event_ids)
+
+        state_event_map = yield self.get_events(
+            [
+                ev_id for group_ids in group_to_ids.values()
+                for ev_id in group_ids.values()
+            ],
+            get_prev_content=False
+        )
+
         defer.returnValue({
-            group: state_map.values()
-            for group, state_map in group_to_state.items()
+            group: [
+                state_event_map[v] for v in event_id_map.values() if v in state_event_map
+            ]
+            for group, event_id_map in group_to_ids.items()
         })
 
     def _store_mult_state_groups_txn(self, txn, events_and_contexts):
@@ -248,8 +267,17 @@ class StateStore(SQLBaseStore):
         groups = set(event_to_groups.values())
         group_to_state = yield self._get_state_for_groups(groups, types)
 
+        state_event_map = yield self.get_events(
+            [ev_id for sd in group_to_state.values() for ev_id in sd.values()],
+            get_prev_content=False
+        )
+
         event_to_state = {
-            event_id: group_to_state[group]
+            event_id: {
+                k: state_event_map[v]
+                for k, v in group_to_state[group].items()
+                if v in state_event_map
+            }
             for event_id, group in event_to_groups.items()
         }
 
@@ -428,20 +456,13 @@ class StateStore(SQLBaseStore):
                     full=(types is None),
                 )
 
-        state_events = yield self._get_events(
-            [ev_id for sd in results.values() for ev_id in sd.values()],
-            get_prev_content=False
-        )
-
-        state_events = {e.event_id: e for e in state_events}
-
         # Remove all the entries with None values. The None values were just
         # used for bookkeeping in the cache.
         for group, state_dict in results.items():
             results[group] = {
-                key: state_events[event_id]
+                key: event_id
                 for key, event_id in state_dict.items()
-                if event_id and event_id in state_events
+                if event_id
             }
 
         defer.returnValue(results)