From 1e25f62ee6a8aaa65c139e264ec2be1f8831eb16 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 30 Mar 2016 12:55:02 +0100
Subject: Use a stream id generator to assign state group ids

---
 synapse/storage/__init__.py |  2 +-
 synapse/storage/events.py   | 90 +++++++++++++++++++++++++--------------------
 synapse/storage/state.py    | 16 ++++----
 3 files changed, 59 insertions(+), 49 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 250ba536ea..aaad38039e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -116,7 +116,7 @@ class DataStore(RoomMemberStore, RoomStore,
         )
 
         self._transaction_id_gen = IdGenerator(db_conn, "sent_transactions", "id")
-        self._state_groups_id_gen = IdGenerator(db_conn, "state_groups", "id")
+        self._state_groups_id_gen = StreamIdGenerator(db_conn, "state_groups", "id")
         self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
         self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id")
         self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5233430028..5f675ab09b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -26,6 +26,7 @@ from synapse.api.constants import EventTypes
 from canonicaljson import encode_canonical_json
 from contextlib import contextmanager
 
+
 import logging
 import math
 import ujson as json
@@ -79,41 +80,57 @@ class EventsStore(SQLBaseStore):
                 len(events_and_contexts)
             )
 
+        state_group_id_manager = self._state_groups_id_gen.get_next_mult(
+            len(events_and_contexts)
+        )
         with stream_ordering_manager as stream_orderings:
-            for (event, _), stream in zip(events_and_contexts, stream_orderings):
-                event.internal_metadata.stream_ordering = stream
-
-            chunks = [
-                events_and_contexts[x:x + 100]
-                for x in xrange(0, len(events_and_contexts), 100)
-            ]
+            with state_group_id_manager as state_group_ids:
+                for (event, context), stream, state_group_id in zip(
+                    events_and_contexts, stream_orderings, state_group_ids
+                ):
+                    event.internal_metadata.stream_ordering = stream
+                    # Assign a state group_id in case a new id is needed for
+                    # this context. In theory we only need to assign this
+                    # for contexts that have current_state and aren't outliers
+                    # but that make the code more complicated. Assigning an ID
+                    # per event only causes the state_group_ids to grow as fast
+                    # as the stream_ordering so in practise shouldn't be a problem.
+                    context.new_state_group_id = state_group_id
+
+                chunks = [
+                    events_and_contexts[x:x + 100]
+                    for x in xrange(0, len(events_and_contexts), 100)
+                ]
 
-            for chunk in chunks:
-                # We can't easily parallelize these since different chunks
-                # might contain the same event. :(
-                yield self.runInteraction(
-                    "persist_events",
-                    self._persist_events_txn,
-                    events_and_contexts=chunk,
-                    backfilled=backfilled,
-                    is_new_state=is_new_state,
-                )
+                for chunk in chunks:
+                    # We can't easily parallelize these since different chunks
+                    # might contain the same event. :(
+                    yield self.runInteraction(
+                        "persist_events",
+                        self._persist_events_txn,
+                        events_and_contexts=chunk,
+                        backfilled=backfilled,
+                        is_new_state=is_new_state,
+                    )
 
     @defer.inlineCallbacks
     @log_function
     def persist_event(self, event, context,
                       is_new_state=True, current_state=None):
+
         try:
             with self._stream_id_gen.get_next() as stream_ordering:
-                event.internal_metadata.stream_ordering = stream_ordering
-                yield self.runInteraction(
-                    "persist_event",
-                    self._persist_event_txn,
-                    event=event,
-                    context=context,
-                    is_new_state=is_new_state,
-                    current_state=current_state,
-                )
+                with self._state_groups_id_gen.get_next() as state_group_id:
+                    event.internal_metadata.stream_ordering = stream_ordering
+                    context.new_state_group_id = state_group_id
+                    yield self.runInteraction(
+                        "persist_event",
+                        self._persist_event_txn,
+                        event=event,
+                        context=context,
+                        is_new_state=is_new_state,
+                        current_state=current_state,
+                    )
         except _RollbackButIsFineException:
             pass
 
@@ -178,7 +195,7 @@ class EventsStore(SQLBaseStore):
 
     @log_function
     def _persist_event_txn(self, txn, event, context,
-                           is_new_state=True, current_state=None):
+                           is_new_state, current_state):
         # We purposefully do this first since if we include a `current_state`
         # key, we *want* to update the `current_state_events` table
         if current_state:
@@ -215,7 +232,7 @@ class EventsStore(SQLBaseStore):
 
     @log_function
     def _persist_events_txn(self, txn, events_and_contexts, backfilled,
-                            is_new_state=True):
+                            is_new_state):
         depth_updates = {}
         for event, context in events_and_contexts:
             # Remove the any existing cache entries for the event_ids
@@ -282,9 +299,7 @@ class EventsStore(SQLBaseStore):
 
             outlier_persisted = have_persisted[event.event_id]
             if not event.internal_metadata.is_outlier() and outlier_persisted:
-                self._store_state_groups_txn(
-                    txn, event, context,
-                )
+                self._store_mult_state_groups_txn(txn, ((event, context),))
 
                 metadata_json = encode_json(
                     event.internal_metadata.get_dict()
@@ -310,19 +325,14 @@ class EventsStore(SQLBaseStore):
 
                 self._update_extremeties(txn, [event])
 
-        events_and_contexts = filter(
-            lambda ec: ec[0] not in to_remove,
-            events_and_contexts
-        )
+        events_and_contexts = [
+            ec for ec in events_and_contexts if ec[0] not in to_remove
+        ]
 
         if not events_and_contexts:
             return
 
-        self._store_mult_state_groups_txn(txn, [
-            (event, context)
-            for event, context in events_and_contexts
-            if not event.internal_metadata.is_outlier()
-        ])
+        self._store_mult_state_groups_txn(txn, events_and_contexts)
 
         self._handle_mult_prev_events(
             txn,
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 02cefdff26..30d1060ecd 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -64,12 +64,12 @@ class StateStore(SQLBaseStore):
             for group, state_map in group_to_state.items()
         })
 
-    def _store_state_groups_txn(self, txn, event, context):
-        return self._store_mult_state_groups_txn(txn, [(event, context)])
-
     def _store_mult_state_groups_txn(self, txn, events_and_contexts):
         state_groups = {}
         for event, context in events_and_contexts:
+            if event.internal_metadata.is_outlier():
+                continue
+
             if context.current_state is None:
                 continue
 
@@ -82,7 +82,8 @@ class StateStore(SQLBaseStore):
             if event.is_state():
                 state_events[(event.type, event.state_key)] = event
 
-            state_group = self._state_groups_id_gen.get_next()
+            state_group = context.new_state_group_id
+
             self._simple_insert_txn(
                 txn,
                 table="state_groups",
@@ -114,11 +115,10 @@ class StateStore(SQLBaseStore):
             table="event_to_state_groups",
             values=[
                 {
-                    "state_group": state_groups[event.event_id],
-                    "event_id": event.event_id,
+                    "state_group": state_group_id,
+                    "event_id": event_id,
                 }
-                for event, context in events_and_contexts
-                if context.current_state is not None
+                for event_id, state_group_id in state_groups.items()
             ],
         )
 
-- 
cgit 1.5.1


From 31a9eceda5cf00b0482baf1c8bf1e138c823f621 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 30 Mar 2016 15:58:20 +0100
Subject: Add a replication stream for state groups

---
 synapse/replication/resource.py    | 36 +++++++++++++++++++++++++++++-------
 synapse/storage/events.py          |  6 +++++-
 synapse/storage/state.py           | 30 ++++++++++++++++++++++++++++++
 tests/replication/test_resource.py | 30 +++++++++++++++++++++++++++---
 4 files changed, 91 insertions(+), 11 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py
index 8c1ae0fbc7..096a79a7a4 100644
--- a/synapse/replication/resource.py
+++ b/synapse/replication/resource.py
@@ -38,6 +38,7 @@ STREAM_NAMES = (
     ("backfill",),
     ("push_rules",),
     ("pushers",),
+    ("state",),
 )
 
 
@@ -123,6 +124,7 @@ class ReplicationResource(Resource):
         backfill_token = yield self.store.get_current_backfill_token()
         push_rules_token, room_stream_token = self.store.get_push_rules_stream_token()
         pushers_token = self.store.get_pushers_stream_token()
+        state_token = self.store.get_state_stream_token()
 
         defer.returnValue(_ReplicationToken(
             room_stream_token,
@@ -133,6 +135,7 @@ class ReplicationResource(Resource):
             backfill_token,
             push_rules_token,
             pushers_token,
+            state_token,
         ))
 
     @request_handler
@@ -156,6 +159,7 @@ class ReplicationResource(Resource):
             yield self.receipts(writer, current_token, limit)
             yield self.push_rules(writer, current_token, limit)
             yield self.pushers(writer, current_token, limit)
+            yield self.state(writer, current_token, limit)
             self.streams(writer, current_token)
 
             logger.info("Replicated %d rows", writer.total)
@@ -205,12 +209,12 @@ class ReplicationResource(Resource):
                 current_token.backfill, current_token.events,
                 limit
             )
-            writer.write_header_and_rows(
-                "events", events_rows, ("position", "internal", "json")
-            )
-            writer.write_header_and_rows(
-                "backfill", backfill_rows, ("position", "internal", "json")
-            )
+            writer.write_header_and_rows("events", events_rows, (
+                "position", "internal", "json", "state_group"
+            ))
+            writer.write_header_and_rows("backfill", backfill_rows, (
+                "position", "internal", "json", "state_group"
+            ))
 
     @defer.inlineCallbacks
     def presence(self, writer, current_token):
@@ -320,6 +324,24 @@ class ReplicationResource(Resource):
                 "position", "user_id", "app_id", "pushkey"
             ))
 
+    @defer.inlineCallbacks
+    def state(self, writer, current_token, limit):
+        current_position = current_token.state
+
+        state = parse_integer(writer.request, "state")
+        if state is not None:
+            state_groups, state_group_state = (
+                yield self.store.get_all_new_state_groups(
+                    state, current_position, limit
+                )
+            )
+            writer.write_header_and_rows("state_groups", state_groups, (
+                "position", "room_id", "event_id"
+            ))
+            writer.write_header_and_rows("state_group_state", state_group_state, (
+                "position", "type", "state_key", "event_id"
+            ))
+
 
 class _Writer(object):
     """Writes the streams as a JSON object as the response to the request"""
@@ -350,7 +372,7 @@ class _Writer(object):
 
 class _ReplicationToken(collections.namedtuple("_ReplicationToken", (
     "events", "presence", "typing", "receipts", "account_data", "backfill",
-    "push_rules", "pushers"
+    "push_rules", "pushers", "state"
 ))):
     __slots__ = []
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5f675ab09b..a4b8995496 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1097,10 +1097,12 @@ class EventsStore(SQLBaseStore):
         new events or as backfilled events"""
         def get_all_new_events_txn(txn):
             sql = (
-                "SELECT e.stream_ordering, ej.internal_metadata, ej.json"
+                "SELECT e.stream_ordering, ej.internal_metadata, ej.json, eg.state_group"
                 " FROM events as e"
                 " JOIN event_json as ej"
                 " ON e.event_id = ej.event_id AND e.room_id = ej.room_id"
+                " LEFT JOIN event_to_state_groups as eg"
+                " ON e.event_id = eg.event_id"
                 " WHERE ? < e.stream_ordering AND e.stream_ordering <= ?"
                 " ORDER BY e.stream_ordering ASC"
                 " LIMIT ?"
@@ -1116,6 +1118,8 @@ class EventsStore(SQLBaseStore):
                 " FROM events as e"
                 " JOIN event_json as ej"
                 " ON e.event_id = ej.event_id AND e.room_id = ej.room_id"
+                " LEFT JOIN event_to_state_groups as eg"
+                " ON e.event_id = eg.event_id"
                 " WHERE ? > e.stream_ordering AND e.stream_ordering >= ?"
                 " ORDER BY e.stream_ordering DESC"
                 " LIMIT ?"
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 30d1060ecd..7fc9a4f264 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -429,3 +429,33 @@ class StateStore(SQLBaseStore):
             }
 
         defer.returnValue(results)
+
+    def get_all_new_state_groups(self, last_id, current_id, limit):
+        def get_all_new_state_groups_txn(txn):
+            sql = (
+                "SELECT id, room_id, event_id FROM state_groups"
+                " WHERE ? < id AND id <= ? ORDER BY id LIMIT ?"
+            )
+            txn.execute(sql, (last_id, current_id, limit))
+            groups = txn.fetchall()
+
+            if not groups:
+                return ([], [])
+
+            lower_bound = groups[0][0]
+            upper_bound = groups[-1][0]
+            sql = (
+                "SELECT state_group, type, state_key, event_id"
+                " FROM state_groups_state"
+                " WHERE ? <= state_group AND state_group <= ?"
+            )
+
+            txn.execute(sql, (lower_bound, upper_bound))
+            state_group_state = txn.fetchall()
+            return (groups, state_group_state)
+        return self.runInteraction(
+            "get_all_new_state_groups", get_all_new_state_groups_txn
+        )
+
+    def get_state_stream_token(self):
+        return self._state_groups_id_gen.get_max_token()
diff --git a/tests/replication/test_resource.py b/tests/replication/test_resource.py
index f4b5fb3328..b1dd7b4a74 100644
--- a/tests/replication/test_resource.py
+++ b/tests/replication/test_resource.py
@@ -58,15 +58,21 @@ class ReplicationResourceCase(unittest.TestCase):
         self.assertEquals(body, {})
 
     @defer.inlineCallbacks
-    def test_events(self):
-        get = self.get(events="-1", timeout="0")
+    def test_events_and_state(self):
+        get = self.get(events="-1", state="-1", timeout="0")
         yield self.hs.get_handlers().room_creation_handler.create_room(
             Requester(self.user, "", False), {}
         )
         code, body = yield get
         self.assertEquals(code, 200)
         self.assertEquals(body["events"]["field_names"], [
-            "position", "internal", "json"
+            "position", "internal", "json", "state_group"
+        ])
+        self.assertEquals(body["state_groups"]["field_names"], [
+            "position", "room_id", "event_id"
+        ])
+        self.assertEquals(body["state_group_state"]["field_names"], [
+            "position", "type", "state_key", "event_id"
         ])
 
     @defer.inlineCallbacks
@@ -132,6 +138,7 @@ class ReplicationResourceCase(unittest.TestCase):
     test_timeout_backfill = _test_timeout("backfill")
     test_timeout_push_rules = _test_timeout("push_rules")
     test_timeout_pushers = _test_timeout("pushers")
+    test_timeout_state = _test_timeout("state")
 
     @defer.inlineCallbacks
     def send_text_message(self, room_id, message):
@@ -182,4 +189,21 @@ class ReplicationResourceCase(unittest.TestCase):
         )
         response_body = json.loads(response_json)
 
+        if response_code == 200:
+            self.check_response(response_body)
+
         defer.returnValue((response_code, response_body))
+
+    def check_response(self, response_body):
+        for name, stream in response_body.items():
+            self.assertIn("field_names", stream)
+            field_names = stream["field_names"]
+            self.assertIn("rows", stream)
+            self.assertTrue(stream["rows"])
+            for row in stream["rows"]:
+                self.assertEquals(
+                    len(row), len(field_names),
+                    "%s: len(row = %r) == len(field_names = %r)" % (
+                        name, row, field_names
+                    )
+                )
-- 
cgit 1.5.1


From 61407986b40e28b590961d364f5618bbe7d44e94 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 30 Mar 2016 16:18:46 +0100
Subject: Add a entry to current_state_resets table when the current state is
 reset

---
 synapse/storage/events.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a4b8995496..bd4d503b6d 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -205,6 +205,15 @@ class EventsStore(SQLBaseStore):
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_room_name_and_aliases, event.room_id)
 
+            # Add an entry to the current_state_resets table to record the point
+            # where we clobbered the current state
+            stream_order = event.internal_metadata.stream_ordering
+            self._simple_insert_txn(
+                txn,
+                table="current_state_resets",
+                values={"event_stream_ordering": stream_order}
+            )
+
             self._simple_delete_txn(
                 txn,
                 table="current_state_events",
-- 
cgit 1.5.1


From 8b8052909f5352e55998b8c7fbb26a8274e75366 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 30 Mar 2016 16:20:07 +0100
Subject: return the state_group for backfill

---
 synapse/storage/events.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a4b8995496..e0ef7f46b2 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1114,7 +1114,8 @@ class EventsStore(SQLBaseStore):
                 new_forward_events = []
 
             sql = (
-                "SELECT -e.stream_ordering, ej.internal_metadata, ej.json"
+                "SELECT -e.stream_ordering, ej.internal_metadata, ej.json,"
+                " eg.state_group"
                 " FROM events as e"
                 " JOIN event_json as ej"
                 " ON e.event_id = ej.event_id AND e.room_id = ej.room_id"
-- 
cgit 1.5.1


From 1fbb094c6fbaab33ef8e17802e37057e83718e7e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 30 Mar 2016 17:19:56 +0100
Subject: Add replication streams for ex outliers and current state resets

---
 synapse/replication/resource.py                  | 17 ++++++-
 synapse/storage/events.py                        | 60 +++++++++++++++++++++++-
 synapse/storage/schema/delta/30/state_stream.sql | 38 +++++++++++++++
 3 files changed, 113 insertions(+), 2 deletions(-)
 create mode 100644 synapse/storage/schema/delta/30/state_stream.sql

(limited to 'synapse/storage')

diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py
index 096a79a7a4..7afa1242d5 100644
--- a/synapse/replication/resource.py
+++ b/synapse/replication/resource.py
@@ -204,7 +204,11 @@ class ReplicationResource(Resource):
                 request_events = current_token.events
             if request_backfill is None:
                 request_backfill = current_token.backfill
-            events_rows, backfill_rows = yield self.store.get_all_new_events(
+            (
+                events_rows, backfill_rows,
+                forward_ex_outliers, backward_ex_outliers,
+                state_resets
+            ) = yield self.store.get_all_new_events(
                 request_backfill, request_events,
                 current_token.backfill, current_token.events,
                 limit
@@ -215,6 +219,17 @@ class ReplicationResource(Resource):
             writer.write_header_and_rows("backfill", backfill_rows, (
                 "position", "internal", "json", "state_group"
             ))
+            writer.write_header_and_rows(
+                "forward_ex_outliers", forward_ex_outliers,
+                ("position", "event_id", "state_group")
+            )
+            writer.write_header_and_rows(
+                "backward_ex_outliers", backward_ex_outliers,
+                ("position", "event_id", "state_group")
+            )
+            writer.write_header_and_rows(
+                "state_resets", state_resets, ("position",)
+            )
 
     @defer.inlineCallbacks
     def presence(self, writer, current_token):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bd4d503b6d..9725a3fed7 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -323,6 +323,18 @@ class EventsStore(SQLBaseStore):
                     (metadata_json, event.event_id,)
                 )
 
+                stream_order = event.internal_metadata.stream_ordering
+                state_group_id = context.state_group or context.new_state_group_id
+                self._simple_insert_txn(
+                    txn,
+                    table="ex_outlier_stream",
+                    values={
+                        "event_stream_ordering": stream_order,
+                        "event_id": event.event_id,
+                        "state_group": state_group_id,
+                    }
+                )
+
                 sql = (
                     "UPDATE events SET outlier = ?"
                     " WHERE event_id = ?"
@@ -1119,8 +1131,34 @@ class EventsStore(SQLBaseStore):
             if last_forward_id != current_forward_id:
                 txn.execute(sql, (last_forward_id, current_forward_id, limit))
                 new_forward_events = txn.fetchall()
+
+                if len(new_forward_events) == limit:
+                    upper_bound = new_forward_events[-1][0]
+                else:
+                    upper_bound = current_forward_id
+
+                sql = (
+                    "SELECT -event_stream_ordering FROM current_state_resets"
+                    " WHERE ? < event_stream_ordering"
+                    " AND event_stream_ordering <= ?"
+                    " ORDER BY event_stream_ordering ASC"
+                )
+                txn.execute(sql, (last_forward_id, upper_bound))
+                state_resets = txn.fetchall()
+
+                sql = (
+                    "SELECT -event_stream_ordering, event_id, state_group"
+                    " FROM ex_outlier_stream"
+                    " WHERE ? > event_stream_ordering"
+                    " AND event_stream_ordering >= ?"
+                    " ORDER BY event_stream_ordering DESC"
+                )
+                txn.execute(sql, (last_forward_id, upper_bound))
+                forward_ex_outliers = txn.fetchall()
             else:
                 new_forward_events = []
+                state_resets = []
+                forward_ex_outliers = []
 
             sql = (
                 "SELECT -e.stream_ordering, ej.internal_metadata, ej.json"
@@ -1136,8 +1174,28 @@ class EventsStore(SQLBaseStore):
             if last_backfill_id != current_backfill_id:
                 txn.execute(sql, (-last_backfill_id, -current_backfill_id, limit))
                 new_backfill_events = txn.fetchall()
+
+                if len(new_backfill_events) == limit:
+                    upper_bound = new_backfill_events[-1][0]
+                else:
+                    upper_bound = current_backfill_id
+
+                sql = (
+                    "SELECT -event_stream_ordering, event_id, state_group"
+                    " FROM ex_outlier_stream"
+                    " WHERE ? > event_stream_ordering"
+                    " AND event_stream_ordering >= ?"
+                    " ORDER BY event_stream_ordering DESC"
+                )
+                txn.execute(sql, (-last_backfill_id, -upper_bound))
+                backward_ex_outliers = txn.fetchall()
             else:
                 new_backfill_events = []
+                backward_ex_outliers = []
 
-            return (new_forward_events, new_backfill_events)
+            return (
+                new_forward_events, new_backfill_events,
+                forward_ex_outliers, backward_ex_outliers,
+                state_resets,
+            )
         return self.runInteraction("get_all_new_events", get_all_new_events_txn)
diff --git a/synapse/storage/schema/delta/30/state_stream.sql b/synapse/storage/schema/delta/30/state_stream.sql
new file mode 100644
index 0000000000..706fe1dcf4
--- /dev/null
+++ b/synapse/storage/schema/delta/30/state_stream.sql
@@ -0,0 +1,38 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * The positions in the event stream_ordering when the current_state was
+ * replaced by the state at the event.
+ */
+
+CREATE TABLE IF NOT EXISTS current_state_resets(
+    event_stream_ordering BIGINT PRIMARY KEY NOT NULL
+);
+
+/* The outlier events that have aquired a state group typically through
+ * backfill. This is tracked separately to the events table, as assigning a
+ * state group change the position of the existing event in the stream
+ * ordering.
+ * However since a stream_ordering is assigned in persist_event for the
+ * (event, state) pair, we can use that stream_ordering to identify when
+ * the new state was assigned for the event.
+ */
+CREATE TABLE IF NOT EXISTS ex_outlier_stream(
+    event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
+    event_id TEXT NOT NULL,
+    state_group BIGINT NOT NULL
+);
-- 
cgit 1.5.1


From 2ec54260350b46c937527bd566b713cf3544f1d2 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Thu, 31 Mar 2016 10:33:02 +0100
Subject: Use a namedtuple rather than tuple unpacking

---
 synapse/replication/resource.py | 16 ++++++----------
 synapse/storage/events.py       | 11 +++++++++--
 2 files changed, 15 insertions(+), 12 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py
index 7afa1242d5..69afcb03d2 100644
--- a/synapse/replication/resource.py
+++ b/synapse/replication/resource.py
@@ -204,31 +204,27 @@ class ReplicationResource(Resource):
                 request_events = current_token.events
             if request_backfill is None:
                 request_backfill = current_token.backfill
-            (
-                events_rows, backfill_rows,
-                forward_ex_outliers, backward_ex_outliers,
-                state_resets
-            ) = yield self.store.get_all_new_events(
+            res = yield self.store.get_all_new_events(
                 request_backfill, request_events,
                 current_token.backfill, current_token.events,
                 limit
             )
-            writer.write_header_and_rows("events", events_rows, (
+            writer.write_header_and_rows("events", res.new_forward_events, (
                 "position", "internal", "json", "state_group"
             ))
-            writer.write_header_and_rows("backfill", backfill_rows, (
+            writer.write_header_and_rows("backfill", res.new_backfill_events, (
                 "position", "internal", "json", "state_group"
             ))
             writer.write_header_and_rows(
-                "forward_ex_outliers", forward_ex_outliers,
+                "forward_ex_outliers", res.forward_ex_outliers,
                 ("position", "event_id", "state_group")
             )
             writer.write_header_and_rows(
-                "backward_ex_outliers", backward_ex_outliers,
+                "backward_ex_outliers", res.backward_ex_outliers,
                 ("position", "event_id", "state_group")
             )
             writer.write_header_and_rows(
-                "state_resets", state_resets, ("position",)
+                "state_resets", res.state_resets, ("position",)
             )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 9725a3fed7..b7ad045e41 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -25,7 +25,7 @@ from synapse.api.constants import EventTypes
 
 from canonicaljson import encode_canonical_json
 from contextlib import contextmanager
-
+from collections import namedtuple
 
 import logging
 import math
@@ -1193,9 +1193,16 @@ class EventsStore(SQLBaseStore):
                 new_backfill_events = []
                 backward_ex_outliers = []
 
-            return (
+            return AllNewEventsResult(
                 new_forward_events, new_backfill_events,
                 forward_ex_outliers, backward_ex_outliers,
                 state_resets,
             )
         return self.runInteraction("get_all_new_events", get_all_new_events_txn)
+
+
+AllNewEventsResult = namedtuple("AllNewEventsResult", [
+    "new_forward_events", "new_backfill_events",
+    "forward_ex_outliers", "backward_ex_outliers",
+    "state_resets"
+])
-- 
cgit 1.5.1


From 76503f95ed2b618a3ff97c6b04868d8e440ef9d4 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 31 Mar 2016 15:00:42 +0100
Subject: Remove the is_new_state argument to persist event.

Move the checks for whether an event is new state inside persist
event itself.

This was harder than expected because there wasn't enough information
passed to persist event to correctly handle invites from remote servers
for new rooms.
---
 synapse/events/__init__.py     |  3 ++
 synapse/handlers/federation.py | 20 ++--------
 synapse/storage/events.py      | 90 +++++++++++++++++++++++-------------------
 3 files changed, 57 insertions(+), 56 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 925a83c645..13154b1723 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -33,6 +33,9 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
+    def is_invite_from_remote(self):
+        return getattr(self, "invite_from_remote", False)
+
 
 def _event_dict_property(key):
     def getter(self):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 267fedf114..4a35344d32 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -102,8 +102,7 @@ class FederationHandler(BaseHandler):
 
     @log_function
     @defer.inlineCallbacks
-    def on_receive_pdu(self, origin, pdu, state=None,
-                       auth_chain=None):
+    def on_receive_pdu(self, origin, pdu, state=None, auth_chain=None):
         """ Called by the ReplicationLayer when we have a new pdu. We need to
         do auth checks and put it through the StateHandler.
         """
@@ -174,11 +173,7 @@ class FederationHandler(BaseHandler):
                     })
                     seen_ids.add(e.event_id)
 
-                yield self._handle_new_events(
-                    origin,
-                    event_infos,
-                    outliers=True
-                )
+                yield self._handle_new_events(origin, event_infos)
 
             try:
                 context, event_stream_id, max_stream_id = yield self._handle_new_event(
@@ -761,6 +756,7 @@ class FederationHandler(BaseHandler):
         event = pdu
 
         event.internal_metadata.outlier = True
+        event.internal_metadata.invite_from_remote = True
 
         event.signatures.update(
             compute_event_signature(
@@ -1069,9 +1065,6 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     @log_function
     def _handle_new_event(self, origin, event, state=None, auth_events=None):
-
-        outlier = event.internal_metadata.is_outlier()
-
         context = yield self._prep_event(
             origin, event,
             state=state,
@@ -1087,14 +1080,12 @@ class FederationHandler(BaseHandler):
         event_stream_id, max_stream_id = yield self.store.persist_event(
             event,
             context=context,
-            is_new_state=not outlier,
         )
 
         defer.returnValue((context, event_stream_id, max_stream_id))
 
     @defer.inlineCallbacks
-    def _handle_new_events(self, origin, event_infos, backfilled=False,
-                           outliers=False):
+    def _handle_new_events(self, origin, event_infos, backfilled=False):
         contexts = yield defer.gatherResults(
             [
                 self._prep_event(
@@ -1113,7 +1104,6 @@ class FederationHandler(BaseHandler):
                 for ev_info, context in itertools.izip(event_infos, contexts)
             ],
             backfilled=backfilled,
-            is_new_state=(not outliers and not backfilled),
         )
 
     @defer.inlineCallbacks
@@ -1176,7 +1166,6 @@ class FederationHandler(BaseHandler):
                 (e, events_to_context[e.event_id])
                 for e in itertools.chain(auth_events, state)
             ],
-            is_new_state=False,
         )
 
         new_event_context = yield self.state_handler.compute_event_context(
@@ -1185,7 +1174,6 @@ class FederationHandler(BaseHandler):
 
         event_stream_id, max_stream_id = yield self.store.persist_event(
             event, new_event_context,
-            is_new_state=True,
             current_state=state,
         )
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index dc3e994de9..b5f9b3b900 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -61,8 +61,7 @@ class EventsStore(SQLBaseStore):
         )
 
     @defer.inlineCallbacks
-    def persist_events(self, events_and_contexts, backfilled=False,
-                       is_new_state=True):
+    def persist_events(self, events_and_contexts, backfilled=False):
         if not events_and_contexts:
             return
 
@@ -110,13 +109,11 @@ class EventsStore(SQLBaseStore):
                         self._persist_events_txn,
                         events_and_contexts=chunk,
                         backfilled=backfilled,
-                        is_new_state=is_new_state,
                     )
 
     @defer.inlineCallbacks
     @log_function
-    def persist_event(self, event, context,
-                      is_new_state=True, current_state=None):
+    def persist_event(self, event, context, current_state=None):
 
         try:
             with self._stream_id_gen.get_next() as stream_ordering:
@@ -128,7 +125,6 @@ class EventsStore(SQLBaseStore):
                         self._persist_event_txn,
                         event=event,
                         context=context,
-                        is_new_state=is_new_state,
                         current_state=current_state,
                     )
         except _RollbackButIsFineException:
@@ -194,8 +190,7 @@ class EventsStore(SQLBaseStore):
         defer.returnValue({e.event_id: e for e in events})
 
     @log_function
-    def _persist_event_txn(self, txn, event, context,
-                           is_new_state, current_state):
+    def _persist_event_txn(self, txn, event, context, current_state):
         # We purposefully do this first since if we include a `current_state`
         # key, we *want* to update the `current_state_events` table
         if current_state:
@@ -236,12 +231,10 @@ class EventsStore(SQLBaseStore):
             txn,
             [(event, context)],
             backfilled=False,
-            is_new_state=is_new_state,
         )
 
     @log_function
-    def _persist_events_txn(self, txn, events_and_contexts, backfilled,
-                            is_new_state):
+    def _persist_events_txn(self, txn, events_and_contexts, backfilled):
         depth_updates = {}
         for event, context in events_and_contexts:
             # Remove the any existing cache entries for the event_ids
@@ -452,10 +445,9 @@ class EventsStore(SQLBaseStore):
             txn, [event for event, _ in events_and_contexts]
         )
 
-        state_events_and_contexts = filter(
-            lambda i: i[0].is_state(),
-            events_and_contexts,
-        )
+        state_events_and_contexts = [
+            ec for ec in events_and_contexts if ec[0].is_state()
+        ]
 
         state_values = []
         for event, context in state_events_and_contexts:
@@ -493,32 +485,50 @@ class EventsStore(SQLBaseStore):
             ],
         )
 
-        if is_new_state:
-            for event, _ in state_events_and_contexts:
-                if not context.rejected:
-                    txn.call_after(
-                        self._get_current_state_for_key.invalidate,
-                        (event.room_id, event.type, event.state_key,)
-                    )
+        for event, _ in state_events_and_contexts:
+            if backfilled:
+                # Backfilled events come before the current state so shouldn't
+                # clobber it.
+                continue
 
-                    if event.type in [EventTypes.Name, EventTypes.Aliases]:
-                        txn.call_after(
-                            self.get_room_name_and_aliases.invalidate,
-                            (event.room_id,)
-                        )
-
-                    self._simple_upsert_txn(
-                        txn,
-                        "current_state_events",
-                        keyvalues={
-                            "room_id": event.room_id,
-                            "type": event.type,
-                            "state_key": event.state_key,
-                        },
-                        values={
-                            "event_id": event.event_id,
-                        }
-                    )
+            if (not event.internal_metadata.is_invite_from_remote()
+                    and event.internal_metadata.is_outlier()):
+                # Outlier events generally shouldn't clobber the current state.
+                # However invites from remote severs for rooms we aren't in
+                # are a bit special: they don't come with any associated
+                # state so are technically an outlier, however all the
+                # client-facing code assumes that they are in the current
+                # state table so we insert the event anyway.
+                continue
+
+            if context.rejected:
+                # If the event failed it's auth checks then it shouldn't
+                # clobbler the current state.
+                continue
+
+            txn.call_after(
+                self._get_current_state_for_key.invalidate,
+                (event.room_id, event.type, event.state_key,)
+            )
+
+            if event.type in [EventTypes.Name, EventTypes.Aliases]:
+                txn.call_after(
+                    self.get_room_name_and_aliases.invalidate,
+                    (event.room_id,)
+                )
+
+            self._simple_upsert_txn(
+                txn,
+                "current_state_events",
+                keyvalues={
+                    "room_id": event.room_id,
+                    "type": event.type,
+                    "state_key": event.state_key,
+                },
+                values={
+                    "event_id": event.event_id,
+                }
+            )
 
         return
 
-- 
cgit 1.5.1


From 5d069291696c62c94d795506dd5a22a3dbd019e1 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 31 Mar 2016 15:09:09 +0100
Subject: Move the check for backfilled outside the for loop

---
 synapse/storage/events.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index b5f9b3b900..83279d65fa 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -485,12 +485,12 @@ class EventsStore(SQLBaseStore):
             ],
         )
 
-        for event, _ in state_events_and_contexts:
-            if backfilled:
-                # Backfilled events come before the current state so shouldn't
-                # clobber it.
-                continue
+        if backfilled:
+            # Backfilled events come before the current state so we don't need
+            # to update the current state table
+            return
 
+        for event, _ in state_events_and_contexts:
             if (not event.internal_metadata.is_invite_from_remote()
                     and event.internal_metadata.is_outlier()):
                 # Outlier events generally shouldn't clobber the current state.
-- 
cgit 1.5.1


From 7753fc65702dc2b98885f462fe0e6ac9f8e27b06 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 10:34:51 +0100
Subject: Fix the invalidation of the names and aliases cache

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 83279d65fa..7468e6e00c 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -198,7 +198,7 @@ class EventsStore(SQLBaseStore):
             txn.call_after(self.get_rooms_for_user.invalidate_all)
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
-            txn.call_after(self.get_room_name_and_aliases, event.room_id)
+            txn.call_after(self.get_room_name_and_aliases.invalidate, (event.room_id,))
 
             # Add an entry to the current_state_resets table to record the point
             # where we clobbered the current state
-- 
cgit 1.5.1


From e36bfbab38def70e0fcc1bafcecb6e666dbbc1ad Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 13:29:05 +0100
Subject: Use a stream id generator for backfilled ids

---
 synapse/storage/__init__.py           | 20 ++++--------
 synapse/storage/account_data.py       |  4 +--
 synapse/storage/events.py             | 19 +++--------
 synapse/storage/presence.py           |  6 ++--
 synapse/storage/push_rule.py          |  2 +-
 synapse/storage/pusher.py             |  2 +-
 synapse/storage/receipts.py           |  6 ++--
 synapse/storage/state.py              |  2 +-
 synapse/storage/stream.py             |  2 +-
 synapse/storage/tags.py               |  6 ++--
 synapse/storage/util/id_generators.py | 61 +++++++++++++++++++++++------------
 11 files changed, 69 insertions(+), 61 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index aaad38039e..f87e907cd8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -88,15 +88,6 @@ class DataStore(RoomMemberStore, RoomStore,
         self.hs = hs
         self.database_engine = hs.database_engine
 
-        cur = db_conn.cursor()
-        try:
-            cur.execute("SELECT MIN(stream_ordering) FROM events",)
-            rows = cur.fetchall()
-            self.min_stream_token = rows[0][0] if rows and rows[0] and rows[0][0] else -1
-            self.min_stream_token = min(self.min_stream_token, -1)
-        finally:
-            cur.close()
-
         self.client_ip_last_seen = Cache(
             name="client_ip_last_seen",
             keylen=4,
@@ -105,6 +96,9 @@ class DataStore(RoomMemberStore, RoomStore,
         self._stream_id_gen = StreamIdGenerator(
             db_conn, "events", "stream_ordering"
         )
+        self._backfill_id_gen = StreamIdGenerator(
+            db_conn, "events", "stream_ordering", direction=-1
+        )
         self._receipts_id_gen = StreamIdGenerator(
             db_conn, "receipts_linearized", "stream_id"
         )
@@ -129,7 +123,7 @@ class DataStore(RoomMemberStore, RoomStore,
             extra_tables=[("deleted_pushers", "stream_id")],
         )
 
-        events_max = self._stream_id_gen.get_max_token()
+        events_max = self._stream_id_gen.get_current_token()
         event_cache_prefill, min_event_val = self._get_cache_dict(
             db_conn, "events",
             entity_column="room_id",
@@ -145,7 +139,7 @@ class DataStore(RoomMemberStore, RoomStore,
             "MembershipStreamChangeCache", events_max,
         )
 
-        account_max = self._account_data_id_gen.get_max_token()
+        account_max = self._account_data_id_gen.get_current_token()
         self._account_data_stream_cache = StreamChangeCache(
             "AccountDataAndTagsChangeCache", account_max,
         )
@@ -156,7 +150,7 @@ class DataStore(RoomMemberStore, RoomStore,
             db_conn, "presence_stream",
             entity_column="user_id",
             stream_column="stream_id",
-            max_value=self._presence_id_gen.get_max_token(),
+            max_value=self._presence_id_gen.get_current_token(),
         )
         self.presence_stream_cache = StreamChangeCache(
             "PresenceStreamChangeCache", min_presence_val,
@@ -167,7 +161,7 @@ class DataStore(RoomMemberStore, RoomStore,
             db_conn, "push_rules_stream",
             entity_column="user_id",
             stream_column="stream_id",
-            max_value=self._push_rules_stream_id_gen.get_max_token()[0],
+            max_value=self._push_rules_stream_id_gen.get_current_token()[0],
         )
 
         self.push_rules_stream_cache = StreamChangeCache(
diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py
index faddefe219..7a7fbf1e52 100644
--- a/synapse/storage/account_data.py
+++ b/synapse/storage/account_data.py
@@ -200,7 +200,7 @@ class AccountDataStore(SQLBaseStore):
                 "add_room_account_data", add_account_data_txn, next_id
             )
 
-        result = self._account_data_id_gen.get_max_token()
+        result = self._account_data_id_gen.get_current_token()
         defer.returnValue(result)
 
     @defer.inlineCallbacks
@@ -239,7 +239,7 @@ class AccountDataStore(SQLBaseStore):
                 "add_user_account_data", add_account_data_txn, next_id
             )
 
-        result = self._account_data_id_gen.get_max_token()
+        result = self._account_data_id_gen.get_current_token()
         defer.returnValue(result)
 
     def _update_max_stream_id(self, txn, next_id):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 83279d65fa..4ab23c1597 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -24,7 +24,6 @@ from synapse.util.logutils import log_function
 from synapse.api.constants import EventTypes
 
 from canonicaljson import encode_canonical_json
-from contextlib import contextmanager
 from collections import namedtuple
 
 import logging
@@ -66,14 +65,9 @@ class EventsStore(SQLBaseStore):
             return
 
         if backfilled:
-            start = self.min_stream_token - 1
-            self.min_stream_token -= len(events_and_contexts) + 1
-            stream_orderings = range(start, self.min_stream_token, -1)
-
-            @contextmanager
-            def stream_ordering_manager():
-                yield stream_orderings
-            stream_ordering_manager = stream_ordering_manager()
+            stream_ordering_manager = self._backfill_id_gen.get_next_mult(
+                len(events_and_contexts)
+            )
         else:
             stream_ordering_manager = self._stream_id_gen.get_next_mult(
                 len(events_and_contexts)
@@ -130,7 +124,7 @@ class EventsStore(SQLBaseStore):
         except _RollbackButIsFineException:
             pass
 
-        max_persisted_id = yield self._stream_id_gen.get_max_token()
+        max_persisted_id = yield self._stream_id_gen.get_current_token()
         defer.returnValue((stream_ordering, max_persisted_id))
 
     @defer.inlineCallbacks
@@ -1117,10 +1111,7 @@ class EventsStore(SQLBaseStore):
 
     def get_current_backfill_token(self):
         """The current minimum token that backfilled events have reached"""
-
-        # TODO: Fix race with the persit_event txn by using one of the
-        # stream id managers
-        return -self.min_stream_token
+        return -self._backfill_id_gen.get_current_token()
 
     def get_all_new_events(self, last_backfill_id, last_forward_id,
                            current_backfill_id, current_forward_id, limit):
diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py
index 4cec31e316..59b4ef5ce6 100644
--- a/synapse/storage/presence.py
+++ b/synapse/storage/presence.py
@@ -68,7 +68,9 @@ class PresenceStore(SQLBaseStore):
                 self._update_presence_txn, stream_orderings, presence_states,
             )
 
-        defer.returnValue((stream_orderings[-1], self._presence_id_gen.get_max_token()))
+        defer.returnValue((
+            stream_orderings[-1], self._presence_id_gen.get_current_token()
+        ))
 
     def _update_presence_txn(self, txn, stream_orderings, presence_states):
         for stream_id, state in zip(stream_orderings, presence_states):
@@ -155,7 +157,7 @@ class PresenceStore(SQLBaseStore):
         defer.returnValue([UserPresenceState(**row) for row in rows])
 
     def get_current_presence_token(self):
-        return self._presence_id_gen.get_max_token()
+        return self._presence_id_gen.get_current_token()
 
     def allow_presence_visible(self, observed_localpart, observer_userid):
         return self._simple_insert(
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 9dbad2fd5f..d2bf7f2aec 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -392,7 +392,7 @@ class PushRuleStore(SQLBaseStore):
         """Get the position of the push rules stream.
         Returns a pair of a stream id for the push_rules stream and the
         room stream ordering it corresponds to."""
-        return self._push_rules_stream_id_gen.get_max_token()
+        return self._push_rules_stream_id_gen.get_current_token()
 
     def have_push_rules_changed_for_user(self, user_id, last_id):
         if not self.push_rules_stream_cache.has_entity_changed(user_id, last_id):
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 87b2ac5773..d1669c778a 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -78,7 +78,7 @@ class PusherStore(SQLBaseStore):
         defer.returnValue(rows)
 
     def get_pushers_stream_token(self):
-        return self._pushers_id_gen.get_max_token()
+        return self._pushers_id_gen.get_current_token()
 
     def get_all_updated_pushers(self, last_id, current_id, limit):
         def get_all_updated_pushers_txn(txn):
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 6b9d848eaa..4befebc8e2 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -31,7 +31,7 @@ class ReceiptsStore(SQLBaseStore):
         super(ReceiptsStore, self).__init__(hs)
 
         self._receipts_stream_cache = StreamChangeCache(
-            "ReceiptsRoomChangeCache", self._receipts_id_gen.get_max_token()
+            "ReceiptsRoomChangeCache", self._receipts_id_gen.get_current_token()
         )
 
     @cached(num_args=2)
@@ -221,7 +221,7 @@ class ReceiptsStore(SQLBaseStore):
         defer.returnValue(results)
 
     def get_max_receipt_stream_id(self):
-        return self._receipts_id_gen.get_max_token()
+        return self._receipts_id_gen.get_current_token()
 
     def insert_linearized_receipt_txn(self, txn, room_id, receipt_type,
                                       user_id, event_id, data, stream_id):
@@ -346,7 +346,7 @@ class ReceiptsStore(SQLBaseStore):
             room_id, receipt_type, user_id, event_ids, data
         )
 
-        max_persisted_id = self._stream_id_gen.get_max_token()
+        max_persisted_id = self._stream_id_gen.get_current_token()
 
         defer.returnValue((stream_id, max_persisted_id))
 
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 7fc9a4f264..8644830657 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -458,4 +458,4 @@ class StateStore(SQLBaseStore):
         )
 
     def get_state_stream_token(self):
-        return self._state_groups_id_gen.get_max_token()
+        return self._state_groups_id_gen.get_current_token()
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index cf84938be5..76bcd9cd00 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -539,7 +539,7 @@ class StreamStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def get_room_events_max_id(self, direction='f'):
-        token = yield self._stream_id_gen.get_max_token()
+        token = yield self._stream_id_gen.get_current_token()
         if direction != 'b':
             defer.returnValue("s%d" % (token,))
         else:
diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py
index a0e6b42b30..9da23f34cb 100644
--- a/synapse/storage/tags.py
+++ b/synapse/storage/tags.py
@@ -30,7 +30,7 @@ class TagsStore(SQLBaseStore):
         Returns:
             A deferred int.
         """
-        return self._account_data_id_gen.get_max_token()
+        return self._account_data_id_gen.get_current_token()
 
     @cached()
     def get_tags_for_user(self, user_id):
@@ -200,7 +200,7 @@ class TagsStore(SQLBaseStore):
 
         self.get_tags_for_user.invalidate((user_id,))
 
-        result = self._account_data_id_gen.get_max_token()
+        result = self._account_data_id_gen.get_current_token()
         defer.returnValue(result)
 
     @defer.inlineCallbacks
@@ -222,7 +222,7 @@ class TagsStore(SQLBaseStore):
 
         self.get_tags_for_user.invalidate((user_id,))
 
-        result = self._account_data_id_gen.get_max_token()
+        result = self._account_data_id_gen.get_current_token()
         defer.returnValue(result)
 
     def _update_revision_txn(self, txn, user_id, room_id, next_id):
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index a02dfc7d58..03f2aa6a5c 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -21,7 +21,7 @@ import threading
 class IdGenerator(object):
     def __init__(self, db_conn, table, column):
         self._lock = threading.Lock()
-        self._next_id = _load_max_id(db_conn, table, column)
+        self._next_id = _load_current_id(db_conn, table, column)
 
     def get_next(self):
         with self._lock:
@@ -29,12 +29,16 @@ class IdGenerator(object):
             return self._next_id
 
 
-def _load_max_id(db_conn, table, column):
+def _load_current_id(db_conn, table, column, direction=1):
     cur = db_conn.cursor()
-    cur.execute("SELECT MAX(%s) FROM %s" % (column, table,))
+    if direction == 1:
+        cur.execute("SELECT MAX(%s) FROM %s" % (column, table,))
+    else:
+        cur.execute("SELECT MIN(%s) FROM %s" % (column, table,))
     val, = cur.fetchone()
     cur.close()
-    return int(val) if val else 1
+    current_id = int(val) if val else direction
+    return (max if direction == 1 else min)(current_id, direction)
 
 
 class StreamIdGenerator(object):
@@ -45,17 +49,30 @@ class StreamIdGenerator(object):
     all ids less than or equal to it have completed. This handles the fact that
     persistence of events can complete out of order.
 
+    :param connection db_conn:  A database connection to use to fetch the
+        initial value of the generator from.
+    :param str table: A database table to read the initial value of the id
+        generator from.
+    :param str column: The column of the database table to read the initial
+        value from the id generator from.
+    :param list extra_tables: List of pairs of database tables and columns to
+        use to source the initial value of the generator from. The value with
+        the largest magnitude is used.
+    :param int direction: which direction the stream ids grow in. +1 to grow
+        upwards, -1 to grow downwards.
+
     Usage:
         with stream_id_gen.get_next() as stream_id:
             # ... persist event ...
     """
-    def __init__(self, db_conn, table, column, extra_tables=[]):
+    def __init__(self, db_conn, table, column, extra_tables=[], direction=1):
         self._lock = threading.Lock()
-        self._current_max = _load_max_id(db_conn, table, column)
+        self._direction = direction
+        self._current = _load_current_id(db_conn, table, column, direction)
         for table, column in extra_tables:
-            self._current_max = max(
-                self._current_max,
-                _load_max_id(db_conn, table, column)
+            self._current = (max if direction > 0 else min)(
+                self._current,
+                _load_current_id(db_conn, table, column, direction)
             )
         self._unfinished_ids = deque()
 
@@ -66,8 +83,8 @@ class StreamIdGenerator(object):
                 # ... persist event ...
         """
         with self._lock:
-            self._current_max += 1
-            next_id = self._current_max
+            self._current += self._direction
+            next_id = self._current
 
             self._unfinished_ids.append(next_id)
 
@@ -88,8 +105,12 @@ class StreamIdGenerator(object):
                 # ... persist events ...
         """
         with self._lock:
-            next_ids = range(self._current_max + 1, self._current_max + n + 1)
-            self._current_max += n
+            next_ids = range(
+                self._current + self._direction,
+                self._current + self._direction * (n + 1),
+                self._direction
+            )
+            self._current += n
 
             for next_id in next_ids:
                 self._unfinished_ids.append(next_id)
@@ -105,15 +126,15 @@ class StreamIdGenerator(object):
 
         return manager()
 
-    def get_max_token(self):
+    def get_current_token(self):
         """Returns the maximum stream id such that all stream ids less than or
         equal to it have been successfully persisted.
         """
         with self._lock:
             if self._unfinished_ids:
-                return self._unfinished_ids[0] - 1
+                return self._unfinished_ids[0] - self._direction
 
-            return self._current_max
+            return self._current
 
 
 class ChainedIdGenerator(object):
@@ -125,7 +146,7 @@ class ChainedIdGenerator(object):
     def __init__(self, chained_generator, db_conn, table, column):
         self.chained_generator = chained_generator
         self._lock = threading.Lock()
-        self._current_max = _load_max_id(db_conn, table, column)
+        self._current_max = _load_current_id(db_conn, table, column)
         self._unfinished_ids = deque()
 
     def get_next(self):
@@ -137,7 +158,7 @@ class ChainedIdGenerator(object):
         with self._lock:
             self._current_max += 1
             next_id = self._current_max
-            chained_id = self.chained_generator.get_max_token()
+            chained_id = self.chained_generator.get_current_token()
 
             self._unfinished_ids.append((next_id, chained_id))
 
@@ -151,7 +172,7 @@ class ChainedIdGenerator(object):
 
         return manager()
 
-    def get_max_token(self):
+    def get_current_token(self):
         """Returns the maximum stream id such that all stream ids less than or
         equal to it have been successfully persisted.
         """
@@ -160,4 +181,4 @@ class ChainedIdGenerator(object):
                 stream_id, chained_id = self._unfinished_ids[0]
                 return (stream_id - 1, chained_id)
 
-            return (self._current_max, self.chained_generator.get_max_token())
+            return (self._current_max, self.chained_generator.get_current_token())
-- 
cgit 1.5.1


From a2866e2e6a8fa60a538a98f62e1733ab062020aa Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 13:50:54 +0100
Subject: Rename direction to step, apply checks consistently

---
 synapse/storage/__init__.py           |  2 +-
 synapse/storage/util/id_generators.py | 30 +++++++++++++++---------------
 2 files changed, 16 insertions(+), 16 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index f87e907cd8..57863bba4d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -97,7 +97,7 @@ class DataStore(RoomMemberStore, RoomStore,
             db_conn, "events", "stream_ordering"
         )
         self._backfill_id_gen = StreamIdGenerator(
-            db_conn, "events", "stream_ordering", direction=-1
+            db_conn, "events", "stream_ordering", step=-1
         )
         self._receipts_id_gen = StreamIdGenerator(
             db_conn, "receipts_linearized", "stream_id"
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index 03f2aa6a5c..310b7dc6ee 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -29,16 +29,16 @@ class IdGenerator(object):
             return self._next_id
 
 
-def _load_current_id(db_conn, table, column, direction=1):
+def _load_current_id(db_conn, table, column, step=1):
     cur = db_conn.cursor()
-    if direction == 1:
+    if step == 1:
         cur.execute("SELECT MAX(%s) FROM %s" % (column, table,))
     else:
         cur.execute("SELECT MIN(%s) FROM %s" % (column, table,))
     val, = cur.fetchone()
     cur.close()
-    current_id = int(val) if val else direction
-    return (max if direction == 1 else min)(current_id, direction)
+    current_id = int(val) if val else step
+    return (max if step > 0 else min)(current_id, step)
 
 
 class StreamIdGenerator(object):
@@ -58,21 +58,21 @@ class StreamIdGenerator(object):
     :param list extra_tables: List of pairs of database tables and columns to
         use to source the initial value of the generator from. The value with
         the largest magnitude is used.
-    :param int direction: which direction the stream ids grow in. +1 to grow
+    :param int step: which direction the stream ids grow in. +1 to grow
         upwards, -1 to grow downwards.
 
     Usage:
         with stream_id_gen.get_next() as stream_id:
             # ... persist event ...
     """
-    def __init__(self, db_conn, table, column, extra_tables=[], direction=1):
+    def __init__(self, db_conn, table, column, extra_tables=[], step=1):
         self._lock = threading.Lock()
-        self._direction = direction
-        self._current = _load_current_id(db_conn, table, column, direction)
+        self._step = step
+        self._current = _load_current_id(db_conn, table, column, step)
         for table, column in extra_tables:
-            self._current = (max if direction > 0 else min)(
+            self._current = (max if step > 0 else min)(
                 self._current,
-                _load_current_id(db_conn, table, column, direction)
+                _load_current_id(db_conn, table, column, step)
             )
         self._unfinished_ids = deque()
 
@@ -83,7 +83,7 @@ class StreamIdGenerator(object):
                 # ... persist event ...
         """
         with self._lock:
-            self._current += self._direction
+            self._current += self._step
             next_id = self._current
 
             self._unfinished_ids.append(next_id)
@@ -106,9 +106,9 @@ class StreamIdGenerator(object):
         """
         with self._lock:
             next_ids = range(
-                self._current + self._direction,
-                self._current + self._direction * (n + 1),
-                self._direction
+                self._current + self._step,
+                self._current + self._step * (n + 1),
+                self._step
             )
             self._current += n
 
@@ -132,7 +132,7 @@ class StreamIdGenerator(object):
         """
         with self._lock:
             if self._unfinished_ids:
-                return self._unfinished_ids[0] - self._direction
+                return self._unfinished_ids[0] - self._step
 
             return self._current
 
-- 
cgit 1.5.1


From 35b5c4ba1b1892fde18f531c96d71aa58de649e1 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 15:07:01 +0100
Subject: use google style doc strings

---
 synapse/storage/util/id_generators.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index 310b7dc6ee..58ea54cf67 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -49,17 +49,18 @@ class StreamIdGenerator(object):
     all ids less than or equal to it have completed. This handles the fact that
     persistence of events can complete out of order.
 
-    :param connection db_conn:  A database connection to use to fetch the
-        initial value of the generator from.
-    :param str table: A database table to read the initial value of the id
-        generator from.
-    :param str column: The column of the database table to read the initial
-        value from the id generator from.
-    :param list extra_tables: List of pairs of database tables and columns to
-        use to source the initial value of the generator from. The value with
-        the largest magnitude is used.
-    :param int step: which direction the stream ids grow in. +1 to grow
-        upwards, -1 to grow downwards.
+    Args:
+        db_conn(connection):  A database connection to use to fetch the
+            initial value of the generator from.
+        table(str): A database table to read the initial value of the id
+            generator from.
+        column(str): The column of the database table to read the initial
+            value from the id generator from.
+        extra_tables(list): List of pairs of database tables and columns to
+            use to source the initial value of the generator from. The value
+            with the largest magnitude is used.
+        step(int): which direction the stream ids grow in. +1 to grow
+            upwards, -1 to grow downwards.
 
     Usage:
         with stream_id_gen.get_next() as stream_id:
-- 
cgit 1.5.1


From 9bc5b4c663ed4bb35ef74a820c108765c7ca0f67 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 15:08:20 +0100
Subject: Assert that the step != 0

---
 synapse/storage/util/id_generators.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index 58ea54cf67..f69f1cdad4 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -67,6 +67,7 @@ class StreamIdGenerator(object):
             # ... persist event ...
     """
     def __init__(self, db_conn, table, column, extra_tables=[], step=1):
+        assert step != 0
         self._lock = threading.Lock()
         self._step = step
         self._current = _load_current_id(db_conn, table, column, step)
-- 
cgit 1.5.1


From 2a37467fa1358eb41513893efe44cbd294dca36c Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 1 Apr 2016 16:08:59 +0100
Subject: Use google style doc strings.

pycharm supports them so there is no need to use the other format.

Might as well convert the existing strings to reduce the risk of
people accidentally cargo culting the wrong doc string format.
---
 setup.cfg                             |  3 ++
 synapse/handlers/_base.py             | 27 +++++++-----
 synapse/handlers/auth.py              | 26 +++++++----
 synapse/handlers/federation.py        | 23 +++++-----
 synapse/handlers/room_member.py       | 48 ++++++++++-----------
 synapse/handlers/sync.py              | 49 +++++++++++++--------
 synapse/http/servlet.py               | 81 ++++++++++++++++++++++-------------
 synapse/notifier.py                   | 15 ++++---
 synapse/push/baserules.py             |  8 ++--
 synapse/rest/client/v2_alpha/sync.py  | 79 ++++++++++++++++++----------------
 synapse/state.py                      | 19 ++++----
 synapse/storage/event_push_actions.py |  5 ++-
 synapse/storage/registration.py       | 15 ++++---
 synapse/storage/state.py              | 13 +++---
 14 files changed, 242 insertions(+), 169 deletions(-)

(limited to 'synapse/storage')

diff --git a/setup.cfg b/setup.cfg
index f8cc13c840..5ebce1c56b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,3 +17,6 @@ ignore =
 [flake8]
 max-line-length = 90
 ignore = W503 ; W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it.
+
+[pep8]
+max-line-length = 90
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 90eabb6eb7..5601ecea6e 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -41,8 +41,9 @@ class BaseHandler(object):
     """
     Common base class for the event handlers.
 
-    :type store: synapse.storage.events.StateStore
-    :type state_handler: synapse.state.StateHandler
+    Attributes:
+        store (synapse.storage.events.StateStore):
+        state_handler (synapse.state.StateHandler):
     """
 
     def __init__(self, hs):
@@ -65,11 +66,12 @@ class BaseHandler(object):
         """ Returns dict of user_id -> list of events that user is allowed to
         see.
 
-        :param (str, bool) user_tuples: (user id, is_peeking) for each
-            user to be checked. is_peeking should be true if:
-              * the user is not currently a member of the room, and:
-              * the user has not been a member of the room since the given
-                events
+        Args:
+            user_tuples (str, bool): (user id, is_peeking) for each user to be
+                checked. is_peeking should be true if:
+                * the user is not currently a member of the room, and:
+                * the user has not been a member of the room since the
+                given events
         """
         forgotten = yield defer.gatherResults([
             self.store.who_forgot_in_room(
@@ -165,13 +167,16 @@ class BaseHandler(object):
         """
         Check which events a user is allowed to see
 
-        :param str user_id: user id to be checked
-        :param [synapse.events.EventBase] events: list of events to be checked
-        :param bool is_peeking should be True if:
+        Args:
+            user_id(str): user id to be checked
+            events([synapse.events.EventBase]): list of events to be checked
+            is_peeking(bool): should be True if:
               * the user is not currently a member of the room, and:
               * the user has not been a member of the room since the given
                 events
-        :rtype [synapse.events.EventBase]
+
+        Returns:
+            [synapse.events.EventBase]
         """
         types = (
             (EventTypes.RoomHistoryVisibility, ""),
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 82d458b424..d5d6faa85f 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -163,9 +163,13 @@ class AuthHandler(BaseHandler):
     def get_session_id(self, clientdict):
         """
         Gets the session ID for a client given the client dictionary
-        :param clientdict: The dictionary sent by the client in the request
-        :return: The string session ID the client sent. If the client did not
-                 send a session ID, returns None.
+
+        Args:
+            clientdict: The dictionary sent by the client in the request
+
+        Returns:
+            str|None: The string session ID the client sent. If the client did
+                not send a session ID, returns None.
         """
         sid = None
         if clientdict and 'auth' in clientdict:
@@ -179,9 +183,11 @@ class AuthHandler(BaseHandler):
         Store a key-value pair into the sessions data associated with this
         request. This data is stored server-side and cannot be modified by
         the client.
-        :param session_id: (string) The ID of this session as returned from check_auth
-        :param key: (string) The key to store the data under
-        :param value: (any) The data to store
+
+        Args:
+            session_id (string): The ID of this session as returned from check_auth
+            key (string): The key to store the data under
+            value (any): The data to store
         """
         sess = self._get_session_info(session_id)
         sess.setdefault('serverdict', {})[key] = value
@@ -190,9 +196,11 @@ class AuthHandler(BaseHandler):
     def get_session_data(self, session_id, key, default=None):
         """
         Retrieve data stored with set_session_data
-        :param session_id: (string) The ID of this session as returned from check_auth
-        :param key: (string) The key to store the data under
-        :param default: (any) Value to return if the key has not been set
+
+        Args:
+            session_id (string): The ID of this session as returned from check_auth
+            key (string): The key to store the data under
+            default (any): Value to return if the key has not been set
         """
         sess = self._get_session_info(session_id)
         return sess.setdefault('serverdict', {}).get(key, default)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 4a35344d32..092802b973 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1706,13 +1706,15 @@ class FederationHandler(BaseHandler):
     def _check_signature(self, event, auth_events):
         """
         Checks that the signature in the event is consistent with its invite.
-        :param event (Event): The m.room.member event to check
-        :param auth_events (dict<(event type, state_key), event>)
 
-        :raises
-            AuthError if signature didn't match any keys, or key has been
+        Args:
+            event (Event): The m.room.member event to check
+            auth_events (dict<(event type, state_key), event>):
+
+        Raises:
+            AuthError: if signature didn't match any keys, or key has been
                 revoked,
-            SynapseError if a transient error meant a key couldn't be checked
+            SynapseError: if a transient error meant a key couldn't be checked
                 for revocation.
         """
         signed = event.content["third_party_invite"]["signed"]
@@ -1754,12 +1756,13 @@ class FederationHandler(BaseHandler):
         """
         Checks whether public_key has been revoked.
 
-        :param public_key (str): base-64 encoded public key.
-        :param url (str): Key revocation URL.
+        Args:
+            public_key (str): base-64 encoded public key.
+            url (str): Key revocation URL.
 
-        :raises
-            AuthError if they key has been revoked.
-            SynapseError if a transient error meant a key couldn't be checked
+        Raises:
+            AuthError: if they key has been revoked.
+            SynapseError: if a transient error meant a key couldn't be checked
                 for revocation.
         """
         try:
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 5fdbd3adcc..01f833c371 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -411,7 +411,7 @@ class RoomMemberHandler(BaseHandler):
             address (str): The third party identifier (e.g. "foo@example.com").
 
         Returns:
-            (str) the matrix ID of the 3pid, or None if it is not recognized.
+            str: the matrix ID of the 3pid, or None if it is not recognized.
         """
         try:
             data = yield self.hs.get_simple_http_client().get_json(
@@ -545,29 +545,29 @@ class RoomMemberHandler(BaseHandler):
         """
         Asks an identity server for a third party invite.
 
-        :param id_server (str): hostname + optional port for the identity server.
-        :param medium (str): The literal string "email".
-        :param address (str): The third party address being invited.
-        :param room_id (str): The ID of the room to which the user is invited.
-        :param inviter_user_id (str): The user ID of the inviter.
-        :param room_alias (str): An alias for the room, for cosmetic
-            notifications.
-        :param room_avatar_url (str): The URL of the room's avatar, for cosmetic
-            notifications.
-        :param room_join_rules (str): The join rules of the email
-            (e.g. "public").
-        :param room_name (str): The m.room.name of the room.
-        :param inviter_display_name (str): The current display name of the
-            inviter.
-        :param inviter_avatar_url (str): The URL of the inviter's avatar.
-
-        :return: A deferred tuple containing:
-            token (str): The token which must be signed to prove authenticity.
-            public_keys ([{"public_key": str, "key_validity_url": str}]):
-                public_key is a base64-encoded ed25519 public key.
-            fallback_public_key: One element from public_keys.
-            display_name (str): A user-friendly name to represent the invited
-                user.
+        Args:
+            id_server (str): hostname + optional port for the identity server.
+            medium (str): The literal string "email".
+            address (str): The third party address being invited.
+            room_id (str): The ID of the room to which the user is invited.
+            inviter_user_id (str): The user ID of the inviter.
+            room_alias (str): An alias for the room, for cosmetic notifications.
+            room_avatar_url (str): The URL of the room's avatar, for cosmetic
+                notifications.
+            room_join_rules (str): The join rules of the email (e.g. "public").
+            room_name (str): The m.room.name of the room.
+            inviter_display_name (str): The current display name of the
+                inviter.
+            inviter_avatar_url (str): The URL of the inviter's avatar.
+
+        Returns:
+            A deferred tuple containing:
+                token (str): The token which must be signed to prove authenticity.
+                public_keys ([{"public_key": str, "key_validity_url": str}]):
+                    public_key is a base64-encoded ed25519 public key.
+                fallback_public_key: One element from public_keys.
+                display_name (str): A user-friendly name to represent the invited
+                    user.
         """
 
         is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 48ab5707e1..20a0626574 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -671,7 +671,8 @@ class SyncHandler(BaseHandler):
     def load_filtered_recents(self, room_id, sync_config, now_token,
                               since_token=None, recents=None, newly_joined_room=False):
         """
-        :returns a Deferred TimelineBatch
+        Returns:
+            a Deferred TimelineBatch
         """
         with Measure(self.clock, "load_filtered_recents"):
             filtering_factor = 2
@@ -838,8 +839,11 @@ class SyncHandler(BaseHandler):
         """
         Get the room state after the given event
 
-        :param synapse.events.EventBase event: event of interest
-        :return: A Deferred map from ((type, state_key)->Event)
+        Args:
+            event(synapse.events.EventBase): event of interest
+
+        Returns:
+            A Deferred map from ((type, state_key)->Event)
         """
         state = yield self.store.get_state_for_event(event.event_id)
         if event.is_state():
@@ -850,9 +854,13 @@ class SyncHandler(BaseHandler):
     @defer.inlineCallbacks
     def get_state_at(self, room_id, stream_position):
         """ Get the room state at a particular stream position
-        :param str room_id: room for which to get state
-        :param StreamToken stream_position: point at which to get state
-        :returns: A Deferred map from ((type, state_key)->Event)
+
+        Args:
+            room_id(str): room for which to get state
+            stream_position(StreamToken): point at which to get state
+
+        Returns:
+            A Deferred map from ((type, state_key)->Event)
         """
         last_events, token = yield self.store.get_recent_events_for_room(
             room_id, end_token=stream_position.room_key, limit=1,
@@ -873,15 +881,18 @@ class SyncHandler(BaseHandler):
         """ Works out the differnce in state between the start of the timeline
         and the previous sync.
 
-        :param str room_id
-        :param TimelineBatch batch: The timeline batch for the room that will
-            be sent to the user.
-        :param sync_config
-        :param str since_token: Token of the end of the previous batch. May be None.
-        :param str now_token: Token of the end of the current batch.
-        :param bool full_state: Whether to force returning the full state.
+        Args:
+            room_id(str):
+            batch(synapse.handlers.sync.TimelineBatch): The timeline batch for
+                the room that will be sent to the user.
+            sync_config(synapse.handlers.sync.SyncConfig):
+            since_token(str|None): Token of the end of the previous batch. May
+                be None.
+            now_token(str): Token of the end of the current batch.
+            full_state(bool): Whether to force returning the full state.
 
-        :returns A new event dictionary
+        Returns:
+             A deferred new event dictionary
         """
         # TODO(mjark) Check if the state events were received by the server
         # after the previous sync, since we need to include those state
@@ -953,11 +964,13 @@ class SyncHandler(BaseHandler):
         Check if the user has just joined the given room (so should
         be given the full state)
 
-        :param sync_config:
-        :param dict[(str,str), synapse.events.FrozenEvent] state_delta: the
-           difference in state since the last sync
+        Args:
+            sync_config(synapse.handlers.sync.SyncConfig):
+            state_delta(dict[(str,str), synapse.events.FrozenEvent]): the
+                difference in state since the last sync
 
-        :returns A deferred Tuple (state_delta, limited)
+        Returns:
+             A deferred Tuple (state_delta, limited)
         """
         join_event = state_delta.get((
             EventTypes.Member, sync_config.user.to_string()), None)
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 1c8bd8666f..e41afeab8e 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -26,14 +26,19 @@ logger = logging.getLogger(__name__)
 def parse_integer(request, name, default=None, required=False):
     """Parse an integer parameter from the request string
 
-    :param request: the twisted HTTP request.
-    :param name (str): the name of the query parameter.
-    :param default: value to use if the parameter is absent, defaults to None.
-    :param required (bool): whether to raise a 400 SynapseError if the
-        parameter is absent, defaults to False.
-    :return: An int value or the default.
-    :raises
-        SynapseError if the parameter is absent and required, or if the
+    Args:
+        request: the twisted HTTP request.
+        name (str): the name of the query parameter.
+        default (int|None): value to use if the parameter is absent, defaults
+            to None.
+        required (bool): whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+
+    Returns:
+        int|None: An int value or the default.
+
+    Raises:
+        SynapseError: if the parameter is absent and required, or if the
             parameter is present and not an integer.
     """
     if name in request.args:
@@ -53,14 +58,19 @@ def parse_integer(request, name, default=None, required=False):
 def parse_boolean(request, name, default=None, required=False):
     """Parse a boolean parameter from the request query string
 
-    :param request: the twisted HTTP request.
-    :param name (str): the name of the query parameter.
-    :param default: value to use if the parameter is absent, defaults to None.
-    :param required (bool): whether to raise a 400 SynapseError if the
-        parameter is absent, defaults to False.
-    :return: A bool value or the default.
-    :raises
-        SynapseError if the parameter is absent and required, or if the
+    Args:
+        request: the twisted HTTP request.
+        name (str): the name of the query parameter.
+        default (bool|None): value to use if the parameter is absent, defaults
+            to None.
+        required (bool): whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+
+    Returns:
+        bool|None: A bool value or the default.
+
+    Raises:
+        SynapseError: if the parameter is absent and required, or if the
             parameter is present and not one of "true" or "false".
     """
 
@@ -88,15 +98,20 @@ def parse_string(request, name, default=None, required=False,
                  allowed_values=None, param_type="string"):
     """Parse a string parameter from the request query string.
 
-    :param request: the twisted HTTP request.
-    :param name (str): the name of the query parameter.
-    :param default: value to use if the parameter is absent, defaults to None.
-    :param required (bool): whether to raise a 400 SynapseError if the
-        parameter is absent, defaults to False.
-    :param allowed_values (list): List of allowed values for the string,
-        or None if any value is allowed, defaults to None
-    :return: A string value or the default.
-    :raises
+    Args:
+        request: the twisted HTTP request.
+        name (str): the name of the query parameter.
+        default (str|None): value to use if the parameter is absent, defaults
+            to None.
+        required (bool): whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+        allowed_values (list[str]): List of allowed values for the string,
+            or None if any value is allowed, defaults to None
+
+    Returns:
+        str|None: A string value or the default.
+
+    Raises:
         SynapseError if the parameter is absent and required, or if the
             parameter is present, must be one of a list of allowed values and
             is not one of those allowed values.
@@ -122,9 +137,13 @@ def parse_string(request, name, default=None, required=False,
 def parse_json_value_from_request(request):
     """Parse a JSON value from the body of a twisted HTTP request.
 
-    :param request: the twisted HTTP request.
-    :returns: The JSON value.
-    :raises
+    Args:
+        request: the twisted HTTP request.
+
+    Returns:
+        The JSON value.
+
+    Raises:
         SynapseError if the request body couldn't be decoded as JSON.
     """
     try:
@@ -143,8 +162,10 @@ def parse_json_value_from_request(request):
 def parse_json_object_from_request(request):
     """Parse a JSON object from the body of a twisted HTTP request.
 
-    :param request: the twisted HTTP request.
-    :raises
+    Args:
+        request: the twisted HTTP request.
+
+    Raises:
         SynapseError if the request body couldn't be decoded as JSON or
             if it wasn't a JSON object.
     """
diff --git a/synapse/notifier.py b/synapse/notifier.py
index f00cd8c588..6af7a8f424 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -503,13 +503,14 @@ class Notifier(object):
     def wait_for_replication(self, callback, timeout):
         """Wait for an event to happen.
 
-        :param callback:
-            Gets called whenever an event happens. If this returns a truthy
-            value then ``wait_for_replication`` returns, otherwise it waits
-            for another event.
-        :param int timeout:
-            How many milliseconds to wait for callback return a truthy value.
-        :returns:
+        Args:
+            callback: Gets called whenever an event happens. If this returns a
+                truthy value then ``wait_for_replication`` returns, otherwise
+                it waits for another event.
+            timeout: How many milliseconds to wait for callback return a truthy
+                value.
+
+        Returns:
             A deferred that resolves with the value returned by the callback.
         """
         listener = _NotificationListener(None)
diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py
index 792af70eb7..6add94beeb 100644
--- a/synapse/push/baserules.py
+++ b/synapse/push/baserules.py
@@ -19,9 +19,11 @@ import copy
 def list_with_base_rules(rawrules):
     """Combine the list of rules set by the user with the default push rules
 
-    :param list rawrules: The rules the user has modified or set.
-    :returns: A new list with the rules set by the user combined with the
-        defaults.
+    Args:
+        rawrules(list): The rules the user has modified or set.
+
+    Returns:
+        A new list with the rules set by the user combined with the defaults.
     """
     ruleslist = []
 
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index c5785d7074..60d3dc4030 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -199,15 +199,17 @@ class SyncRestServlet(RestServlet):
         """
         Encode the joined rooms in a sync result
 
-        :param list[synapse.handlers.sync.JoinedSyncResult] rooms: list of sync
-            results for rooms this user is joined to
-        :param int time_now: current time - used as a baseline for age
-            calculations
-        :param int token_id: ID of the user's auth token - used for namespacing
-            of transaction IDs
-
-        :return: the joined rooms list, in our response format
-        :rtype: dict[str, dict[str, object]]
+        Args:
+            rooms(list[synapse.handlers.sync.JoinedSyncResult]): list of sync
+                results for rooms this user is joined to
+            time_now(int): current time - used as a baseline for age
+                calculations
+            token_id(int): ID of the user's auth token - used for namespacing
+                of transaction IDs
+
+        Returns:
+            dict[str, dict[str, object]]: the joined rooms list, in our
+                response format
         """
         joined = {}
         for room in rooms:
@@ -221,15 +223,17 @@ class SyncRestServlet(RestServlet):
         """
         Encode the invited rooms in a sync result
 
-        :param list[synapse.handlers.sync.InvitedSyncResult] rooms: list of
-             sync results for rooms this user is joined to
-        :param int time_now: current time - used as a baseline for age
-            calculations
-        :param int token_id: ID of the user's auth token - used for namespacing
+        Args:
+            rooms(list[synapse.handlers.sync.InvitedSyncResult]): list of
+                sync results for rooms this user is joined to
+            time_now(int): current time - used as a baseline for age
+                calculations
+            token_id(int): ID of the user's auth token - used for namespacing
             of transaction IDs
 
-        :return: the invited rooms list, in our response format
-        :rtype: dict[str, dict[str, object]]
+        Returns:
+            dict[str, dict[str, object]]: the invited rooms list, in our
+                response format
         """
         invited = {}
         for room in rooms:
@@ -251,15 +255,17 @@ class SyncRestServlet(RestServlet):
         """
         Encode the archived rooms in a sync result
 
-        :param list[synapse.handlers.sync.ArchivedSyncResult] rooms: list of
-             sync results for rooms this user is joined to
-        :param int time_now: current time - used as a baseline for age
-            calculations
-        :param int token_id: ID of the user's auth token - used for namespacing
-            of transaction IDs
-
-        :return: the invited rooms list, in our response format
-        :rtype: dict[str, dict[str, object]]
+        Args:
+            rooms (list[synapse.handlers.sync.ArchivedSyncResult]): list of
+                sync results for rooms this user is joined to
+            time_now(int): current time - used as a baseline for age
+                calculations
+            token_id(int): ID of the user's auth token - used for namespacing
+                of transaction IDs
+
+        Returns:
+            dict[str, dict[str, object]]: The invited rooms list, in our
+                response format
         """
         joined = {}
         for room in rooms:
@@ -272,17 +278,18 @@ class SyncRestServlet(RestServlet):
     @staticmethod
     def encode_room(room, time_now, token_id, joined=True):
         """
-        :param JoinedSyncResult|ArchivedSyncResult room: sync result for a
-            single room
-        :param int time_now: current time - used as a baseline for age
-            calculations
-        :param int token_id: ID of the user's auth token - used for namespacing
-            of transaction IDs
-        :param joined: True if the user is joined to this room - will mean
-            we handle ephemeral events
-
-        :return: the room, encoded in our response format
-        :rtype: dict[str, object]
+        Args:
+            room (JoinedSyncResult|ArchivedSyncResult): sync result for a
+                single room
+            time_now (int): current time - used as a baseline for age
+                calculations
+            token_id (int): ID of the user's auth token - used for namespacing
+                of transaction IDs
+            joined (bool): True if the user is joined to this room - will mean
+                we handle ephemeral events
+
+        Returns:
+            dict[str, object]: the room, encoded in our response format
         """
         def serialize(event):
             # TODO(mjark): Respect formatting requirements in the filter.
diff --git a/synapse/state.py b/synapse/state.py
index 41d32e664a..4a9e148de7 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -86,7 +86,8 @@ class StateHandler(object):
         If `event_type` is specified, then the method returns only the one
         event (or None) with that `event_type` and `state_key`.
 
-        :returns map from (type, state_key) to event
+        Returns:
+            map from (type, state_key) to event
         """
         event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
 
@@ -176,10 +177,11 @@ class StateHandler(object):
         """ Given a list of event_ids this method fetches the state at each
         event, resolves conflicts between them and returns them.
 
-        :returns a Deferred tuple of (`state_group`, `state`, `prev_state`).
-        `state_group` is the name of a state group if one and only one is
-        involved. `state` is a map from (type, state_key) to event, and
-        `prev_state` is a list of event ids.
+        Returns:
+            a Deferred tuple of (`state_group`, `state`, `prev_state`).
+            `state_group` is the name of a state group if one and only one is
+            involved. `state` is a map from (type, state_key) to event, and
+            `prev_state` is a list of event ids.
         """
         logger.debug("resolve_state_groups event_ids %s", event_ids)
 
@@ -251,9 +253,10 @@ class StateHandler(object):
 
     def _resolve_events(self, state_sets, event_type=None, state_key=""):
         """
-        :returns a tuple (new_state, prev_states). new_state is a map
-        from (type, state_key) to event. prev_states is a list of event_ids.
-        :rtype: (dict[(str, str), synapse.events.FrozenEvent], list[str])
+        Returns
+            (dict[(str, str), synapse.events.FrozenEvent], list[str]): a tuple
+            (new_state, prev_states). new_state is a map from (type, state_key)
+            to event. prev_states is a list of event_ids.
         """
         with Measure(self.clock, "state._resolve_events"):
             state = {}
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index dc5830450a..3933b6e2c5 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -26,8 +26,9 @@ logger = logging.getLogger(__name__)
 class EventPushActionsStore(SQLBaseStore):
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):
         """
-        :param event: the event set actions for
-        :param tuples: list of tuples of (user_id, actions)
+        Args:
+            event: the event set actions for
+            tuples: list of tuples of (user_id, actions)
         """
         values = []
         for uid, actions in tuples:
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index bd4eb88a92..d46a963bb8 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -458,12 +458,15 @@ class RegistrationStore(SQLBaseStore):
         """
         Gets the 3pid's guest access token if exists, else saves access_token.
 
-        :param medium (str): Medium of the 3pid. Must be "email".
-        :param address (str): 3pid address.
-        :param access_token (str): The access token to persist if none is
-            already persisted.
-        :param inviter_user_id (str): User ID of the inviter.
-        :return (deferred str): Whichever access token is persisted at the end
+        Args:
+            medium (str): Medium of the 3pid. Must be "email".
+            address (str): 3pid address.
+            access_token (str): The access token to persist if none is
+                already persisted.
+            inviter_user_id (str): User ID of the inviter.
+
+        Returns:
+            deferred str: Whichever access token is persisted at the end
             of this function call.
         """
         def insert(txn):
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 7fc9a4f264..f84fd0e30a 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -249,11 +249,14 @@ class StateStore(SQLBaseStore):
         """
         Get the state dict corresponding to a particular event
 
-        :param str event_id: event whose state should be returned
-        :param list[(str, str)]|None types: List of (type, state_key) tuples
-            which are used to filter the state fetched. May be None, which
-            matches any key
-        :return: a deferred dict from (type, state_key) -> state_event
+        Args:
+            event_id(str): event whose state should be returned
+            types(list[(str, str)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. May be None, which
+                matches any key
+
+        Returns:
+            A deferred dict from (type, state_key) -> state_event
         """
         state_map = yield self.get_state_for_events([event_id], types)
         defer.returnValue(state_map[event_id])
-- 
cgit 1.5.1


From d76d89323c4b962b4f3ff72a3c9b40f2d2d347b3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 1 Apr 2016 17:39:32 +0100
Subject: Use computed prev event ids

---
 synapse/handlers/_base.py           | 29 +++++++++++++++++------------
 synapse/handlers/message.py         |  6 +++++-
 synapse/handlers/room_member.py     |  3 +++
 synapse/storage/event_federation.py | 16 ++++++++++++++++
 4 files changed, 41 insertions(+), 13 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 2d8ff79a9e..242afa1564 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -199,20 +199,25 @@ class BaseHandler(object):
             )
 
     @defer.inlineCallbacks
-    def _create_new_client_event(self, builder):
-        latest_ret = yield self.store.get_latest_event_ids_and_hashes_in_room(
-            builder.room_id,
-        )
-
-        if latest_ret:
-            depth = max([d for _, _, d in latest_ret]) + 1
+    def _create_new_client_event(self, builder, prev_event_ids=None):
+        if prev_event_ids:
+            prev_events = yield self.store.add_event_hashes(prev_event_ids)
+            prev_max_depth = yield self.store.get_max_depth_of_events(prev_event_ids)
+            depth = prev_max_depth + 1
         else:
-            depth = 1
+            latest_ret = yield self.store.get_latest_event_ids_and_hashes_in_room(
+                builder.room_id,
+            )
 
-        prev_events = [
-            (event_id, prev_hashes)
-            for event_id, prev_hashes, _ in latest_ret
-        ]
+            if latest_ret:
+                depth = max([d for _, _, d in latest_ret]) + 1
+            else:
+                depth = 1
+
+            prev_events = [
+                (event_id, prev_hashes)
+                for event_id, prev_hashes, _ in latest_ret
+            ]
 
         builder.prev_events = prev_events
         builder.depth = depth
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 0bb111d047..10608c0dd9 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -176,7 +176,7 @@ class MessageHandler(BaseHandler):
         defer.returnValue(chunk)
 
     @defer.inlineCallbacks
-    def create_event(self, event_dict, token_id=None, txn_id=None):
+    def create_event(self, event_dict, token_id=None, txn_id=None, prev_event_ids=None):
         """
         Given a dict from a client, create a new event.
 
@@ -187,6 +187,9 @@ class MessageHandler(BaseHandler):
 
         Args:
             event_dict (dict): An entire event
+            token_id (str)
+            txn_id (str)
+            prev_event_ids (list): The prev event ids to use when creating the event
 
         Returns:
             Tuple of created event (FrozenEvent), Context
@@ -225,6 +228,7 @@ class MessageHandler(BaseHandler):
 
         event, context = yield self._create_new_client_event(
             builder=builder,
+            prev_event_ids=prev_event_ids,
         )
         defer.returnValue((event, context))
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 09b8f6217a..7c1bb8cfe4 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -98,6 +98,7 @@ class RoomMemberHandler(BaseHandler):
     @defer.inlineCallbacks
     def _local_membership_update(
         self, requester, target, room_id, membership,
+        prev_event_ids,
         txn_id=None,
         ratelimit=True,
     ):
@@ -120,6 +121,7 @@ class RoomMemberHandler(BaseHandler):
             },
             token_id=requester.access_token_id,
             txn_id=txn_id,
+            prev_event_ids=prev_event_ids,
         )
 
         yield self.handle_new_client_event(
@@ -268,6 +270,7 @@ class RoomMemberHandler(BaseHandler):
             membership=effective_membership_state,
             txn_id=txn_id,
             ratelimit=ratelimit,
+            prev_event_ids=latest_event_ids,
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 3489315e0d..0827946207 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -163,6 +163,22 @@ class EventFederationStore(SQLBaseStore):
             room_id,
         )
 
+    @defer.inlineCallbacks
+    def get_max_depth_of_events(self, event_ids):
+        sql = (
+            "SELECT MAX(depth) FROM events WHERE event_id IN (%s)"
+        ) % (",".join(["?"] * len(event_ids)),)
+
+        rows = yield self._execute(
+            "get_max_depth_of_events", None,
+            sql, *event_ids
+        )
+
+        if rows:
+            defer.returnValue(rows[0][0])
+        else:
+            defer.returnValue(1)
+
     def _get_min_depth_interaction(self, txn, room_id):
         min_depth = self._simple_select_one_onecol_txn(
             txn,
-- 
cgit 1.5.1


From 7426c86eb88a7abef9af7ba544ccd709b25e8304 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 3 Apr 2016 00:31:57 +0100
Subject: add a persistent cache of URL lookups, and fix up the in-memory one
 to work

---
 synapse/http/client.py                             |  6 +-
 synapse/rest/media/v1/preview_url_resource.py      | 64 ++++++++++++++++++----
 synapse/storage/media_repository.py                | 54 +++++++++++++++++-
 .../delta/30/local_media_repository_url_cache.sql  | 27 +++++++++
 4 files changed, 137 insertions(+), 14 deletions(-)
 create mode 100644 synapse/storage/schema/delta/30/local_media_repository_url_cache.sql

(limited to 'synapse/storage')

diff --git a/synapse/http/client.py b/synapse/http/client.py
index b21bf17378..f42a36ffa6 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -251,8 +251,8 @@ class SimpleHttpClient(object):
             url (str): The URL to GET
             output_stream (file): File to write the response body to.
         Returns:
-            A (int,dict) tuple of the file length and a dict of the response
-            headers.
+            A (int,dict,string,int) tuple of the file length, dict of the response
+            headers, absolute URI of the response and HTTP response code.
         """
 
         response = yield self.request(
@@ -287,7 +287,7 @@ class SimpleHttpClient(object):
             logger.exception("Failed to download body")
             raise
 
-        defer.returnValue((length, headers, response.request.absoluteURI))
+        defer.returnValue((length, headers, response.request.absoluteURI, response.code))
 
 
 # XXX: FIXME: This is horribly copy-pasted from matrixfederationclient.
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 162e09ba71..86341cc4cc 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -37,6 +37,8 @@ class PreviewUrlResource(BaseMediaResource):
     def __init__(self, hs, filepaths):
         BaseMediaResource.__init__(self, hs, filepaths)
         self.client = SpiderHttpClient(hs)
+
+        # simple memory cache mapping urls to OG metadata
         self.cache = ExpiringCache(
             cache_name = "url_previews",
             clock = self.clock,
@@ -56,17 +58,41 @@ class PreviewUrlResource(BaseMediaResource):
             # XXX: if get_user_by_req fails, what should we do in an async render?
             requester = yield self.auth.get_user_by_req(request)
             url = request.args.get("url")[0]
-
-            if self.cache:
-                og = self.cache.get(url)
-                respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
-                return
+            ts = request.args.get("ts")[0] if "ts" in request.args else self.clock.time_msec()
 
             # TODO: keep track of whether there's an ongoing request for this preview
             # and block and return their details if there is one.
 
+            # first check the memory cache - good to handle all the clients on this
+            # HS thundering away to preview the same URL at the same time.
+            try:
+                og = self.cache[url]
+                respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
+                return
+            except:
+                pass
+
+            # then check the URL cache in the DB (which will also provide us with
+            # historical previews, if we have any)
+            cache_result = yield self.store.get_url_cache(url, ts)
+            if (
+                cache_result and
+                cache_result["download_ts"] + cache_result["expires"] > ts and
+                cache_result["response_code"] / 100 == 2
+            ):
+                respond_with_json_bytes(
+                    request, 200, cache_result["og"].encode('utf-8'),
+                    send_cors=True
+                )
+                return
+
             media_info = yield self._download_url(url, requester.user)
 
+            # FIXME: we should probably update our cache now anyway, so that
+            # even if the OG calculation raises, we don't keep hammering on the
+            # remote server.  For now, leave it uncached to aid debugging OG
+            # calculation problems
+
             logger.debug("got media_info of '%s'" % media_info)
 
             if self._is_media(media_info['media_type']):
@@ -105,10 +131,21 @@ class PreviewUrlResource(BaseMediaResource):
                 logger.warn("Failed to find any OG data in %s", url)
                 og = {}
 
-            if self.cache:
-                self.cache[url] = og
+            logger.debug("Calculated OG for %s as %s" % (url, og));
+
+            # store OG in ephemeral in-memory cache
+            self.cache[url] = og
 
-            logger.warn(og);
+            # store OG in history-aware DB cache
+            yield self.store.store_url_cache(
+                url,
+                media_info["response_code"],
+                media_info["etag"],
+                media_info["expires"],
+                json.dumps(og),
+                media_info["filesystem_id"],
+                media_info["created_ts"],
+            )
 
             respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
         except:
@@ -187,6 +224,9 @@ class PreviewUrlResource(BaseMediaResource):
                     og['og:image'] = self._rebase_url(images[0].attrib['src'], media_info['uri'])
 
         # pre-cache the image for posterity
+        # FIXME: it might be cleaner to use the same flow as the main /preview_url request itself
+        # and benefit from the same caching etc.  But for now we just rely on the caching
+        # of the master request to speed things up.
         if 'og:image' in og and og['og:image']:
             image_info = yield self._download_url(og['og:image'], requester.user)
 
@@ -226,7 +266,6 @@ class PreviewUrlResource(BaseMediaResource):
                 text = text.strip()[:500]
                 og['og:description'] = text if text else None
 
-        # TODO: persist a cache mapping { url, etag } -> { og, mxc of url (if we bother keeping it around), age }
         # TODO: delete the url downloads to stop diskfilling, as we only ever cared about its OG
         defer.returnValue(og);
 
@@ -256,7 +295,7 @@ class PreviewUrlResource(BaseMediaResource):
         try:
             with open(fname, "wb") as f:
                 logger.debug("Trying to get url '%s'" % url)
-                length, headers, uri = yield self.client.get_file(
+                length, headers, uri, code = yield self.client.get_file(
                     url, output_stream=f, max_size=self.max_spider_size,
                 )
                 # FIXME: pass through 404s and other error messages nicely
@@ -311,6 +350,11 @@ class PreviewUrlResource(BaseMediaResource):
             "filesystem_id": file_id,
             "filename": fname,
             "uri": uri,
+            "response_code": code,
+            # FIXME: we should calculate a proper expiration based on the
+            # Cache-Control and Expire headers.  But for now, assume 1 hour.
+            "expires": 60 * 60 * 1000,
+            "etag": headers["ETag"] if "ETag" in headers else None,
         })
 
     def _is_media(self, content_type):
diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py
index 9d3ba32478..bb002081ae 100644
--- a/synapse/storage/media_repository.py
+++ b/synapse/storage/media_repository.py
@@ -25,7 +25,7 @@ class MediaRepositoryStore(SQLBaseStore):
     def get_local_media(self, media_id):
         """Get the metadata for a local piece of media
         Returns:
-            None if the meia_id doesn't exist.
+            None if the media_id doesn't exist.
         """
         return self._simple_select_one(
             "local_media_repository",
@@ -50,6 +50,58 @@ class MediaRepositoryStore(SQLBaseStore):
             desc="store_local_media",
         )
 
+    def get_url_cache(self, url, ts):
+        """Get the media_id and ts for a cached URL as of the given timestamp
+        Returns:
+            None if the URL isn't cached.
+        """
+        def get_url_cache_txn(txn):
+            # get the most recently cached result (relative to the given ts)
+            sql = (
+                "SELECT response_code, etag, expires, og, media_id, max(download_ts)"
+                " FROM local_media_repository_url_cache"
+                " WHERE url = ? AND download_ts <= ?"
+            )
+            txn.execute(sql, (url, ts))
+            row = txn.fetchone()
+
+            if not row[3]:
+                # ...or if we've requested a timestamp older than the oldest
+                # copy in the cache, return the oldest copy (if any)
+                sql = (
+                    "SELECT response_code, etag, expires, og, media_id, min(download_ts)"
+                    " FROM local_media_repository_url_cache"
+                    " WHERE url = ? AND download_ts > ?"
+                )
+                txn.execute(sql, (url, ts))
+                row = txn.fetchone()
+
+            if not row[3]:
+                return None
+
+            return dict(zip((
+                'response_code', 'etag', 'expires', 'og', 'media_id', 'download_ts'
+            ), row))
+
+        return self.runInteraction(
+            "get_url_cache", get_url_cache_txn
+        )
+
+    def store_url_cache(self, url, response_code, etag, expires, og, media_id, download_ts):
+        return self._simple_insert(
+            "local_media_repository_url_cache",
+            {
+                "url": url,
+                "response_code": response_code,
+                "etag": etag,
+                "expires": expires,
+                "og": og,
+                "media_id": media_id,
+                "download_ts": download_ts,
+            },
+            desc="store_url_cache",
+        )
+
     def get_local_media_thumbnails(self, media_id):
         return self._simple_select_list(
             "local_media_repository_thumbnails",
diff --git a/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql b/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql
new file mode 100644
index 0000000000..9efb4280eb
--- /dev/null
+++ b/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql
@@ -0,0 +1,27 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE local_media_repository_url_cache(
+    url TEXT,              -- the URL being cached
+    response_code INTEGER, -- the HTTP response code of this download attempt
+    etag TEXT,             -- the etag header of this response
+    expires INTEGER,       -- the number of ms this response was valid for
+    og TEXT,               -- cache of the OG metadata of this URL as JSON
+    media_id TEXT,         -- the media_id, if any, of the URL's content in the repo
+    download_ts BIGINT     -- the timestamp of this download attempt
+);
+
+CREATE INDEX local_media_repository_url_cache_by_url_download_ts
+    ON local_media_repository_url_cache(url, download_ts);
-- 
cgit 1.5.1


From 8b98a7e8c37f0fae09f33a6d93953584288ed394 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 3 Apr 2016 12:56:29 +0100
Subject: pep8

---
 synapse/http/client.py                        |  14 ++-
 synapse/rest/media/v1/media_repository.py     |   1 -
 synapse/rest/media/v1/preview_url_resource.py | 127 +++++++++++++++-----------
 synapse/storage/media_repository.py           |   3 +-
 4 files changed, 85 insertions(+), 60 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/http/client.py b/synapse/http/client.py
index f42a36ffa6..442b4bb73d 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -15,7 +15,9 @@
 from OpenSSL import SSL
 from OpenSSL.SSL import VERIFY_NONE
 
-from synapse.api.errors import CodeMessageException
+from synapse.api.errors import (
+    CodeMessageException, SynapseError, Codes,
+)
 from synapse.util.logcontext import preserve_context_over_fn
 import synapse.metrics
 
@@ -268,7 +270,7 @@ class SimpleHttpClient(object):
         if 'Content-Length' in headers and headers['Content-Length'] > max_size:
             logger.warn("Requested URL is too large > %r bytes" % (self.max_size,))
             # XXX: do we want to explicitly drop the connection here somehow? if so, how?
-            raise # what should we be raising here?
+            raise  # what should we be raising here?
 
         if response.code > 299:
             logger.warn("Got %d when downloading %s" % (response.code, url))
@@ -331,6 +333,7 @@ def _readBodyToFile(response, stream, max_size):
     response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size))
     return d
 
+
 class CaptchaServerHttpClient(SimpleHttpClient):
     """
     Separate HTTP client for talking to google's captcha servers
@@ -360,6 +363,7 @@ class CaptchaServerHttpClient(SimpleHttpClient):
             # twisted dislikes google's response, no content length.
             defer.returnValue(e.response)
 
+
 class SpiderHttpClient(SimpleHttpClient):
     """
     Separate HTTP client for spidering arbitrary URLs.
@@ -376,8 +380,10 @@ class SpiderHttpClient(SimpleHttpClient):
             connectTimeout=15,
             contextFactory=hs.get_http_client_context_factory()
         )), [('gzip', GzipDecoder)])
-        # Look like Chrome for now
-        #self.user_agent = ("Mozilla/5.0 (%s) (KHTML, like Gecko) Chrome Safari" % hs.version_string)
+        # We could look like Chrome:
+        # self.user_agent = ("Mozilla/5.0 (%s) (KHTML, like Gecko)
+        #                   Chrome Safari" % hs.version_string)
+
 
 def encode_urlencode_args(args):
     return {k: encode_urlencode_arg(v) for k, v in args.items()}
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 8f3491b91c..11f672aeab 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -80,4 +80,3 @@ class MediaRepositoryResource(Resource):
         self.putChild("thumbnail", ThumbnailResource(hs, filepaths))
         self.putChild("identicon", IdenticonResource())
         self.putChild("preview_url", PreviewUrlResource(hs, filepaths))
-
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 7c69c01a6c..29db5c7fce 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -13,25 +13,31 @@
 # limitations under the License.
 
 from .base_resource import BaseMediaResource
-from synapse.api.errors import Codes
-from twisted.web.resource import Resource
+
 from twisted.web.server import NOT_DONE_YET
 from twisted.internet import defer
 from lxml import html
 from urlparse import urlparse, urlunparse
+
+from synapse.api.errors import Codes
 from synapse.util.stringutils import random_string
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.http.client import SpiderHttpClient
-from synapse.http.server import request_handler, respond_with_json, respond_with_json_bytes
+from synapse.http.server import (
+    request_handler, respond_with_json, respond_with_json_bytes
+)
 from synapse.util.async import ObservableDeferred
+from synapse.util.stringutils import is_ascii
 
 import os
 import re
+import cgi
 import ujson as json
 
 import logging
 logger = logging.getLogger(__name__)
 
+
 class PreviewUrlResource(BaseMediaResource):
     isLeaf = True
 
@@ -41,9 +47,10 @@ class PreviewUrlResource(BaseMediaResource):
 
         # simple memory cache mapping urls to OG metadata
         self.cache = ExpiringCache(
-            cache_name = "url_previews",
-            clock = self.clock,
-            expiry_ms = 60*60*1000, # don't spider URLs more often than once an hour
+            cache_name="url_previews",
+            clock=self.clock,
+            # don't spider URLs more often than once an hour
+            expiry_ms=60 * 60 * 1000,
         )
         self.cache.start()
 
@@ -56,12 +63,15 @@ class PreviewUrlResource(BaseMediaResource):
     @request_handler
     @defer.inlineCallbacks
     def _async_render_GET(self, request):
-        
+
         try:
             # XXX: if get_user_by_req fails, what should we do in an async render?
             requester = yield self.auth.get_user_by_req(request)
             url = request.args.get("url")[0]
-            ts = int(request.args.get("ts")[0]) if "ts" in request.args else self.clock.time_msec()
+            if "ts" in request.args:
+                ts = int(request.args.get("ts")[0])
+            else:
+                ts = self.clock.time_msec()
 
             # first check the memory cache - good to handle all the clients on this
             # HS thundering away to preview the same URL at the same time.
@@ -98,7 +108,7 @@ class PreviewUrlResource(BaseMediaResource):
 
                 @download.addBoth
                 def callback(media_info):
-                    del self.downloads[key]
+                    del self.downloads[url]
                     return media_info
             media_info = yield download.observe()
 
@@ -111,13 +121,15 @@ class PreviewUrlResource(BaseMediaResource):
 
             if self._is_media(media_info['media_type']):
                 dims = yield self._generate_local_thumbnails(
-                        media_info['filesystem_id'], media_info
-                      )
+                    media_info['filesystem_id'], media_info
+                )
 
                 og = {
-                    "og:description" : media_info['download_name'],
-                    "og:image" : "mxc://%s/%s" % (self.server_name, media_info['filesystem_id']),
-                    "og:image:type" : media_info['media_type'],
+                    "og:description": media_info['download_name'],
+                    "og:image": "mxc://%s/%s" % (
+                        self.server_name, media_info['filesystem_id']
+                    ),
+                    "og:image:type": media_info['media_type'],
                 }
 
                 if dims:
@@ -138,14 +150,14 @@ class PreviewUrlResource(BaseMediaResource):
                     file = open(media_info['filename'])
                     body = file.read()
                     file.close()
-                    tree = html.fromstring(body.decode('utf-8','ignore'))
+                    tree = html.fromstring(body.decode('utf-8', 'ignore'))
                     og = yield self._calc_og(tree, media_info, requester)
 
             else:
                 logger.warn("Failed to find any OG data in %s", url)
                 og = {}
 
-            logger.debug("Calculated OG for %s as %s" % (url, og));
+            logger.debug("Calculated OG for %s as %s" % (url, og))
 
             # store OG in ephemeral in-memory cache
             self.cache[url] = og
@@ -181,28 +193,20 @@ class PreviewUrlResource(BaseMediaResource):
         # suck our tree into lxml and define our OG response.
 
         # if we see any image URLs in the OG response, then spider them
-        # (although the client could choose to do this by asking for previews of those URLs to avoid DoSing the server)
-
-        # "og:type"        : "article"
-        # "og:url"         : "https://twitter.com/matrixdotorg/status/684074366691356672"
-        # "og:title"       : "Matrix on Twitter"
-        # "og:image"       : "https://pbs.twimg.com/profile_images/500400952029888512/yI0qtFi7_400x400.png"
-        # "og:description" : "Synapse 0.12 is out! Lots of polishing, performance &amp;amp; bugfixes: /sync API, /r0 prefix, fulltext search, 3PID invites https://t.co/5alhXLLEGP"
-        # "og:site_name"   : "Twitter"
-        
-        # or:
+        # (although the client could choose to do this by asking for previews of those
+        # URLs to avoid DoSing the server)
 
         # "og:type"         : "video",
         # "og:url"          : "https://www.youtube.com/watch?v=LXDBoHyjmtw",
         # "og:site_name"    : "YouTube",
         # "og:video:type"   : "application/x-shockwave-flash",
-        # "og:description"  : " ",
+        # "og:description"  : "Fun stuff happening here",
         # "og:title"        : "RemoteJam - Matrix team hack for Disrupt Europe Hackathon",
         # "og:image"        : "https://i.ytimg.com/vi/LXDBoHyjmtw/maxresdefault.jpg",
         # "og:video:url"    : "http://www.youtube.com/v/LXDBoHyjmtw?version=3&autohide=1",
         # "og:video:width"  : "1280"
         # "og:video:height" : "720",
-        # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3&autohide=1",
+        # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3",
 
         og = {}
         for tag in tree.xpath("//*/meta[starts-with(@property, 'og:')]"):
@@ -210,64 +214,76 @@ class PreviewUrlResource(BaseMediaResource):
 
         # TODO: grab article: meta tags too, e.g.:
 
-        # <meta property="article:publisher" content="https://www.facebook.com/thethudonline" />
-        # <meta property="article:author" content="https://www.facebook.com/thethudonline" />
-        # <meta property="article:tag" content="baby" />
-        # <meta property="article:section" content="Breaking News" />
-        # <meta property="article:published_time" content="2016-03-31T19:58:24+00:00" />
-        # <meta property="article:modified_time" content="2016-04-01T18:31:53+00:00" />
+        # "article:publisher" : "https://www.facebook.com/thethudonline" />
+        # "article:author" content="https://www.facebook.com/thethudonline" />
+        # "article:tag" content="baby" />
+        # "article:section" content="Breaking News" />
+        # "article:published_time" content="2016-03-31T19:58:24+00:00" />
+        # "article:modified_time" content="2016-04-01T18:31:53+00:00" />
 
         if 'og:title' not in og:
             # do some basic spidering of the HTML
             title = tree.xpath("(//title)[1] | (//h1)[1] | (//h2)[1] | (//h3)[1]")
             og['og:title'] = title[0].text.strip() if title else None
 
-
         if 'og:image' not in og:
             # TODO: extract a favicon failing all else
-            meta_image = tree.xpath("//*/meta[translate(@itemprop, 'IMAGE', 'image')='image']/@content");
+            meta_image = tree.xpath(
+                "//*/meta[translate(@itemprop, 'IMAGE', 'image')='image']/@content"
+            )
             if meta_image:
                 og['og:image'] = self._rebase_url(meta_image[0], media_info['uri'])
             else:
                 # TODO: consider inlined CSS styles as well as width & height attribs
                 images = tree.xpath("//img[@src][number(@width)>10][number(@height)>10]")
-                images = sorted(images, key=lambda i: (-1 * int(i.attrib['width']) * int(i.attrib['height'])))
+                images = sorted(images, key=lambda i: (
+                    -1 * int(i.attrib['width']) * int(i.attrib['height'])
+                ))
                 if not images:
                     images = tree.xpath("//img[@src]")
                 if images:
                     og['og:image'] = images[0].attrib['src']
 
         # pre-cache the image for posterity
-        # FIXME: it might be cleaner to use the same flow as the main /preview_url request itself
-        # and benefit from the same caching etc.  But for now we just rely on the caching
-        # of the master request to speed things up.
+        # FIXME: it might be cleaner to use the same flow as the main /preview_url request
+        # itself and benefit from the same caching etc.  But for now we just rely on the
+        # caching on the master request to speed things up.
         if 'og:image' in og and og['og:image']:
-            image_info = yield self._download_url(self._rebase_url(og['og:image'], media_info['uri']), requester.user)
+            image_info = yield self._download_url(
+                self._rebase_url(og['og:image'], media_info['uri']), requester.user
+            )
 
             if self._is_media(image_info['media_type']):
                 # TODO: make sure we don't choke on white-on-transparent images
                 dims = yield self._generate_local_thumbnails(
-                        image_info['filesystem_id'], image_info
-                      )
+                    image_info['filesystem_id'], image_info
+                )
                 if dims:
                     og["og:image:width"] = dims['width']
                     og["og:image:height"] = dims['height']
                 else:
                     logger.warn("Couldn't get dims for %s" % og["og:image"])
 
-                og["og:image"] = "mxc://%s/%s" % (self.server_name, image_info['filesystem_id'])
+                og["og:image"] = "mxc://%s/%s" % (
+                    self.server_name, image_info['filesystem_id']
+                )
                 og["og:image:type"] = image_info['media_type']
             else:
                 del og["og:image"]
 
         if 'og:description' not in og:
-            meta_description = tree.xpath("//*/meta[translate(@name, 'DESCRIPTION', 'description')='description']/@content");
+            meta_description = tree.xpath(
+                "//*/meta"
+                "[translate(@name, 'DESCRIPTION', 'description')='description']"
+                "/@content")
             if meta_description:
                 og['og:description'] = meta_description[0]
             else:
-                # text_nodes = tree.xpath("//h1/text() | //h2/text() | //h3/text() | //p/text() | //div/text() | //span/text() | //a/text()")                
-                text_nodes = tree.xpath("//text()[not(ancestor::header | ancestor::nav | ancestor::aside | " +
-                                        "ancestor::footer | ancestor::script | ancestor::style)]" +
+                # text_nodes = tree.xpath("//h1/text() | //h2/text() | //h3/text() | "
+                #    "//p/text() | //div/text() | //span/text() | //a/text()")
+                text_nodes = tree.xpath("//text()[not(ancestor::header | ancestor::nav | "
+                                        "ancestor::aside | ancestor::footer | "
+                                        "ancestor::script | ancestor::style)]" +
                                         "[ancestor::body]")
                 text = ''
                 for text_node in text_nodes:
@@ -280,15 +296,16 @@ class PreviewUrlResource(BaseMediaResource):
                 text = text.strip()[:500]
                 og['og:description'] = text if text else None
 
-        # TODO: delete the url downloads to stop diskfilling, as we only ever cared about its OG
-        defer.returnValue(og);
+        # TODO: delete the url downloads to stop diskfilling,
+        # as we only ever cared about its OG
+        defer.returnValue(og)
 
     def _rebase_url(self, url, base):
         base = list(urlparse(base))
         url = list(urlparse(url))
-        if not url[0]: # fix up schema
+        if not url[0]:  # fix up schema
             url[0] = base[0] or "http"
-        if not url[1]: # fix up hostname
+        if not url[1]:  # fix up hostname
             url[1] = base[1]
             if not url[2].startswith('/'):
                 url[2] = re.sub(r'/[^/]+$', '/', base[2]) + url[2]
@@ -377,6 +394,8 @@ class PreviewUrlResource(BaseMediaResource):
 
     def _is_html(self, content_type):
         content_type = content_type.lower()
-        if (content_type.startswith("text/html") or
-            content_type.startswith("application/xhtml")):
+        if (
+            content_type.startswith("text/html") or
+            content_type.startswith("application/xhtml")
+        ):
             return True
diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py
index bb002081ae..c9dd20eed8 100644
--- a/synapse/storage/media_repository.py
+++ b/synapse/storage/media_repository.py
@@ -87,7 +87,8 @@ class MediaRepositoryStore(SQLBaseStore):
             "get_url_cache", get_url_cache_txn
         )
 
-    def store_url_cache(self, url, response_code, etag, expires, og, media_id, download_ts):
+    def store_url_cache(self, url, response_code, etag, expires, og, media_id,
+                        download_ts):
         return self._simple_insert(
             "local_media_repository_url_cache",
             {
-- 
cgit 1.5.1


From 3d76b7cb2ba05fbf17be0a6647f39c419f428c16 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 4 Apr 2016 15:52:01 +0100
Subject: Store invites in a separate table.

---
 synapse/handlers/room_member.py             |   2 +-
 synapse/storage/events.py                   |  13 +---
 synapse/storage/prepare_database.py         |   2 +-
 synapse/storage/roommember.py               | 111 ++++++++++++++++++++++------
 synapse/storage/schema/delta/31/invites.sql |  28 +++++++
 5 files changed, 124 insertions(+), 32 deletions(-)
 create mode 100644 synapse/storage/schema/delta/31/invites.sql

(limited to 'synapse/storage')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 7c1bb8cfe4..98e346d48e 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -400,7 +400,7 @@ class RoomMemberHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_inviter(self, user_id, room_id):
-        invite = yield self.store.get_room_member(user_id=user_id, room_id=room_id)
+        invite = yield self.store.get_inviter(user_id=user_id, room_id=room_id)
         if invite:
             defer.returnValue(UserID.from_string(invite.sender))
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index c4dc3b3d51..5d299a1132 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -367,7 +367,8 @@ class EventsStore(SQLBaseStore):
                 event
                 for event, _ in events_and_contexts
                 if event.type == EventTypes.Member
-            ]
+            ],
+            backfilled=backfilled,
         )
 
         def event_dict(event):
@@ -485,14 +486,8 @@ class EventsStore(SQLBaseStore):
             return
 
         for event, _ in state_events_and_contexts:
-            if (not event.internal_metadata.is_invite_from_remote()
-                    and event.internal_metadata.is_outlier()):
-                # Outlier events generally shouldn't clobber the current state.
-                # However invites from remote severs for rooms we aren't in
-                # are a bit special: they don't come with any associated
-                # state so are technically an outlier, however all the
-                # client-facing code assumes that they are in the current
-                # state table so we insert the event anyway.
+            if event.internal_metadata.is_outlier():
+                # Outlier events shouldn't clobber the current state.
                 continue
 
             if context.rejected:
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 3f29aad1e8..4099387ba7 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 30
+SCHEMA_VERSION = 31
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 430b49c12e..4c026b33ae 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -36,7 +36,7 @@ RoomsForUser = namedtuple(
 
 class RoomMemberStore(SQLBaseStore):
 
-    def _store_room_members_txn(self, txn, events):
+    def _store_room_members_txn(self, txn, events, backfilled):
         """Store a room member in the database.
         """
         self._simple_insert_many_txn(
@@ -62,6 +62,41 @@ class RoomMemberStore(SQLBaseStore):
                 self._membership_stream_cache.entity_has_changed,
                 event.state_key, event.internal_metadata.stream_ordering
             )
+            txn.call_after(
+                self.get_invited_rooms_for_user.invalidate, (event.state_key,)
+            )
+
+            is_mine = self.hs.is_mine_id(event.state_key)
+            is_new_state = not backfilled and (
+                not event.internal_metadata.is_outlier()
+                or event.internal_metadata.is_invite_from_remote()
+            )
+            if is_new_state and is_mine:
+                if event.membership == Membership.INVITE:
+                    self._simple_insert_txn(
+                        txn,
+                        table="invites",
+                        values={
+                            "event_id": event.event_id,
+                            "invitee": event.state_key,
+                            "inviter": event.sender,
+                            "room_id": event.room_id,
+                            "stream_id": event.internal_metadata.stream_ordering,
+                        }
+                    )
+                else:
+                    sql = (
+                        "UPDATE invites SET stream_id = ?, replaced_by = ? WHERE"
+                        " room_id = ? AND invitee = ? AND locally_rejected is NULL"
+                        " AND replaced_by is NULL"
+                    )
+
+                    txn.execute(sql, (
+                        event.internal_metadata.stream_ordering,
+                        event.event_id,
+                        event.room_id,
+                        event.state_key,
+                    ))
 
     def get_room_member(self, user_id, room_id):
         """Retrieve the current state of a room member.
@@ -127,6 +162,14 @@ class RoomMemberStore(SQLBaseStore):
             user_id, [Membership.INVITE]
         )
 
+    @defer.inlineCallbacks
+    def get_inviter(self, user_id, room_id):
+        invites = yield self.get_invited_rooms_for_user(user_id)
+        for invite in invites:
+            if invite.room_id == room_id:
+                defer.returnValue(invite)
+        defer.returnValue(None)
+
     def get_leave_and_ban_events_for_user(self, user_id):
         """ Get all the leave events for a user
         Args:
@@ -163,29 +206,55 @@ class RoomMemberStore(SQLBaseStore):
 
     def _get_rooms_for_user_where_membership_is_txn(self, txn, user_id,
                                                     membership_list):
-        where_clause = "user_id = ? AND (%s) AND forgotten = 0" % (
-            " OR ".join(["membership = ?" for _ in membership_list]),
-        )
 
-        args = [user_id]
-        args.extend(membership_list)
+        do_invite = Membership.INVITE in membership_list
+        membership_list = [m for m in membership_list if m != Membership.INVITE]
 
-        sql = (
-            "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering"
-            " FROM current_state_events as c"
-            " INNER JOIN room_memberships as m"
-            " ON m.event_id = c.event_id"
-            " INNER JOIN events as e"
-            " ON e.event_id = c.event_id"
-            " AND m.room_id = c.room_id"
-            " AND m.user_id = c.state_key"
-            " WHERE %s"
-        ) % (where_clause,)
+        results = []
+        if membership_list:
+            where_clause = "user_id = ? AND (%s) AND forgotten = 0" % (
+                " OR ".join(["membership = ?" for _ in membership_list]),
+            )
+
+            args = [user_id]
+            args.extend(membership_list)
+
+            sql = (
+                "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering"
+                " FROM current_state_events as c"
+                " INNER JOIN room_memberships as m"
+                " ON m.event_id = c.event_id"
+                " INNER JOIN events as e"
+                " ON e.event_id = c.event_id"
+                " AND m.room_id = c.room_id"
+                " AND m.user_id = c.state_key"
+                " WHERE %s"
+            ) % (where_clause,)
+
+            txn.execute(sql, args)
+            results = [
+                RoomsForUser(**r) for r in self.cursor_to_dict(txn)
+            ]
+
+        if do_invite:
+            sql = (
+                "SELECT i.room_id, inviter, i.event_id, e.stream_ordering"
+                " FROM invites as i"
+                " INNER JOIN events as e USING (event_id)"
+                " WHERE invitee = ? AND locally_rejected is NULL"
+                " AND replaced_by is NULL"
+            )
+
+            txn.execute(sql, (user_id,))
+            results.extend(RoomsForUser(
+                room_id=r["room_id"],
+                sender=r["inviter"],
+                event_id=r["event_id"],
+                stream_ordering=r["stream_ordering"],
+                membership=Membership.INVITE,
+            ) for r in self.cursor_to_dict(txn))
 
-        txn.execute(sql, args)
-        return [
-            RoomsForUser(**r) for r in self.cursor_to_dict(txn)
-        ]
+        return results
 
     @cached(max_entries=5000)
     def get_joined_hosts_for_room(self, room_id):
diff --git a/synapse/storage/schema/delta/31/invites.sql b/synapse/storage/schema/delta/31/invites.sql
new file mode 100644
index 0000000000..4f6fb9ea63
--- /dev/null
+++ b/synapse/storage/schema/delta/31/invites.sql
@@ -0,0 +1,28 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE TABLE invites(
+    stream_id BIGINT NOT NULL,
+    inviter TEXT NOT NULL,
+    invitee TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    locally_rejected TEXT,
+    replaced_by TEXT
+);
+
+CREATE INDEX invites_id ON invites(stream_id);
+CREATE INDEX invites_for_user_idx ON invites(invitee, locally_rejected, replaced_by, room_id);
-- 
cgit 1.5.1


From 92ab45a330c2d6c4e896786135e93b6cabfad1ea Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 4 Apr 2016 17:07:43 +0100
Subject: Add upgrade path, rename table

---
 synapse/storage/roommember.py               |  6 +++---
 synapse/storage/schema/delta/31/invites.sql | 20 +++++++++++++++++---
 2 files changed, 20 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 4c026b33ae..abe5942744 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -75,7 +75,7 @@ class RoomMemberStore(SQLBaseStore):
                 if event.membership == Membership.INVITE:
                     self._simple_insert_txn(
                         txn,
-                        table="invites",
+                        table="local_invites",
                         values={
                             "event_id": event.event_id,
                             "invitee": event.state_key,
@@ -86,7 +86,7 @@ class RoomMemberStore(SQLBaseStore):
                     )
                 else:
                     sql = (
-                        "UPDATE invites SET stream_id = ?, replaced_by = ? WHERE"
+                        "UPDATE local_invites SET stream_id = ?, replaced_by = ? WHERE"
                         " room_id = ? AND invitee = ? AND locally_rejected is NULL"
                         " AND replaced_by is NULL"
                     )
@@ -239,7 +239,7 @@ class RoomMemberStore(SQLBaseStore):
         if do_invite:
             sql = (
                 "SELECT i.room_id, inviter, i.event_id, e.stream_ordering"
-                " FROM invites as i"
+                " FROM local_invites as i"
                 " INNER JOIN events as e USING (event_id)"
                 " WHERE invitee = ? AND locally_rejected is NULL"
                 " AND replaced_by is NULL"
diff --git a/synapse/storage/schema/delta/31/invites.sql b/synapse/storage/schema/delta/31/invites.sql
index 4f6fb9ea63..1c83430da4 100644
--- a/synapse/storage/schema/delta/31/invites.sql
+++ b/synapse/storage/schema/delta/31/invites.sql
@@ -14,7 +14,7 @@
  */
 
 
-CREATE TABLE invites(
+CREATE TABLE local_invites(
     stream_id BIGINT NOT NULL,
     inviter TEXT NOT NULL,
     invitee TEXT NOT NULL,
@@ -24,5 +24,19 @@ CREATE TABLE invites(
     replaced_by TEXT
 );
 
-CREATE INDEX invites_id ON invites(stream_id);
-CREATE INDEX invites_for_user_idx ON invites(invitee, locally_rejected, replaced_by, room_id);
+-- Insert all invites for local users into new `invites` table
+INSERT INTO local_invites SELECT
+    stream_ordering as stream_id,
+    sender as inviter,
+    state_key as invitee,
+    event_id,
+    room_id,
+    NULL as locally_rejected,
+    NULL as replaced_by
+FROM events
+NATURAL JOIN current_state_events
+NATURAL JOIN room_memberships
+WHERE membership = 'invite'  AND state_key IN (SELECT name FROM users);
+
+CREATE INDEX local_invites_id ON local_invites(stream_id);
+CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);
-- 
cgit 1.5.1


From 0c53d750e7145f57ed97c544efdb846cc9e37b67 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 4 Apr 2016 18:02:48 +0100
Subject: Docs and indents

---
 synapse/handlers/room_member.py             |  5 ++++-
 synapse/storage/roommember.py               | 18 ++++++++++++++++--
 synapse/storage/schema/delta/31/invites.sql | 22 +++++++++++-----------
 3 files changed, 31 insertions(+), 14 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 98e346d48e..f1c3e90ecd 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -400,7 +400,10 @@ class RoomMemberHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_inviter(self, user_id, room_id):
-        invite = yield self.store.get_inviter(user_id=user_id, room_id=room_id)
+        invite = yield self.store.get_invite_for_user_in_room(
+            user_id=user_id,
+            room_id=room_id,
+        )
         if invite:
             defer.returnValue(UserID.from_string(invite.sender))
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index abe5942744..36456a75fc 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -66,11 +66,15 @@ class RoomMemberStore(SQLBaseStore):
                 self.get_invited_rooms_for_user.invalidate, (event.state_key,)
             )
 
-            is_mine = self.hs.is_mine_id(event.state_key)
+            # We update the local_invites table only if the event is "current",
+            # i.e., its something that has just happened.
+            # The only current event that can also be an outlier is if its an
+            # invite that has come in across federation.
             is_new_state = not backfilled and (
                 not event.internal_metadata.is_outlier()
                 or event.internal_metadata.is_invite_from_remote()
             )
+            is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
                 if event.membership == Membership.INVITE:
                     self._simple_insert_txn(
@@ -163,7 +167,17 @@ class RoomMemberStore(SQLBaseStore):
         )
 
     @defer.inlineCallbacks
-    def get_inviter(self, user_id, room_id):
+    def get_invite_for_user_in_room(self, user_id, room_id):
+        """Gets the invite for the given user and room
+
+        Args:
+            user_id (str)
+            room_id (str)
+
+        Returns:
+            Deferred: Resolves to either a RoomsForUser or None if no invite was
+                found.
+        """
         invites = yield self.get_invited_rooms_for_user(user_id)
         for invite in invites:
             if invite.room_id == room_id:
diff --git a/synapse/storage/schema/delta/31/invites.sql b/synapse/storage/schema/delta/31/invites.sql
index 1c83430da4..2c57846d5a 100644
--- a/synapse/storage/schema/delta/31/invites.sql
+++ b/synapse/storage/schema/delta/31/invites.sql
@@ -26,17 +26,17 @@ CREATE TABLE local_invites(
 
 -- Insert all invites for local users into new `invites` table
 INSERT INTO local_invites SELECT
-    stream_ordering as stream_id,
-    sender as inviter,
-    state_key as invitee,
-    event_id,
-    room_id,
-    NULL as locally_rejected,
-    NULL as replaced_by
-FROM events
-NATURAL JOIN current_state_events
-NATURAL JOIN room_memberships
-WHERE membership = 'invite'  AND state_key IN (SELECT name FROM users);
+        stream_ordering as stream_id,
+        sender as inviter,
+        state_key as invitee,
+        event_id,
+        room_id,
+        NULL as locally_rejected,
+        NULL as replaced_by
+    FROM events
+    NATURAL JOIN current_state_events
+    NATURAL JOIN room_memberships
+    WHERE membership = 'invite'  AND state_key IN (SELECT name FROM users);
 
 CREATE INDEX local_invites_id ON local_invites(stream_id);
 CREATE INDEX local_invites_for_user_idx ON local_invites(invitee, locally_rejected, replaced_by, room_id);
-- 
cgit 1.5.1


From df727f212606b771b1410c8e322fb8a99d159de4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 5 Apr 2016 11:13:24 +0100
Subject: Fix stuck invites

If rejecting a remote invite fails with an error response don't fail
the entire request; instead mark the invite as locally rejected.

This fixes the bug where users can get stuck invites which they can
neither accept nor reject.
---
 synapse/handlers/federation.py  | 34 +++++++++++++++++++++++-----------
 synapse/handlers/room_member.py | 18 ++++++++++++++----
 synapse/storage/__init__.py     |  3 ++-
 synapse/storage/roommember.py   | 19 +++++++++++++++++++
 4 files changed, 58 insertions(+), 16 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 4049c01d26..19769eecd7 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -784,13 +784,19 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
-        origin, event = yield self._make_and_verify_event(
-            target_hosts,
-            room_id,
-            user_id,
-            "leave"
-        )
-        signed_event = self._sign_event(event)
+        try:
+            origin, event = yield self._make_and_verify_event(
+                target_hosts,
+                room_id,
+                user_id,
+                "leave"
+            )
+            signed_event = self._sign_event(event)
+        except SynapseError:
+            raise
+        except CodeMessageException as e:
+            logger.warn("Failed to reject invite: %s", e)
+            raise SynapseError(500, "Failed to reject invite")
 
         # Try the host we successfully got a response to /make_join/
         # request first.
@@ -800,10 +806,16 @@ class FederationHandler(BaseHandler):
         except ValueError:
             pass
 
-        yield self.replication_layer.send_leave(
-            target_hosts,
-            signed_event
-        )
+        try:
+            yield self.replication_layer.send_leave(
+                target_hosts,
+                signed_event
+            )
+        except SynapseError:
+            raise
+        except CodeMessageException as e:
+            logger.warn("Failed to reject invite: %s", e)
+            raise SynapseError(500, "Failed to reject invite")
 
         context = yield self.state_handler.compute_event_context(event)
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index f1c3e90ecd..6c7409215a 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -258,10 +258,20 @@ class RoomMemberHandler(BaseHandler):
                 else:
                     # send the rejection to the inviter's HS.
                     remote_room_hosts = remote_room_hosts + [inviter.domain]
-                    ret = yield self.reject_remote_invite(
-                        target.to_string(), room_id, remote_room_hosts
-                    )
-                    defer.returnValue(ret)
+
+                    try:
+                        ret = yield self.reject_remote_invite(
+                            target.to_string(), room_id, remote_room_hosts
+                        )
+                        defer.returnValue(ret)
+                    except SynapseError as e:
+                        logger.warn("Failed to reject invite: %s", e)
+
+                        yield self.store.locally_reject_invite(
+                            target.to_string(), room_id
+                        )
+
+                        defer.returnValue({})
 
         yield self._local_membership_update(
             requester=requester,
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 57863bba4d..07916b292d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -94,7 +94,8 @@ class DataStore(RoomMemberStore, RoomStore,
         )
 
         self._stream_id_gen = StreamIdGenerator(
-            db_conn, "events", "stream_ordering"
+            db_conn, "events", "stream_ordering",
+            extra_tables=[("local_invites", "stream_id")]
         )
         self._backfill_id_gen = StreamIdGenerator(
             db_conn, "events", "stream_ordering", step=-1
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 36456a75fc..66e7a40e3c 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -102,6 +102,25 @@ class RoomMemberStore(SQLBaseStore):
                         event.state_key,
                     ))
 
+    @defer.inlineCallbacks
+    def locally_reject_invite(self, user_id, room_id):
+        sql = (
+            "UPDATE local_invites SET stream_id = ?, locally_rejected = ? WHERE"
+            " room_id = ? AND invitee = ? AND locally_rejected is NULL"
+            " AND replaced_by is NULL"
+        )
+
+        def f(txn, stream_ordering):
+            txn.execute(sql, (
+                stream_ordering,
+                True,
+                room_id,
+                user_id,
+            ))
+
+        with self._stream_id_gen.get_next() as stream_ordering:
+            yield self.runInteraction("locally_reject_invite", f, stream_ordering)
+
     def get_room_member(self, user_id, room_id):
         """Retrieve the current state of a room member.
 
-- 
cgit 1.5.1


From a1e0d316ea354fce07939073d9afc9c5d1013939 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 6 Apr 2016 13:05:19 +0100
Subject: Move _get_cache_dict into the SQLBaseStore

---
 synapse/storage/__init__.py | 33 ---------------------------------
 synapse/storage/_base.py    | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 33 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 07916b292d..045ae6c03f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -177,39 +177,6 @@ class DataStore(RoomMemberStore, RoomStore,
         self.__presence_on_startup = None
         return active_on_startup
 
-    def _get_cache_dict(self, db_conn, table, entity_column, stream_column, max_value):
-        # Fetch a mapping of room_id -> max stream position for "recent" rooms.
-        # It doesn't really matter how many we get, the StreamChangeCache will
-        # do the right thing to ensure it respects the max size of cache.
-        sql = (
-            "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s"
-            " WHERE %(stream)s > ? - 100000"
-            " GROUP BY %(entity)s"
-        ) % {
-            "table": table,
-            "entity": entity_column,
-            "stream": stream_column,
-        }
-
-        sql = self.database_engine.convert_param_style(sql)
-
-        txn = db_conn.cursor()
-        txn.execute(sql, (int(max_value),))
-        rows = txn.fetchall()
-        txn.close()
-
-        cache = {
-            row[0]: int(row[1])
-            for row in rows
-        }
-
-        if cache:
-            min_val = min(cache.values())
-        else:
-            min_val = max_value
-
-        return cache, min_val
-
     def _get_active_presence(self, db_conn):
         """Fetch non-offline presence from the database so that we can register
         the appropriate time outs.
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index b75b79df36..04d7fcf6d6 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -816,6 +816,40 @@ class SQLBaseStore(object):
             self._next_stream_id += 1
             return i
 
+    def _get_cache_dict(self, db_conn, table, entity_column, stream_column,
+                        max_value):
+        # Fetch a mapping of room_id -> max stream position for "recent" rooms.
+        # It doesn't really matter how many we get, the StreamChangeCache will
+        # do the right thing to ensure it respects the max size of cache.
+        sql = (
+            "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s"
+            " WHERE %(stream)s > ? - 100000"
+            " GROUP BY %(entity)s"
+        ) % {
+            "table": table,
+            "entity": entity_column,
+            "stream": stream_column,
+        }
+
+        sql = self.database_engine.convert_param_style(sql)
+
+        txn = db_conn.cursor()
+        txn.execute(sql, (int(max_value),))
+        rows = txn.fetchall()
+        txn.close()
+
+        cache = {
+            row[0]: int(row[1])
+            for row in rows
+        }
+
+        if cache:
+            min_val = min(cache.values())
+        else:
+            min_val = max_value
+
+        return cache, min_val
+
 
 class _RollbackButIsFineException(Exception):
     """ This exception is used to rollback a transaction without implying
-- 
cgit 1.5.1


From 87f2dec8d475f038beb138bc56e3ef76fcb83ec6 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 6 Apr 2016 13:08:05 +0100
Subject: Make the cache objects be per instance rather than being global

---
 synapse/storage/receipts.py        |  4 ++--
 synapse/storage/registration.py    |  2 +-
 synapse/storage/state.py           |  4 ++--
 synapse/util/caches/descriptors.py | 45 ++++++++++++++++++++------------------
 4 files changed, 29 insertions(+), 26 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 4befebc8e2..7fdd84bbdc 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -160,8 +160,8 @@ class ReceiptsStore(SQLBaseStore):
             "content": content,
         }])
 
-    @cachedList(cache=get_linearized_receipts_for_room.cache, list_name="room_ids",
-                num_args=3, inlineCallbacks=True)
+    @cachedList(cached_method_name="get_linearized_receipts_for_room",
+                list_name="room_ids", num_args=3, inlineCallbacks=True)
     def _get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None):
         if not room_ids:
             defer.returnValue({})
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index d46a963bb8..1f71773aaa 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -319,7 +319,7 @@ class RegistrationStore(SQLBaseStore):
 
         defer.returnValue(res if res else False)
 
-    @cachedList(cache=is_guest.cache, list_name="user_ids", num_args=1,
+    @cachedList(cached_method_name="is_guest", list_name="user_ids", num_args=1,
                 inlineCallbacks=True)
     def are_guests(self, user_ids):
         sql = "SELECT name, is_guest FROM users WHERE name IN (%s)" % (
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index e9f9406014..c5d2a3a6df 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -273,8 +273,8 @@ class StateStore(SQLBaseStore):
             desc="_get_state_group_for_event",
         )
 
-    @cachedList(cache=_get_state_group_for_event.cache, list_name="event_ids",
-                num_args=1, inlineCallbacks=True)
+    @cachedList(cached_method_name="_get_state_group_for_event",
+                list_name="event_ids", num_args=1, inlineCallbacks=True)
     def _get_state_group_for_events(self, event_ids):
         """Returns mapping event_id -> state_group
         """
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 35544b19fd..758f5982b0 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -167,7 +167,8 @@ class CacheDescriptor(object):
                 % (orig.__name__,)
             )
 
-        self.cache = Cache(
+    def __get__(self, obj, objtype=None):
+        cache = Cache(
             name=self.orig.__name__,
             max_entries=self.max_entries,
             keylen=self.num_args,
@@ -175,14 +176,12 @@ class CacheDescriptor(object):
             tree=self.tree,
         )
 
-    def __get__(self, obj, objtype=None):
-
         @functools.wraps(self.orig)
         def wrapped(*args, **kwargs):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
             cache_key = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names)
             try:
-                cached_result_d = self.cache.get(cache_key)
+                cached_result_d = cache.get(cache_key)
 
                 observer = cached_result_d.observe()
                 if DEBUG_CACHES:
@@ -204,7 +203,7 @@ class CacheDescriptor(object):
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated
                 # while the SELECT is executing (SYN-369)
-                sequence = self.cache.sequence
+                sequence = cache.sequence
 
                 ret = defer.maybeDeferred(
                     preserve_context_over_fn,
@@ -213,20 +212,21 @@ class CacheDescriptor(object):
                 )
 
                 def onErr(f):
-                    self.cache.invalidate(cache_key)
+                    cache.invalidate(cache_key)
                     return f
 
                 ret.addErrback(onErr)
 
                 ret = ObservableDeferred(ret, consumeErrors=True)
-                self.cache.update(sequence, cache_key, ret)
+                cache.update(sequence, cache_key, ret)
 
                 return preserve_context_over_deferred(ret.observe())
 
-        wrapped.invalidate = self.cache.invalidate
-        wrapped.invalidate_all = self.cache.invalidate_all
-        wrapped.invalidate_many = self.cache.invalidate_many
-        wrapped.prefill = self.cache.prefill
+        wrapped.invalidate = cache.invalidate
+        wrapped.invalidate_all = cache.invalidate_all
+        wrapped.invalidate_many = cache.invalidate_many
+        wrapped.prefill = cache.prefill
+        wrapped.cache = cache
 
         obj.__dict__[self.orig.__name__] = wrapped
 
@@ -240,11 +240,12 @@ class CacheListDescriptor(object):
     the list of missing keys to the wrapped fucntion.
     """
 
-    def __init__(self, orig, cache, list_name, num_args=1, inlineCallbacks=False):
+    def __init__(self, orig, cached_method_name, list_name, num_args=1,
+                 inlineCallbacks=False):
         """
         Args:
             orig (function)
-            cache (Cache)
+            method_name (str); The name of the chached method.
             list_name (str): Name of the argument which is the bulk lookup list
             num_args (int)
             inlineCallbacks (bool): Whether orig is a generator that should
@@ -263,7 +264,7 @@ class CacheListDescriptor(object):
         self.arg_names = inspect.getargspec(orig).args[1:num_args + 1]
         self.list_pos = self.arg_names.index(self.list_name)
 
-        self.cache = cache
+        self.cached_method_name = cached_method_name
 
         self.sentinel = object()
 
@@ -277,11 +278,13 @@ class CacheListDescriptor(object):
         if self.list_name not in self.arg_names:
             raise Exception(
                 "Couldn't see arguments %r for %r."
-                % (self.list_name, cache.name,)
+                % (self.list_name, cached_method_name,)
             )
 
     def __get__(self, obj, objtype=None):
 
+        cache = getattr(obj, self.cached_method_name).cache
+
         @functools.wraps(self.orig)
         def wrapped(*args, **kwargs):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
@@ -297,14 +300,14 @@ class CacheListDescriptor(object):
                 key[self.list_pos] = arg
 
                 try:
-                    res = self.cache.get(tuple(key)).observe()
+                    res = cache.get(tuple(key)).observe()
                     res.addCallback(lambda r, arg: (arg, r), arg)
                     cached[arg] = res
                 except KeyError:
                     missing.append(arg)
 
             if missing:
-                sequence = self.cache.sequence
+                sequence = cache.sequence
                 args_to_call = dict(arg_dict)
                 args_to_call[self.list_name] = missing
 
@@ -327,10 +330,10 @@ class CacheListDescriptor(object):
 
                     key = list(keyargs)
                     key[self.list_pos] = arg
-                    self.cache.update(sequence, tuple(key), observer)
+                    cache.update(sequence, tuple(key), observer)
 
                     def invalidate(f, key):
-                        self.cache.invalidate(key)
+                        cache.invalidate(key)
                         return f
                     observer.addErrback(invalidate, tuple(key))
 
@@ -370,7 +373,7 @@ def cachedInlineCallbacks(max_entries=1000, num_args=1, lru=False, tree=False):
     )
 
 
-def cachedList(cache, list_name, num_args=1, inlineCallbacks=False):
+def cachedList(cached_method_name, list_name, num_args=1, inlineCallbacks=False):
     """Creates a descriptor that wraps a function in a `CacheListDescriptor`.
 
     Used to do batch lookups for an already created cache. A single argument
@@ -400,7 +403,7 @@ def cachedList(cache, list_name, num_args=1, inlineCallbacks=False):
     """
     return lambda orig: CacheListDescriptor(
         orig,
-        cache=cache,
+        cached_method_name=cached_method_name,
         list_name=list_name,
         num_args=num_args,
         inlineCallbacks=inlineCallbacks,
-- 
cgit 1.5.1


From 8aab9d87fa6739345810f0edf3982fe7f898ee30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 6 Apr 2016 14:08:18 +0100
Subject: Don't require config to create database

---
 scripts/synapse_port_db                            | 13 ++---
 synapse/app/homeserver.py                          | 15 +++--
 synapse/storage/engines/__init__.py                |  6 +-
 synapse/storage/engines/postgres.py                |  8 +--
 synapse/storage/engines/sqlite3.py                 | 13 +----
 synapse/storage/prepare_database.py                | 64 +++++++---------------
 .../schema/delta/14/upgrade_appservice_db.py       |  6 +-
 synapse/storage/schema/delta/20/pushers.py         |  6 +-
 synapse/storage/schema/delta/25/fts.py             |  6 +-
 synapse/storage/schema/delta/27/ts.py              |  6 +-
 synapse/storage/schema/delta/30/as_users.py        |  4 +-
 tests/storage/test_base.py                         |  2 +-
 tests/utils.py                                     |  6 +-
 13 files changed, 69 insertions(+), 86 deletions(-)

(limited to 'synapse/storage')

diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index a2a0f364cf..253a6ef6c7 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -19,6 +19,7 @@ from twisted.enterprise import adbapi
 
 from synapse.storage._base import LoggingTransaction, SQLBaseStore
 from synapse.storage.engines import create_engine
+from synapse.storage.prepare_database import prepare_database
 
 import argparse
 import curses
@@ -37,6 +38,7 @@ BOOLEAN_COLUMNS = {
     "rooms": ["is_public"],
     "event_edges": ["is_state"],
     "presence_list": ["accepted"],
+    "presence_stream": ["currently_active"],
 }
 
 
@@ -292,7 +294,7 @@ class Porter(object):
             }
         )
 
-        database_engine.prepare_database(db_conn)
+        prepare_database(db_conn, database_engine, config=None)
 
         db_conn.commit()
 
@@ -309,8 +311,8 @@ class Porter(object):
                 **self.postgres_config["args"]
             )
 
-            sqlite_engine = create_engine(FakeConfig(sqlite_config))
-            postgres_engine = create_engine(FakeConfig(postgres_config))
+            sqlite_engine = create_engine(sqlite_config)
+            postgres_engine = create_engine(postgres_config)
 
             self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
             self.postgres_store = Store(postgres_db_pool, postgres_engine)
@@ -792,8 +794,3 @@ if __name__ == "__main__":
     if end_error_exec_info:
         exc_type, exc_value, exc_traceback = end_error_exec_info
         traceback.print_exception(exc_type, exc_value, exc_traceback)
-
-
-class FakeConfig:
-    def __init__(self, database_config):
-        self.database_config = database_config
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index fcdc8e6e10..2b4473b9ac 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -33,7 +33,7 @@ from synapse.python_dependencies import (
 from synapse.rest import ClientRestResource
 from synapse.storage.engines import create_engine, IncorrectDatabaseSetup
 from synapse.storage import are_all_users_on_domain
-from synapse.storage.prepare_database import UpgradeDatabaseException
+from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
 
 from synapse.server import HomeServer
 
@@ -245,7 +245,7 @@ class SynapseHomeServer(HomeServer):
         except IncorrectDatabaseSetup as e:
             quit_with_error(e.message)
 
-    def get_db_conn(self):
+    def get_db_conn(self, run_new_connection=True):
         # Any param beginning with cp_ is a parameter for adbapi, and should
         # not be passed to the database engine.
         db_params = {
@@ -254,7 +254,8 @@ class SynapseHomeServer(HomeServer):
         }
         db_conn = self.database_engine.module.connect(**db_params)
 
-        self.database_engine.on_new_connection(db_conn)
+        if run_new_connection:
+            self.database_engine.on_new_connection(db_conn)
         return db_conn
 
 
@@ -386,7 +387,7 @@ def setup(config_options):
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
 
-    database_engine = create_engine(config)
+    database_engine = create_engine(config.database_config)
     config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
 
     hs = SynapseHomeServer(
@@ -402,8 +403,10 @@ def setup(config_options):
     logger.info("Preparing database: %s...", config.database_config['name'])
 
     try:
-        db_conn = hs.get_db_conn()
-        database_engine.prepare_database(db_conn)
+        db_conn = hs.get_db_conn(run_new_connection=False)
+        prepare_database(db_conn, database_engine, config=config)
+        database_engine.on_new_connection(db_conn)
+
         hs.run_startup_checks(db_conn, database_engine)
 
         db_conn.commit()
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index a48230b93f..7bb5de1fe7 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -26,13 +26,13 @@ SUPPORTED_MODULE = {
 }
 
 
-def create_engine(config):
-    name = config.database_config["name"]
+def create_engine(database_config):
+    name = database_config["name"]
     engine_class = SUPPORTED_MODULE.get(name, None)
 
     if engine_class:
         module = importlib.import_module(name)
-        return engine_class(module, config=config)
+        return engine_class(module)
 
     raise RuntimeError(
         "Unsupported database engine '%s'" % (name,)
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index a09685b4df..c2290943b4 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -13,18 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.prepare_database import prepare_database
-
 from ._base import IncorrectDatabaseSetup
 
 
 class PostgresEngine(object):
     single_threaded = False
 
-    def __init__(self, database_module, config):
+    def __init__(self, database_module):
         self.module = database_module
         self.module.extensions.register_type(self.module.extensions.UNICODE)
-        self.config = config
 
     def check_database(self, txn):
         txn.execute("SHOW SERVER_ENCODING")
@@ -44,9 +41,6 @@ class PostgresEngine(object):
             self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ
         )
 
-    def prepare_database(self, db_conn):
-        prepare_database(db_conn, self, config=self.config)
-
     def is_deadlock(self, error):
         if isinstance(error, self.module.DatabaseError):
             return error.pgcode in ["40001", "40P01"]
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 522b905949..14203aa500 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.prepare_database import (
-    prepare_database, prepare_sqlite3_database
-)
+from synapse.storage.prepare_database import prepare_database
 
 import struct
 
@@ -23,9 +21,8 @@ import struct
 class Sqlite3Engine(object):
     single_threaded = True
 
-    def __init__(self, database_module, config):
+    def __init__(self, database_module):
         self.module = database_module
-        self.config = config
 
     def check_database(self, txn):
         pass
@@ -34,13 +31,9 @@ class Sqlite3Engine(object):
         return sql
 
     def on_new_connection(self, db_conn):
-        self.prepare_database(db_conn)
+        prepare_database(db_conn, self, config=None)
         db_conn.create_function("rank", 1, _rank)
 
-    def prepare_database(self, db_conn):
-        prepare_sqlite3_database(db_conn)
-        prepare_database(db_conn, self, config=self.config)
-
     def is_deadlock(self, error):
         return False
 
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 4099387ba7..00833422af 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -53,6 +53,9 @@ class UpgradeDatabaseException(PrepareDatabaseException):
 def prepare_database(db_conn, database_engine, config):
     """Prepares a database for usage. Will either create all necessary tables
     or upgrade from an older schema version.
+
+    If `config` is None then prepare_database will assert that no upgrade is
+    necessary, *or* will create a fresh database if the database is empty.
     """
     try:
         cur = db_conn.cursor()
@@ -60,13 +63,18 @@ def prepare_database(db_conn, database_engine, config):
 
         if version_info:
             user_version, delta_files, upgraded = version_info
-            _upgrade_existing_database(
-                cur, user_version, delta_files, upgraded, database_engine, config
-            )
-        else:
-            _setup_new_database(cur, database_engine, config)
 
-        # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
+            if config is None:
+                if user_version != SCHEMA_VERSION:
+                    # If we don't pass in a config file then we are expecting to
+                    # have already upgraded the DB.
+                    raise UpgradeDatabaseException("Database needs to be upgraded")
+            else:
+                _upgrade_existing_database(
+                    cur, user_version, delta_files, upgraded, database_engine, config
+                )
+        else:
+            _setup_new_database(cur, database_engine)
 
         cur.close()
         db_conn.commit()
@@ -75,7 +83,7 @@ def prepare_database(db_conn, database_engine, config):
         raise
 
 
-def _setup_new_database(cur, database_engine, config):
+def _setup_new_database(cur, database_engine):
     """Sets up the database by finding a base set of "full schemas" and then
     applying any necessary deltas.
 
@@ -148,12 +156,13 @@ def _setup_new_database(cur, database_engine, config):
         applied_delta_files=[],
         upgraded=False,
         database_engine=database_engine,
-        config=config,
+        config=None,
+        is_empty=True,
     )
 
 
 def _upgrade_existing_database(cur, current_version, applied_delta_files,
-                               upgraded, database_engine, config):
+                               upgraded, database_engine, config, is_empty=False):
     """Upgrades an existing database.
 
     Delta files can either be SQL stored in *.sql files, or python modules
@@ -246,7 +255,9 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
                         module_name, absolute_path, python_file
                     )
                 logger.debug("Running script %s", relative_path)
-                module.run_upgrade(cur, database_engine, config=config)
+                module.run_create(cur, database_engine)
+                if not is_empty:
+                    module.run_upgrade(cur, database_engine, config=config)
             elif ext == ".pyc":
                 # Sometimes .pyc files turn up anyway even though we've
                 # disabled their generation; e.g. from distribution package
@@ -361,36 +372,3 @@ def _get_or_create_schema_state(txn, database_engine):
         return current_version, applied_deltas, upgraded
 
     return None
-
-
-def prepare_sqlite3_database(db_conn):
-    """This function should be called before `prepare_database` on sqlite3
-    databases.
-
-    Since we changed the way we store the current schema version and handle
-    updates to schemas, we need a way to upgrade from the old method to the
-    new. This only affects sqlite databases since they were the only ones
-    supported at the time.
-    """
-    with db_conn:
-        schema_path = os.path.join(
-            dir_path, "schema", "schema_version.sql",
-        )
-        create_schema = read_schema(schema_path)
-        db_conn.executescript(create_schema)
-
-        c = db_conn.execute("SELECT * FROM schema_version")
-        rows = c.fetchall()
-        c.close()
-
-        if not rows:
-            c = db_conn.execute("PRAGMA user_version")
-            row = c.fetchone()
-            c.close()
-
-            if row and row[0]:
-                db_conn.execute(
-                    "REPLACE INTO schema_version (version, upgraded)"
-                    " VALUES (?,?)",
-                    (row[0], False)
-                )
diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
index 5c40a77757..8755bb2e49 100644
--- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py
+++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
@@ -18,7 +18,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def run_upgrade(cur, *args, **kwargs):
+def run_create(cur, *args, **kwargs):
     cur.execute("SELECT id, regex FROM application_services_regex")
     for row in cur.fetchall():
         try:
@@ -35,3 +35,7 @@ def run_upgrade(cur, *args, **kwargs):
                 "UPDATE application_services_regex SET regex=? WHERE id=?",
                 (new_regex, row[0])
             )
+
+
+def run_upgrade(*args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/delta/20/pushers.py b/synapse/storage/schema/delta/20/pushers.py
index 29164732af..147496a38b 100644
--- a/synapse/storage/schema/delta/20/pushers.py
+++ b/synapse/storage/schema/delta/20/pushers.py
@@ -27,7 +27,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def run_upgrade(cur, database_engine, *args, **kwargs):
+def run_create(cur, database_engine, *args, **kwargs):
     logger.info("Porting pushers table...")
     cur.execute("""
         CREATE TABLE IF NOT EXISTS pushers2 (
@@ -74,3 +74,7 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
     cur.execute("DROP TABLE pushers")
     cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
     logger.info("Moved %d pushers to new table", count)
+
+
+def run_upgrade(*args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py
index d3ff2b1779..4269ac69ad 100644
--- a/synapse/storage/schema/delta/25/fts.py
+++ b/synapse/storage/schema/delta/25/fts.py
@@ -43,7 +43,7 @@ SQLITE_TABLE = (
 )
 
 
-def run_upgrade(cur, database_engine, *args, **kwargs):
+def run_create(cur, database_engine, *args, **kwargs):
     if isinstance(database_engine, PostgresEngine):
         for statement in get_statements(POSTGRES_TABLE.splitlines()):
             cur.execute(statement)
@@ -76,3 +76,7 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
         sql = database_engine.convert_param_style(sql)
 
         cur.execute(sql, ("event_search", progress_json))
+
+
+def run_upgrade(*args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/delta/27/ts.py b/synapse/storage/schema/delta/27/ts.py
index f8c16391a2..71b12a2731 100644
--- a/synapse/storage/schema/delta/27/ts.py
+++ b/synapse/storage/schema/delta/27/ts.py
@@ -27,7 +27,7 @@ ALTER_TABLE = (
 )
 
 
-def run_upgrade(cur, database_engine, *args, **kwargs):
+def run_create(cur, database_engine, *args, **kwargs):
     for statement in get_statements(ALTER_TABLE.splitlines()):
         cur.execute(statement)
 
@@ -55,3 +55,7 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
         sql = database_engine.convert_param_style(sql)
 
         cur.execute(sql, ("event_origin_server_ts", progress_json))
+
+
+def run_upgrade(*args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py
index 4f6e9dd540..b417e3ac08 100644
--- a/synapse/storage/schema/delta/30/as_users.py
+++ b/synapse/storage/schema/delta/30/as_users.py
@@ -18,7 +18,7 @@ from synapse.storage.appservice import ApplicationServiceStore
 logger = logging.getLogger(__name__)
 
 
-def run_upgrade(cur, database_engine, config, *args, **kwargs):
+def run_create(cur, database_engine, *args, **kwargs):
     # NULL indicates user was not registered by an appservice.
     try:
         cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
@@ -26,6 +26,8 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs):
         # Maybe we already added the column? Hope so...
         pass
 
+
+def run_upgrade(cur, database_engine, config, *args, **kwargs):
     cur.execute("SELECT name FROM users")
     rows = cur.fetchall()
 
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index 2e33beb07c..afbefb2e2d 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -53,7 +53,7 @@ class SQLBaseStoreTestCase(unittest.TestCase):
             "test",
             db_pool=self.db_pool,
             config=config,
-            database_engine=create_engine(config),
+            database_engine=create_engine(config.database_config),
         )
 
         self.datastore = SQLBaseStore(hs)
diff --git a/tests/utils.py b/tests/utils.py
index 52405502e9..c179df31ee 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -64,7 +64,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         hs = HomeServer(
             name, db_pool=db_pool, config=config,
             version_string="Synapse/tests",
-            database_engine=create_engine(config),
+            database_engine=create_engine(config.database_config),
             get_db_conn=db_pool.get_db_conn,
             **kargs
         )
@@ -73,7 +73,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         hs = HomeServer(
             name, db_pool=None, datastore=datastore, config=config,
             version_string="Synapse/tests",
-            database_engine=create_engine(config),
+            database_engine=create_engine(config.database_config),
             **kargs
         )
 
@@ -298,7 +298,7 @@ class SQLiteMemoryDbPool(ConnectionPool, object):
         return conn
 
     def create_engine(self):
-        return create_engine(self.config)
+        return create_engine(self.config.database_config)
 
 
 class MemoryDataStore(object):
-- 
cgit 1.5.1


From 75fb9ac1be0fada60cdde38153ac0e3fe2b19a0a Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 6 Apr 2016 14:12:51 +0100
Subject: Add a slaved events store class

Add a test to check that get_room_names_and_aliases does the same
thing on both the master and on the slave data store.
---
 synapse/replication/slave/__init__.py              |  14 ++
 synapse/replication/slave/storage/__init__.py      |  14 ++
 synapse/replication/slave/storage/_base.py         |  28 +++
 .../slave/storage/_slaved_id_tracker.py            |  30 ++++
 synapse/replication/slave/storage/events.py        | 198 +++++++++++++++++++++
 synapse/storage/events.py                          |   4 +-
 tests/replication/slave/__init__.py                |  14 ++
 tests/replication/slave/storage/__init__.py        |  14 ++
 tests/replication/slave/storage/_base.py           |  57 ++++++
 tests/replication/slave/storage/test_events.py     | 114 ++++++++++++
 10 files changed, 485 insertions(+), 2 deletions(-)
 create mode 100644 synapse/replication/slave/__init__.py
 create mode 100644 synapse/replication/slave/storage/__init__.py
 create mode 100644 synapse/replication/slave/storage/_base.py
 create mode 100644 synapse/replication/slave/storage/_slaved_id_tracker.py
 create mode 100644 synapse/replication/slave/storage/events.py
 create mode 100644 tests/replication/slave/__init__.py
 create mode 100644 tests/replication/slave/storage/__init__.py
 create mode 100644 tests/replication/slave/storage/_base.py
 create mode 100644 tests/replication/slave/storage/test_events.py

(limited to 'synapse/storage')

diff --git a/synapse/replication/slave/__init__.py b/synapse/replication/slave/__init__.py
new file mode 100644
index 0000000000..b7df13c9ee
--- /dev/null
+++ b/synapse/replication/slave/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/synapse/replication/slave/storage/__init__.py b/synapse/replication/slave/storage/__init__.py
new file mode 100644
index 0000000000..b7df13c9ee
--- /dev/null
+++ b/synapse/replication/slave/storage/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py
new file mode 100644
index 0000000000..46e43ce1c7
--- /dev/null
+++ b/synapse/replication/slave/storage/_base.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.storage._base import SQLBaseStore
+from twisted.internet import defer
+
+
+class BaseSlavedStore(SQLBaseStore):
+    def __init__(self, db_conn, hs):
+        super(BaseSlavedStore, self).__init__(hs)
+
+    def stream_positions(self):
+        return {}
+
+    def process_replication(self, result):
+        return defer.succeed(None)
diff --git a/synapse/replication/slave/storage/_slaved_id_tracker.py b/synapse/replication/slave/storage/_slaved_id_tracker.py
new file mode 100644
index 0000000000..24b5c79d4a
--- /dev/null
+++ b/synapse/replication/slave/storage/_slaved_id_tracker.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.storage.util.id_generators import _load_current_id
+
+
+class SlavedIdTracker(object):
+    def __init__(self, db_conn, table, column, extra_tables=[], step=1):
+        self.step = step
+        self._current = _load_current_id(db_conn, table, column, step)
+        for table, column in extra_tables:
+            self.advance(_load_current_id(db_conn, table, column))
+
+    def advance(self, new_id):
+        self._current = (max if self.step > 0 else min)(self._current, new_id)
+
+    def get_current_token(self):
+        return self._current
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
new file mode 100644
index 0000000000..68b924e37b
--- /dev/null
+++ b/synapse/replication/slave/storage/events.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from ._base import BaseSlavedStore
+from ._slaved_id_tracker import SlavedIdTracker
+
+from synapse.api.constants import EventTypes
+from synapse.events import FrozenEvent
+from synapse.storage import DataStore
+from synapse.storage.room import RoomStore
+from synapse.storage.roommember import RoomMemberStore
+from synapse.storage.event_federation import EventFederationStore
+from synapse.storage.state import StateStore
+from synapse.util.caches.stream_change_cache import StreamChangeCache
+
+import ujson as json
+
+# So, um, we want to borrow a load of functions intended for reading from
+# a DataStore, but we don't want to take functions that either write to the
+# DataStore or are cached and don't have cache invalidation logic.
+#
+# Rather than write duplicate versions of those functions, or lift them to
+# a common base class, we going to grab the underlying __func__ object from
+# the method descriptor on the DataStore and chuck them into our class.
+
+
+class SlavedEventStore(BaseSlavedStore):
+
+    def __init__(self, db_conn, hs):
+        super(SlavedEventStore, self).__init__(db_conn, hs)
+        self._stream_id_gen = SlavedIdTracker(
+            db_conn, "events", "stream_ordering",
+        )
+        self._backfill_id_gen = SlavedIdTracker(
+            db_conn, "events", "stream_ordering", step=-1
+        )
+        events_max = self._stream_id_gen.get_current_token()
+        event_cache_prefill, min_event_val = self._get_cache_dict(
+            db_conn, "events",
+            entity_column="room_id",
+            stream_column="stream_ordering",
+            max_value=events_max,
+        )
+        self._events_stream_cache = StreamChangeCache(
+            "EventsRoomStreamChangeCache", min_event_val,
+            prefilled_cache=event_cache_prefill,
+        )
+
+    # Cached functions can't be accessed through a class instance so we need
+    # to reach inside the __dict__ to extract them.
+    get_room_name_and_aliases = RoomStore.__dict__["get_room_name_and_aliases"]
+    get_rooms_for_user = RoomMemberStore.__dict__["get_rooms_for_user"]
+    get_users_in_room = RoomMemberStore.__dict__["get_users_in_room"]
+    get_latest_event_ids_in_room = EventFederationStore.__dict__[
+        "get_latest_event_ids_in_room"
+    ]
+    _get_current_state_for_key = StateStore.__dict__[
+        "_get_current_state_for_key"
+    ]
+
+    get_current_state = DataStore.get_current_state.__func__
+    get_current_state_for_key = DataStore.get_current_state_for_key.__func__
+    _get_rooms_for_user_where_membership_is_txn = (
+        DataStore._get_rooms_for_user_where_membership_is_txn.__func__
+    )
+    get_rooms_for_user_where_membership_is = (
+        DataStore.get_rooms_for_user_where_membership_is.__func__
+    )
+    get_membership_changes_for_user = (
+        DataStore.get_membership_changes_for_user.__func__
+    )
+    get_room_events_max_id = DataStore.get_room_events_max_id.__func__
+    get_room_events_stream_for_room = (
+        DataStore.get_room_events_stream_for_room.__func__
+    )
+    _set_before_and_after = DataStore._set_before_and_after
+
+    _get_events = DataStore._get_events.__func__
+    _get_events_from_cache = DataStore._get_events_from_cache.__func__
+
+    _invalidate_get_event_cache = DataStore._invalidate_get_event_cache.__func__
+    _parse_events_txn = DataStore._parse_events_txn.__func__
+    _get_events_txn = DataStore._get_events_txn.__func__
+    _fetch_events_txn = DataStore._fetch_events_txn.__func__
+    _fetch_event_rows = DataStore._fetch_event_rows.__func__
+    _get_event_from_row_txn = DataStore._get_event_from_row_txn.__func__
+
+    def stream_positions(self):
+        result = super(SlavedEventStore, self).stream_positions()
+        result["events"] = self._stream_id_gen.get_current_token()
+        result["backfilled"] = self._backfill_id_gen.get_current_token()
+        return result
+
+    def process_replication(self, result):
+        state_resets = set(
+            r[0] for r in result.get("state_resets", {"rows": []})["rows"]
+        )
+
+        stream = result.get("events")
+        if stream:
+            self._stream_id_gen.advance(stream["position"])
+            for row in stream["rows"]:
+                self._process_replication_row(
+                    row, backfilled=False, state_resets=state_resets
+                )
+
+        stream = result.get("backfill")
+        if stream:
+            self._backfill_id_gen.advance(stream["position"])
+            for row in stream["rows"]:
+                self._process_replication_row(
+                    row, backfilled=True, state_resets=state_resets
+                )
+
+        stream = result.get("forward_ex_outliers")
+        if stream:
+            for row in stream["rows"]:
+                event_id = row[1]
+                self._invalidate_get_event_cache(event_id)
+
+        stream = result.get("backward_ex_outliers")
+        if stream:
+            for row in stream["rows"]:
+                event_id = row[1]
+                self._invalidate_get_event_cache(event_id)
+
+        return super(SlavedEventStore, self).process_replication(result)
+
+    def _process_replication_row(self, row, backfilled, state_resets):
+        position = row[0]
+        internal = json.loads(row[1])
+        event_json = json.loads(row[2])
+
+        event = FrozenEvent(event_json, internal_metadata_dict=internal)
+        self._invalidate_caches_for_event(
+            event, backfilled, reset_state=position in state_resets
+        )
+
+    def _invalidate_caches_for_event(self, event, backfilled, reset_state):
+        if reset_state:
+            self._get_current_state_for_key.invalidate_all()
+            self.get_rooms_for_user.invalidate_all()
+            self.get_users_in_room.invalidate((event.room_id,))
+            # self.get_joined_hosts_for_room.invalidate((event.room_id,))
+            self.get_room_name_and_aliases.invalidate((event.room_id,))
+
+        self._invalidate_get_event_cache(event.event_id)
+
+        if not backfilled:
+            self._events_stream_cache.entity_has_changed(
+                event.room_id, event.internal_metadata.stream_ordering
+            )
+
+        # self.get_unread_event_push_actions_by_room_for_user.invalidate_many(
+        #     (event.room_id,)
+        # )
+
+        if event.type == EventTypes.Redaction:
+            self._invalidate_get_event_cache(event.redacts)
+
+        if event.type == EventTypes.Member:
+            self.get_rooms_for_user.invalidate((event.state_key,))
+            # self.get_joined_hosts_for_room.invalidate((event.room_id,))
+            self.get_users_in_room.invalidate((event.room_id,))
+            # self._membership_stream_cache.entity_has_changed(
+            #    event.state_key, event.internal_metadata.stream_ordering
+            # )
+
+        if not event.is_state():
+            return
+
+        if backfilled:
+            return
+
+        if (not event.internal_metadata.is_invite_from_remote()
+                and event.internal_metadata.is_outlier()):
+            return
+
+        self._get_current_state_for_key.invalidate((
+            event.room_id, event.type, event.state_key
+        ))
+
+        if event.type in [EventTypes.Name, EventTypes.Aliases]:
+            self.get_room_name_and_aliases.invalidate(
+                (event.room_id,)
+            )
+            pass
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5d299a1132..ee87a71719 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1134,7 +1134,7 @@ class EventsStore(SQLBaseStore):
                     upper_bound = current_forward_id
 
                 sql = (
-                    "SELECT -event_stream_ordering FROM current_state_resets"
+                    "SELECT event_stream_ordering FROM current_state_resets"
                     " WHERE ? < event_stream_ordering"
                     " AND event_stream_ordering <= ?"
                     " ORDER BY event_stream_ordering ASC"
@@ -1143,7 +1143,7 @@ class EventsStore(SQLBaseStore):
                 state_resets = txn.fetchall()
 
                 sql = (
-                    "SELECT -event_stream_ordering, event_id, state_group"
+                    "SELECT event_stream_ordering, event_id, state_group"
                     " FROM ex_outlier_stream"
                     " WHERE ? > event_stream_ordering"
                     " AND event_stream_ordering >= ?"
diff --git a/tests/replication/slave/__init__.py b/tests/replication/slave/__init__.py
new file mode 100644
index 0000000000..b7df13c9ee
--- /dev/null
+++ b/tests/replication/slave/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/replication/slave/storage/__init__.py b/tests/replication/slave/storage/__init__.py
new file mode 100644
index 0000000000..b7df13c9ee
--- /dev/null
+++ b/tests/replication/slave/storage/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
new file mode 100644
index 0000000000..0f525a8943
--- /dev/null
+++ b/tests/replication/slave/storage/_base.py
@@ -0,0 +1,57 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+from tests import unittest
+
+from synapse.replication.slave.storage.events import SlavedEventStore
+
+from mock import Mock, NonCallableMock
+from tests.utils import setup_test_homeserver
+from synapse.replication.resource import ReplicationResource
+
+
+class BaseSlavedStoreTestCase(unittest.TestCase):
+    @defer.inlineCallbacks
+    def setUp(self):
+        self.hs = yield setup_test_homeserver(
+            "blue",
+            http_client=None,
+            replication_layer=Mock(),
+            ratelimiter=NonCallableMock(spec_set=[
+                "send_message",
+            ]),
+        )
+        self.hs.get_ratelimiter().send_message.return_value = (True, 0)
+
+        self.replication = ReplicationResource(self.hs)
+
+        self.master_store = self.hs.get_datastore()
+        self.slaved_store = SlavedEventStore(self.hs.get_db_conn(), self.hs)
+        self.event_id = 0
+
+    @defer.inlineCallbacks
+    def replicate(self):
+        streams = self.slaved_store.stream_positions()
+        result = yield self.replication.replicate(streams, 100)
+        yield self.slaved_store.process_replication(result)
+
+    @defer.inlineCallbacks
+    def check(self, method, args, expected_result=None):
+        master_result = yield getattr(self.master_store, method)(*args)
+        slaved_result = yield getattr(self.slaved_store, method)(*args)
+        self.assertEqual(master_result, slaved_result)
+        if expected_result is not None:
+            self.assertEqual(master_result, expected_result)
+            self.assertEqual(slaved_result, expected_result)
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
new file mode 100644
index 0000000000..c30c7c6063
--- /dev/null
+++ b/tests/replication/slave/storage/test_events.py
@@ -0,0 +1,114 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import BaseSlavedStoreTestCase
+
+from synapse.types import UserID
+from synapse.events import FrozenEvent
+from synapse.events.snapshot import EventContext
+
+from twisted.internet import defer
+
+USER_ID = "@feeling:blue"
+USER = UserID.from_string(USER_ID)
+OUTLIER = {"outlier": True}
+ROOM_ID = "!room:blue"
+
+
+class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
+
+    @defer.inlineCallbacks
+    def test_room_name_and_aliases(self):
+        create = yield self.persist(type="m.room.create", key="", creator=USER_ID)
+        yield self.persist(type="m.room.member", key=USER_ID, membership="join")
+        yield self.persist(type="m.room.name", key="", name="name1")
+        yield self.persist(
+            type="m.room.aliases", key="blue", aliases=["#1:blue"]
+        )
+        yield self.replicate()
+        yield self.check(
+            "get_room_name_and_aliases", (ROOM_ID,), ("name1", ["#1:blue"])
+        )
+
+        # Set the room name.
+        yield self.persist(type="m.room.name", key="", name="name2")
+        yield self.replicate()
+        yield self.check(
+            "get_room_name_and_aliases", (ROOM_ID,), ("name2", ["#1:blue"])
+        )
+
+        # Set the room aliases.
+        yield self.persist(
+            type="m.room.aliases", key="blue", aliases=["#2:blue"]
+        )
+        yield self.replicate()
+        yield self.check(
+            "get_room_name_and_aliases", (ROOM_ID,), ("name2", ["#2:blue"])
+        )
+
+        # Leave and join the room clobbering the state.
+        yield self.persist(type="m.room.member", key=USER_ID, membership="leave")
+        yield self.persist(
+            type="m.room.member", key=USER_ID, membership="join",
+            reset_state=[create]
+        )
+        yield self.replicate()
+
+        yield self.check(
+            "get_room_name_and_aliases", (ROOM_ID,), (None, [])
+        )
+
+    event_id = 0
+
+    @defer.inlineCallbacks
+    def persist(
+            self, sender=USER_ID, room_id=ROOM_ID, type={}, key=None,
+            internal={},
+            state=None, reset_state=False, backfill=False,
+            depth=None, prev_events=[], auth_events=[], prev_state=[],
+            **content
+    ):
+        if depth is None:
+            depth = self.event_id
+
+        event_dict = {
+            "sender": sender,
+            "type": type,
+            "content": content,
+            "event_id": "$%d:blue" % (self.event_id,),
+            "room_id": room_id,
+            "depth": depth,
+            "origin_server_ts": self.event_id,
+            "prev_events": prev_events,
+            "auth_events": auth_events,
+        }
+        if key is not None:
+            event_dict["state_key"] = key
+            event_dict["prev_state"] = prev_state
+
+        event = FrozenEvent(event_dict, internal_metadata_dict=internal)
+
+        self.event_id += 1
+
+        context = EventContext(current_state=state)
+
+        if backfill:
+            yield self.master_store.persist_events(
+                [(event, context)], backfilled=True
+            )
+        else:
+            yield self.master_store.persist_event(
+                event, context, current_state=reset_state
+            )
+        defer.returnValue(event)
-- 
cgit 1.5.1


From 7e2c89a37f3a5261f43b4d472b36219ac41dfb16 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 6 Apr 2016 15:42:15 +0100
Subject: Make pushers use the event_push_actions table instead of listening on
 an event stream & running the rules again. Sytest passes, but remaining to
 do:

 * Make badges work again
 * Remove old, unused code
---
 synapse/handlers/_base.py                  |   8 +-
 synapse/handlers/federation.py             |   8 +-
 synapse/push/bulk_push_rule_evaluator.py   |  25 +++-
 synapse/push/httppusher.py                 | 204 +++++++++++++++++++++++------
 synapse/push/push_tools.py                 |  66 ++++++++++
 synapse/push/pusher.py                     |  10 ++
 synapse/push/pusherpool.py                 |  75 ++++++-----
 synapse/storage/event_push_actions.py      |  48 +++++++
 synapse/storage/events.py                  |  12 ++
 synapse/storage/pusher.py                  |  81 ++++++++----
 synapse/storage/registration.py            |  20 ---
 synapse/storage/roommember.py              |   1 +
 synapse/storage/schema/delta/31/pushers.py |  75 +++++++++++
 13 files changed, 503 insertions(+), 130 deletions(-)
 create mode 100644 synapse/push/push_tools.py
 create mode 100644 synapse/push/pusher.py
 create mode 100644 synapse/storage/schema/delta/31/pushers.py

(limited to 'synapse/storage')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index c77afe7f51..9c92ea01ed 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -21,7 +21,7 @@ from synapse.api.constants import Membership, EventTypes
 from synapse.types import UserID, RoomAlias, Requester
 from synapse.push.action_generator import ActionGenerator
 
-from synapse.util.logcontext import PreserveLoggingContext
+from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
 
 import logging
 
@@ -377,6 +377,12 @@ class BaseHandler(object):
             event, context=context
         )
 
+        # this intentionally does not yield: we don't care about the result
+        # and don't need to wait for it.
+        preserve_fn(self.hs.get_pusherpool().on_new_notifications)(
+            event_stream_id, max_stream_id
+        )
+
         destinations = set()
         for k, s in context.current_state.items():
             try:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 026ebe52be..fc5e0b0590 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -26,7 +26,7 @@ from synapse.api.errors import (
 from synapse.api.constants import EventTypes, Membership, RejectedReason
 from synapse.events.validator import EventValidator
 from synapse.util import unwrapFirstError
-from synapse.util.logcontext import PreserveLoggingContext
+from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
 from synapse.util.logutils import log_function
 from synapse.util.async import run_on_reactor
 from synapse.util.frozenutils import unfreeze
@@ -1094,6 +1094,12 @@ class FederationHandler(BaseHandler):
             context=context,
         )
 
+        # this intentionally does not yield: we don't care about the result
+        # and don't need to wait for it.
+        preserve_fn(self.hs.get_pusherpool().on_new_notifications)(
+            event_stream_id, max_stream_id
+        )
+
         defer.returnValue((context, event_stream_id, max_stream_id))
 
     @defer.inlineCallbacks
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 76d7eb7ce0..7f94591dcb 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -70,11 +70,17 @@ def _get_rules(room_id, user_ids, store):
 
 @defer.inlineCallbacks
 def evaluator_for_room_id(room_id, hs, store):
-    results = yield store.get_receipts_for_room(room_id, "m.read")
-    user_ids = [
-        row["user_id"] for row in results
-        if hs.is_mine_id(row["user_id"])
-    ]
+    users_with_pushers = yield store.get_users_with_pushers_in_room(room_id)
+    receipts = yield store.get_receipts_for_room(room_id, "m.read")
+
+    # any users with pushers must be ours: they have pushers
+    user_ids = set(users_with_pushers)
+    for r in receipts:
+        if hs.is_mine_id(r['user_id']):
+            user_ids.add(r['user_id'])
+
+    user_ids = list(user_ids)
+
     rules_by_user = yield _get_rules(room_id, user_ids, store)
 
     defer.returnValue(BulkPushRuleEvaluator(
@@ -101,10 +107,15 @@ class BulkPushRuleEvaluator:
     def action_for_event_by_user(self, event, handler, current_state):
         actions_by_user = {}
 
-        users_dict = yield self.store.are_guests(self.rules_by_user.keys())
+        # None of these users can be peeking since this list of users comes
+        # from the set of users in the room, so we know for sure they're all
+        # actually in the room.
+        user_tuples = [
+            (u, False) for u in self.rules_by_user.keys()
+        ]
 
         filtered_by_user = yield handler.filter_events_for_clients(
-            users_dict.items(), [event], {event.event_id: current_state}
+            user_tuples, [event], {event.event_id: current_state}
         )
 
         room_members = yield self.store.get_users_in_room(self.room_id)
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 9be4869360..d695885649 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -13,60 +13,188 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.push import Pusher, PusherConfigException
+from synapse.push import PusherConfigException
 
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 
 import logging
+import push_rule_evaluator
+import push_tools
 
 logger = logging.getLogger(__name__)
 
 
-class HttpPusher(Pusher):
-    def __init__(self, _hs, user_id, app_id,
-                 app_display_name, device_display_name, pushkey, pushkey_ts,
-                 data, last_token, last_success, failing_since):
-        super(HttpPusher, self).__init__(
-            _hs,
-            user_id,
-            app_id,
-            app_display_name,
-            device_display_name,
-            pushkey,
-            pushkey_ts,
-            data,
-            last_token,
-            last_success,
-            failing_since
+class HttpPusher(object):
+    INITIAL_BACKOFF_SEC = 1  # in seconds because that's what Twisted takes
+    MAX_BACKOFF_SEC = 60 * 60
+
+    # This one's in ms because we compare it against the clock
+    GIVE_UP_AFTER_MS = 24 * 60 * 60 * 1000
+
+    def __init__(self, hs, pusherdict):
+        self.hs = hs
+        self.store = self.hs.get_datastore()
+        self.clock = self.hs.get_clock()
+        self.user_id = pusherdict['user_name']
+        self.app_id = pusherdict['app_id']
+        self.app_display_name = pusherdict['app_display_name']
+        self.device_display_name = pusherdict['device_display_name']
+        self.pushkey = pusherdict['pushkey']
+        self.pushkey_ts = pusherdict['ts']
+        self.data = pusherdict['data']
+        self.last_stream_ordering = pusherdict['last_stream_ordering']
+        self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
+        self.failing_since = pusherdict['failing_since']
+        self.timed_call = None
+
+        # This is the highest stream ordering we know it's safe to process.
+        # When new events arrive, we'll be given a window of new events: we
+        # should honour this rather than just looking for anything higher
+        # because of potential out-of-order event serialisation. This starts
+        # off as None though as we don't know any better.
+        self.max_stream_ordering = None
+
+        if 'data' not in pusherdict:
+            raise PusherConfigException(
+                "No 'data' key for HTTP pusher"
+            )
+        self.data = pusherdict['data']
+
+        self.name = "%s/%s/%s" % (
+            pusherdict['user_name'],
+            pusherdict['app_id'],
+            pusherdict['pushkey'],
         )
-        if 'url' not in data:
+
+        if 'url' not in self.data:
             raise PusherConfigException(
                 "'url' required in data for HTTP pusher"
             )
-        self.url = data['url']
-        self.http_client = _hs.get_simple_http_client()
+        self.url = self.data['url']
+        self.http_client = hs.get_simple_http_client()
         self.data_minus_url = {}
         self.data_minus_url.update(self.data)
         del self.data_minus_url['url']
 
+    def on_started(self):
+        self._process()
+
+    def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
+        self.max_stream_ordering = max_stream_ordering
+        self._process()
+
+    def on_timer(self):
+        self._process()
+
+    def on_stop(self):
+        if self.timed_call:
+            self.timed_call.cancel()
+
     @defer.inlineCallbacks
-    def _build_notification_dict(self, event, tweaks, badge):
-        # we probably do not want to push for every presence update
-        # (we may want to be able to set up notifications when specific
-        # people sign in, but we'd want to only deliver the pertinent ones)
-        # Actually, presence events will not get this far now because we
-        # need to filter them out in the main Pusher code.
-        if 'event_id' not in event:
-            defer.returnValue(None)
+    def _process(self):
+        unprocessed = yield self.store.get_unread_push_actions_for_user_in_range(
+            self.user_id, self.last_stream_ordering, self.max_stream_ordering
+        )
+
+        for push_action in unprocessed:
+            processed = yield self._process_one(push_action)
+            if processed:
+                self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
+                self.last_stream_ordering = push_action['stream_ordering']
+                self.store.update_pusher_last_stream_ordering_and_success(
+                    self.app_id, self.pushkey, self.user_id,
+                    self.last_stream_ordering,
+                    self.clock.time_msec()
+                )
+                self.failing_since = None
+                yield self.store.update_pusher_failing_since(
+                    self.app_id, self.pushkey, self.user_id,
+                    self.failing_since
+                )
+            else:
+                self.failing_since = self.clock.time_msec()
+                yield self.store.update_pusher_failing_since(
+                    self.app_id, self.pushkey, self.user_id,
+                    self.failing_since
+                )
+
+                if (
+                    self.failing_since and
+                    self.failing_since <
+                    self.clock.time_msec() - HttpPusher.GIVE_UP_AFTER
+                ):
+                    # we really only give up so that if the URL gets
+                    # fixed, we don't suddenly deliver a load
+                    # of old notifications.
+                    logger.warn("Giving up on a notification to user %s, "
+                                "pushkey %s",
+                                self.user_id, self.pushkey)
+                    self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
+                    self.last_stream_ordering = push_action['stream_ordering']
+                    yield self.store.update_pusher_last_stream_ordering(
+                        self.app_id,
+                        self.pushkey,
+                        self.user_id,
+                        self.last_stream_ordering
+                    )
+
+                    self.failing_since = None
+                    yield self.store.update_pusher_failing_since(
+                        self.app_id,
+                        self.pushkey,
+                        self.user_id,
+                        self.failing_since
+                    )
+                else:
+                    logger.info("Push failed: delaying for %ds", self.backoff_delay)
+                    self.timed_call = reactor.callLater(self.backoff_delay, self.on_timer)
+                    self.backoff_delay = min(self.backoff_delay, self.MAX_BACKOFF_SEC)
+                    break
+
+    @defer.inlineCallbacks
+    def _process_one(self, push_action):
+        if 'notify' not in push_action['actions']:
+            defer.returnValue(True)
 
-        ctx = yield self.get_context_for_event(event)
+        tweaks = push_rule_evaluator.PushRuleEvaluator.tweaks_for_actions(push_action['actions'])
+        badge = yield push_tools.get_badge_count(self.hs, self.user_id)
+
+        event = yield self.store.get_event(push_action['event_id'], allow_none=True)
+        if event is None:
+            defer.returnValue(True)  # It's been redacted
+        rejected = yield self.dispatch_push(event, tweaks, badge)
+        if rejected is False:
+            defer.returnValue(False)
+
+        if isinstance(rejected, list) or isinstance(rejected, tuple):
+            for pk in rejected:
+                if pk != self.pushkey:
+                    # for sanity, we only remove the pushkey if it
+                    # was the one we actually sent...
+                    logger.warn(
+                        ("Ignoring rejected pushkey %s because we"
+                         " didn't send it"), pk
+                    )
+                else:
+                    logger.info(
+                        "Pushkey %s was rejected: removing",
+                        pk
+                    )
+                    yield self.hs.get_pusherpool().remove_pusher(
+                        self.app_id, pk, self.user_id
+                    )
+        defer.returnValue(True)
+
+    @defer.inlineCallbacks
+    def _build_notification_dict(self, event, tweaks, badge):
+        ctx = yield push_tools.get_context_for_event(self.hs, event)
 
         d = {
             'notification': {
-                'id': event['event_id'],
-                'room_id': event['room_id'],
-                'type': event['type'],
-                'sender': event['user_id'],
+                'id': event.event_id,
+                'room_id': event.room_id,
+                'type': event.type,
+                'sender': event.user_id,
                 'counts': {  # -- we don't mark messages as read yet so
                              # we have no way of knowing
                     # Just set the badge to 1 until we have read receipts
@@ -84,11 +212,11 @@ class HttpPusher(Pusher):
                 ]
             }
         }
-        if event['type'] == 'm.room.member':
-            d['notification']['membership'] = event['content']['membership']
-            d['notification']['user_is_target'] = event['state_key'] == self.user_id
+        if event.type == 'm.room.member':
+            d['notification']['membership'] = event.content['membership']
+            d['notification']['user_is_target'] = event.state_key == self.user_id
         if 'content' in event:
-            d['notification']['content'] = event['content']
+            d['notification']['content'] = event.content
 
         if len(ctx['aliases']):
             d['notification']['room_alias'] = ctx['aliases'][0]
diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py
new file mode 100644
index 0000000000..e1e61e49e8
--- /dev/null
+++ b/synapse/push/push_tools.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+
+@defer.inlineCallbacks
+def get_badge_count(hs, user_id):
+    invites, joins = yield defer.gatherResults([
+        hs.get_datastore().get_invited_rooms_for_user(user_id),
+        hs.get_datastore().get_rooms_for_user(user_id),
+    ], consumeErrors=True)
+
+    my_receipts_by_room = yield hs.get_datastore().get_receipts_for_user(
+        user_id, "m.read",
+    )
+
+    badge = len(invites)
+
+    for r in joins:
+        if r.room_id in my_receipts_by_room:
+            last_unread_event_id = my_receipts_by_room[r.room_id]
+
+            notifs = yield (
+                hs.get_datastore().get_unread_event_push_actions_by_room_for_user(
+                    r.room_id, user_id, last_unread_event_id
+                )
+            )
+            badge += notifs["notify_count"]
+    defer.returnValue(badge)
+
+
+@defer.inlineCallbacks
+def get_context_for_event(hs, ev):
+    name_aliases = yield hs.get_datastore().get_room_name_and_aliases(
+        ev.room_id
+    )
+
+    ctx = {'aliases': name_aliases[1]}
+    if name_aliases[0] is not None:
+        ctx['name'] = name_aliases[0]
+
+    their_member_events_for_room = yield hs.get_datastore().get_current_state(
+        room_id=ev.room_id,
+        event_type='m.room.member',
+        state_key=ev.user_id
+    )
+    for mev in their_member_events_for_room:
+        if mev.content['membership'] == 'join' and 'displayname' in mev.content:
+            dn = mev.content['displayname']
+            if dn is not None:
+                ctx['sender_display_name'] = dn
+
+    defer.returnValue(ctx)
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
new file mode 100644
index 0000000000..4960837504
--- /dev/null
+++ b/synapse/push/pusher.py
@@ -0,0 +1,10 @@
+from httppusher import HttpPusher
+
+PUSHER_TYPES = {
+    'http': HttpPusher
+}
+
+
+def create_pusher(hs, pusherdict):
+    if pusherdict['kind'] in PUSHER_TYPES:
+        return PUSHER_TYPES[pusherdict['kind']](hs, pusherdict)
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 0b463c6fdb..b67ad455ea 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -16,9 +16,10 @@
 
 from twisted.internet import defer
 
-from .httppusher import HttpPusher
+import pusher
 from synapse.push import PusherConfigException
 from synapse.util.logcontext import preserve_fn
+from synapse.util.async import run_on_reactor
 
 import logging
 
@@ -48,7 +49,7 @@ class PusherPool:
         # will then get pulled out of the database,
         # recreated, added and started: this means we have only one
         # code path adding pushers.
-        self._create_pusher({
+        pusher.create_pusher(self.hs, {
             "user_name": user_id,
             "kind": kind,
             "app_id": app_id,
@@ -58,10 +59,18 @@ class PusherPool:
             "ts": time_now_msec,
             "lang": lang,
             "data": data,
-            "last_token": None,
+            "last_stream_ordering": None,
             "last_success": None,
             "failing_since": None
         })
+
+        # create the pusher setting last_stream_ordering to the current maximum
+        # stream ordering in event_push_actions, so it will process
+        # pushes from this point onwards.
+        last_stream_ordering = (
+            yield self.store.get_latest_push_action_stream_ordering()
+        )
+
         yield self.store.add_pusher(
             user_id=user_id,
             access_token=access_token,
@@ -73,6 +82,7 @@ class PusherPool:
             pushkey_ts=time_now_msec,
             lang=lang,
             data=data,
+            last_stream_ordering=last_stream_ordering,
             profile_tag=profile_tag,
         )
         yield self._refresh_pusher(app_id, pushkey, user_id)
@@ -106,26 +116,19 @@ class PusherPool:
                 )
                 yield self.remove_pusher(p['app_id'], p['pushkey'], p['user_name'])
 
-    def _create_pusher(self, pusherdict):
-        if pusherdict['kind'] == 'http':
-            return HttpPusher(
-                self.hs,
-                user_id=pusherdict['user_name'],
-                app_id=pusherdict['app_id'],
-                app_display_name=pusherdict['app_display_name'],
-                device_display_name=pusherdict['device_display_name'],
-                pushkey=pusherdict['pushkey'],
-                pushkey_ts=pusherdict['ts'],
-                data=pusherdict['data'],
-                last_token=pusherdict['last_token'],
-                last_success=pusherdict['last_success'],
-                failing_since=pusherdict['failing_since']
-            )
-        else:
-            raise PusherConfigException(
-                "Unknown pusher type '%s' for user %s" %
-                (pusherdict['kind'], pusherdict['user_name'])
+    @defer.inlineCallbacks
+    def on_new_notifications(self, min_stream_id, max_stream_id):
+        yield run_on_reactor()
+        try:
+            users_affected = yield self.store.get_push_action_users_in_range(
+                min_stream_id, max_stream_id
             )
+            for u in users_affected:
+                if u in self.pushers:
+                    for p in self.pushers[u].values():
+                        p.on_new_notifications(min_stream_id, max_stream_id)
+        except:
+            logger.exception("Exception in pusher on_new_notifications")
 
     @defer.inlineCallbacks
     def _refresh_pusher(self, app_id, pushkey, user_id):
@@ -146,30 +149,34 @@ class PusherPool:
         logger.info("Starting %d pushers", len(pushers))
         for pusherdict in pushers:
             try:
-                p = self._create_pusher(pusherdict)
+                p = pusher.create_pusher(self.hs, pusherdict)
             except PusherConfigException:
                 logger.exception("Couldn't start a pusher: caught PusherConfigException")
                 continue
             if p:
-                fullid = "%s:%s:%s" % (
+                appid_pushkey = "%s:%s" % (
                     pusherdict['app_id'],
                     pusherdict['pushkey'],
-                    pusherdict['user_name']
                 )
-                if fullid in self.pushers:
-                    self.pushers[fullid].stop()
-                self.pushers[fullid] = p
-                preserve_fn(p.start)()
+                byuser = self.pushers.setdefault(pusherdict['user_name'], {})
+
+                if appid_pushkey in byuser:
+                    byuser[appid_pushkey].on_stop()
+                byuser[appid_pushkey] = p
+                preserve_fn(p.on_started)()
 
         logger.info("Started pushers")
 
     @defer.inlineCallbacks
     def remove_pusher(self, app_id, pushkey, user_id):
-        fullid = "%s:%s:%s" % (app_id, pushkey, user_id)
-        if fullid in self.pushers:
-            logger.info("Stopping pusher %s", fullid)
-            self.pushers[fullid].stop()
-            del self.pushers[fullid]
+        appid_pushkey = "%s:%s" % (app_id, pushkey)
+
+        byuser = self.pushers.get(user_id, {})
+
+        if appid_pushkey in byuser:
+            logger.info("Stopping pusher %s / %s", user_id, appid_pushkey)
+            byuser[appid_pushkey].on_stop()
+            del byuser[appid_pushkey]
         yield self.store.delete_pusher_by_app_id_pushkey_user_id(
             app_id, pushkey, user_id
         )
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 3933b6e2c5..5f61743e34 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -100,6 +100,54 @@ class EventPushActionsStore(SQLBaseStore):
         )
         defer.returnValue(ret)
 
+    @defer.inlineCallbacks
+    def get_push_action_users_in_range(self, min_stream_ordering, max_stream_ordering):
+        def f(txn):
+            sql = (
+                "SELECT DISTINCT(user_id) FROM event_push_actions WHERE"
+                " stream_ordering >= ? AND stream_ordering >= ?"
+            )
+            txn.execute(sql, (min_stream_ordering, max_stream_ordering))
+            return [r[0] for r in txn.fetchall()]
+        ret = yield self.runInteraction("get_push_action_users_in_range", f)
+        defer.returnValue(ret)
+
+    @defer.inlineCallbacks
+    def get_unread_push_actions_for_user_in_range(self, user_id,
+                                                  min_stream_ordering,
+                                                  max_stream_ordering=None):
+        def f(txn):
+            sql = (
+                "SELECT event_id, stream_ordering, actions"
+                " FROM event_push_actions"
+                " WHERE user_id = ? AND stream_ordering > ?"
+            )
+            args = [user_id, min_stream_ordering]
+            if max_stream_ordering is not None:
+                sql += " AND stream_ordering <= ?"
+                args.append(max_stream_ordering)
+            sql += " ORDER BY stream_ordering ASC"
+            txn.execute(sql, args)
+            return txn.fetchall()
+        ret = yield self.runInteraction("get_unread_push_actions_for_user_in_range", f)
+        defer.returnValue([
+            {
+                "event_id": row[0],
+                "stream_ordering": row[1],
+                "actions": json.loads(row[2]),
+            } for row in ret
+        ])
+
+    @defer.inlineCallbacks
+    def get_latest_push_action_stream_ordering(self):
+        def f(txn):
+            txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
+            return txn.fetchone()
+        result = yield self.runInteraction(
+                "get_latest_push_action_stream_ordering", f
+        )
+        defer.returnValue(result[0] or 0)
+
     def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
         # Sad that we have to blow away the cache for the whole room here
         txn.call_after(
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5d299a1132..ceae8715ce 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -61,6 +61,17 @@ class EventsStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def persist_events(self, events_and_contexts, backfilled=False):
+        """
+        Write events to the database
+        Args:
+            events_and_contexts: list of tuples of (event, context)
+            backfilled: ?
+
+        Returns: Tuple of stream_orderings where the first is the minimum and
+            last is the maximum stream ordering assigned to the events when
+            persisting.
+
+        """
         if not events_and_contexts:
             return
 
@@ -191,6 +202,7 @@ class EventsStore(SQLBaseStore):
             txn.call_after(self._get_current_state_for_key.invalidate_all)
             txn.call_after(self.get_rooms_for_user.invalidate_all)
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
+            txn.call_after(self.get_users_with_pushers_in_room.invalidate, (event.room_id,))
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_room_name_and_aliases.invalidate, (event.room_id,))
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index d1669c778a..f7886dd1bb 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -18,6 +18,8 @@ from twisted.internet import defer
 
 from canonicaljson import encode_canonical_json
 
+from synapse.util.caches.descriptors import cachedInlineCallbacks
+
 import logging
 import simplejson as json
 import types
@@ -107,31 +109,46 @@ class PusherStore(SQLBaseStore):
             "get_all_updated_pushers", get_all_updated_pushers_txn
         )
 
+    @cachedInlineCallbacks(num_args=1)
+    def get_users_with_pushers_in_room(self, room_id):
+        users = yield self.get_users_in_room(room_id)
+
+        result = yield self._simple_select_many_batch(
+            'pushers', 'user_name', users, ['user_name']
+        )
+
+        defer.returnValue([r['user_name'] for r in result])
+
     @defer.inlineCallbacks
     def add_pusher(self, user_id, access_token, kind, app_id,
                    app_display_name, device_display_name,
-                   pushkey, pushkey_ts, lang, data, profile_tag=""):
-        with self._pushers_id_gen.get_next() as stream_id:
-            yield self._simple_upsert(
-                "pushers",
-                dict(
-                    app_id=app_id,
-                    pushkey=pushkey,
-                    user_name=user_id,
-                ),
-                dict(
-                    access_token=access_token,
-                    kind=kind,
-                    app_display_name=app_display_name,
-                    device_display_name=device_display_name,
-                    ts=pushkey_ts,
-                    lang=lang,
-                    data=encode_canonical_json(data),
-                    profile_tag=profile_tag,
-                    id=stream_id,
-                ),
-                desc="add_pusher",
-            )
+                   pushkey, pushkey_ts, lang, data, last_stream_ordering,
+                   profile_tag=""):
+        def f(txn):
+            txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
+            with self._pushers_id_gen.get_next() as stream_id:
+                return self._simple_upsert_txn(
+                    txn,
+                    "pushers",
+                    dict(
+                        app_id=app_id,
+                        pushkey=pushkey,
+                        user_name=user_id,
+                    ),
+                    dict(
+                        access_token=access_token,
+                        kind=kind,
+                        app_display_name=app_display_name,
+                        device_display_name=device_display_name,
+                        ts=pushkey_ts,
+                        lang=lang,
+                        data=encode_canonical_json(data),
+                        last_stream_ordering=last_stream_ordering,
+                        profile_tag=profile_tag,
+                        id=stream_id,
+                    ),
+                )
+        defer.returnValue((yield self.runInteraction("add_pusher", f)))
 
     @defer.inlineCallbacks
     def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id):
@@ -153,22 +170,28 @@ class PusherStore(SQLBaseStore):
             )
 
     @defer.inlineCallbacks
-    def update_pusher_last_token(self, app_id, pushkey, user_id, last_token):
+    def update_pusher_last_stream_ordering(self, app_id, pushkey, user_id,
+                                           last_stream_ordering):
         yield self._simple_update_one(
             "pushers",
             {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_id},
-            {'last_token': last_token},
-            desc="update_pusher_last_token",
+            {'last_stream_ordering': last_stream_ordering},
+            desc="update_pusher_last_stream_ordering",
         )
 
     @defer.inlineCallbacks
-    def update_pusher_last_token_and_success(self, app_id, pushkey, user_id,
-                                             last_token, last_success):
+    def update_pusher_last_stream_ordering_and_success(self, app_id, pushkey,
+                                                       user_id,
+                                                       last_stream_ordering,
+                                                       last_success):
         yield self._simple_update_one(
             "pushers",
             {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_id},
-            {'last_token': last_token, 'last_success': last_success},
-            desc="update_pusher_last_token_and_success",
+            {
+                'last_stream_ordering': last_stream_ordering,
+                'last_success': last_success
+            },
+            desc="update_pusher_last_stream_ordering_and_success",
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index d46a963bb8..701dd2f656 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -319,26 +319,6 @@ class RegistrationStore(SQLBaseStore):
 
         defer.returnValue(res if res else False)
 
-    @cachedList(cache=is_guest.cache, list_name="user_ids", num_args=1,
-                inlineCallbacks=True)
-    def are_guests(self, user_ids):
-        sql = "SELECT name, is_guest FROM users WHERE name IN (%s)" % (
-            ",".join("?" for _ in user_ids),
-        )
-
-        rows = yield self._execute(
-            "are_guests", self.cursor_to_dict, sql, *user_ids
-        )
-
-        result = {user_id: False for user_id in user_ids}
-
-        result.update({
-            row["name"]: bool(row["is_guest"])
-            for row in rows
-        })
-
-        defer.returnValue(result)
-
     def _query_for_auth(self, txn, token):
         sql = (
             "SELECT users.name, users.is_guest, access_tokens.id as token_id"
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 66e7a40e3c..22a690aa8d 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -58,6 +58,7 @@ class RoomMemberStore(SQLBaseStore):
             txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,))
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
+            txn.call_after(self.get_users_with_pushers_in_room.invalidate, (event.room_id,))
             txn.call_after(
                 self._membership_stream_cache.entity_has_changed,
                 event.state_key, event.internal_metadata.stream_ordering
diff --git a/synapse/storage/schema/delta/31/pushers.py b/synapse/storage/schema/delta/31/pushers.py
new file mode 100644
index 0000000000..7e0e385fb5
--- /dev/null
+++ b/synapse/storage/schema/delta/31/pushers.py
@@ -0,0 +1,75 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Change the last_token to last_stream_ordering now that pushers no longer
+# listen on an event stream but instead select out of the event_push_actions
+# table.
+
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def token_to_stream_ordering(token):
+    return int(token[1:].split('_')[0])
+
+
+def run_upgrade(cur, database_engine, *args, **kwargs):
+    logger.info("Porting pushers table, delta 31...")
+    cur.execute("""
+        CREATE TABLE IF NOT EXISTS pushers2 (
+          id BIGINT PRIMARY KEY,
+          user_name TEXT NOT NULL,
+          access_token BIGINT DEFAULT NULL,
+          profile_tag VARCHAR(32) NOT NULL,
+          kind VARCHAR(8) NOT NULL,
+          app_id VARCHAR(64) NOT NULL,
+          app_display_name VARCHAR(64) NOT NULL,
+          device_display_name VARCHAR(128) NOT NULL,
+          pushkey TEXT NOT NULL,
+          ts BIGINT NOT NULL,
+          lang VARCHAR(8),
+          data TEXT,
+          last_stream_ordering INTEGER,
+          last_success BIGINT,
+          failing_since BIGINT,
+          UNIQUE (app_id, pushkey, user_name)
+        )
+    """)
+    cur.execute("""SELECT
+        id, user_name, access_token, profile_tag, kind,
+        app_id, app_display_name, device_display_name,
+        pushkey, ts, lang, data, last_token, last_success,
+        failing_since
+        FROM pushers
+    """)
+    count = 0
+    for row in cur.fetchall():
+        row = list(row)
+        row[12] = token_to_stream_ordering(row[12])
+        cur.execute(database_engine.convert_param_style("""
+            INSERT into pushers2 (
+            id, user_name, access_token, profile_tag, kind,
+            app_id, app_display_name, device_display_name,
+            pushkey, ts, lang, data, last_stream_ordering, last_success,
+            failing_since
+            ) values (%s)""" % (','.join(['?' for _ in range(len(row))]))),
+            row
+        )
+        count += 1
+    cur.execute("DROP TABLE pushers")
+    cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
+    logger.info("Moved %d pushers to new table", count)
-- 
cgit 1.5.1


From 0fd1cd24003b54e475985cf90db4223c3098375d Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 6 Apr 2016 16:50:47 +0100
Subject: pep8

---
 synapse/storage/event_push_actions.py | 2 +-
 synapse/storage/events.py             | 4 +++-
 synapse/storage/registration.py       | 2 +-
 synapse/storage/roommember.py         | 4 +++-
 4 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 5f61743e34..4d72e4a85e 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -144,7 +144,7 @@ class EventPushActionsStore(SQLBaseStore):
             txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
             return txn.fetchone()
         result = yield self.runInteraction(
-                "get_latest_push_action_stream_ordering", f
+            "get_latest_push_action_stream_ordering", f
         )
         defer.returnValue(result[0] or 0)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ceae8715ce..5be5bc01b1 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -202,7 +202,9 @@ class EventsStore(SQLBaseStore):
             txn.call_after(self._get_current_state_for_key.invalidate_all)
             txn.call_after(self.get_rooms_for_user.invalidate_all)
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
-            txn.call_after(self.get_users_with_pushers_in_room.invalidate, (event.room_id,))
+            txn.call_after(
+                self.get_users_with_pushers_in_room.invalidate, (event.room_id,)
+            )
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_room_name_and_aliases.invalidate, (event.room_id,))
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 701dd2f656..7af0cae6a5 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
 from synapse.api.errors import StoreError, Codes
 
 from ._base import SQLBaseStore
-from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
+from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
 
 class RegistrationStore(SQLBaseStore):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 22a690aa8d..088ad0f914 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -58,7 +58,9 @@ class RoomMemberStore(SQLBaseStore):
             txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,))
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
-            txn.call_after(self.get_users_with_pushers_in_room.invalidate, (event.room_id,))
+            txn.call_after(
+                self.get_users_with_pushers_in_room.invalidate, (event.room_id,)
+            )
             txn.call_after(
                 self._membership_stream_cache.entity_has_changed,
                 event.state_key, event.internal_metadata.stream_ordering
-- 
cgit 1.5.1


From 92e3071623c34350bf072bb77e089d5d6d5f41c2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 15:39:53 +0100
Subject: Send badge count pushes.

Also fix bugs with retrying.
---
 synapse/handlers/receipts.py | 21 +++++++++++++++++----
 synapse/push/httppusher.py   | 45 ++++++++++++++++++++++++++++----------------
 synapse/push/pusherpool.py   | 20 +++++++++++++++++++-
 synapse/storage/receipts.py  |  9 ++++++---
 4 files changed, 71 insertions(+), 24 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index 935c339707..26b0368080 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -80,6 +80,9 @@ class ReceiptsHandler(BaseHandler):
     def _handle_new_receipts(self, receipts):
         """Takes a list of receipts, stores them and informs the notifier.
         """
+        min_batch_id = None
+        max_batch_id = None
+
         for receipt in receipts:
             room_id = receipt["room_id"]
             receipt_type = receipt["receipt_type"]
@@ -97,10 +100,20 @@ class ReceiptsHandler(BaseHandler):
 
             stream_id, max_persisted_id = res
 
-            with PreserveLoggingContext():
-                self.notifier.on_new_event(
-                    "receipt_key", max_persisted_id, rooms=[room_id]
-                )
+            if min_batch_id is None or stream_id < min_batch_id:
+                min_batch_id = stream_id
+            if max_batch_id is None or max_persisted_id > max_batch_id:
+                max_batch_id = max_persisted_id
+
+        affected_room_ids = list(set([r["room_id"] for r in receipts]))
+
+        with PreserveLoggingContext():
+            self.notifier.on_new_event(
+                "receipt_key", max_batch_id, rooms=affected_room_ids
+            )
+            self.hs.get_pusherpool().on_new_receipts(
+                min_batch_id, max_batch_id, affected_room_ids
+            )
 
             defer.returnValue(True)
 
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index d695885649..0d5450bc01 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -76,15 +76,25 @@ class HttpPusher(object):
         self.data_minus_url.update(self.data)
         del self.data_minus_url['url']
 
+    @defer.inlineCallbacks
     def on_started(self):
-        self._process()
+        yield self._process()
 
+    @defer.inlineCallbacks
     def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
         self.max_stream_ordering = max_stream_ordering
-        self._process()
+        yield self._process()
+
+    @defer.inlineCallbacks
+    def on_new_receipts(self, min_stream_id, max_stream_id):
+        # We could check the receipts are actually m.read receipts here,
+        # but currently that's the only type of receipt anyway...
+        badge = yield push_tools.get_badge_count(self.hs, self.user_id)
+        yield self.send_badge(badge)
 
+    @defer.inlineCallbacks
     def on_timer(self):
-        self._process()
+        yield self._process()
 
     def on_stop(self):
         if self.timed_call:
@@ -106,22 +116,24 @@ class HttpPusher(object):
                     self.last_stream_ordering,
                     self.clock.time_msec()
                 )
-                self.failing_since = None
-                yield self.store.update_pusher_failing_since(
-                    self.app_id, self.pushkey, self.user_id,
-                    self.failing_since
-                )
+                if self.failing_since:
+                    self.failing_since = None
+                    yield self.store.update_pusher_failing_since(
+                        self.app_id, self.pushkey, self.user_id,
+                        self.failing_since
+                    )
             else:
-                self.failing_since = self.clock.time_msec()
-                yield self.store.update_pusher_failing_since(
-                    self.app_id, self.pushkey, self.user_id,
-                    self.failing_since
-                )
+                if not self.failing_since:
+                    self.failing_since = self.clock.time_msec()
+                    yield self.store.update_pusher_failing_since(
+                        self.app_id, self.pushkey, self.user_id,
+                        self.failing_since
+                    )
 
                 if (
                     self.failing_since and
                     self.failing_since <
-                    self.clock.time_msec() - HttpPusher.GIVE_UP_AFTER
+                    self.clock.time_msec() - HttpPusher.GIVE_UP_AFTER_MS
                 ):
                     # we really only give up so that if the URL gets
                     # fixed, we don't suddenly deliver a load
@@ -148,7 +160,7 @@ class HttpPusher(object):
                 else:
                     logger.info("Push failed: delaying for %ds", self.backoff_delay)
                     self.timed_call = reactor.callLater(self.backoff_delay, self.on_timer)
-                    self.backoff_delay = min(self.backoff_delay, self.MAX_BACKOFF_SEC)
+                    self.backoff_delay = min(self.backoff_delay * 2, self.MAX_BACKOFF_SEC)
                     break
 
     @defer.inlineCallbacks
@@ -191,7 +203,8 @@ class HttpPusher(object):
 
         d = {
             'notification': {
-                'id': event.event_id,
+                'id': event.event_id,  # deprecated: remove soon
+                'event_id': event.event_id,
                 'room_id': event.room_id,
                 'type': event.type,
                 'sender': event.user_id,
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index b67ad455ea..7b1ce81e9a 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -126,10 +126,28 @@ class PusherPool:
             for u in users_affected:
                 if u in self.pushers:
                     for p in self.pushers[u].values():
-                        p.on_new_notifications(min_stream_id, max_stream_id)
+                        yield p.on_new_notifications(min_stream_id, max_stream_id)
         except:
             logger.exception("Exception in pusher on_new_notifications")
 
+    @defer.inlineCallbacks
+    def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
+        yield run_on_reactor()
+        try:
+            # Need to subtract 1 from the minimum because the lower bound here
+            # is not inclusive
+            updated_receipts = yield self.store.get_all_updated_receipts(
+                min_stream_id - 1, max_stream_id
+            )
+            # This returns a tuple, user_id is at index 3
+            users_affected = set([r[3] for r in updated_receipts])
+            for u in users_affected:
+                if u in self.pushers:
+                    for p in self.pushers[u].values():
+                        yield p.on_new_receipts(min_stream_id, max_stream_id)
+        except:
+            logger.exception("Exception in pusher on_new_receipts")
+
     @defer.inlineCallbacks
     def _refresh_pusher(self, app_id, pushkey, user_id):
         resultlist = yield self.store.get_pushers_by_app_id_and_pushkey(
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 4befebc8e2..59d1ac0314 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -390,16 +390,19 @@ class ReceiptsStore(SQLBaseStore):
             }
         )
 
-    def get_all_updated_receipts(self, last_id, current_id, limit):
+    def get_all_updated_receipts(self, last_id, current_id, limit=None):
         def get_all_updated_receipts_txn(txn):
             sql = (
                 "SELECT stream_id, room_id, receipt_type, user_id, event_id, data"
                 " FROM receipts_linearized"
                 " WHERE ? < stream_id AND stream_id <= ?"
                 " ORDER BY stream_id ASC"
-                " LIMIT ?"
             )
-            txn.execute(sql, (last_id, current_id, limit))
+            args = [last_id, current_id]
+            if limit is not None:
+                sql += " LIMIT ?"
+                args.append(limit)
+            txn.execute(sql, args)
 
             return txn.fetchall()
         return self.runInteraction(
-- 
cgit 1.5.1


From 2d5c693fd3c72800980f906b8255e3619ac524e2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 16:43:54 +0100
Subject: Fix port script for changes merged from develop

---
 synapse/storage/schema/delta/31/pushers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/pushers.py b/synapse/storage/schema/delta/31/pushers.py
index 7e0e385fb5..d07bab012f 100644
--- a/synapse/storage/schema/delta/31/pushers.py
+++ b/synapse/storage/schema/delta/31/pushers.py
@@ -27,7 +27,7 @@ def token_to_stream_ordering(token):
     return int(token[1:].split('_')[0])
 
 
-def run_upgrade(cur, database_engine, *args, **kwargs):
+def run_create(cur, database_engine, *args, **kwargs):
     logger.info("Porting pushers table, delta 31...")
     cur.execute("""
         CREATE TABLE IF NOT EXISTS pushers2 (
@@ -73,3 +73,6 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
     cur.execute("DROP TABLE pushers")
     cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
     logger.info("Moved %d pushers to new table", count)
+
+def run_upgrade(cur, database_engine, *args, **kwargs):
+    pass
-- 
cgit 1.5.1


From 05d044aac396de9dff64ffb47e8b9a3f43ad0919 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 16:45:38 +0100
Subject: pep8

---
 synapse/storage/schema/delta/31/pushers.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/pushers.py b/synapse/storage/schema/delta/31/pushers.py
index d07bab012f..93367fa09e 100644
--- a/synapse/storage/schema/delta/31/pushers.py
+++ b/synapse/storage/schema/delta/31/pushers.py
@@ -74,5 +74,6 @@ def run_create(cur, database_engine, *args, **kwargs):
     cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
     logger.info("Moved %d pushers to new table", count)
 
+
 def run_upgrade(cur, database_engine, *args, **kwargs):
     pass
-- 
cgit 1.5.1


From ceb599e789ef34a3733a99d17701a75fc5409d17 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 7 Apr 2016 16:26:52 +0100
Subject: Add tests for redactions

---
 synapse/replication/slave/storage/events.py    |  4 +-
 synapse/storage/util/id_generators.py          |  2 +-
 tests/replication/slave/storage/_base.py       |  2 +-
 tests/replication/slave/storage/test_events.py | 51 +++++++++++++++++++++++++-
 4 files changed, 54 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index 707ddd248a..cfc728a038 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -69,6 +69,7 @@ class SlavedEventStore(BaseSlavedStore):
         "_get_current_state_for_key"
     ]
 
+    get_event = DataStore.get_event.__func__
     get_current_state = DataStore.get_current_state.__func__
     get_current_state_for_key = DataStore.get_current_state_for_key.__func__
     get_rooms_for_user_where_membership_is = (
@@ -103,7 +104,7 @@ class SlavedEventStore(BaseSlavedStore):
     def stream_positions(self):
         result = super(SlavedEventStore, self).stream_positions()
         result["events"] = self._stream_id_gen.get_current_token()
-        result["backfilled"] = self._backfill_id_gen.get_current_token()
+        result["backfill"] = self._backfill_id_gen.get_current_token()
         return result
 
     def process_replication(self, result):
@@ -145,7 +146,6 @@ class SlavedEventStore(BaseSlavedStore):
         position = row[0]
         internal = json.loads(row[1])
         event_json = json.loads(row[2])
-
         event = FrozenEvent(event_json, internal_metadata_dict=internal)
         self._invalidate_caches_for_event(
             event, backfilled, reset_state=position in state_resets
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index f69f1cdad4..46cf93ff87 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -112,7 +112,7 @@ class StreamIdGenerator(object):
                 self._current + self._step * (n + 1),
                 self._step
             )
-            self._current += n
+            self._current += n * self._step
 
             for next_id in next_ids:
                 self._unfinished_ids.append(next_id)
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 0f525a8943..983caafe8a 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -51,7 +51,7 @@ class BaseSlavedStoreTestCase(unittest.TestCase):
     def check(self, method, args, expected_result=None):
         master_result = yield getattr(self.master_store, method)(*args)
         slaved_result = yield getattr(self.slaved_store, method)(*args)
-        self.assertEqual(master_result, slaved_result)
         if expected_result is not None:
             self.assertEqual(master_result, expected_result)
             self.assertEqual(slaved_result, expected_result)
+        self.assertEqual(master_result, slaved_result)
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index 9af62702b3..baa4a26eb5 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -205,13 +205,59 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
             [join3]
         )
 
+    @defer.inlineCallbacks
+    def test_redactions(self):
+        yield self.persist(type="m.room.create", key="", creator=USER_ID)
+        yield self.persist(type="m.room.member", key=USER_ID, membership="join")
+
+        msg = yield self.persist(
+            type="m.room.message", msgtype="m.text", body="Hello"
+        )
+        yield self.replicate()
+        yield self.check("get_event", [msg.event_id], msg)
+
+        redaction = yield self.persist(
+            type="m.room.redaction", redacts=msg.event_id
+        )
+        yield self.replicate()
+
+        msg_dict = msg.get_dict()
+        msg_dict["content"] = {}
+        msg_dict["unsigned"]["redacted_by"] = redaction.event_id
+        msg_dict["unsigned"]["redacted_because"] = redaction
+        redacted = FrozenEvent(msg_dict, msg.internal_metadata.get_dict())
+        yield self.check("get_event", [msg.event_id], redacted)
+
+    @defer.inlineCallbacks
+    def test_backfilled_redactions(self):
+        yield self.persist(type="m.room.create", key="", creator=USER_ID)
+        yield self.persist(type="m.room.member", key=USER_ID, membership="join")
+
+        msg = yield self.persist(
+            type="m.room.message", msgtype="m.text", body="Hello"
+        )
+        yield self.replicate()
+        yield self.check("get_event", [msg.event_id], msg)
+
+        redaction = yield self.persist(
+            type="m.room.redaction", redacts=msg.event_id, backfill=True
+        )
+        yield self.replicate()
+
+        msg_dict = msg.get_dict()
+        msg_dict["content"] = {}
+        msg_dict["unsigned"]["redacted_by"] = redaction.event_id
+        msg_dict["unsigned"]["redacted_because"] = redaction
+        redacted = FrozenEvent(msg_dict, msg.internal_metadata.get_dict())
+        yield self.check("get_event", [msg.event_id], redacted)
+
     event_id = 0
 
     @defer.inlineCallbacks
     def persist(
         self, sender=USER_ID, room_id=ROOM_ID, type={}, key=None, internal={},
         state=None, reset_state=False, backfill=False,
-        depth=None, prev_events=[], auth_events=[], prev_state=[],
+        depth=None, prev_events=[], auth_events=[], prev_state=[], redacts=None,
         **content
     ):
         """
@@ -236,6 +282,9 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
             event_dict["state_key"] = key
             event_dict["prev_state"] = prev_state
 
+        if redacts is not None:
+            event_dict["redacts"] = redacts
+
         event = FrozenEvent(event_dict, internal_metadata_dict=internal)
 
         self.event_id += 1
-- 
cgit 1.5.1


From 3fb35cbd6fc52905c88344fd3ea55a4ee1d1c478 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 17:33:37 +0100
Subject: Oops, inequality fail

---
 synapse/storage/event_push_actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 4d72e4a85e..355478957d 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -105,7 +105,7 @@ class EventPushActionsStore(SQLBaseStore):
         def f(txn):
             sql = (
                 "SELECT DISTINCT(user_id) FROM event_push_actions WHERE"
-                " stream_ordering >= ? AND stream_ordering >= ?"
+                " stream_ordering >= ? AND stream_ordering <= ?"
             )
             txn.execute(sql, (min_stream_ordering, max_stream_ordering))
             return [r[0] for r in txn.fetchall()]
-- 
cgit 1.5.1


From 4836864f5681fbcca34a4c40384a7c4c8309b4e2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 17:38:48 +0100
Subject: generate id in the main thread

---
 synapse/storage/pusher.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index f7886dd1bb..b314e3ab4f 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -124,9 +124,9 @@ class PusherStore(SQLBaseStore):
                    app_display_name, device_display_name,
                    pushkey, pushkey_ts, lang, data, last_stream_ordering,
                    profile_tag=""):
-        def f(txn):
-            txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
-            with self._pushers_id_gen.get_next() as stream_id:
+        with self._pushers_id_gen.get_next() as stream_id:
+            def f(txn):
+                txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
                 return self._simple_upsert_txn(
                     txn,
                     "pushers",
-- 
cgit 1.5.1


From d9f38561c8855fa6893868069f0ec00d802618df Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 7 Apr 2016 17:45:01 +0100
Subject: Literally a dictionary

---
 synapse/storage/pusher.py | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index b314e3ab4f..b34a30a8fb 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -130,23 +130,23 @@ class PusherStore(SQLBaseStore):
                 return self._simple_upsert_txn(
                     txn,
                     "pushers",
-                    dict(
-                        app_id=app_id,
-                        pushkey=pushkey,
-                        user_name=user_id,
-                    ),
-                    dict(
-                        access_token=access_token,
-                        kind=kind,
-                        app_display_name=app_display_name,
-                        device_display_name=device_display_name,
-                        ts=pushkey_ts,
-                        lang=lang,
-                        data=encode_canonical_json(data),
-                        last_stream_ordering=last_stream_ordering,
-                        profile_tag=profile_tag,
-                        id=stream_id,
-                    ),
+                    {
+                        "app_id": app_id,
+                        "pushkey": pushkey,
+                        "user_name": user_id,
+                    },
+                    {
+                        "access_token": access_token,
+                        "kind": kind,
+                        "app_display_name": app_display_name,
+                        "device_display_name": device_display_name,
+                        "ts": pushkey_ts,
+                        "lang": lang,
+                        "data": encode_canonical_json(data),
+                        "last_stream_ordering": last_stream_ordering,
+                        "profile_tag": profile_tag,
+                        "id": stream_id,
+                    },
                 )
         defer.returnValue((yield self.runInteraction("add_pusher", f)))
 
-- 
cgit 1.5.1


From 7e2f971c08250cf432d43dd6244faefb2074ff8c Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Fri, 8 Apr 2016 14:01:56 +0100
Subject: Remove some unused functions (#711)

* Remove some unused functions

* get_room_events_stream is only used in tests

* is_exclusive_room might actually be something we want
---
 synapse/appservice/api.py           |   5 -
 synapse/handlers/message.py         |  29 ------
 synapse/handlers/room_member.py     |  13 ---
 synapse/storage/_base.py            |   6 --
 synapse/storage/prepare_database.py |  12 ---
 synapse/storage/presence.py         |  10 --
 synapse/storage/roommember.py       |  33 -------
 synapse/storage/stream.py           |  90 ------------------
 synapse/util/__init__.py            |   3 -
 synapse/util/ratelimitutils.py      |  14 ---
 synapse/util/stringutils.py         |   4 -
 tests/storage/test_presence.py      |  27 ------
 tests/storage/test_redaction.py     |  51 +---------
 tests/storage/test_roommember.py    |   7 --
 tests/storage/test_stream.py        | 185 ------------------------------------
 15 files changed, 4 insertions(+), 485 deletions(-)
 delete mode 100644 tests/storage/test_stream.py

(limited to 'synapse/storage')

diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index bc90605324..6da6a1b62e 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -100,11 +100,6 @@ class ApplicationServiceApi(SimpleHttpClient):
             logger.warning("push_bulk to %s threw exception %s", uri, ex)
         defer.returnValue(False)
 
-    @defer.inlineCallbacks
-    def push(self, service, event, txn_id=None):
-        response = yield self.push_bulk(service, [event], txn_id)
-        defer.returnValue(response)
-
     def _serialize(self, events):
         time_now = self.clock.time_msec()
         return [
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index fa78d4acec..f51feda2f4 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -44,35 +44,6 @@ class MessageHandler(BaseHandler):
         self.validator = EventValidator()
         self.snapshot_cache = SnapshotCache()
 
-    @defer.inlineCallbacks
-    def get_message(self, msg_id=None, room_id=None, sender_id=None,
-                    user_id=None):
-        """ Retrieve a message.
-
-        Args:
-            msg_id (str): The message ID to obtain.
-            room_id (str): The room where the message resides.
-            sender_id (str): The user ID of the user who sent the message.
-            user_id (str): The user ID of the user making this request.
-        Returns:
-            The message, or None if no message exists.
-        Raises:
-            SynapseError if something went wrong.
-        """
-        yield self.auth.check_joined_room(room_id, user_id)
-
-        # Pull out the message from the db
-#        msg = yield self.store.get_message(
-#            room_id=room_id,
-#            msg_id=msg_id,
-#            user_id=sender_id
-#        )
-
-        # TODO (erikj): Once we work out the correct c-s api we need to think
-        # on how to do this.
-
-        defer.returnValue(None)
-
     @defer.inlineCallbacks
     def get_messages(self, requester, room_id=None, pagin_config=None,
                      as_client_event=True):
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 753c75d9c1..b69f36aefe 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -392,19 +392,6 @@ class RoomMemberHandler(BaseHandler):
             and guest_access.content["guest_access"] == "can_join"
         )
 
-    def _should_do_dance(self, current_state, inviter, room_hosts=None):
-        # TODO: Shouldn't this be remote_room_host?
-        room_hosts = room_hosts or []
-
-        is_host_in_room = self.is_host_in_room(current_state)
-        if is_host_in_room:
-            return False, room_hosts
-
-        if inviter and not self.hs.is_mine(inviter):
-            room_hosts.append(inviter.domain)
-
-        return True, room_hosts
-
     @defer.inlineCallbacks
     def lookup_room_alias(self, room_alias):
         """
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 04d7fcf6d6..1e27c2c0ce 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -810,12 +810,6 @@ class SQLBaseStore(object):
 
         return txn.execute(sql, keyvalues.values())
 
-    def get_next_stream_id(self):
-        with self._next_stream_id_lock:
-            i = self._next_stream_id
-            self._next_stream_id += 1
-            return i
-
     def _get_cache_dict(self, db_conn, table, entity_column, stream_column,
                         max_value):
         # Fetch a mapping of room_id -> max stream position for "recent" rooms.
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 00833422af..57f14fd12b 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -30,18 +30,6 @@ SCHEMA_VERSION = 31
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
 
-def read_schema(path):
-    """ Read the named database schema.
-
-    Args:
-        path: Path of the database schema.
-    Returns:
-        A string containing the database schema.
-    """
-    with open(path) as schema_file:
-        return schema_file.read()
-
-
 class PrepareDatabaseException(Exception):
     pass
 
diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py
index 59b4ef5ce6..07f5fae8dd 100644
--- a/synapse/storage/presence.py
+++ b/synapse/storage/presence.py
@@ -176,16 +176,6 @@ class PresenceStore(SQLBaseStore):
             desc="disallow_presence_visible",
         )
 
-    def is_presence_visible(self, observed_localpart, observer_userid):
-        return self._simple_select_one(
-            table="presence_allow_inbound",
-            keyvalues={"observed_user_id": observed_localpart,
-                       "observer_user_id": observer_userid},
-            retcols=["observed_user_id"],
-            allow_none=True,
-            desc="is_presence_visible",
-        )
-
     def add_presence_list_pending(self, observer_localpart, observed_userid):
         return self._simple_insert(
             table="presence_list",
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 66e7a40e3c..77518e893f 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -121,26 +121,6 @@ class RoomMemberStore(SQLBaseStore):
         with self._stream_id_gen.get_next() as stream_ordering:
             yield self.runInteraction("locally_reject_invite", f, stream_ordering)
 
-    def get_room_member(self, user_id, room_id):
-        """Retrieve the current state of a room member.
-
-        Args:
-            user_id (str): The member's user ID.
-            room_id (str): The room the member is in.
-        Returns:
-            Deferred: Results in a MembershipEvent or None.
-        """
-        return self.runInteraction(
-            "get_room_member",
-            self._get_members_events_txn,
-            room_id,
-            user_id=user_id,
-        ).addCallback(
-            self._get_events
-        ).addCallback(
-            lambda events: events[0] if events else None
-        )
-
     @cached(max_entries=5000)
     def get_users_in_room(self, room_id):
         def f(txn):
@@ -203,19 +183,6 @@ class RoomMemberStore(SQLBaseStore):
                 defer.returnValue(invite)
         defer.returnValue(None)
 
-    def get_leave_and_ban_events_for_user(self, user_id):
-        """ Get all the leave events for a user
-        Args:
-            user_id (str): The user ID.
-        Returns:
-            A deferred list of event objects.
-        """
-        return self.get_rooms_for_user_where_membership_is(
-            user_id, (Membership.LEAVE, Membership.BAN)
-        ).addCallback(lambda leaves: self._get_events([
-            leave.event_id for leave in leaves
-        ]))
-
     def get_rooms_for_user_where_membership_is(self, user_id, membership_list):
         """ Get all the rooms for this user where the membership for this user
         matches one in the membership list.
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 76bcd9cd00..95b12559a6 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -303,96 +303,6 @@ class StreamStore(SQLBaseStore):
 
         defer.returnValue(ret)
 
-    def get_room_events_stream(
-        self,
-        user_id,
-        from_key,
-        to_key,
-        limit=0,
-        is_guest=False,
-        room_ids=None
-    ):
-        room_ids = room_ids or []
-        room_ids = [r for r in room_ids]
-        if is_guest:
-            current_room_membership_sql = (
-                "SELECT c.room_id FROM history_visibility AS h"
-                " INNER JOIN current_state_events AS c"
-                " ON h.event_id = c.event_id"
-                " WHERE c.room_id IN (%s)"
-                " AND h.history_visibility = 'world_readable'" % (
-                    ",".join(map(lambda _: "?", room_ids))
-                )
-            )
-            current_room_membership_args = room_ids
-        else:
-            current_room_membership_sql = (
-                "SELECT m.room_id FROM room_memberships as m "
-                " INNER JOIN current_state_events as c"
-                " ON m.event_id = c.event_id AND c.state_key = m.user_id"
-                " WHERE m.user_id = ? AND m.membership = 'join'"
-            )
-            current_room_membership_args = [user_id]
-
-        # We also want to get any membership events about that user, e.g.
-        # invites or leave notifications.
-        membership_sql = (
-            "SELECT m.event_id FROM room_memberships as m "
-            "INNER JOIN current_state_events as c ON m.event_id = c.event_id "
-            "WHERE m.user_id = ? "
-        )
-        membership_args = [user_id]
-
-        if limit:
-            limit = max(limit, MAX_STREAM_SIZE)
-        else:
-            limit = MAX_STREAM_SIZE
-
-        # From and to keys should be integers from ordering.
-        from_id = RoomStreamToken.parse_stream_token(from_key)
-        to_id = RoomStreamToken.parse_stream_token(to_key)
-
-        if from_key == to_key:
-            return defer.succeed(([], to_key))
-
-        sql = (
-            "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE "
-            "(e.outlier = ? AND (room_id IN (%(current)s)) OR "
-            "(event_id IN (%(invites)s))) "
-            "AND e.stream_ordering > ? AND e.stream_ordering <= ? "
-            "ORDER BY stream_ordering ASC LIMIT %(limit)d "
-        ) % {
-            "current": current_room_membership_sql,
-            "invites": membership_sql,
-            "limit": limit
-        }
-
-        def f(txn):
-            args = ([False] + current_room_membership_args + membership_args +
-                    [from_id.stream, to_id.stream])
-            txn.execute(sql, args)
-
-            rows = self.cursor_to_dict(txn)
-
-            ret = self._get_events_txn(
-                txn,
-                [r["event_id"] for r in rows],
-                get_prev_content=True
-            )
-
-            self._set_before_and_after(ret, rows)
-
-            if rows:
-                key = "s%d" % max(r["stream_ordering"] for r in rows)
-            else:
-                # Assume we didn't get anything because there was nothing to
-                # get.
-                key = to_key
-
-            return ret, key
-
-        return self.runInteraction("get_room_events_stream", f)
-
     @defer.inlineCallbacks
     def paginate_room_events(self, room_id, from_key, to_key=None,
                              direction='b', limit=-1):
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 3b9da5b34a..b462495eb8 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -49,9 +49,6 @@ class Clock(object):
         l.start(msec / 1000.0, now=False)
         return l
 
-    def stop_looping_call(self, loop):
-        loop.stop()
-
     def call_later(self, delay, callback, *args, **kwargs):
         """Call something later
 
diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py
index 4076eed269..1101881a2d 100644
--- a/synapse/util/ratelimitutils.py
+++ b/synapse/util/ratelimitutils.py
@@ -100,20 +100,6 @@ class _PerHostRatelimiter(object):
         self.current_processing = set()
         self.request_times = []
 
-    def is_empty(self):
-        time_now = self.clock.time_msec()
-        self.request_times[:] = [
-            r for r in self.request_times
-            if time_now - r < self.window_size
-        ]
-
-        return not (
-            self.ready_request_queue
-            or self.sleeping_requests
-            or self.current_processing
-            or self.request_times
-        )
-
     @contextlib.contextmanager
     def ratelimit(self):
         # `contextlib.contextmanager` takes a generator and turns it into a
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index b490bb8725..a100f151d4 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -21,10 +21,6 @@ _string_with_symbols = (
 )
 
 
-def origin_from_ucid(ucid):
-    return ucid.split("@", 1)[1]
-
-
 def random_string(length):
     return ''.join(random.choice(string.ascii_letters) for _ in xrange(length))
 
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index ec78f007ca..63203cea35 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -34,33 +34,6 @@ class PresenceStoreTestCase(unittest.TestCase):
         self.u_apple = UserID.from_string("@apple:test")
         self.u_banana = UserID.from_string("@banana:test")
 
-    @defer.inlineCallbacks
-    def test_visibility(self):
-        self.assertFalse((yield self.store.is_presence_visible(
-            observed_localpart=self.u_apple.localpart,
-            observer_userid=self.u_banana.to_string(),
-        )))
-
-        yield self.store.allow_presence_visible(
-            observed_localpart=self.u_apple.localpart,
-            observer_userid=self.u_banana.to_string(),
-        )
-
-        self.assertTrue((yield self.store.is_presence_visible(
-            observed_localpart=self.u_apple.localpart,
-            observer_userid=self.u_banana.to_string(),
-        )))
-
-        yield self.store.disallow_presence_visible(
-            observed_localpart=self.u_apple.localpart,
-            observer_userid=self.u_banana.to_string(),
-        )
-
-        self.assertFalse((yield self.store.is_presence_visible(
-            observed_localpart=self.u_apple.localpart,
-            observer_userid=self.u_banana.to_string(),
-        )))
-
     @defer.inlineCallbacks
     def test_presence_list(self):
         self.assertEquals(
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 5880409867..6afaca3a61 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -110,22 +110,10 @@ class RedactionTestCase(unittest.TestCase):
             self.room1, self.u_alice, Membership.JOIN
         )
 
-        start = yield self.store.get_room_events_max_id()
-
         msg_event = yield self.inject_message(self.room1, self.u_alice, u"t")
 
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_alice.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
         # Check event has not been redacted:
-        event = results[0]
+        event = yield self.store.get_event(msg_event.event_id)
 
         self.assertObjectHasAttributes(
             {
@@ -144,17 +132,7 @@ class RedactionTestCase(unittest.TestCase):
             self.room1, msg_event.event_id, self.u_alice, reason
         )
 
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_alice.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
-        # Check redaction
-
-        event = results[0]
+        event = yield self.store.get_event(msg_event.event_id)
 
         self.assertEqual(msg_event.event_id, event.event_id)
 
@@ -184,25 +162,12 @@ class RedactionTestCase(unittest.TestCase):
             self.room1, self.u_alice, Membership.JOIN
         )
 
-        start = yield self.store.get_room_events_max_id()
-
         msg_event = yield self.inject_room_member(
             self.room1, self.u_bob, Membership.JOIN,
             extra_content={"blue": "red"},
         )
 
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_alice.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
-        # Check event has not been redacted:
-        event = results[0]
+        event = yield self.store.get_event(msg_event.event_id)
 
         self.assertObjectHasAttributes(
             {
@@ -221,17 +186,9 @@ class RedactionTestCase(unittest.TestCase):
             self.room1, msg_event.event_id, self.u_alice, reason
         )
 
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_alice.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
         # Check redaction
 
-        event = results[0]
+        event = yield self.store.get_event(msg_event.event_id)
 
         self.assertTrue("redacted_because" in event.unsigned)
 
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index b029ff0584..997090fe35 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -70,13 +70,6 @@ class RoomMemberStoreTestCase(unittest.TestCase):
     def test_one_member(self):
         yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)
 
-        self.assertEquals(
-            Membership.JOIN,
-            (yield self.store.get_room_member(
-                user_id=self.u_alice.to_string(),
-                room_id=self.room.to_string(),
-            )).membership
-        )
         self.assertEquals(
             [self.u_alice.to_string()],
             [m.user_id for m in (
diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py
deleted file mode 100644
index da322152c7..0000000000
--- a/tests/storage/test_stream.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from tests import unittest
-from twisted.internet import defer
-
-from synapse.api.constants import EventTypes, Membership
-from synapse.types import UserID, RoomID
-from tests.storage.event_injector import EventInjector
-
-from tests.utils import setup_test_homeserver
-
-from mock import Mock
-
-
-class StreamStoreTestCase(unittest.TestCase):
-
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(
-            resource_for_federation=Mock(),
-            http_client=None,
-        )
-
-        self.store = hs.get_datastore()
-        self.event_builder_factory = hs.get_event_builder_factory()
-        self.event_injector = EventInjector(hs)
-        self.handlers = hs.get_handlers()
-        self.message_handler = self.handlers.message_handler
-
-        self.u_alice = UserID.from_string("@alice:test")
-        self.u_bob = UserID.from_string("@bob:test")
-
-        self.room1 = RoomID.from_string("!abc123:test")
-        self.room2 = RoomID.from_string("!xyx987:test")
-
-    @defer.inlineCallbacks
-    def test_event_stream_get_other(self):
-        # Both bob and alice joins the room
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN
-        )
-
-        # Initial stream key:
-        start = yield self.store.get_room_events_max_id()
-
-        yield self.event_injector.inject_message(self.room1, self.u_alice, u"test")
-
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_bob.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
-        event = results[0]
-
-        self.assertObjectHasAttributes(
-            {
-                "type": EventTypes.Message,
-                "user_id": self.u_alice.to_string(),
-                "content": {"body": "test", "msgtype": "message"},
-            },
-            event,
-        )
-
-    @defer.inlineCallbacks
-    def test_event_stream_get_own(self):
-        # Both bob and alice joins the room
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN
-        )
-
-        # Initial stream key:
-        start = yield self.store.get_room_events_max_id()
-
-        yield self.event_injector.inject_message(self.room1, self.u_alice, u"test")
-
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_alice.to_string(),
-            start,
-            end,
-        )
-
-        self.assertEqual(1, len(results))
-
-        event = results[0]
-
-        self.assertObjectHasAttributes(
-            {
-                "type": EventTypes.Message,
-                "user_id": self.u_alice.to_string(),
-                "content": {"body": "test", "msgtype": "message"},
-            },
-            event,
-        )
-
-    @defer.inlineCallbacks
-    def test_event_stream_join_leave(self):
-        # Both bob and alice joins the room
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN
-        )
-
-        # Then bob leaves again.
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_bob, Membership.LEAVE
-        )
-
-        # Initial stream key:
-        start = yield self.store.get_room_events_max_id()
-
-        yield self.event_injector.inject_message(self.room1, self.u_alice, u"test")
-
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_bob.to_string(),
-            start,
-            end,
-        )
-
-        # We should not get the message, as it happened *after* bob left.
-        self.assertEqual(0, len(results))
-
-    @defer.inlineCallbacks
-    def test_event_stream_prev_content(self):
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN
-        )
-
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
-
-        start = yield self.store.get_room_events_max_id()
-
-        yield self.event_injector.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN,
-        )
-
-        end = yield self.store.get_room_events_max_id()
-
-        results, _ = yield self.store.get_room_events_stream(
-            self.u_bob.to_string(),
-            start,
-            end,
-        )
-
-        # We should not get the message, as it happened *after* bob left.
-        self.assertEqual(1, len(results))
-
-        event = results[0]
-
-        self.assertTrue(
-            "prev_content" in event.unsigned,
-            msg="No prev_content key"
-        )
-- 
cgit 1.5.1


From ed3979df5faac6d63990f4230662ff8cdcf59584 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 8 Apr 2016 15:29:59 +0100
Subject: Fix invite pushes

 * If the event is an invite event, add the invitee to list of user we run push rules for (if they have a pusher etc)
 * Move invite_for_me to be higher prio than member events otherwise member events matches them
 * Spell override right
---
 synapse/push/action_generator.py         |  6 +--
 synapse/push/baserules.py                | 72 ++++++++++++++++----------------
 synapse/push/bulk_push_rule_evaluator.py | 12 +++++-
 synapse/storage/pusher.py                |  7 ++++
 4 files changed, 58 insertions(+), 39 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py
index 84efcdd184..59e512f507 100644
--- a/synapse/push/action_generator.py
+++ b/synapse/push/action_generator.py
@@ -15,7 +15,7 @@
 
 from twisted.internet import defer
 
-from .bulk_push_rule_evaluator import evaluator_for_room_id
+from .bulk_push_rule_evaluator import evaluator_for_event
 
 import logging
 
@@ -35,8 +35,8 @@ class ActionGenerator:
 
     @defer.inlineCallbacks
     def handle_push_actions_for_event(self, event, context, handler):
-        bulk_evaluator = yield evaluator_for_room_id(
-            event.room_id, self.hs, self.store
+        bulk_evaluator = yield evaluator_for_event(
+            event, self.hs, self.store
         )
 
         actions_by_user = yield bulk_evaluator.action_for_event_by_user(
diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py
index 6add94beeb..8a174feeaf 100644
--- a/synapse/push/baserules.py
+++ b/synapse/push/baserules.py
@@ -79,7 +79,7 @@ def make_base_append_rules(kind, modified_base_rules):
     rules = []
 
     if kind == 'override':
-        rules = BASE_APPEND_OVRRIDE_RULES
+        rules = BASE_APPEND_OVERRIDE_RULES
     elif kind == 'underride':
         rules = BASE_APPEND_UNDERRIDE_RULES
     elif kind == 'content':
@@ -148,7 +148,7 @@ BASE_PREPEND_OVERRIDE_RULES = [
 ]
 
 
-BASE_APPEND_OVRRIDE_RULES = [
+BASE_APPEND_OVERRIDE_RULES = [
     {
         'rule_id': 'global/override/.m.rule.suppress_notices',
         'conditions': [
@@ -163,6 +163,40 @@ BASE_APPEND_OVRRIDE_RULES = [
             'dont_notify',
         ]
     },
+    # NB. .m.rule.invite_for_me must be higher prio than .m.rule.member_event
+    # otherwise invites will be matched by .m.rule.member_event
+    {
+        'rule_id': 'global/underride/.m.rule.invite_for_me',
+        'conditions': [
+            {
+                'kind': 'event_match',
+                'key': 'type',
+                'pattern': 'm.room.member',
+                '_id': '_member',
+            },
+            {
+                'kind': 'event_match',
+                'key': 'content.membership',
+                'pattern': 'invite',
+                '_id': '_invite_member',
+            },
+            {
+                'kind': 'event_match',
+                'key': 'state_key',
+                'pattern_type': 'user_id'
+            },
+        ],
+        'actions': [
+            'notify',
+            {
+                'set_tweak': 'sound',
+                'value': 'default'
+            }, {
+                'set_tweak': 'highlight',
+                'value': False
+            }
+        ]
+    },
     # Will we sometimes want to know about people joining and leaving?
     # Perhaps: if so, this could be expanded upon. Seems the most usual case
     # is that we don't though. We add this override rule so that even if
@@ -251,38 +285,6 @@ BASE_APPEND_UNDERRIDE_RULES = [
             }
         ]
     },
-    {
-        'rule_id': 'global/underride/.m.rule.invite_for_me',
-        'conditions': [
-            {
-                'kind': 'event_match',
-                'key': 'type',
-                'pattern': 'm.room.member',
-                '_id': '_member',
-            },
-            {
-                'kind': 'event_match',
-                'key': 'content.membership',
-                'pattern': 'invite',
-                '_id': '_invite_member',
-            },
-            {
-                'kind': 'event_match',
-                'key': 'state_key',
-                'pattern_type': 'user_id'
-            },
-        ],
-        'actions': [
-            'notify',
-            {
-                'set_tweak': 'sound',
-                'value': 'default'
-            }, {
-                'set_tweak': 'highlight',
-                'value': False
-            }
-        ]
-    },
     {
         'rule_id': 'global/underride/.m.rule.message',
         'conditions': [
@@ -315,7 +317,7 @@ for r in BASE_PREPEND_OVERRIDE_RULES:
     r['default'] = True
     BASE_RULE_IDS.add(r['rule_id'])
 
-for r in BASE_APPEND_OVRRIDE_RULES:
+for r in BASE_APPEND_OVERRIDE_RULES:
     r['priority_class'] = PRIORITY_CLASS_MAP['override']
     r['default'] = True
     BASE_RULE_IDS.add(r['rule_id'])
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 7f94591dcb..49216f0c15 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -69,7 +69,8 @@ def _get_rules(room_id, user_ids, store):
 
 
 @defer.inlineCallbacks
-def evaluator_for_room_id(room_id, hs, store):
+def evaluator_for_event(event, hs, store):
+    room_id = event.room_id
     users_with_pushers = yield store.get_users_with_pushers_in_room(room_id)
     receipts = yield store.get_receipts_for_room(room_id, "m.read")
 
@@ -79,6 +80,15 @@ def evaluator_for_room_id(room_id, hs, store):
         if hs.is_mine_id(r['user_id']):
             user_ids.add(r['user_id'])
 
+    # if this event is an invite event, we may need to run rules for the user
+    # who's been invited, otherwise they won't get told they've been invited
+    if event.type == 'm.room.member' and event.content['membership'] == 'invite':
+        invited_user = event.state_key
+        if invited_user and hs.is_mine_id(invited_user):
+            has_pusher = yield store.user_has_pusher(invited_user)
+            if has_pusher:
+                user_ids.add(invited_user)
+
     user_ids = list(user_ids)
 
     rules_by_user = yield _get_rules(room_id, user_ids, store)
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index b34a30a8fb..19888a8e76 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -49,6 +49,13 @@ class PusherStore(SQLBaseStore):
 
         return rows
 
+    @defer.inlineCallbacks
+    def user_has_pusher(self, user_id):
+        ret = yield self._simple_select_one_onecol(
+            "pushers", {"user_name": user_id}, "id", allow_none=True
+        )
+        defer.returnValue(ret is not None)
+
     @defer.inlineCallbacks
     def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
         def r(txn):
-- 
cgit 1.5.1


From fb83f6a1fcb8125ee1ddca81269b3593c727e84d Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 8 Apr 2016 18:55:38 +0100
Subject: fix SQL based on PR feedback

---
 synapse/storage/media_repository.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py
index c9dd20eed8..6bc53bbb6d 100644
--- a/synapse/storage/media_repository.py
+++ b/synapse/storage/media_repository.py
@@ -58,9 +58,10 @@ class MediaRepositoryStore(SQLBaseStore):
         def get_url_cache_txn(txn):
             # get the most recently cached result (relative to the given ts)
             sql = (
-                "SELECT response_code, etag, expires, og, media_id, max(download_ts)"
+                "SELECT response_code, etag, expires, og, media_id, download_ts"
                 " FROM local_media_repository_url_cache"
                 " WHERE url = ? AND download_ts <= ?"
+                " ORDER BY download_ts DESC LIMIT 1"
             )
             txn.execute(sql, (url, ts))
             row = txn.fetchone()
@@ -69,9 +70,10 @@ class MediaRepositoryStore(SQLBaseStore):
                 # ...or if we've requested a timestamp older than the oldest
                 # copy in the cache, return the oldest copy (if any)
                 sql = (
-                    "SELECT response_code, etag, expires, og, media_id, min(download_ts)"
+                    "SELECT response_code, etag, expires, og, media_id, download_ts"
                     " FROM local_media_repository_url_cache"
                     " WHERE url = ? AND download_ts > ?"
+                    " ORDER BY download_ts ASC LIMIT 1"
                 )
                 txn.execute(sql, (url, ts))
                 row = txn.fetchone()
-- 
cgit 1.5.1


From 2460d904bd1e910884c1854a8c5d57df8aa4b765 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 8 Apr 2016 19:04:29 +0100
Subject: fix error checking for new SQL

---
 synapse/storage/media_repository.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py
index 6bc53bbb6d..a820fcf07f 100644
--- a/synapse/storage/media_repository.py
+++ b/synapse/storage/media_repository.py
@@ -66,7 +66,7 @@ class MediaRepositoryStore(SQLBaseStore):
             txn.execute(sql, (url, ts))
             row = txn.fetchone()
 
-            if not row[3]:
+            if not row:
                 # ...or if we've requested a timestamp older than the oldest
                 # copy in the cache, return the oldest copy (if any)
                 sql = (
@@ -78,7 +78,7 @@ class MediaRepositoryStore(SQLBaseStore):
                 txn.execute(sql, (url, ts))
                 row = txn.fetchone()
 
-            if not row[3]:
+            if not row:
                 return None
 
             return dict(zip((
-- 
cgit 1.5.1


From 6ff7a79308ccd1b57109b0ed52e9ebbc9949ec8f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 8 Apr 2016 19:09:02 +0100
Subject: move local_media_repository_url_cache.sql to schema v31

---
 .../delta/30/local_media_repository_url_cache.sql  | 27 ----------------------
 .../delta/31/local_media_repository_url_cache.sql  | 27 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 27 deletions(-)
 delete mode 100644 synapse/storage/schema/delta/30/local_media_repository_url_cache.sql
 create mode 100644 synapse/storage/schema/delta/31/local_media_repository_url_cache.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql b/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql
deleted file mode 100644
index 9efb4280eb..0000000000
--- a/synapse/storage/schema/delta/30/local_media_repository_url_cache.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE local_media_repository_url_cache(
-    url TEXT,              -- the URL being cached
-    response_code INTEGER, -- the HTTP response code of this download attempt
-    etag TEXT,             -- the etag header of this response
-    expires INTEGER,       -- the number of ms this response was valid for
-    og TEXT,               -- cache of the OG metadata of this URL as JSON
-    media_id TEXT,         -- the media_id, if any, of the URL's content in the repo
-    download_ts BIGINT     -- the timestamp of this download attempt
-);
-
-CREATE INDEX local_media_repository_url_cache_by_url_download_ts
-    ON local_media_repository_url_cache(url, download_ts);
diff --git a/synapse/storage/schema/delta/31/local_media_repository_url_cache.sql b/synapse/storage/schema/delta/31/local_media_repository_url_cache.sql
new file mode 100644
index 0000000000..9efb4280eb
--- /dev/null
+++ b/synapse/storage/schema/delta/31/local_media_repository_url_cache.sql
@@ -0,0 +1,27 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE local_media_repository_url_cache(
+    url TEXT,              -- the URL being cached
+    response_code INTEGER, -- the HTTP response code of this download attempt
+    etag TEXT,             -- the etag header of this response
+    expires INTEGER,       -- the number of ms this response was valid for
+    og TEXT,               -- cache of the OG metadata of this URL as JSON
+    media_id TEXT,         -- the media_id, if any, of the URL's content in the repo
+    download_ts BIGINT     -- the timestamp of this download attempt
+);
+
+CREATE INDEX local_media_repository_url_cache_by_url_download_ts
+    ON local_media_repository_url_cache(url, download_ts);
-- 
cgit 1.5.1


From 8a76094965635eb2afd501038ea3c09f88416ce2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 11 Apr 2016 18:00:03 +0100
Subject: Add get endpoint for pushers

As per https://github.com/matrix-org/matrix-doc/pull/308
---
 synapse/rest/client/v1/pusher.py | 35 ++++++++++++++++++++++++++++++++++-
 synapse/storage/pusher.py        | 19 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 9881f068c3..5c88a9ba09 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -27,14 +27,47 @@ logger = logging.getLogger(__name__)
 
 
 class PusherRestServlet(ClientV1RestServlet):
-    PATTERNS = client_path_patterns("/pushers/set$")
+    PATTERNS = client_path_patterns("/pushers(/set)?$")
 
     def __init__(self, hs):
         super(PusherRestServlet, self).__init__(hs)
         self.notifier = hs.get_notifier()
 
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        if request.postpath != ["pushers"]:
+            defer.returnValue((405, {}))
+
+        requester = yield self.auth.get_user_by_req(request)
+        user = requester.user
+
+        pushers = yield self.hs.get_datastore().get_pushers_by_app_user_id(
+            user.to_string()
+        )
+
+        allowed_keys = [
+            "app_display_name",
+            "app_id",
+            "data",
+            "device_display_name",
+            "kind",
+            "lang",
+            "profile_tag",
+            "pushkey",
+        ]
+
+        for p in pushers:
+            for k, v in p.items():
+                if k not in allowed_keys:
+                    del p[k]
+
+        defer.returnValue((200, {"pushers": pushers}))
+
     @defer.inlineCallbacks
     def on_POST(self, request):
+        if request.postpath != ["pushers", "set"]:
+            defer.returnValue((405, {}))
+
         requester = yield self.auth.get_user_by_req(request)
         user = requester.user
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 19888a8e76..11feab72de 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -75,6 +75,25 @@ class PusherStore(SQLBaseStore):
 
         defer.returnValue(rows)
 
+    @defer.inlineCallbacks
+    def get_pushers_by_app_user_id(self, user_id):
+        def r(txn):
+            sql = (
+                "SELECT * FROM pushers"
+                " WHERE user_name = ?"
+            )
+
+            txn.execute(sql, (user_id,))
+            rows = self.cursor_to_dict(txn)
+
+            return self._decode_pushers_rows(rows)
+
+        result = yield self.runInteraction(
+            "get_pushers_by_user_id", r
+        )
+
+        defer.returnValue(result)
+
     @defer.inlineCallbacks
     def get_all_pushers(self):
         def get_pushers(txn):
-- 
cgit 1.5.1


From cccf86dd05a72f87aaf6fe288c700697954a2144 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 12 Apr 2016 11:19:10 +0100
Subject: Check if we've already backfilled events

---
 synapse/handlers/federation.py | 27 ++++++++++++++++++++-------
 synapse/storage/events.py      | 16 ++++++++++++++++
 2 files changed, 36 insertions(+), 7 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 83dab32bcb..a38300ec6a 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -294,6 +294,15 @@ class FederationHandler(BaseHandler):
             extremities=extremities,
         )
 
+        seen_events = yield self.store.have_events_in_timeline(
+            set(e.event_id for e in events)
+        )
+
+        events = [e for e in events if e.event_id not in seen_events]
+
+        if not events:
+            defer.returnValue([])
+
         event_map = {e.event_id: e for e in events}
 
         event_ids = set(e.event_id for e in events)
@@ -353,6 +362,7 @@ class FederationHandler(BaseHandler):
         for a in auth_events.values():
             if a.event_id in seen_events:
                 continue
+            a.internal_metadata.outlier = True
             ev_infos.append({
                 "event": a,
                 "auth_events": {
@@ -373,6 +383,11 @@ class FederationHandler(BaseHandler):
                 }
             })
 
+        yield self._handle_new_events(
+            dest, ev_infos,
+            backfilled=True,
+        )
+
         events.sort(key=lambda e: e.depth)
 
         for event in events:
@@ -383,10 +398,9 @@ class FederationHandler(BaseHandler):
                 "event": event,
             })
 
-        yield self._handle_new_events(
-            dest, ev_infos,
-            backfilled=True,
-        )
+            yield self._handle_new_event(
+                dest, event
+            )
 
         defer.returnValue(events)
 
@@ -458,11 +472,12 @@ class FederationHandler(BaseHandler):
             # TODO: Should we try multiple of these at a time?
             for dom in domains:
                 try:
-                    events = yield self.backfill(
+                    yield self.backfill(
                         dom, room_id,
                         limit=100,
                         extremities=[e for e in extremities.keys()]
                     )
+                    defer.returnValue(True)
                 except SynapseError as e:
                     logger.info(
                         "Failed to backfill from %s because %s",
@@ -488,8 +503,6 @@ class FederationHandler(BaseHandler):
                     )
                     continue
 
-                if events:
-                    defer.returnValue(True)
             defer.returnValue(False)
 
         success = yield try_backfill(likely_domains)
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 308a2c9b02..f847c161af 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -543,6 +543,22 @@ class EventsStore(SQLBaseStore):
             (event.event_id, event.redacts)
         )
 
+    @defer.inlineCallbacks
+    def have_events_in_timeline(self, event_ids):
+        """Given a list of event ids, check if we have already processed and
+        stored them as non outliers.
+        """
+        rows = yield self._simple_select_many_batch(
+            table="events",
+            retcols=("event_id",),
+            column="event_id",
+            iterable=list(event_ids),
+            keyvalues={"outlier": False},
+            desc="have_events_in_timeline",
+        )
+
+        defer.returnValue(set(r["event_id"] for r in rows))
+
     def have_events(self, event_ids):
         """Given a list of event ids, check if we have already processed them.
 
-- 
cgit 1.5.1


From 762ada1e079350df46fc8e1c923b7c5beeb533c0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 12 Apr 2016 11:58:04 +0100
Subject: Add back backfilled parameter that was removed

---
 synapse/handlers/federation.py | 4 +++-
 synapse/storage/events.py      | 7 ++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 654fd3829f..cb56733b92 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1090,7 +1090,8 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def _handle_new_event(self, origin, event, state=None, auth_events=None):
+    def _handle_new_event(self, origin, event, state=None, auth_events=None,
+                          backfilled=False):
         context = yield self._prep_event(
             origin, event,
             state=state,
@@ -1106,6 +1107,7 @@ class FederationHandler(BaseHandler):
         event_stream_id, max_stream_id = yield self.store.persist_event(
             event,
             context=context,
+            backfilled=backfilled,
         )
 
         # this intentionally does not yield: we don't care about the result
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f847c161af..21487724ed 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -118,7 +118,7 @@ class EventsStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     @log_function
-    def persist_event(self, event, context, current_state=None):
+    def persist_event(self, event, context, current_state=None, backfilled=False):
 
         try:
             with self._stream_id_gen.get_next() as stream_ordering:
@@ -131,6 +131,7 @@ class EventsStore(SQLBaseStore):
                         event=event,
                         context=context,
                         current_state=current_state,
+                        backfilled=backfilled,
                     )
         except _RollbackButIsFineException:
             pass
@@ -195,7 +196,7 @@ class EventsStore(SQLBaseStore):
         defer.returnValue({e.event_id: e for e in events})
 
     @log_function
-    def _persist_event_txn(self, txn, event, context, current_state):
+    def _persist_event_txn(self, txn, event, context, current_state, backfilled=False):
         # We purposefully do this first since if we include a `current_state`
         # key, we *want* to update the `current_state_events` table
         if current_state:
@@ -238,7 +239,7 @@ class EventsStore(SQLBaseStore):
         return self._persist_events_txn(
             txn,
             [(event, context)],
-            backfilled=False,
+            backfilled=backfilled,
         )
 
     @log_function
-- 
cgit 1.5.1


From 7b39bcdaae70e3a167d38bb1cdf8e6d4d95f0cca Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 12 Apr 2016 13:35:08 +0100
Subject: Mis-named function

---
 synapse/rest/client/v1/pusher.py | 2 +-
 synapse/storage/pusher.py        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 321eadbc24..d792dac648 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -37,7 +37,7 @@ class PushersRestServlet(ClientV1RestServlet):
         requester = yield self.auth.get_user_by_req(request)
         user = requester.user
 
-        pushers = yield self.hs.get_datastore().get_pushers_by_app_user_id(
+        pushers = yield self.hs.get_datastore().get_pushers_by_user_id(
             user.to_string()
         )
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 11feab72de..90ec06809a 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -76,7 +76,7 @@ class PusherStore(SQLBaseStore):
         defer.returnValue(rows)
 
     @defer.inlineCallbacks
-    def get_pushers_by_app_user_id(self, user_id):
+    def get_pushers_by_user_id(self, user_id):
         def r(txn):
             sql = (
                 "SELECT * FROM pushers"
-- 
cgit 1.5.1


From 44891b4a0aaa28a3b202d22ef14c6c3b2bca40a5 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 12 Apr 2016 13:47:17 +0100
Subject: Tidy up get_pusher functions

Decodes pushers rows on the main thread rather than the db thread and uses _simple_select_list. Also do the same to the function I copied and factor out the duplication into a helper function.
---
 synapse/storage/pusher.py | 62 +++++++++++++++++++++++------------------------
 1 file changed, 30 insertions(+), 32 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 90ec06809a..774eda0f3d 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -56,43 +56,41 @@ class PusherStore(SQLBaseStore):
         )
         defer.returnValue(ret is not None)
 
-    @defer.inlineCallbacks
     def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
-        def r(txn):
-            sql = (
-                "SELECT * FROM pushers"
-                " WHERE app_id = ? AND pushkey = ?"
-            )
-
-            txn.execute(sql, (app_id, pushkey,))
-            rows = self.cursor_to_dict(txn)
-
-            return self._decode_pushers_rows(rows)
-
-        rows = yield self.runInteraction(
-            "get_pushers_by_app_id_and_pushkey", r
-        )
-
-        defer.returnValue(rows)
+        return self.get_pushers_by({
+            "app_id": app_id,
+            "pushkey": pushkey,
+        })
 
-    @defer.inlineCallbacks
     def get_pushers_by_user_id(self, user_id):
-        def r(txn):
-            sql = (
-                "SELECT * FROM pushers"
-                " WHERE user_name = ?"
-            )
-
-            txn.execute(sql, (user_id,))
-            rows = self.cursor_to_dict(txn)
-
-            return self._decode_pushers_rows(rows)
+        return self.get_pushers_by({
+            "user_name": user_id,
+        })
 
-        result = yield self.runInteraction(
-            "get_pushers_by_user_id", r
+    @defer.inlineCallbacks
+    def get_pushers_by(self, keyvalues):
+        ret = yield self._simple_select_list(
+            "pushers", keyvalues,
+            [
+                "id",
+                "user_name",
+                "access_token",
+                "profile_tag",
+                "kind",
+                "app_id",
+                "app_display_name",
+                "device_display_name",
+                "pushkey",
+                "ts",
+                "lang",
+                "data",
+                "last_stream_ordering",
+                "last_success",
+                "failing_since",
+
+            ]
         )
-
-        defer.returnValue(result)
+        defer.returnValue(self._decode_pushers_rows(ret))
 
     @defer.inlineCallbacks
     def get_all_pushers(self):
-- 
cgit 1.5.1


From c1267d04c5fdd900ea917682056b1239495f90ee Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 12 Apr 2016 13:55:32 +0100
Subject: Oops, forgot the desc.

---
 synapse/storage/pusher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 774eda0f3d..e0b80d64f4 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -88,7 +88,7 @@ class PusherStore(SQLBaseStore):
                 "last_success",
                 "failing_since",
 
-            ]
+            ], desc="get_pushers_by"
         )
         defer.returnValue(self._decode_pushers_rows(ret))
 
-- 
cgit 1.5.1


From 7984ffdc6a1850f6e0eb46562c53a10ce31e09b0 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 12 Apr 2016 13:55:57 +0100
Subject: Unneccessarywhitespaceisunnecessary

---
 synapse/storage/pusher.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index e0b80d64f4..e64c0dce0a 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -87,7 +87,6 @@ class PusherStore(SQLBaseStore):
                 "last_stream_ordering",
                 "last_success",
                 "failing_since",
-
             ], desc="get_pushers_by"
         )
         defer.returnValue(self._decode_pushers_rows(ret))
-- 
cgit 1.5.1


From 54d08c88686eee0f2a411a6f79b9e374d6f49d01 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 13 Apr 2016 14:16:45 +0100
Subject: Only return unread notifications

Make get_unread_push_actions_for_user_in_range only return unread event actions, being more true to its name. Done in two separate sql queries to get actions after a read receipt and those in a room wiht no receipt at all. SQL queries by Erik.
---
 synapse/storage/event_push_actions.py | 59 +++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 355478957d..3269834e2f 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -116,26 +116,67 @@ class EventPushActionsStore(SQLBaseStore):
     def get_unread_push_actions_for_user_in_range(self, user_id,
                                                   min_stream_ordering,
                                                   max_stream_ordering=None):
-        def f(txn):
+        def get_after_receipt(txn):
             sql = (
-                "SELECT event_id, stream_ordering, actions"
-                " FROM event_push_actions"
-                " WHERE user_id = ? AND stream_ordering > ?"
+                "SELECT ep.event_id, ep.stream_ordering, ep.actions "
+                "FROM event_push_actions AS ep, ("
+                "   SELECT room_id, user_id,"
+                "       max(topological_ordering) as topological_ordering,"
+                "       max(stream_ordering) as stream_ordering"
+                "       FROM events"
+                "   NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'"
+                "   GROUP BY room_id, user_id"
+                ") AS rl "
+                "WHERE"
+                "   ep.room_id = rl.room_id"
+                "   AND ("
+                "       ep.topological_ordering > rl.topological_ordering"
+                "       OR ("
+                "           ep.topological_ordering = rl.topological_ordering"
+                "           AND ep.stream_ordering > ?"
+                "       )"
+                "   )"
+                "   AND ep.user_id = ?"
+                "   AND ep.user_id = rl.user_id"
             )
-            args = [user_id, min_stream_ordering]
+            args = [min_stream_ordering, user_id]
             if max_stream_ordering is not None:
-                sql += " AND stream_ordering <= ?"
+                sql += " AND ep.stream_ordering <= ?"
                 args.append(max_stream_ordering)
-            sql += " ORDER BY stream_ordering ASC"
+            sql += " ORDER BY ep.stream_ordering ASC"
             txn.execute(sql, args)
             return txn.fetchall()
-        ret = yield self.runInteraction("get_unread_push_actions_for_user_in_range", f)
+        after_read_receipt = yield self.runInteraction(
+            "get_unread_push_actions_for_user_in_range", get_after_receipt
+        )
+
+        def get_no_receipt(txn):
+            sql = (
+                "SELECT ep.event_id, ep.stream_ordering, ep.actions "
+                "FROM event_push_actions AS ep "
+                "WHERE ep.room_id not in ("
+                "   SELECT room_id FROM events NATURAL JOIN receipts_linearized"
+                "   WHERE receipt_type = 'm.read' AND user_id = ? "
+                "   GROUP BY room_id"
+                ") AND ep.user_id = ? AND ep.stream_ordering > ?"
+            )
+            args = [user_id, user_id, min_stream_ordering]
+            if max_stream_ordering is not None:
+                sql += " AND ep.stream_ordering <= ?"
+                args.append(max_stream_ordering)
+            sql += " ORDER BY ep.stream_ordering ASC"
+            txn.execute(sql, args)
+            return txn.fetchall()
+        no_read_receipt = yield self.runInteraction(
+            "get_unread_push_actions_for_user_in_range", get_no_receipt
+        )
+
         defer.returnValue([
             {
                 "event_id": row[0],
                 "stream_ordering": row[1],
                 "actions": json.loads(row[2]),
-            } for row in ret
+            } for row in after_read_receipt+no_read_receipt
         ])
 
     @defer.inlineCallbacks
-- 
cgit 1.5.1


From 767fc0b73963743f1d2ed7c06026d0e333b62a13 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 13 Apr 2016 14:23:27 +0100
Subject: pep8

---
 synapse/storage/event_push_actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 3269834e2f..88737b7a6c 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -176,7 +176,7 @@ class EventPushActionsStore(SQLBaseStore):
                 "event_id": row[0],
                 "stream_ordering": row[1],
                 "actions": json.loads(row[2]),
-            } for row in after_read_receipt+no_read_receipt
+            } for row in after_read_receipt + no_read_receipt
         ])
 
     @defer.inlineCallbacks
-- 
cgit 1.5.1


From a2546b9082b54bd7fc6e0580da4afa84d571fce8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 14 Apr 2016 11:08:31 +0100
Subject: Fix query for get_unread_push_actions_for_user_in_range

---
 synapse/storage/event_push_actions.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 88737b7a6c..86a98b6f11 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -133,9 +133,10 @@ class EventPushActionsStore(SQLBaseStore):
                 "       ep.topological_ordering > rl.topological_ordering"
                 "       OR ("
                 "           ep.topological_ordering = rl.topological_ordering"
-                "           AND ep.stream_ordering > ?"
+                "           AND ep.stream_ordering > rl.stream_ordering"
                 "       )"
                 "   )"
+                "   AND ep.stream_ordering > ?"
                 "   AND ep.user_id = ?"
                 "   AND ep.user_id = rl.user_id"
             )
-- 
cgit 1.5.1


From 96bcfb29c7b8cb04f4b887c518016cb968645cb9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 14 Apr 2016 11:26:33 +0100
Subject: Add index

---
 synapse/storage/schema/delta/31/pushers_index.sql | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 synapse/storage/schema/delta/31/pushers_index.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/pushers_index.sql b/synapse/storage/schema/delta/31/pushers_index.sql
new file mode 100644
index 0000000000..9027bccc69
--- /dev/null
+++ b/synapse/storage/schema/delta/31/pushers_index.sql
@@ -0,0 +1,18 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ CREATE INDEX event_push_actions_stream_ordering on event_push_actions(
+     stream_ordering, user_id
+ );
-- 
cgit 1.5.1


From d213d69fe301a1166516c2f56c50d7a379deaf6e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 14 Apr 2016 11:36:23 +0100
Subject: Add desc arg

---
 synapse/storage/pusher.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index e64c0dce0a..e5755c0aea 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -137,7 +137,11 @@ class PusherStore(SQLBaseStore):
         users = yield self.get_users_in_room(room_id)
 
         result = yield self._simple_select_many_batch(
-            'pushers', 'user_name', users, ['user_name']
+            table='pushers',
+            column='user_name',
+            iterable=users,
+            retcols=['user_name'],
+            desc='get_users_with_pushers_in_room'
         )
 
         defer.returnValue([r['user_name'] for r in result])
-- 
cgit 1.5.1


From 07d765209dea12229e70a09784e647611acabcda Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 19 Apr 2016 14:24:36 +0100
Subject: First bits of emailpusher

Mostly logic of when to send an email
---
 synapse/push/emailpusher.py                        | 214 +++++++++++++++++++++
 synapse/push/pusher.py                             |   4 +-
 synapse/storage/event_push_actions.py              |  57 +++++-
 synapse/storage/events.py                          |   2 +
 synapse/storage/pusher.py                          |  27 +++
 synapse/storage/schema/delta/31/events.sql         |  16 ++
 .../storage/schema/delta/31/pusher_throttle.sql    |  23 +++
 7 files changed, 335 insertions(+), 8 deletions(-)
 create mode 100644 synapse/push/emailpusher.py
 create mode 100644 synapse/storage/schema/delta/31/events.sql
 create mode 100644 synapse/storage/schema/delta/31/pusher_throttle.sql

(limited to 'synapse/storage')

diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
new file mode 100644
index 0000000000..f9954df392
--- /dev/null
+++ b/synapse/push/emailpusher.py
@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer, reactor
+
+import logging
+
+from synapse.util.metrics import Measure
+from synapse.util.async import run_on_reactor
+
+logger = logging.getLogger(__name__)
+
+# The amount of time we always wait before ever emailing about a notification
+# (to give the user a chance to respond to other push or notice the window)
+DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000
+
+THROTTLE_START_MS = 2 * 60 * 1000
+THROTTLE_MAX_MS = (2 * 60 * 1000) * (2**11)  # ~3 days
+
+# If no event triggers a notification for this long after the previous,
+# the throttle is released.
+THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2**11)  # ~3 days
+
+
+class EmailPusher(object):
+    """
+    A pusher that sends email notifications about events (approximately)
+    when they happen.
+    This shares quite a bit of code with httpusher: it would be good to
+    factor out the common parts
+    """
+    def __init__(self, hs, pusherdict):
+        self.hs = hs
+        self.store = self.hs.get_datastore()
+        self.clock = self.hs.get_clock()
+        self.pusher_id = pusherdict['id']
+        self.user_id = pusherdict['user_name']
+        self.app_id = pusherdict['app_id']
+        self.email = pusherdict['pushkey']
+        self.last_stream_ordering = pusherdict['last_stream_ordering']
+        self.timed_call = None
+        self.throttle_params = None
+
+        # See httppusher
+        self.max_stream_ordering = None
+
+    @defer.inlineCallbacks
+    def on_started(self):
+        self.throttle_params = yield self.store.get_throttle_params_by_room(
+            self.pusher_id
+        )
+        yield self._process()
+
+    @defer.inlineCallbacks
+    def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
+        with Measure(self.clock, "push.on_new_notifications"):
+            self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
+            yield self._process()
+
+    @defer.inlineCallbacks
+    def on_timer(self):
+        self.timed_call = None
+        with Measure(self.clock, "push.on_timer"):
+            yield self._process()
+
+    @defer.inlineCallbacks
+    def _process(self):
+        last_notifs = yield self.store.get_time_of_latest_push_action_by_room_for_user(
+            self.user_id
+        )
+
+        unprocessed = yield self.store.get_unread_push_actions_for_user_in_range(
+            self.user_id, self.last_stream_ordering, self.max_stream_ordering
+        )
+
+        soonest_due_at = None
+
+        for push_action in unprocessed:
+            received_at = push_action['received_ts']
+            if received_at is None:
+                received_at = 0
+            notif_ready_at = received_at + DELAY_BEFORE_MAIL_MS
+
+            room_ready_at = self.room_ready_to_notify_at(
+                push_action['room_id'], self.get_room_last_notif_ts(
+                    last_notifs, push_action['room_id']
+                )
+            )
+
+            should_notify_at = max(notif_ready_at, room_ready_at)
+
+            if should_notify_at < self.clock.time_msec():
+                # one of our notifications is ready for sending, so we send
+                # *one* email updating the user on their notifications,
+                # we then consider all previously outstanding notifications
+                # to be delivered.
+                yield self.send_notification(push_action)
+
+                yield self.save_last_stream_ordering_and_success(max([
+                    ea['stream_ordering'] for ea in unprocessed
+                ]))
+                yield self.sent_notif_update_throttle(
+                    push_action['room_id'], push_action
+                )
+            else:
+                if soonest_due_at is None or should_notify_at < soonest_due_at:
+                    soonest_due_at = should_notify_at
+
+        if self.timed_call is not None:
+            self.timed_call.cancel()
+            self.timed_call = None
+
+        if soonest_due_at is not None:
+            self.timed_call = reactor.callLater(
+                self.seconds_until(soonest_due_at), self.on_timer
+            )
+
+    @defer.inlineCallbacks
+    def save_last_stream_ordering_and_success(self, last_stream_ordering):
+        self.last_stream_ordering = last_stream_ordering
+        yield self.store.update_pusher_last_stream_ordering_and_success(
+            self.app_id, self.email, self.user_id,
+            last_stream_ordering, self.clock.time_msec()
+        )
+
+    def seconds_until(self, ts_msec):
+        return (ts_msec - self.clock.time_msec()) / 1000
+
+    def get_room_last_notif_ts(self, last_notif_by_room, room_id):
+        if room_id in last_notif_by_room:
+            return last_notif_by_room[room_id]
+        else:
+            return 0
+
+    def get_room_throttle_ms(self, room_id):
+        if room_id in self.throttle_params:
+            return self.throttle_params[room_id]["throttle_ms"]
+        else:
+            return 0
+
+    def get_room_last_sent_ts(self, room_id):
+        if room_id in self.throttle_params:
+            return self.throttle_params[room_id]["last_sent_ts"]
+        else:
+            return 0
+
+    def room_ready_to_notify_at(self, room_id, last_notif_time):
+        """
+        Determines whether throttling should prevent us from sending an email
+        for the given room
+        Returns: True if we should send, False if we should not
+        """
+        last_sent_ts = self.get_room_last_sent_ts(room_id)
+        throttle_ms = self.get_room_throttle_ms(room_id)
+
+        may_send_at = last_sent_ts + throttle_ms
+        return may_send_at
+
+    @defer.inlineCallbacks
+    def sent_notif_update_throttle(self, room_id, notified_push_action):
+        # We have sent a notification, so update the throttle accordingly.
+        # If the event that triggered the notif happened more than
+        # THROTTLE_RESET_AFTER_MS after the previous one that triggered a
+        # notif, we release the throttle. Otherwise, the throttle is increased.
+        time_of_previous_notifs = yield self.store.get_time_of_last_push_action_before(
+            notified_push_action['stream_ordering']
+        )
+
+        time_of_this_notifs = notified_push_action['received_ts']
+
+        if time_of_previous_notifs is not None and time_of_this_notifs is not None:
+            gap = time_of_this_notifs - time_of_previous_notifs
+        else:
+            # if we don't know the arrival time of one of the notifs (it was not
+            # stored prior to email notification code) then assume a gap of
+            # zero which will just not reset the throttle
+            gap = 0
+
+        current_throttle_ms = self.get_room_throttle_ms(room_id)
+
+        if gap > THROTTLE_RESET_AFTER_MS:
+            new_throttle_ms = THROTTLE_START_MS
+        else:
+            if current_throttle_ms == 0:
+                new_throttle_ms = THROTTLE_START_MS
+            else:
+                new_throttle_ms = min(
+                    current_throttle_ms * 2,
+                    THROTTLE_MAX_MS
+                )
+        self.throttle_params[room_id] = {
+            "last_sent_ts": self.clock.time_msec(),
+            "throttle_ms": new_throttle_ms
+        }
+        yield self.store.set_throttle_params(
+            self.pusher_id, room_id, self.throttle_params[room_id]
+        )
+
+    @defer.inlineCallbacks
+    def send_notification(self, push_action):
+        yield run_on_reactor()
+        logger.error("sending notif email for user %r", self.user_id)
\ No newline at end of file
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index 4960837504..f7c3021fcc 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -1,7 +1,9 @@
 from httppusher import HttpPusher
+from emailpusher import EmailPusher
 
 PUSHER_TYPES = {
-    'http': HttpPusher
+    'http': HttpPusher,
+    'email': EmailPusher,
 }
 
 
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 86a98b6f11..ad512b2f07 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -118,15 +118,17 @@ class EventPushActionsStore(SQLBaseStore):
                                                   max_stream_ordering=None):
         def get_after_receipt(txn):
             sql = (
-                "SELECT ep.event_id, ep.stream_ordering, ep.actions "
+                "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, "
+                "e.received_ts "
                 "FROM event_push_actions AS ep, ("
-                "   SELECT room_id, user_id,"
-                "       max(topological_ordering) as topological_ordering,"
-                "       max(stream_ordering) as stream_ordering"
+                "   SELECT room_id, user_id, "
+                "       max(topological_ordering) as topological_ordering, "
+                "       max(stream_ordering) as stream_ordering "
                 "       FROM events"
                 "   NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'"
                 "   GROUP BY room_id, user_id"
                 ") AS rl "
+                "NATURAL JOIN events e "
                 "WHERE"
                 "   ep.room_id = rl.room_id"
                 "   AND ("
@@ -153,8 +155,10 @@ class EventPushActionsStore(SQLBaseStore):
 
         def get_no_receipt(txn):
             sql = (
-                "SELECT ep.event_id, ep.stream_ordering, ep.actions "
+                "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, "
+                "e.received_ts "
                 "FROM event_push_actions AS ep "
+                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
                 "WHERE ep.room_id not in ("
                 "   SELECT room_id FROM events NATURAL JOIN receipts_linearized"
                 "   WHERE receipt_type = 'm.read' AND user_id = ? "
@@ -175,11 +179,30 @@ class EventPushActionsStore(SQLBaseStore):
         defer.returnValue([
             {
                 "event_id": row[0],
-                "stream_ordering": row[1],
-                "actions": json.loads(row[2]),
+                "room_id": row[1],
+                "stream_ordering": row[2],
+                "actions": json.loads(row[3]),
+                "received_ts": row[4],
             } for row in after_read_receipt + no_read_receipt
         ])
 
+    @defer.inlineCallbacks
+    def get_time_of_last_push_action_before(self, stream_ordering):
+        def f(txn):
+            sql = (
+                "SELECT e.received_ts "
+                "FROM event_push_actions AS ep "
+                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
+                "WHERE ep.stream_ordering > ? "
+                "ORDER BY ep.stream_ordering ASC "
+                "LIMIT 1"
+            )
+            txn.execute(sql, (stream_ordering,))
+            return txn.fetchone()
+        result = yield self.runInteraction("get_time_of_last_push_action_before", f)
+        defer.returnValue(result[0] if result is not None else None)
+
+
     @defer.inlineCallbacks
     def get_latest_push_action_stream_ordering(self):
         def f(txn):
@@ -190,6 +213,26 @@ class EventPushActionsStore(SQLBaseStore):
         )
         defer.returnValue(result[0] or 0)
 
+    @defer.inlineCallbacks
+    def get_time_of_latest_push_action_by_room_for_user(self, user_id):
+        """
+        Returns only the received_ts of the last notification in each of the
+        user's rooms, in a dict by room_id
+        """
+        def f(txn):
+            txn.execute(
+                "SELECT ep.room_id, MAX(e.received_ts) "
+                "FROM event_push_actions AS ep "
+                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
+                "GROUP BY ep.room_id"
+            )
+            return txn.fetchall()
+        result = yield self.runInteraction(
+            "get_time_of_latest_push_action_by_room_for_user", f
+        )
+
+        defer.returnValue({row[0]: row[1] for row in result})
+
     def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
         # Sad that we have to blow away the cache for the whole room here
         txn.call_after(
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 21487724ed..dd58e001dc 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -55,6 +55,7 @@ class EventsStore(SQLBaseStore):
 
     def __init__(self, hs):
         super(EventsStore, self).__init__(hs)
+        self._clock = hs.get_clock()
         self.register_background_update_handler(
             self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
         )
@@ -427,6 +428,7 @@ class EventsStore(SQLBaseStore):
                     "outlier": event.internal_metadata.is_outlier(),
                     "content": encode_json(event.content).decode("UTF-8"),
                     "origin_server_ts": int(event.origin_server_ts),
+                    "received_ts": self._clock.time_msec(),
                 }
                 for event, _ in events_and_contexts
             ],
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index e5755c0aea..caef9b59a5 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -230,3 +230,30 @@ class PusherStore(SQLBaseStore):
             {'failing_since': failing_since},
             desc="update_pusher_failing_since",
         )
+
+    @defer.inlineCallbacks
+    def get_throttle_params_by_room(self, pusher_id):
+        res = yield self._simple_select_list(
+            "pusher_throttle",
+            {"pusher": pusher_id},
+            ["room_id", "last_sent_ts", "throttle_ms"],
+            desc="get_throttle_params_by_room"
+        )
+
+        params_by_room = {}
+        for row in res:
+            params_by_room[row["room_id"]] = {
+                "last_sent_ts": row["last_sent_ts"],
+                "throttle_ms": row["throttle_ms"]
+            }
+
+        defer.returnValue(params_by_room)
+
+    @defer.inlineCallbacks
+    def set_throttle_params(self, pusher_id, room_id, params):
+        yield self._simple_upsert(
+            "pusher_throttle",
+            {"pusher": pusher_id, "room_id": room_id},
+            params,
+            desc="set_throttle_params"
+        )
\ No newline at end of file
diff --git a/synapse/storage/schema/delta/31/events.sql b/synapse/storage/schema/delta/31/events.sql
new file mode 100644
index 0000000000..1dd0f9e170
--- /dev/null
+++ b/synapse/storage/schema/delta/31/events.sql
@@ -0,0 +1,16 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE events ADD COLUMN received_ts BIGINT;
diff --git a/synapse/storage/schema/delta/31/pusher_throttle.sql b/synapse/storage/schema/delta/31/pusher_throttle.sql
new file mode 100644
index 0000000000..d86d30c13c
--- /dev/null
+++ b/synapse/storage/schema/delta/31/pusher_throttle.sql
@@ -0,0 +1,23 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE TABLE pusher_throttle(
+    pusher BIGINT NOT NULL,
+    room_id TEXT NOT NULL,
+    last_sent_ts BIGINT,
+    throttle_ms BIGINT,
+    PRIMARY KEY (pusher, room_id)
+);
-- 
cgit 1.5.1


From 61c7edfd34abdb9eaa7c8d3dd3dbef95b60de5de Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 19 Apr 2016 17:22:03 +0100
Subject: Add cache to _get_state_groups_from_groups

---
 synapse/storage/state.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index c5d2a3a6df..5b743db67a 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -174,6 +174,12 @@ class StateStore(SQLBaseStore):
             return [r[0] for r in results]
         return self.runInteraction("get_current_state_for_key", f)
 
+    @cached(num_args=2, lru=True, max_entries=1000)
+    def _get_state_group_from_group(self, group, types):
+        raise NotImplementedError()
+
+    @cachedList(cached_method_name="_get_state_group_from_group",
+                list_name="groups", num_args=2, inlineCallbacks=True)
     def _get_state_groups_from_groups(self, groups, types):
         """Returns dictionary state_group -> (dict of (type, state_key) -> event id)
         """
@@ -201,18 +207,23 @@ class StateStore(SQLBaseStore):
             txn.execute(sql, args)
             rows = self.cursor_to_dict(txn)
 
-            results = {}
+            results = {group: {} for group in groups}
             for row in rows:
                 key = (row["type"], row["state_key"])
-                results.setdefault(row["state_group"], {})[key] = row["event_id"]
+                results[row["state_group"]][key] = row["event_id"]
             return results
 
+        results = {}
+
         chunks = [groups[i:i + 100] for i in xrange(0, len(groups), 100)]
         for chunk in chunks:
-            return self.runInteraction(
+            res = yield self.runInteraction(
                 "_get_state_groups_from_groups",
                 f, chunk
             )
+            results.update(res)
+
+        defer.returnValue(results)
 
     @defer.inlineCallbacks
     def get_state_for_events(self, event_ids, types):
@@ -359,6 +370,8 @@ class StateStore(SQLBaseStore):
         a `state_key` of None matches all state_keys. If `types` is None then
         all events are returned.
         """
+        if types:
+            types = frozenset(types)
         results = {}
         missing_groups = []
         if types is not None:
-- 
cgit 1.5.1


From f63bd4ff4704c9f7b6e23c76720dbd955a60c058 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 20 Apr 2016 13:02:01 +0100
Subject: Send a rather basic email notif

Also pep8 fixes
---
 synapse/config/emailconfig.py         | 62 +++++++++++++++++++++++++++++++++++
 synapse/config/homeserver.py          |  3 +-
 synapse/push/emailpusher.py           | 32 +++++++++++++-----
 synapse/push/mailer.py                | 48 +++++++++++++++++++++++++++
 synapse/storage/event_push_actions.py |  1 -
 synapse/storage/pusher.py             |  2 +-
 6 files changed, 136 insertions(+), 12 deletions(-)
 create mode 100644 synapse/config/emailconfig.py
 create mode 100644 synapse/push/mailer.py

(limited to 'synapse/storage')

diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
new file mode 100644
index 0000000000..978826627b
--- /dev/null
+++ b/synapse/config/emailconfig.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file can't be called email.py because if it is, we cannot:
+import email.utils
+
+from ._base import Config
+
+
+class EmailConfig(Config):
+    """
+    Email Configuration
+    """
+
+    def read_config(self, config):
+        email_config = config.get("email", None)
+        if email_config:
+            self.email_enable_notifs = email_config.get("enable_notifs", True)
+            if (
+                "smtp_host" not in email_config or
+                "smtp_port" not in email_config or
+                "notif_from" not in email_config
+            ):
+                raise RuntimeError(
+                    "You must set smtp_host, smtp_port and notif_from "
+                    "to send email notifications"
+                )
+
+            self.email_smtp_host = email_config["smtp_host"]
+            self.email_smtp_port = email_config["smtp_port"]
+            self.email_notif_from = email_config["notif_from"]
+
+            # make sure it's valid
+            parsed = email.utils.parseaddr(self.email_notif_from)
+            if parsed[1] == '':
+                raise RuntimeError("Invalid notif_from address")
+        else:
+            self.email_enable_notifs = False
+            self.email_smtp_host = None
+            self.email_smtp_port = None
+            self.email_notif_from = None
+
+    def default_config(self, config_dir_path, server_name, **kwargs):
+        return """
+        # Enable sending emails for notification events
+        #email_config:
+        #   enable_notifs: false
+        #   smtp_host: "localhost"
+        #   smtp_port: 25
+        """
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 9a80ac39ec..fc2445484c 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -31,13 +31,14 @@ from .cas import CasConfig
 from .password import PasswordConfig
 from .jwt import JWTConfig
 from .ldap import LDAPConfig
+from .emailconfig import EmailConfig
 
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                        RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
                        VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig,
                        AppServiceConfig, KeyConfig, SAML2Config, CasConfig,
-                       JWTConfig, LDAPConfig, PasswordConfig,):
+                       JWTConfig, LDAPConfig, PasswordConfig, EmailConfig,):
     pass
 
 
diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
index 74e3a70562..820c8f8467 100644
--- a/synapse/push/emailpusher.py
+++ b/synapse/push/emailpusher.py
@@ -18,9 +18,10 @@ from twisted.internet import defer, reactor
 import logging
 
 from synapse.util.metrics import Measure
-from synapse.util.async import run_on_reactor
 from synapse.util.logcontext import LoggingContext
 
+from mailer import Mailer
+
 logger = logging.getLogger(__name__)
 
 # The amount of time we always wait before ever emailing about a notification
@@ -28,11 +29,11 @@ logger = logging.getLogger(__name__)
 DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000
 
 THROTTLE_START_MS = 2 * 60 * 1000
-THROTTLE_MAX_MS = (2 * 60 * 1000) * (2**11)  # ~3 days
+THROTTLE_MAX_MS = (2 * 60 * 1000) * (2 ** 11)  # ~3 days
 
 # If no event triggers a notification for this long after the previous,
 # the throttle is released.
-THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2**11)  # ~3 days
+THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2 ** 11)  # ~3 days
 
 
 class EmailPusher(object):
@@ -59,12 +60,22 @@ class EmailPusher(object):
 
         self.processing = False
 
+        if self.hs.config.email_enable_notifs:
+            self.mailer = Mailer(
+                self.store,
+                self.hs.config.email_smtp_host, self.hs.config.email_smtp_port,
+                self.hs.config.email_notif_from,
+            )
+        else:
+            self.mailer = None
+
     @defer.inlineCallbacks
     def on_started(self):
-        self.throttle_params = yield self.store.get_throttle_params_by_room(
-            self.pusher_id
-        )
-        yield self._process()
+        if self.mailer is not None:
+            self.throttle_params = yield self.store.get_throttle_params_by_room(
+                self.pusher_id
+            )
+            yield self._process()
 
     def on_stop(self):
         if self.timed_call:
@@ -102,6 +113,7 @@ class EmailPusher(object):
                 finally:
                     self.processing = False
 
+    @defer.inlineCallbacks
     def _unsafe_process(self):
         """
         Main logic of the push loop without the wrapper function that sets
@@ -241,5 +253,7 @@ class EmailPusher(object):
 
     @defer.inlineCallbacks
     def send_notification(self, push_action):
-        yield run_on_reactor()
-        logger.error("sending notif email for user %r", self.user_id)
\ No newline at end of file
+        logger.info("Sending notif email for user %r", self.user_id)
+        yield self.mailer.send_notification_mail(
+            self.user_id, self.email, push_action
+        )
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
new file mode 100644
index 0000000000..93d3866ec7
--- /dev/null
+++ b/synapse/push/mailer.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import smtplib
+import email.utils
+import email.mime.multipart
+from email.mime.text import MIMEText
+
+
+class Mailer(object):
+    def __init__(self, store, smtp_host, smtp_port, notif_from):
+        self.store = store
+        self.smtp_host = smtp_host
+        self.smtp_port = smtp_port
+        self.notif_from = notif_from
+
+    @defer.inlineCallbacks
+    def send_notification_mail(self, user_id, email_address, push_action):
+        raw_from = email.utils.parseaddr(self.notif_from)[1]
+        raw_to = email.utils.parseaddr(email_address)[1]
+
+        if raw_to == '':
+            raise RuntimeError("Invalid 'to' address")
+
+        plainText = "yo dawg, you got notifications!"
+
+        text_part = MIMEText(plainText, "plain")
+        text_part['Subject'] = "New Matrix Notifications"
+        text_part['From'] = self.notif_from
+        text_part['To'] = email_address
+
+        smtp = smtplib.SMTP(self.smtp_host, self.smtp_port)
+        smtp.sendmail(raw_from, raw_to, text_part.as_string())
+        smtp.quit()
\ No newline at end of file
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index ad512b2f07..f2af8bdb36 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -202,7 +202,6 @@ class EventPushActionsStore(SQLBaseStore):
         result = yield self.runInteraction("get_time_of_last_push_action_before", f)
         defer.returnValue(result[0] if result is not None else None)
 
-
     @defer.inlineCallbacks
     def get_latest_push_action_stream_ordering(self):
         def f(txn):
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index caef9b59a5..5fb47d418a 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -256,4 +256,4 @@ class PusherStore(SQLBaseStore):
             {"pusher": pusher_id, "room_id": room_id},
             params,
             desc="set_throttle_params"
-        )
\ No newline at end of file
+        )
-- 
cgit 1.5.1


From c877f0f0345f1ff6d329af2920d7d1a6b5659a86 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 16:41:39 +0100
Subject: Optimise event_search in postgres

---
 synapse/storage/room.py                          | 16 ++++--
 synapse/storage/schema/delta/31/search_update.py | 65 ++++++++++++++++++++++++
 synapse/storage/search.py                        | 61 +++++++++++++++++++++-
 3 files changed, 137 insertions(+), 5 deletions(-)
 create mode 100644 synapse/storage/schema/delta/31/search_update.py

(limited to 'synapse/storage')

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 9be977f387..70aa64fb31 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -169,20 +169,28 @@ class RoomStore(SQLBaseStore):
     def _store_event_search_txn(self, txn, event, key, value):
         if isinstance(self.database_engine, PostgresEngine):
             sql = (
-                "INSERT INTO event_search (event_id, room_id, key, vector)"
-                " VALUES (?,?,?,to_tsvector('english', ?))"
+                "INSERT INTO event_search"
+                " (event_id, room_id, key, vector, stream_ordering, origin_server_ts)"
+                " VALUES (?,?,?,to_tsvector('english', ?),?,?)"
+            )
+            txn.execute(
+                sql,
+                (
+                    event.event_id, event.room_id, key, value,
+                    event.internal_metadata.stream_ordering,
+                    event.origin_server_ts,
+                )
             )
         elif isinstance(self.database_engine, Sqlite3Engine):
             sql = (
                 "INSERT INTO event_search (event_id, room_id, key, value)"
                 " VALUES (?,?,?,?)"
             )
+            txn.execute(sql, (event.event_id, event.room_id, key, value,))
         else:
             # This should be unreachable.
             raise Exception("Unrecognized database engine")
 
-        txn.execute(sql, (event.event_id, event.room_id, key, value,))
-
     @cachedInlineCallbacks()
     def get_room_name_and_aliases(self, room_id):
         def f(txn):
diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
new file mode 100644
index 0000000000..46a3795d12
--- /dev/null
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -0,0 +1,65 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.storage.engines import PostgresEngine
+from synapse.storage.prepare_database import get_statements
+
+import logging
+import ujson
+
+logger = logging.getLogger(__name__)
+
+
+ALTER_TABLE = """
+ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
+ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
+
+CREATE INDEX event_search_room_order ON event_search(
+    room_id, origin_server_ts, stream_ordering
+);
+CREATE INDEX event_search_order ON event_search(origin_server_ts, stream_ordering);
+"""
+
+
+def run_create(cur, database_engine, *args, **kwargs):
+    if not isinstance(database_engine, PostgresEngine):
+        return
+
+    for statement in get_statements(ALTER_TABLE.splitlines()):
+        cur.execute(statement)
+
+    cur.execute("SELECT MIN(stream_ordering) FROM events")
+    rows = cur.fetchall()
+    min_stream_id = rows[0][0]
+
+    cur.execute("SELECT MAX(stream_ordering) FROM events")
+    rows = cur.fetchall()
+    max_stream_id = rows[0][0]
+
+    if min_stream_id is not None and max_stream_id is not None:
+        progress = {
+            "target_min_stream_id_inclusive": min_stream_id,
+            "max_stream_id_exclusive": max_stream_id + 1,
+            "rows_inserted": 0,
+        }
+        progress_json = ujson.dumps(progress)
+
+        sql = (
+            "INSERT into background_updates (update_name, progress_json)"
+            " VALUES (?, ?)"
+        )
+
+        sql = database_engine.convert_param_style(sql)
+
+        cur.execute(sql, ("event_search_order", progress_json))
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 59ac7f424c..375057fa3e 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -29,12 +29,17 @@ logger = logging.getLogger(__name__)
 class SearchStore(BackgroundUpdateStore):
 
     EVENT_SEARCH_UPDATE_NAME = "event_search"
+    EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order"
 
     def __init__(self, hs):
         super(SearchStore, self).__init__(hs)
         self.register_background_update_handler(
             self.EVENT_SEARCH_UPDATE_NAME, self._background_reindex_search
         )
+        self.register_background_update_handler(
+            self.EVENT_SEARCH_ORDER_UPDATE_NAME,
+            self._background_reindex_search_order
+        )
 
     @defer.inlineCallbacks
     def _background_reindex_search(self, progress, batch_size):
@@ -131,6 +136,61 @@ class SearchStore(BackgroundUpdateStore):
 
         defer.returnValue(result)
 
+    @defer.inlineCallbacks
+    def _background_reindex_search_order(self, progress, batch_size):
+        target_min_stream_id = progress["target_min_stream_id_inclusive"]
+        max_stream_id = progress["max_stream_id_exclusive"]
+        rows_inserted = progress.get("rows_inserted", 0)
+
+        INSERT_CLUMP_SIZE = 1000
+
+        def reindex_search_txn(txn):
+            sql = (
+                "SELECT stream_ordering, origin_server_ts, event_id FROM events"
+                " INNER JOIN event_search USING (room_id, event_id)"
+                " WHERE ? <= stream_ordering AND stream_ordering < ?"
+                " ORDER BY stream_ordering DESC"
+                " LIMIT ?"
+            )
+
+            txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
+
+            rows = txn.fetchall()
+            if not rows:
+                return 0
+
+            min_stream_id = rows[-1][0]
+
+            sql = (
+                "UPDATE event_search SET stream_ordering = ?, origin_server_ts = ?"
+                " WHERE event_id = ?"
+            )
+
+            for index in range(0, len(rows), INSERT_CLUMP_SIZE):
+                clump = rows[index:index + INSERT_CLUMP_SIZE]
+                txn.executemany(sql, clump)
+
+            progress = {
+                "target_min_stream_id_inclusive": target_min_stream_id,
+                "max_stream_id_exclusive": min_stream_id,
+                "rows_inserted": rows_inserted + len(rows)
+            }
+
+            self._background_update_progress_txn(
+                txn, self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
+            )
+
+            return len(rows)
+
+        result = yield self.runInteraction(
+            self.EVENT_SEARCH_ORDER_UPDATE_NAME, reindex_search_txn
+        )
+
+        if not result:
+            yield self._end_background_update(self.EVENT_SEARCH_ORDER_UPDATE_NAME)
+
+        defer.returnValue(result)
+
     @defer.inlineCallbacks
     def search_msgs(self, room_ids, search_term, keys):
         """Performs a full text search over events with given keys.
@@ -310,7 +370,6 @@ class SearchStore(BackgroundUpdateStore):
                 "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
                 " origin_server_ts, stream_ordering, room_id, event_id"
                 " FROM event_search"
-                " NATURAL JOIN events"
                 " WHERE vector @@ to_tsquery('english', ?) AND "
             )
             args = [search_query, search_query] + args
-- 
cgit 1.5.1


From b743c1237e68e75056b83ea4ab93ba2e1ec44b7e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:12:04 +0100
Subject: Add missing run_upgrade

---
 synapse/storage/schema/delta/31/search_update.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
index 46a3795d12..989e990dbd 100644
--- a/synapse/storage/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -63,3 +63,7 @@ def run_create(cur, database_engine, *args, **kwargs):
         sql = database_engine.convert_param_style(sql)
 
         cur.execute(sql, ("event_search_order", progress_json))
+
+
+def run_upgrade(cur, database_engine, *args, **kwargs):
+    pass
-- 
cgit 1.5.1


From 51bb339ab2130ab29ee9fcfec48d8e62f46c75f6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:16:11 +0100
Subject: Create index concurrently

---
 synapse/storage/schema/delta/31/search_update.py |  6 +-----
 synapse/storage/search.py                        | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
index 989e990dbd..470ae0c005 100644
--- a/synapse/storage/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -24,11 +24,6 @@ logger = logging.getLogger(__name__)
 ALTER_TABLE = """
 ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
 ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
-
-CREATE INDEX event_search_room_order ON event_search(
-    room_id, origin_server_ts, stream_ordering
-);
-CREATE INDEX event_search_order ON event_search(origin_server_ts, stream_ordering);
 """
 
 
@@ -52,6 +47,7 @@ def run_create(cur, database_engine, *args, **kwargs):
             "target_min_stream_id_inclusive": min_stream_id,
             "max_stream_id_exclusive": max_stream_id + 1,
             "rows_inserted": 0,
+            "have_added_indexes": False,
         }
         progress_json = ujson.dumps(progress)
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 375057fa3e..548e9eeaef 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -141,10 +141,21 @@ class SearchStore(BackgroundUpdateStore):
         target_min_stream_id = progress["target_min_stream_id_inclusive"]
         max_stream_id = progress["max_stream_id_exclusive"]
         rows_inserted = progress.get("rows_inserted", 0)
+        have_added_index = progress['have_added_indexes']
 
         INSERT_CLUMP_SIZE = 1000
 
         def reindex_search_txn(txn):
+            if not have_added_index:
+                txn.execute(
+                    "CREATE INDEX CONCURRENTLY event_search_room_order ON event_search("
+                    "room_id, origin_server_ts, stream_ordering)"
+                )
+                txn.execute(
+                    "CREATE INDEX CONCURRENTLY event_search_order ON event_search("
+                    "origin_server_ts, stream_ordering)"
+                )
+
             sql = (
                 "SELECT stream_ordering, origin_server_ts, event_id FROM events"
                 " INNER JOIN event_search USING (room_id, event_id)"
@@ -173,7 +184,8 @@ class SearchStore(BackgroundUpdateStore):
             progress = {
                 "target_min_stream_id_inclusive": target_min_stream_id,
                 "max_stream_id_exclusive": min_stream_id,
-                "rows_inserted": rows_inserted + len(rows)
+                "rows_inserted": rows_inserted + len(rows),
+                "have_added_index": True,
             }
 
             self._background_update_progress_txn(
-- 
cgit 1.5.1


From 129e4034870956126daf520b41f74a51040ddbf9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:19:25 +0100
Subject: Create index must be on a conn

---
 synapse/storage/search.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 548e9eeaef..05641fb579 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -143,19 +143,26 @@ class SearchStore(BackgroundUpdateStore):
         rows_inserted = progress.get("rows_inserted", 0)
         have_added_index = progress['have_added_indexes']
 
-        INSERT_CLUMP_SIZE = 1000
-
-        def reindex_search_txn(txn):
-            if not have_added_index:
-                txn.execute(
+        if not have_added_index:
+            def create_index(conn):
+                conn.rollback()
+                conn.set_session(autocommit=True)
+                c = conn.cursor()
+                c.execute(
                     "CREATE INDEX CONCURRENTLY event_search_room_order ON event_search("
                     "room_id, origin_server_ts, stream_ordering)"
                 )
-                txn.execute(
+                c.execute(
                     "CREATE INDEX CONCURRENTLY event_search_order ON event_search("
                     "origin_server_ts, stream_ordering)"
                 )
+                conn.set_session(autocommit=False)
+
+            yield self.runWithConnection(create_index)
 
+        INSERT_CLUMP_SIZE = 1000
+
+        def reindex_search_txn(txn):
             sql = (
                 "SELECT stream_ordering, origin_server_ts, event_id FROM events"
                 " INNER JOIN event_search USING (room_id, event_id)"
-- 
cgit 1.5.1


From 3b0fa77f5050bb54dccc5140a76b171d6603f2e7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:37:42 +0100
Subject: Fix SQL statement

---
 synapse/storage/search.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 05641fb579..f7730f5d7c 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -164,10 +164,10 @@ class SearchStore(BackgroundUpdateStore):
 
         def reindex_search_txn(txn):
             sql = (
-                "SELECT stream_ordering, origin_server_ts, event_id FROM events"
+                "SELECT e.stream_ordering, e.origin_server_ts, event_id FROM events as e"
                 " INNER JOIN event_search USING (room_id, event_id)"
-                " WHERE ? <= stream_ordering AND stream_ordering < ?"
-                " ORDER BY stream_ordering DESC"
+                " WHERE ? <= e.stream_ordering AND e.stream_ordering < ?"
+                " ORDER BY e.stream_ordering DESC"
                 " LIMIT ?"
             )
 
-- 
cgit 1.5.1


From e395eb1108abac2ada1f846d08285a84fd5042a2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:39:24 +0100
Subject: Update progress when creating index

---
 synapse/storage/search.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index f7730f5d7c..1baed674a8 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -158,6 +158,13 @@ class SearchStore(BackgroundUpdateStore):
                 )
                 conn.set_session(autocommit=False)
 
+                pg = dict(progress)
+                pg["have_added_indexes"] = True
+
+                self._background_update_progress_txn(
+                    conn.cursor(), self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
+                )
+
             yield self.runWithConnection(create_index)
 
         INSERT_CLUMP_SIZE = 1000
-- 
cgit 1.5.1


From 26db18bc90e745e1eb054e2ffd6969918a6253c4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:45:56 +0100
Subject: Need to do _background_update_progress_txn in actual transaction

---
 synapse/storage/search.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 1baed674a8..fd40b44a9a 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -158,14 +158,16 @@ class SearchStore(BackgroundUpdateStore):
                 )
                 conn.set_session(autocommit=False)
 
-                pg = dict(progress)
-                pg["have_added_indexes"] = True
+            yield self.runWithConnection(create_index)
 
-                self._background_update_progress_txn(
-                    conn.cursor(), self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
-                )
+            pg = dict(progress)
+            pg["have_added_indexes"] = True
 
-            yield self.runWithConnection(create_index)
+            yield self.runInteraction(
+                self.EVENT_SEARCH_ORDER_UPDATE_NAME,
+                self._background_update_progress_txn,
+                self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress,
+            )
 
         INSERT_CLUMP_SIZE = 1000
 
-- 
cgit 1.5.1


From b57dcb4b51d31914756412ec00d94646bc3b4c79 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 17:49:00 +0100
Subject: Typo

---
 synapse/storage/search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index fd40b44a9a..dc47425c23 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -201,7 +201,7 @@ class SearchStore(BackgroundUpdateStore):
                 "target_min_stream_id_inclusive": target_min_stream_id,
                 "max_stream_id_exclusive": min_stream_id,
                 "rows_inserted": rows_inserted + len(rows),
-                "have_added_index": True,
+                "have_added_indexes": True,
             }
 
             self._background_update_progress_txn(
-- 
cgit 1.5.1


From 8fae3d7b1eea87b48db96f1671d850a4a247e926 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 18:01:49 +0100
Subject: Use special UPDATE syntax

---
 synapse/storage/schema/delta/31/search_update.py |  4 +--
 synapse/storage/search.py                        | 32 +++++++++---------------
 2 files changed, 14 insertions(+), 22 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
index 470ae0c005..2c15edd1a4 100644
--- a/synapse/storage/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -22,8 +22,8 @@ logger = logging.getLogger(__name__)
 
 
 ALTER_TABLE = """
-ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
-ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
+ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT DEFAULT 0;
+ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT DEFAULT 0;
 """
 
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index dc47425c23..813e1e90ac 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -169,34 +169,26 @@ class SearchStore(BackgroundUpdateStore):
                 self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress,
             )
 
-        INSERT_CLUMP_SIZE = 1000
-
         def reindex_search_txn(txn):
-            sql = (
-                "SELECT e.stream_ordering, e.origin_server_ts, event_id FROM events as e"
-                " INNER JOIN event_search USING (room_id, event_id)"
-                " WHERE ? <= e.stream_ordering AND e.stream_ordering < ?"
-                " ORDER BY e.stream_ordering DESC"
+            events_sql = (
+                "SELECT stream_ordering, origin_server_ts, event_id FROM events"
+                " WHERE ? <= stream_ordering AND stream_ordering < ?"
+                " ORDER BY stream_ordering DESC"
                 " LIMIT ?"
             )
 
-            txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
+            sql = (
+                "UPDATE event_search AS es SET es.stream_ordering = e.stream_ordering,"
+                " es.origin_server_ts = e.origin_server_ts"
+                " FROM (%s) AS e"
+                " WHERE e.event_id = es.event_id"
+                " RETURNING es.stream_ordering"
+            ) % (events_sql,)
 
+            txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
             rows = txn.fetchall()
-            if not rows:
-                return 0
-
             min_stream_id = rows[-1][0]
 
-            sql = (
-                "UPDATE event_search SET stream_ordering = ?, origin_server_ts = ?"
-                " WHERE event_id = ?"
-            )
-
-            for index in range(0, len(rows), INSERT_CLUMP_SIZE):
-                clump = rows[index:index + INSERT_CLUMP_SIZE]
-                txn.executemany(sql, clump)
-
             progress = {
                 "target_min_stream_id_inclusive": target_min_stream_id,
                 "max_stream_id_exclusive": min_stream_id,
-- 
cgit 1.5.1


From 3ddbb1687ced5eb9b9a87367fcd6b754b8d0c5dc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 18:02:36 +0100
Subject: Fix query

---
 synapse/storage/search.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 813e1e90ac..dd3486783d 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -178,8 +178,8 @@ class SearchStore(BackgroundUpdateStore):
             )
 
             sql = (
-                "UPDATE event_search AS es SET es.stream_ordering = e.stream_ordering,"
-                " es.origin_server_ts = e.origin_server_ts"
+                "UPDATE event_search AS es SET stream_ordering = e.stream_ordering,"
+                " origin_server_ts = e.origin_server_ts"
                 " FROM (%s) AS e"
                 " WHERE e.event_id = es.event_id"
                 " RETURNING es.stream_ordering"
-- 
cgit 1.5.1


From ae571810f2283c1825da62af0e931a0e40f74168 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Apr 2016 18:09:48 +0100
Subject: Order NULLs first

---
 synapse/storage/schema/delta/31/search_update.py |  4 ++--
 synapse/storage/search.py                        | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
index 2c15edd1a4..470ae0c005 100644
--- a/synapse/storage/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -22,8 +22,8 @@ logger = logging.getLogger(__name__)
 
 
 ALTER_TABLE = """
-ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT DEFAULT 0;
-ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT DEFAULT 0;
+ALTER TABLE event_search ADD COLUMN origin_server_ts BIGINT;
+ALTER TABLE event_search ADD COLUMN stream_ordering BIGINT;
 """
 
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index dd3486783d..2c71db8c96 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -148,13 +148,16 @@ class SearchStore(BackgroundUpdateStore):
                 conn.rollback()
                 conn.set_session(autocommit=True)
                 c = conn.cursor()
+
+                # We create with NULLS FIRST so that when we search *backwards*
+                # we get the ones with non null origin_server_ts *first*
                 c.execute(
                     "CREATE INDEX CONCURRENTLY event_search_room_order ON event_search("
-                    "room_id, origin_server_ts, stream_ordering)"
+                    "room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
                 )
                 c.execute(
                     "CREATE INDEX CONCURRENTLY event_search_order ON event_search("
-                    "origin_server_ts, stream_ordering)"
+                    "origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)"
                 )
                 conn.set_session(autocommit=False)
 
@@ -434,7 +437,15 @@ class SearchStore(BackgroundUpdateStore):
 
         # We add an arbitrary limit here to ensure we don't try to pull the
         # entire table from the database.
-        sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
+        if isinstance(self.database_engine, PostgresEngine):
+            sql += (
+                " ORDER BY origin_server_ts DESC NULLS LAST,"
+                " stream_ordering DESC NULLS LAST LIMIT ?"
+            )
+        elif isinstance(self.database_engine, Sqlite3Engine):
+            sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
+        else:
+            raise Exception("Unrecognized database engine")
 
         args.append(limit)
 
-- 
cgit 1.5.1


From 183cacac90ca237b448da244270d55920470389b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 22 Apr 2016 09:37:16 +0100
Subject: Simplify query and handle finishing correctly

---
 synapse/storage/background_updates.py |  3 ++-
 synapse/storage/search.py             | 30 ++++++++++++++----------------
 2 files changed, 16 insertions(+), 17 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index 49904046cf..66a995157d 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -173,11 +173,12 @@ class BackgroundUpdateStore(SQLBaseStore):
 
         logger.info(
             "Updating %r. Updated %r items in %rms."
-            " (total_rate=%r/ms, current_rate=%r/ms, total_updated=%r)",
+            " (total_rate=%r/ms, current_rate=%r/ms, total_updated=%r, batch_size=%r)",
             update_name, items_updated, duration_ms,
             performance.total_items_per_ms(),
             performance.average_items_per_ms(),
             performance.total_item_count,
+            batch_size,
         )
 
         performance.update(items_updated, duration_ms)
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 2c71db8c96..0224299625 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -169,28 +169,26 @@ class SearchStore(BackgroundUpdateStore):
             yield self.runInteraction(
                 self.EVENT_SEARCH_ORDER_UPDATE_NAME,
                 self._background_update_progress_txn,
-                self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress,
+                self.EVENT_SEARCH_ORDER_UPDATE_NAME, pg,
             )
 
         def reindex_search_txn(txn):
-            events_sql = (
-                "SELECT stream_ordering, origin_server_ts, event_id FROM events"
-                " WHERE ? <= stream_ordering AND stream_ordering < ?"
-                " ORDER BY stream_ordering DESC"
-                " LIMIT ?"
-            )
-
             sql = (
                 "UPDATE event_search AS es SET stream_ordering = e.stream_ordering,"
                 " origin_server_ts = e.origin_server_ts"
-                " FROM (%s) AS e"
+                " FROM events AS e"
                 " WHERE e.event_id = es.event_id"
+                " AND ? <= e.stream_ordering AND e.stream_ordering < ?"
                 " RETURNING es.stream_ordering"
-            ) % (events_sql,)
+            )
 
-            txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
+            min_stream_id = max_stream_id - batch_size
+            txn.execute(sql, (min_stream_id, max_stream_id))
             rows = txn.fetchall()
-            min_stream_id = rows[-1][0]
+
+            if min_stream_id < target_min_stream_id:
+                # We've recached the end.
+                return len(rows), False
 
             progress = {
                 "target_min_stream_id_inclusive": target_min_stream_id,
@@ -203,16 +201,16 @@ class SearchStore(BackgroundUpdateStore):
                 txn, self.EVENT_SEARCH_ORDER_UPDATE_NAME, progress
             )
 
-            return len(rows)
+            return len(rows), True
 
-        result = yield self.runInteraction(
+        num_rows, finished = yield self.runInteraction(
             self.EVENT_SEARCH_ORDER_UPDATE_NAME, reindex_search_txn
         )
 
-        if not result:
+        if not finished:
             yield self._end_background_update(self.EVENT_SEARCH_ORDER_UPDATE_NAME)
 
-        defer.returnValue(result)
+        defer.returnValue(num_rows)
 
     @defer.inlineCallbacks
     def search_msgs(self, room_ids, search_term, keys):
-- 
cgit 1.5.1


From 871357d539bfbaf5552a098de2253600bf5f3a51 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 27 Apr 2016 11:54:13 +0100
Subject: Check that somethign has happend before running the selects

---
 synapse/storage/events.py   | 10 ++++++++--
 synapse/storage/pusher.py   |  3 +++
 synapse/storage/receipts.py |  3 +++
 3 files changed, 14 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 21487724ed..0307b2af3c 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1143,6 +1143,12 @@ class EventsStore(SQLBaseStore):
                            current_backfill_id, current_forward_id, limit):
         """Get all the new events that have arrived at the server either as
         new events or as backfilled events"""
+        have_backfill_events = last_backfill_id != current_backfill_id
+        have_forward_events = last_forward_id != current_forward_id
+
+        if not have_backfill_events and not have_forward_events:
+            return defer.succeed(AllNewEventsResult([], [], [], [], []))
+
         def get_all_new_events_txn(txn):
             sql = (
                 "SELECT e.stream_ordering, ej.internal_metadata, ej.json, eg.state_group"
@@ -1155,7 +1161,7 @@ class EventsStore(SQLBaseStore):
                 " ORDER BY e.stream_ordering ASC"
                 " LIMIT ?"
             )
-            if last_forward_id != current_forward_id:
+            if have_forward_events:
                 txn.execute(sql, (last_forward_id, current_forward_id, limit))
                 new_forward_events = txn.fetchall()
 
@@ -1199,7 +1205,7 @@ class EventsStore(SQLBaseStore):
                 " ORDER BY e.stream_ordering DESC"
                 " LIMIT ?"
             )
-            if last_backfill_id != current_backfill_id:
+            if have_backfill_events:
                 txn.execute(sql, (-last_backfill_id, -current_backfill_id, limit))
                 new_backfill_events = txn.fetchall()
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index e5755c0aea..11feb3eb11 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -106,6 +106,9 @@ class PusherStore(SQLBaseStore):
         return self._pushers_id_gen.get_current_token()
 
     def get_all_updated_pushers(self, last_id, current_id, limit):
+        if last_id == current_id:
+            return defer.succeed(([], []))
+
         def get_all_updated_pushers_txn(txn):
             sql = (
                 "SELECT id, user_name, access_token, profile_tag, kind,"
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 3b8805593e..935fc503d9 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -391,6 +391,9 @@ class ReceiptsStore(SQLBaseStore):
         )
 
     def get_all_updated_receipts(self, last_id, current_id, limit=None):
+        if last_id == current_id:
+            return defer.succeed([])
+
         def get_all_updated_receipts_txn(txn):
             sql = (
                 "SELECT stream_id, room_id, receipt_type, user_id, event_id, data"
-- 
cgit 1.5.1


From 50ad8005e4b3e8054f990f34d2d7735b09dc8c19 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 29 Apr 2016 19:16:15 +0100
Subject: Put spaces at start of line

---
 synapse/storage/event_push_actions.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index f2af8bdb36..5b9a4ca60d 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -155,13 +155,13 @@ class EventPushActionsStore(SQLBaseStore):
 
         def get_no_receipt(txn):
             sql = (
-                "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, "
-                "e.received_ts "
-                "FROM event_push_actions AS ep "
-                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
-                "WHERE ep.room_id not in ("
+                "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions,"
+                " e.received_ts"
+                " FROM event_push_actions AS ep"
+                " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
+                " WHERE ep.room_id not in ("
                 "   SELECT room_id FROM events NATURAL JOIN receipts_linearized"
-                "   WHERE receipt_type = 'm.read' AND user_id = ? "
+                "   WHERE receipt_type = 'm.read' AND user_id = ?"
                 "   GROUP BY room_id"
                 ") AND ep.user_id = ? AND ep.stream_ordering > ?"
             )
@@ -190,12 +190,12 @@ class EventPushActionsStore(SQLBaseStore):
     def get_time_of_last_push_action_before(self, stream_ordering):
         def f(txn):
             sql = (
-                "SELECT e.received_ts "
-                "FROM event_push_actions AS ep "
-                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
-                "WHERE ep.stream_ordering > ? "
-                "ORDER BY ep.stream_ordering ASC "
-                "LIMIT 1"
+                "SELECT e.received_ts"
+                " FROM event_push_actions AS ep"
+                " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
+                " WHERE ep.stream_ordering > ?"
+                " ORDER BY ep.stream_ordering ASC"
+                " LIMIT 1"
             )
             txn.execute(sql, (stream_ordering,))
             return txn.fetchone()
@@ -220,10 +220,10 @@ class EventPushActionsStore(SQLBaseStore):
         """
         def f(txn):
             txn.execute(
-                "SELECT ep.room_id, MAX(e.received_ts) "
-                "FROM event_push_actions AS ep "
-                "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id "
-                "GROUP BY ep.room_id"
+                "SELECT ep.room_id, MAX(e.received_ts)"
+                " FROM event_push_actions AS ep"
+                " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
+                " GROUP BY ep.room_id"
             )
             return txn.fetchall()
         result = yield self.runInteraction(
-- 
cgit 1.5.1


From 60f44c098d14754d0159c2ccc2c888e7ca970427 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 29 Apr 2016 19:17:10 +0100
Subject: Remove unnecessary if

---
 synapse/storage/event_push_actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 5b9a4ca60d..c6625a7b08 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -200,7 +200,7 @@ class EventPushActionsStore(SQLBaseStore):
             txn.execute(sql, (stream_ordering,))
             return txn.fetchone()
         result = yield self.runInteraction("get_time_of_last_push_action_before", f)
-        defer.returnValue(result[0] if result is not None else None)
+        defer.returnValue(result[0] if result else None)
 
     @defer.inlineCallbacks
     def get_latest_push_action_stream_ordering(self):
-- 
cgit 1.5.1


From 8f99cd5996a11bb1cc50dce35542b2591426fdbc Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 29 Apr 2016 19:27:03 +0100
Subject: Oops, actually specify the user id

---
 synapse/storage/event_push_actions.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index c6625a7b08..974ceb3389 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -223,7 +223,9 @@ class EventPushActionsStore(SQLBaseStore):
                 "SELECT ep.room_id, MAX(e.received_ts)"
                 " FROM event_push_actions AS ep"
                 " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
-                " GROUP BY ep.room_id"
+                " WHERE ep.user_id = ?"
+                " GROUP BY ep.room_id",
+                (user_id,)
             )
             return txn.fetchall()
         result = yield self.runInteraction(
-- 
cgit 1.5.1


From b0a1036d93eaec50839c95bf0dc621826342ea62 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 29 Apr 2016 19:28:56 +0100
Subject: Use explicit join

---
 synapse/storage/event_push_actions.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 974ceb3389..b710101b15 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -127,9 +127,9 @@ class EventPushActionsStore(SQLBaseStore):
                 "       FROM events"
                 "   NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'"
                 "   GROUP BY room_id, user_id"
-                ") AS rl "
-                "NATURAL JOIN events e "
-                "WHERE"
+                ") AS rl"
+                " INNER JOIN events AS e USING (room_id, event_id)"
+                " WHERE"
                 "   ep.room_id = rl.room_id"
                 "   AND ("
                 "       ep.topological_ordering > rl.topological_ordering"
-- 
cgit 1.5.1


From 35b7b8e4bccd8cfbe44b2ffee97cdee0a48701ba Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 29 Apr 2016 20:10:34 +0100
Subject: Remove unused function

---
 synapse/push/emailpusher.py           |  4 ----
 synapse/storage/event_push_actions.py | 22 ----------------------
 2 files changed, 26 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
index 8b105b85c8..c10deded06 100644
--- a/synapse/push/emailpusher.py
+++ b/synapse/push/emailpusher.py
@@ -122,10 +122,6 @@ class EmailPusher(object):
         up logging, measures and guards against multiple instances of it
         being run.
         """
-        last_notifs = yield self.store.get_time_of_latest_push_action_by_room_for_user(
-            self.user_id
-        )
-
         unprocessed = yield self.store.get_unread_push_actions_for_user_in_range(
             self.user_id, self.last_stream_ordering, self.max_stream_ordering
         )
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index b710101b15..85290d2a90 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -212,28 +212,6 @@ class EventPushActionsStore(SQLBaseStore):
         )
         defer.returnValue(result[0] or 0)
 
-    @defer.inlineCallbacks
-    def get_time_of_latest_push_action_by_room_for_user(self, user_id):
-        """
-        Returns only the received_ts of the last notification in each of the
-        user's rooms, in a dict by room_id
-        """
-        def f(txn):
-            txn.execute(
-                "SELECT ep.room_id, MAX(e.received_ts)"
-                " FROM event_push_actions AS ep"
-                " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id"
-                " WHERE ep.user_id = ?"
-                " GROUP BY ep.room_id",
-                (user_id,)
-            )
-            return txn.fetchall()
-        result = yield self.runInteraction(
-            "get_time_of_latest_push_action_by_room_for_user", f
-        )
-
-        defer.returnValue({row[0]: row[1] for row in result})
-
     def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
         # Sad that we have to blow away the cache for the whole room here
         txn.call_after(
-- 
cgit 1.5.1


From 183f23f10d32ece4503f18e70296c88a1625ecff Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 3 May 2016 14:22:33 +0100
Subject: Delete old pushers

---
 synapse/storage/event_push_actions.py | 12 ++++++++++++
 synapse/storage/receipts.py           | 26 +++++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 86a98b6f11..312c0071f1 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -201,6 +201,18 @@ class EventPushActionsStore(SQLBaseStore):
             (room_id, event_id)
         )
 
+    def _remove_push_actions_before_txn(self, txn, room_id, user_id,
+                                        topological_ordering):
+        txn.call_after(
+            self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
+            (room_id, user_id, )
+        )
+        txn.execute(
+            "DELETE FROM event_push_actions"
+            " WHERE room_id = ? AND user_id = ? AND topological_ordering < ?",
+            (room_id, user_id, topological_ordering,)
+        )
+
 
 def _action_has_highlight(actions):
     for action in actions:
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 935fc503d9..cd1b611a0c 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -244,6 +244,15 @@ class ReceiptsStore(SQLBaseStore):
             (user_id, room_id, receipt_type)
         )
 
+        res = self._simple_select_one_txn(
+            txn,
+            table="events",
+            retcols=["topological_ordering", "stream_ordering"],
+            keyvalues={"event_id": event_id},
+        )
+        topological_ordering = int(res["topological_ordering"])
+        stream_ordering = int(res["stream_ordering"])
+
         # We don't want to clobber receipts for more recent events, so we
         # have to compare orderings of existing receipts
         sql = (
@@ -256,15 +265,6 @@ class ReceiptsStore(SQLBaseStore):
         results = txn.fetchall()
 
         if results:
-            res = self._simple_select_one_txn(
-                txn,
-                table="events",
-                retcols=["topological_ordering", "stream_ordering"],
-                keyvalues={"event_id": event_id},
-            )
-            topological_ordering = int(res["topological_ordering"])
-            stream_ordering = int(res["stream_ordering"])
-
             for to, so, _ in results:
                 if int(to) > topological_ordering:
                     return False
@@ -294,6 +294,14 @@ class ReceiptsStore(SQLBaseStore):
             }
         )
 
+        if receipt_type == "m.read":
+            self._remove_push_actions_before_txn(
+                txn,
+                room_id=room_id,
+                user_id=user_id,
+                topological_ordering=topological_ordering,
+            )
+
         return True
 
     @defer.inlineCallbacks
-- 
cgit 1.5.1


From a438a6d2bca7fae3a4f1fa38b72b821c742cdc8a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 3 May 2016 16:01:24 +0100
Subject: Implement basic ignore user

---
 synapse/api/filtering.py        |  8 +++++---
 synapse/handlers/_base.py       | 26 +++++++++++++++++++++++---
 synapse/handlers/sync.py        | 22 +++++++++++++++++++---
 synapse/storage/account_data.py | 31 +++++++++++++++++++++++++++++++
 4 files changed, 78 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index cd699ef27f..4f5a4281fa 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -15,6 +15,8 @@
 from synapse.api.errors import SynapseError
 from synapse.types import UserID, RoomID
 
+from twisted.internet import defer
+
 import ujson as json
 
 
@@ -24,10 +26,10 @@ class Filtering(object):
         super(Filtering, self).__init__()
         self.store = hs.get_datastore()
 
+    @defer.inlineCallbacks
     def get_user_filter(self, user_localpart, filter_id):
-        result = self.store.get_user_filter(user_localpart, filter_id)
-        result.addCallback(FilterCollection)
-        return result
+        result = yield self.store.get_user_filter(user_localpart, filter_id)
+        defer.returnValue(FilterCollection(result))
 
     def add_user_filter(self, user_localpart, user_filter):
         self.check_valid_filter(user_filter)
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 13a675b208..0912274e1a 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -84,7 +84,7 @@ class BaseHandler(object):
             events ([synapse.events.EventBase]): list of events to filter
         """
         forgotten = yield defer.gatherResults([
-            self.store.who_forgot_in_room(
+            preserve_fn(self.store.who_forgot_in_room)(
                 room_id,
             )
             for room_id in frozenset(e.room_id for e in events)
@@ -95,13 +95,33 @@ class BaseHandler(object):
             row["event_id"] for rows in forgotten for row in rows
         )
 
-        def allowed(event, user_id, is_peeking):
+        # Maps user_id -> account data content
+        ignore_dict_content = yield defer.gatherResults([
+            preserve_fn(self.store.get_global_account_data_by_type_for_user)(
+                user_id, "m.ignored_user_list"
+            ).addCallback(lambda d, u: (u, d), user_id)
+            for user_id, is_peeking in user_tuples
+        ]).addCallback(dict)
+
+        # FIXME: This will explode if people upload something incorrect.
+        ignore_dict = {
+            user_id: frozenset(
+                content.get("ignored_users", {}).keys() if content else []
+            )
+            for user_id, content in ignore_dict_content.items()
+        }
+
+        def allowed(event, user_id, is_peeking, ignore_list):
             """
             Args:
                 event (synapse.events.EventBase): event to check
                 user_id (str)
                 is_peeking (bool)
+                ignore_list (list): list of users to ignore
             """
+            if not event.is_state() and event.sender in ignore_list:
+                return False
+
             state = event_id_to_state[event.event_id]
 
             # get the room_visibility at the time of the event.
@@ -186,7 +206,7 @@ class BaseHandler(object):
             user_id: [
                 event
                 for event in events
-                if allowed(event, user_id, is_peeking)
+                if allowed(event, user_id, is_peeking, ignore_dict.get(user_id, []))
             ]
             for user_id, is_peeking in user_tuples
         })
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 231140b655..27c3c1b525 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -247,6 +247,10 @@ class SyncHandler(BaseHandler):
             sync_config.user.to_string()
         )
 
+        ignored_users = account_data.get(
+            "m.ignored_user_list", {}
+        ).get("ignored_users", {}).keys()
+
         joined = []
         invited = []
         archived = []
@@ -267,6 +271,8 @@ class SyncHandler(BaseHandler):
                 )
                 joined.append(room_result)
             elif event.membership == Membership.INVITE:
+                if event.sender in ignored_users:
+                    return
                 invite = yield self.store.get_event(event.event_id)
                 invited.append(InvitedSyncResult(
                     room_id=event.room_id,
@@ -515,6 +521,15 @@ class SyncHandler(BaseHandler):
                 sync_config.user
             )
 
+        ignored_account_data = yield self.store.get_global_account_data_by_type_for_user(
+            user_id, "m.ignored_user_list"
+        )
+
+        if ignored_account_data:
+            ignored_users = ignored_account_data.get("ignored_users", {}).keys()
+        else:
+            ignored_users = frozenset()
+
         # Get a list of membership change events that have happened.
         rooms_changed = yield self.store.get_membership_changes_for_user(
             user_id, since_token.room_key, now_token.room_key
@@ -549,9 +564,10 @@ class SyncHandler(BaseHandler):
             # Only bother if we're still currently invited
             should_invite = non_joins[-1].membership == Membership.INVITE
             if should_invite:
-                room_sync = InvitedSyncResult(room_id, invite=non_joins[-1])
-                if room_sync:
-                    invited.append(room_sync)
+                if event.sender not in ignored_users:
+                    room_sync = InvitedSyncResult(room_id, invite=non_joins[-1])
+                    if room_sync:
+                        invited.append(room_sync)
 
             # Always include leave/ban events. Just take the last one.
             # TODO: How do we handle ban -> leave in same batch?
diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py
index 7a7fbf1e52..cc0b92bc89 100644
--- a/synapse/storage/account_data.py
+++ b/synapse/storage/account_data.py
@@ -16,6 +16,8 @@
 from ._base import SQLBaseStore
 from twisted.internet import defer
 
+from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
+
 import ujson as json
 import logging
 
@@ -24,6 +26,7 @@ logger = logging.getLogger(__name__)
 
 class AccountDataStore(SQLBaseStore):
 
+    @cached()
     def get_account_data_for_user(self, user_id):
         """Get all the client account_data for a user.
 
@@ -60,6 +63,28 @@ class AccountDataStore(SQLBaseStore):
             "get_account_data_for_user", get_account_data_for_user_txn
         )
 
+    @cachedInlineCallbacks(num_args=2)
+    def get_global_account_data_by_type_for_user(self, user_id, data_type):
+        """
+        Returns:
+            Deferred: A dict
+        """
+        result = yield self._simple_select_one_onecol(
+            table="account_data",
+            keyvalues={
+                "user_id": user_id,
+                "account_data_type": data_type,
+            },
+            retcol="content",
+            desc="get_global_account_data_by_type_for_user",
+            allow_none=True,
+        )
+
+        if result:
+            defer.returnValue(json.loads(result))
+        else:
+            defer.returnValue(None)
+
     def get_account_data_for_room(self, user_id, room_id):
         """Get all the client account_data for a user for a room.
 
@@ -193,6 +218,7 @@ class AccountDataStore(SQLBaseStore):
                 self._account_data_stream_cache.entity_has_changed,
                 user_id, next_id,
             )
+            txn.call_after(self.get_account_data_for_user.invalidate, (user_id,))
             self._update_max_stream_id(txn, next_id)
 
         with self._account_data_id_gen.get_next() as next_id:
@@ -232,6 +258,11 @@ class AccountDataStore(SQLBaseStore):
                 self._account_data_stream_cache.entity_has_changed,
                 user_id, next_id,
             )
+            txn.call_after(self.get_account_data_for_user.invalidate, (user_id,))
+            txn.call_after(
+                self.get_global_account_data_by_type_for_user.invalidate,
+                (user_id, account_data_type,)
+            )
             self._update_max_stream_id(txn, next_id)
 
         with self._account_data_id_gen.get_next() as next_id:
-- 
cgit 1.5.1


From 984d4a2c0f59039a623b6a6f1945ff697f004c27 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 May 2016 11:28:10 +0100
Subject: Add /report endpoint

---
 synapse/rest/__init__.py                     |  2 +
 synapse/rest/client/v2_alpha/report_event.py | 59 ++++++++++++++++++++++++++++
 synapse/storage/prepare_database.py          |  2 +-
 synapse/storage/room.py                      | 14 +++++++
 synapse/storage/schema/delta/32/reports.sql  | 23 +++++++++++
 5 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 synapse/rest/client/v2_alpha/report_event.py
 create mode 100644 synapse/storage/schema/delta/32/reports.sql

(limited to 'synapse/storage')

diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 6688fa8fa0..e805cb9111 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -44,6 +44,7 @@ from synapse.rest.client.v2_alpha import (
     tokenrefresh,
     tags,
     account_data,
+    report_event,
 )
 
 from synapse.http.server import JsonResource
@@ -86,3 +87,4 @@ class ClientRestResource(JsonResource):
         tokenrefresh.register_servlets(hs, client_resource)
         tags.register_servlets(hs, client_resource)
         account_data.register_servlets(hs, client_resource)
+        report_event.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py
new file mode 100644
index 0000000000..412e5b1903
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/report_event.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from ._base import client_v2_patterns
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReportEventRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/rooms/(?P<room_id>[^/]*)/report$"
+    )
+
+    def __init__(self, hs):
+        super(ReportEventRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        body = parse_json_object_from_request(request)
+
+        event_id = body["event_id"]
+
+        yield self.store.add_event_report(
+            room_id=room_id,
+            event_id=event_id,
+            user_id=user_id,
+            reason=body.get("reason"),
+            content=body,
+        )
+
+        defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+    ReportEventRestServlet(hs).register(http_server)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 57f14fd12b..c8487c8838 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 31
+SCHEMA_VERSION = 32
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 70aa64fb31..ceced7d516 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -23,6 +23,7 @@ from .engines import PostgresEngine, Sqlite3Engine
 
 import collections
 import logging
+import ujson as json
 
 logger = logging.getLogger(__name__)
 
@@ -221,3 +222,16 @@ class RoomStore(SQLBaseStore):
                     aliases.extend(e.content['aliases'])
 
         defer.returnValue((name, aliases))
+
+    def add_event_report(self, room_id, event_id, user_id, reason, content):
+        return self._simple_insert(
+            table="event_reports",
+            values={
+                "room_id": room_id,
+                "event_id": event_id,
+                "user_id": user_id,
+                "reason": reason,
+                "content": json.dumps(content),
+            },
+            desc="add_event_report"
+        )
diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql
new file mode 100644
index 0000000000..06bf0d9b5a
--- /dev/null
+++ b/synapse/storage/schema/delta/32/reports.sql
@@ -0,0 +1,23 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE TABLE event_reports(
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    reason TEXT,
+    content TEXT
+);
-- 
cgit 1.5.1


From 80be39646467c46a52530cd0839746810ad32b62 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 4 May 2016 13:19:59 +0100
Subject: Correct SQL statement for postgres

In standard sql, join binds tighter than comma, so we were joining on the wrong table. Postgres follows the standard (apparently).
---
 synapse/storage/event_push_actions.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 85290d2a90..6f316f7d24 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -120,14 +120,15 @@ class EventPushActionsStore(SQLBaseStore):
             sql = (
                 "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, "
                 "e.received_ts "
-                "FROM event_push_actions AS ep, ("
+                "FROM ("
                 "   SELECT room_id, user_id, "
                 "       max(topological_ordering) as topological_ordering, "
                 "       max(stream_ordering) as stream_ordering "
                 "       FROM events"
                 "   NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'"
                 "   GROUP BY room_id, user_id"
-                ") AS rl"
+                ") AS rl,"
+                " event_push_actions AS ep"
                 " INNER JOIN events AS e USING (room_id, event_id)"
                 " WHERE"
                 "   ep.room_id = rl.room_id"
-- 
cgit 1.5.1


From 8e6a163f2762b3f62ae9b350c5050bc2318ec268 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 May 2016 15:19:12 +0100
Subject: Add timestamp and auto incrementing ID

---
 synapse/rest/client/v2_alpha/report_event.py | 2 ++
 synapse/storage/__init__.py                  | 1 +
 synapse/storage/room.py                      | 6 +++++-
 synapse/storage/schema/delta/32/reports.sql  | 2 ++
 4 files changed, 10 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py
index 9c1c9662c9..8903e12405 100644
--- a/synapse/rest/client/v2_alpha/report_event.py
+++ b/synapse/rest/client/v2_alpha/report_event.py
@@ -33,6 +33,7 @@ class ReportEventRestServlet(RestServlet):
         super(ReportEventRestServlet, self).__init__()
         self.hs = hs
         self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
@@ -48,6 +49,7 @@ class ReportEventRestServlet(RestServlet):
             user_id=user_id,
             reason=body.get("reason"),
             content=body,
+            received_ts=self.clock.time_msec(),
         )
 
         defer.returnValue((200, {}))
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 045ae6c03f..7122b0cbb1 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -114,6 +114,7 @@ class DataStore(RoomMemberStore, RoomStore,
         self._state_groups_id_gen = StreamIdGenerator(db_conn, "state_groups", "id")
         self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
         self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id")
+        self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id")
         self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
         self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id")
         self._push_rules_stream_id_gen = ChainedIdGenerator(
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index ceced7d516..26933e593a 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -223,10 +223,14 @@ class RoomStore(SQLBaseStore):
 
         defer.returnValue((name, aliases))
 
-    def add_event_report(self, room_id, event_id, user_id, reason, content):
+    def add_event_report(self, room_id, event_id, user_id, reason, content,
+                         received_ts):
+        next_id = self._event_reports_id_gen.get_next()
         return self._simple_insert(
             table="event_reports",
             values={
+                "id": next_id,
+                "received_ts": received_ts,
                 "room_id": room_id,
                 "event_id": event_id,
                 "user_id": user_id,
diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql
index 06bf0d9b5a..3f25027457 100644
--- a/synapse/storage/schema/delta/32/reports.sql
+++ b/synapse/storage/schema/delta/32/reports.sql
@@ -15,6 +15,8 @@
 
 
 CREATE TABLE event_reports(
+    id BIGINT NOT NULL,
+    received_ts BIGINT NOT NULL,
     room_id TEXT NOT NULL,
     event_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
-- 
cgit 1.5.1


From fcd1eb642dc8b73e372b78bb788e3a17a5e40ace Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 May 2016 16:51:51 +0100
Subject: Add primary key

---
 synapse/storage/schema/delta/32/reports.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql
index 3f25027457..d13609776f 100644
--- a/synapse/storage/schema/delta/32/reports.sql
+++ b/synapse/storage/schema/delta/32/reports.sql
@@ -15,7 +15,7 @@
 
 
 CREATE TABLE event_reports(
-    id BIGINT NOT NULL,
+    id BIGINT NOT NULL PRIMARY KEY,
     received_ts BIGINT NOT NULL,
     room_id TEXT NOT NULL,
     event_id TEXT NOT NULL,
-- 
cgit 1.5.1


From 1f0f5ffa1e22dcbe2b0bb605ccaf12bf571dc624 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 5 May 2016 09:51:03 +0100
Subject: Add bulk fetch storage API

---
 synapse/handlers/_base.py       | 10 +++-------
 synapse/handlers/sync.py        |  2 +-
 synapse/storage/account_data.py | 25 ++++++++++++++++++++++---
 3 files changed, 26 insertions(+), 11 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 0912274e1a..81f7929b50 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -95,13 +95,9 @@ class BaseHandler(object):
             row["event_id"] for rows in forgotten for row in rows
         )
 
-        # Maps user_id -> account data content
-        ignore_dict_content = yield defer.gatherResults([
-            preserve_fn(self.store.get_global_account_data_by_type_for_user)(
-                user_id, "m.ignored_user_list"
-            ).addCallback(lambda d, u: (u, d), user_id)
-            for user_id, is_peeking in user_tuples
-        ]).addCallback(dict)
+        ignore_dict_content = yield self.store.get_global_account_data_by_type_for_users(
+            "m.ignored_user_list", user_ids=[user_id for user_id, _ in user_tuples]
+        )
 
         # FIXME: This will explode if people upload something incorrect.
         ignore_dict = {
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 27c3c1b525..0bb1913285 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -522,7 +522,7 @@ class SyncHandler(BaseHandler):
             )
 
         ignored_account_data = yield self.store.get_global_account_data_by_type_for_user(
-            user_id, "m.ignored_user_list"
+            "m.ignored_user_list", user_id=user_id,
         )
 
         if ignored_account_data:
diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py
index cc0b92bc89..ec7e8d40d2 100644
--- a/synapse/storage/account_data.py
+++ b/synapse/storage/account_data.py
@@ -16,7 +16,7 @@
 from ._base import SQLBaseStore
 from twisted.internet import defer
 
-from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
+from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
 
 import ujson as json
 import logging
@@ -64,7 +64,7 @@ class AccountDataStore(SQLBaseStore):
         )
 
     @cachedInlineCallbacks(num_args=2)
-    def get_global_account_data_by_type_for_user(self, user_id, data_type):
+    def get_global_account_data_by_type_for_user(self, data_type, user_id):
         """
         Returns:
             Deferred: A dict
@@ -85,6 +85,25 @@ class AccountDataStore(SQLBaseStore):
         else:
             defer.returnValue(None)
 
+    @cachedList(cached_method_name="get_global_account_data_by_type_for_user",
+                num_args=2, list_name="user_ids", inlineCallbacks=True)
+    def get_global_account_data_by_type_for_users(self, data_type, user_ids):
+        rows = yield self._simple_select_many_batch(
+            table="account_data",
+            column="user_id",
+            iterable=user_ids,
+            keyvalues={
+                "account_data_type": data_type,
+            },
+            retcols=("user_id", "content",),
+            desc="get_global_account_data_by_type_for_users",
+        )
+
+        defer.returnValue({
+            row["user_id"]: json.loads(row["content"]) if row["content"] else None
+            for row in rows
+        })
+
     def get_account_data_for_room(self, user_id, room_id):
         """Get all the client account_data for a user for a room.
 
@@ -261,7 +280,7 @@ class AccountDataStore(SQLBaseStore):
             txn.call_after(self.get_account_data_for_user.invalidate, (user_id,))
             txn.call_after(
                 self.get_global_account_data_by_type_for_user.invalidate,
-                (user_id, account_data_type,)
+                (account_data_type, user_id,)
             )
             self._update_max_stream_id(txn, next_id)
 
-- 
cgit 1.5.1


From 9c272da05fcf51534aaa877647bc3b82bf841cf3 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 5 May 2016 13:42:44 +0100
Subject: Add an openidish mechanism for proving to third parties that you own
 a given user_id

---
 synapse/federation/federation_server.py    |  5 ++
 synapse/federation/transport/server.py     | 47 ++++++++++++++-
 synapse/rest/__init__.py                   |  2 +
 synapse/rest/client/v2_alpha/openid.py     | 96 ++++++++++++++++++++++++++++++
 synapse/storage/__init__.py                |  4 +-
 synapse/storage/openid.py                  | 32 ++++++++++
 synapse/storage/schema/delta/32/openid.sql |  9 +++
 7 files changed, 193 insertions(+), 2 deletions(-)
 create mode 100644 synapse/rest/client/v2_alpha/openid.py
 create mode 100644 synapse/storage/openid.py
 create mode 100644 synapse/storage/schema/delta/32/openid.sql

(limited to 'synapse/storage')

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 429ab6ddec..f1d231b9d8 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -387,6 +387,11 @@ class FederationServer(FederationBase):
             "events": [ev.get_pdu_json(time_now) for ev in missing_events],
         })
 
+    @log_function
+    def on_openid_userinfo(self, token):
+        ts_now_ms = self._clock.time_msec()
+        return self.store.get_user_id_for_open_id_token(token, ts_now_ms)
+
     @log_function
     def _get_persisted_pdu(self, origin, event_id, do_auth=True):
         """ Get a PDU from the database with given origin and id.
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 3e552b6c44..5b6c7d11dd 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from synapse.api.urls import FEDERATION_PREFIX as PREFIX
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.server import JsonResource
-from synapse.http.servlet import parse_json_object_from_request
+from synapse.http.servlet import parse_json_object_from_request, parse_string
 from synapse.util.ratelimitutils import FederationRateLimiter
 
 import functools
@@ -448,6 +448,50 @@ class On3pidBindServlet(BaseFederationServlet):
         return code
 
 
+class OpenIdUserInfo(BaseFederationServlet):
+    """
+    Exchange a bearer token for information about a user.
+
+    The response format should be compatible with:
+        http://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse
+
+    GET /openid/userinfo?access_token=ABDEFGH HTTP/1.1
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+
+    {
+        "sub": "@userpart:example.org",
+    }
+    """
+
+    PATH = "/openid/userinfo"
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        token = parse_string(request, "access_token")
+        if token is None:
+            defer.returnValue((401, {
+                "errcode": "M_MISSING_TOKEN", "error": "Access Token required"
+            }))
+            return
+
+        user_id = yield self.handler.on_openid_userinfo(token)
+
+        if user_id is None:
+            defer.returnValue((401, {
+                "errcode": "M_UNKNOWN_TOKEN",
+                "error": "Access Token unknown or expired"
+            }))
+
+        defer.returnValue((200, {"sub": user_id}))
+
+    # Avoid doing remote HS authorization checks which are done by default by
+    # BaseFederationServlet.
+    def _wrap(self, code):
+        return code
+
+
 SERVLET_CLASSES = (
     FederationSendServlet,
     FederationPullServlet,
@@ -468,6 +512,7 @@ SERVLET_CLASSES = (
     FederationClientKeysClaimServlet,
     FederationThirdPartyInviteExchangeServlet,
     On3pidBindServlet,
+    OpenIdUserInfo,
 )
 
 
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index e805cb9111..8b223e032b 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -45,6 +45,7 @@ from synapse.rest.client.v2_alpha import (
     tags,
     account_data,
     report_event,
+    openid,
 )
 
 from synapse.http.server import JsonResource
@@ -88,3 +89,4 @@ class ClientRestResource(JsonResource):
         tags.register_servlets(hs, client_resource)
         account_data.register_servlets(hs, client_resource)
         report_event.register_servlets(hs, client_resource)
+        openid.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/openid.py b/synapse/rest/client/v2_alpha/openid.py
new file mode 100644
index 0000000000..ddea750323
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/openid.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ._base import client_v2_patterns
+
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.api.errors import AuthError
+from synapse.util.stringutils import random_string
+
+from twisted.internet import defer
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class IdTokenServlet(RestServlet):
+    """
+    Get a bearer token that may be passed to a third party to confirm ownership
+    of a matrix user id.
+
+    The format of the response could be made compatible with the format given
+    in http://openid.net/specs/openid-connect-core-1_0.html#TokenResponse
+
+    But instead of returning a signed "id_token" the response contains the
+    name of the issuing matrix homeserver. This means that for now the third
+    party will need to check the validity of the "id_token" against the
+    federation /openid/userinfo endpoint of the homeserver.
+
+    Request:
+
+    POST /user/{user_id}/openid/token?access_token=... HTTP/1.1
+
+    {}
+
+    Response:
+
+    HTTP/1.1 200 OK
+    {
+        "access_token": "ABDEFGH",
+        "token_type": "Bearer",
+        "matrix_server_name": "example.com",
+        "expires_in": 3600,
+    }
+    """
+    PATTERNS = client_v2_patterns(
+        "/user/(?P<user_id>[^/]*)/openid/token"
+    )
+
+    EXPIRES_MS = 3600 * 1000
+
+    def __init__(self, hs):
+        super(IdTokenServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+        self.server_name = hs.config.server_name
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        if user_id != requester.user.to_string():
+            raise AuthError(403, "Cannot request tokens for other users.")
+
+        # Parse the request body to make sure it's JSON, but ignore the contents
+        # for now.
+        parse_json_object_from_request(request)
+
+        token = random_string(24)
+        ts_valid_until_ms = self.clock.time_msec() + self.EXPIRES_MS
+
+        yield self.store.insert_open_id_token(token, ts_valid_until_ms, user_id)
+
+        defer.returnValue((200, {
+            "access_token": token,
+            "token_type": "Bearer",
+            "matrix_server_name": self.server_name,
+            "expires_in": self.EXPIRES_MS / 1000,
+        }))
+
+
+def register_servlets(hs, http_server):
+    IdTokenServlet(hs).register(http_server)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 7122b0cbb1..d970fde9e8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -44,6 +44,7 @@ from .receipts import ReceiptsStore
 from .search import SearchStore
 from .tags import TagsStore
 from .account_data import AccountDataStore
+from .openid import OpenIdStore
 
 from .util.id_generators import IdGenerator, StreamIdGenerator, ChainedIdGenerator
 
@@ -81,7 +82,8 @@ class DataStore(RoomMemberStore, RoomStore,
                 SearchStore,
                 TagsStore,
                 AccountDataStore,
-                EventPushActionsStore
+                EventPushActionsStore,
+                OpenIdStore,
                 ):
 
     def __init__(self, db_conn, hs):
diff --git a/synapse/storage/openid.py b/synapse/storage/openid.py
new file mode 100644
index 0000000000..5dabb607bd
--- /dev/null
+++ b/synapse/storage/openid.py
@@ -0,0 +1,32 @@
+from ._base import SQLBaseStore
+
+
+class OpenIdStore(SQLBaseStore):
+    def insert_open_id_token(self, token, ts_valid_until_ms, user_id):
+        return self._simple_insert(
+            table="open_id_tokens",
+            values={
+                "token": token,
+                "ts_valid_until_ms": ts_valid_until_ms,
+                "user_id": user_id,
+            },
+            desc="insert_open_id_token"
+        )
+
+    def get_user_id_for_open_id_token(self, token, ts_now_ms):
+        def get_user_id_for_token_txn(txn):
+            sql = (
+                "SELECT user_id FROM open_id_tokens"
+                " WHERE token = ? AND ? <= ts_valid_until_ms"
+            )
+
+            txn.execute(sql, (token, ts_now_ms))
+
+            rows = txn.fetchall()
+            if not rows:
+                return None
+            else:
+                return rows[0][0]
+        return self.runInteraction(
+            "get_user_id_for_token", get_user_id_for_token_txn
+        )
diff --git a/synapse/storage/schema/delta/32/openid.sql b/synapse/storage/schema/delta/32/openid.sql
new file mode 100644
index 0000000000..36f37b11c8
--- /dev/null
+++ b/synapse/storage/schema/delta/32/openid.sql
@@ -0,0 +1,9 @@
+
+CREATE TABLE open_id_tokens (
+    token TEXT NOT NULL PRIMARY KEY,
+    ts_valid_until_ms bigint NOT NULL,
+    user_id TEXT NOT NULL,
+    UNIQUE (token)
+);
+
+CREATE index open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms);
-- 
cgit 1.5.1


From 56b5e83e36f22f3eab1ebf7a46b9f23f0c1a3e8d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 11:20:18 +0100
Subject: Reduce database inserts when sending transactions

---
 synapse/handlers/presence.py    |   2 +-
 synapse/storage/transactions.py | 157 +++++++++++++++++++++++++++++-----------
 2 files changed, 114 insertions(+), 45 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index d0c8f1328b..639567953a 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -168,7 +168,7 @@ class PresenceHandler(BaseHandler):
         # The initial delay is to allow disconnected clients a chance to
         # reconnect before we treat them as offline.
         self.clock.call_later(
-            0 * 1000,
+            30 * 1000,
             self.clock.looping_call,
             self._handle_timeouts,
             5000,
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index d338dfcf0a..17fc601983 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -16,16 +16,54 @@
 from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached
 
+from twisted.internet import defer, reactor
+
 from canonicaljson import encode_canonical_json
+
+from collections import namedtuple
+
+import itertools
 import logging
 
 logger = logging.getLogger(__name__)
 
 
+_TransactionRow = namedtuple(
+    "_TransactionRow", (
+        "id", "transaction_id", "destination", "ts", "response_code",
+        "response_json",
+    )
+)
+
+_UpdateTransactionRow = namedtuple(
+    "_TransactionRow", (
+        "response_code", "response_json",
+    )
+)
+
+
 class TransactionStore(SQLBaseStore):
     """A collection of queries for handling PDUs.
     """
 
+    def __init__(self, hs):
+        super(TransactionStore, self).__init__(hs)
+
+        # New transactions that are currently in flights
+        self.inflight_transactions = {}
+
+        # Newly delievered transactions that *weren't* persisted while in flight
+        self.new_delivered_transactions = {}
+
+        # Newly delivered transactions that *were* persisted while in flight
+        self.update_delivered_transactions = {}
+
+        reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns)
+        hs.get_clock().looping_call(
+            self._persist_in_mem_txns,
+            1000,
+        )
+
     def get_received_txn_response(self, transaction_id, origin):
         """For an incoming transaction from a given origin, check if we have
         already responded to it. If so, return the response code and response
@@ -108,17 +146,28 @@ class TransactionStore(SQLBaseStore):
             list: A list of previous transaction ids.
         """
 
-        return self.runInteraction(
-            "prep_send_transaction",
-            self._prep_send_transaction,
-            transaction_id, destination, origin_server_ts
+        auto_id = self._transaction_id_gen.get_next()
+
+        txn_row = _TransactionRow(
+            id=auto_id,
+            transaction_id=transaction_id,
+            destination=destination,
+            ts=origin_server_ts,
+            response_code=0,
+            response_json=None,
         )
 
-    def _prep_send_transaction(self, txn, transaction_id, destination,
-                               origin_server_ts):
+        self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row
+
+        # TODO: Fetch prev_txns
 
-        next_id = self._transaction_id_gen.get_next()
+        return self.runInteraction(
+            "prep_send_transaction",
+            self._get_prevs_txn,
+            destination,
+        )
 
+    def _get_prevs_txn(self, txn, destination):
         # First we find out what the prev_txns should be.
         # Since we know that we are only sending one transaction at a time,
         # we can simply take the last one.
@@ -133,23 +182,6 @@ class TransactionStore(SQLBaseStore):
 
         prev_txns = [r["transaction_id"] for r in results]
 
-        # Actually add the new transaction to the sent_transactions table.
-
-        self._simple_insert_txn(
-            txn,
-            table="sent_transactions",
-            values={
-                "id": next_id,
-                "transaction_id": transaction_id,
-                "destination": destination,
-                "ts": origin_server_ts,
-                "response_code": 0,
-                "response_json": None,
-            }
-        )
-
-        # TODO Update the tx id -> pdu id mapping
-
         return prev_txns
 
     def delivered_txn(self, transaction_id, destination, code, response_dict):
@@ -161,27 +193,21 @@ class TransactionStore(SQLBaseStore):
             code (int)
             response_json (str)
         """
-        return self.runInteraction(
-            "delivered_txn",
-            self._delivered_txn,
-            transaction_id, destination, code,
-            buffer(encode_canonical_json(response_dict)),
-        )
 
-    def _delivered_txn(self, txn, transaction_id, destination,
-                       code, response_json):
-        self._simple_update_one_txn(
-            txn,
-            table="sent_transactions",
-            keyvalues={
-                "transaction_id": transaction_id,
-                "destination": destination,
-            },
-            updatevalues={
-                "response_code": code,
-                "response_json": None,  # For now, don't persist response_json
-            }
-        )
+        txn_row = self.inflight_transactions.get(
+            destination, {}
+        ).pop(transaction_id, None)
+
+        if txn_row:
+            d = self.new_delivered_transactions.setdefault(destination, {})
+            d[transaction_id] = txn_row._replace(
+                response_code=code,
+                response_json=None,  # For now, don't persist response
+            )
+        else:
+            d = self.update_delivered_transactions.setdefault(destination, {})
+            # For now, don't persist response
+            d[transaction_id] = _UpdateTransactionRow(code, None)
 
     def get_transactions_after(self, transaction_id, destination):
         """Get all transactions after a given local transaction_id.
@@ -305,3 +331,46 @@ class TransactionStore(SQLBaseStore):
 
         txn.execute(query, (self._clock.time_msec(),))
         return self.cursor_to_dict(txn)
+
+    @defer.inlineCallbacks
+    def _persist_in_mem_txns(self):
+        try:
+            inflight = self.inflight_transactions
+            new_delivered = self.new_delivered_transactions
+            update_delivered = self.update_delivered_transactions
+
+            self.inflight_transactions = {}
+            self.new_delivered_transactions = {}
+            self.update_delivered_transactions = {}
+
+            full_rows = [
+                row._asdict()
+                for txn_map in itertools.chain(inflight.values(), new_delivered.values())
+                for row in txn_map.values()
+            ]
+
+            def f(txn):
+                self._simple_insert_many_txn(
+                    txn=txn,
+                    table="sent_transactions",
+                    values=full_rows
+                )
+
+                for dest, txn_map in update_delivered.items():
+                    for txn_id, update_row in txn_map.items():
+                        self._simple_update_one_txn(
+                            txn,
+                            table="sent_transactions",
+                            keyvalues={
+                                "transaction_id": txn_id,
+                                "destination": dest,
+                            },
+                            updatevalues={
+                                "response_code": update_row.response_code,
+                                "response_json": None,  # For now, don't persist response
+                            }
+                        )
+
+            yield self.runInteraction("_persist_in_mem_txns", f)
+        except:
+            logger.exception("Failed to persist transactions!")
-- 
cgit 1.5.1


From 1d275dba69272f6df5a79871dc4b8d31e71514d0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 11:25:58 +0100
Subject: Don't needlessly enter transaction

---
 synapse/storage/transactions.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 17fc601983..ba1969b243 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -159,10 +159,8 @@ class TransactionStore(SQLBaseStore):
 
         self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row
 
-        # TODO: Fetch prev_txns
-
         return self.runInteraction(
-            "prep_send_transaction",
+            "_get_prevs_txn",
             self._get_prevs_txn,
             destination,
         )
@@ -350,11 +348,12 @@ class TransactionStore(SQLBaseStore):
             ]
 
             def f(txn):
-                self._simple_insert_many_txn(
-                    txn=txn,
-                    table="sent_transactions",
-                    values=full_rows
-                )
+                if full_rows:
+                    self._simple_insert_many_txn(
+                        txn=txn,
+                        table="sent_transactions",
+                        values=full_rows
+                    )
 
                 for dest, txn_map in update_delivered.items():
                     for txn_id, update_row in txn_map.items():
@@ -371,6 +370,7 @@ class TransactionStore(SQLBaseStore):
                             }
                         )
 
-            yield self.runInteraction("_persist_in_mem_txns", f)
+            if full_rows or update_delivered:
+                yield self.runInteraction("_persist_in_mem_txns", f)
         except:
             logger.exception("Failed to persist transactions!")
-- 
cgit 1.5.1


From d13459636fab9637a43c950d17f883ca77b832f7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 11:30:55 +0100
Subject: Pull prev txn from in memory

---
 synapse/storage/transactions.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index ba1969b243..6c7481a728 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -58,6 +58,8 @@ class TransactionStore(SQLBaseStore):
         # Newly delivered transactions that *were* persisted while in flight
         self.update_delivered_transactions = {}
 
+        self.last_transaction = {}
+
         reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns)
         hs.get_clock().looping_call(
             self._persist_in_mem_txns,
@@ -159,11 +161,15 @@ class TransactionStore(SQLBaseStore):
 
         self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row
 
-        return self.runInteraction(
-            "_get_prevs_txn",
-            self._get_prevs_txn,
-            destination,
-        )
+        prev_txn = self.last_transaction.get(destination)
+        if prev_txn:
+            return defer.succeed(prev_txn)
+        else:
+            return self.runInteraction(
+                "_get_prevs_txn",
+                self._get_prevs_txn,
+                destination,
+            )
 
     def _get_prevs_txn(self, txn, destination):
         # First we find out what the prev_txns should be.
@@ -196,6 +202,8 @@ class TransactionStore(SQLBaseStore):
             destination, {}
         ).pop(transaction_id, None)
 
+        self.last_transaction[destination] = transaction_id
+
         if txn_row:
             d = self.new_delivered_transactions.setdefault(destination, {})
             d[transaction_id] = txn_row._replace(
-- 
cgit 1.5.1


From b6e0be701eb8ae36236242549ef356397f5f1d06 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 14:31:38 +0100
Subject: Queue events for persistence

---
 synapse/storage/events.py | 155 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 144 insertions(+), 11 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 0307b2af3c..07e873fce4 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -19,12 +19,14 @@ from twisted.internet import defer, reactor
 from synapse.events import FrozenEvent, USE_FROZEN_DICTS
 from synapse.events.utils import prune_event
 
+from synapse.util.async import ObservableDeferred
 from synapse.util.logcontext import preserve_fn, PreserveLoggingContext
 from synapse.util.logutils import log_function
 from synapse.api.constants import EventTypes
 
 from canonicaljson import encode_canonical_json
-from collections import namedtuple
+from collections import deque, namedtuple
+
 
 import logging
 import math
@@ -50,6 +52,80 @@ EVENT_QUEUE_ITERATIONS = 3  # No. times we block waiting for requests for events
 EVENT_QUEUE_TIMEOUT_S = 0.1  # Timeout when waiting for requests for events
 
 
+class _EventPeristenceQueue(object):
+    """Queues up events so that they can be persisted in bulk with only one
+    concurrent transaction per room.
+    """
+
+    _EventPersistQueueItem = namedtuple("_EventPersistQueueItem", (
+        "events_and_contexts", "current_state", "backfilled", "deferred",
+    ))
+
+    def __init__(self):
+        self._event_persist_queues = {}
+        self._currently_persisting_rooms = set()
+
+    def add_to_queue(self, room_id, events_and_contexts, backfilled, current_state):
+        """Add events to the queue, with the given persist_event options.
+        """
+        queue = self._event_persist_queues.setdefault(room_id, deque())
+        if queue:
+            end_item = queue[-1]
+            if end_item.current_state or current_state:
+                # We perist events with current_state set to True one at a time
+                pass
+            if end_item.backfilled == backfilled:
+                end_item.events_and_contexts.extend(events_and_contexts)
+                return end_item.deferred.observe()
+
+        deferred = ObservableDeferred(defer.Deferred())
+
+        queue.append(self._EventPersistQueueItem(
+            events_and_contexts=events_and_contexts,
+            backfilled=backfilled,
+            current_state=current_state,
+            deferred=deferred,
+        ))
+
+        return deferred.observe()
+
+    def handle_queue(self, room_id, callback):
+        """Attempts to handle the queue for a room if not already being handled.
+
+        The given callback will be invoked with a 'queue' arg, which is a
+        generator over _EventPersistQueueItem's. The queue will finish if there
+        are no longer any items in the room queue.
+
+        This function should therefore be called whenever anything is added
+        to the queue.
+
+        If another callback is currently handling the queue then it will not be
+        invoked.
+        """
+
+        if room_id in self._currently_persisting_rooms:
+            return
+
+        self._currently_persisting_rooms.add(room_id)
+
+        try:
+            callback(self._get_drainining_queue(room_id))
+        finally:
+            self._currently_persisting_rooms.discard(room_id)
+
+    def _get_drainining_queue(self, room_id):
+        queue = self._event_persist_queues.pop(room_id, None)
+        if not queue:
+            return
+
+        try:
+            while True:
+                yield queue.popleft()
+        except IndexError:
+            # Queue has been drained.
+            pass
+
+
 class EventsStore(SQLBaseStore):
     EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
 
@@ -59,19 +135,80 @@ class EventsStore(SQLBaseStore):
             self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
         )
 
-    @defer.inlineCallbacks
+        self._event_persist_queue = _EventPeristenceQueue()
+
     def persist_events(self, events_and_contexts, backfilled=False):
         """
         Write events to the database
         Args:
             events_and_contexts: list of tuples of (event, context)
             backfilled: ?
+        """
+        partitioned = {}
+        for event, ctx in events_and_contexts:
+            partitioned.setdefault(event.room_id, []).append((event, ctx))
+
+        deferreds = []
+        for room_id, evs_ctxs in partitioned.items():
+            d = self._event_persist_queue.add_to_queue(
+                room_id, evs_ctxs,
+                backfilled=backfilled,
+                current_state=None,
+            )
+            deferreds.append(d)
 
-        Returns: Tuple of stream_orderings where the first is the minimum and
-            last is the maximum stream ordering assigned to the events when
-            persisting.
+        for room_id in partitioned.keys():
+            self._maybe_start_persisting(room_id)
 
-        """
+        return defer.gatherResults(deferreds, consumeErrors=True)
+
+    @defer.inlineCallbacks
+    @log_function
+    def persist_event(self, event, context, current_state=None, backfilled=False):
+        deferred = self._event_persist_queue.add_to_queue(
+            event.room_id, [(event, context)],
+            backfilled=backfilled,
+            current_state=current_state,
+        )
+
+        self._maybe_start_persisting(event.room_id)
+
+        yield deferred
+
+        max_persisted_id = yield self._stream_id_gen.get_current_token()
+        defer.returnValue((event.internal_metadata.stream_ordering, max_persisted_id))
+
+    def _maybe_start_persisting(self, room_id):
+        @defer.inlineCallbacks
+        def persisting_queue(queue):
+            for item in queue:
+                try:
+                    ret = None
+                    if item.current_state:
+                        for event, context in item.events_and_contexts:
+                            # There should only ever be one item in
+                            # events_and_contexts when current_state is
+                            # not None
+                            yield self._persist_event(
+                                event, context,
+                                current_state=item.current_state,
+                                backfilled=item.backfilled,
+                            )
+                    else:
+                        yield self._persist_events(
+                            item.events_and_contexts,
+                            backfilled=item.backfilled,
+                        )
+                    logger.info("Resolving with ret: %r", ret)
+                    item.deferred.callback(ret)
+                except Exception as e:
+                    logger.exception("Failed to persist events")
+                    item.deferred.errback(e)
+
+        self._event_persist_queue.handle_queue(room_id, persisting_queue)
+
+    @defer.inlineCallbacks
+    def _persist_events(self, events_and_contexts, backfilled=False):
         if not events_and_contexts:
             return
 
@@ -118,8 +255,7 @@ class EventsStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     @log_function
-    def persist_event(self, event, context, current_state=None, backfilled=False):
-
+    def _persist_event(self, event, context, current_state=None, backfilled=False):
         try:
             with self._stream_id_gen.get_next() as stream_ordering:
                 with self._state_groups_id_gen.get_next() as state_group_id:
@@ -136,9 +272,6 @@ class EventsStore(SQLBaseStore):
         except _RollbackButIsFineException:
             pass
 
-        max_persisted_id = yield self._stream_id_gen.get_current_token()
-        defer.returnValue((stream_ordering, max_persisted_id))
-
     @defer.inlineCallbacks
     def get_event(self, event_id, check_redacted=True,
                   get_prev_content=False, allow_rejected=False,
-- 
cgit 1.5.1


From fd85b167ecb650451bf8bbfc68b771f607bb1d9d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 15:38:42 +0100
Subject: Pull loop one level up

---
 synapse/storage/events.py | 77 +++++++++++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 36 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 07e873fce4..9f8f0a0823 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -19,7 +19,7 @@ from twisted.internet import defer, reactor
 from synapse.events import FrozenEvent, USE_FROZEN_DICTS
 from synapse.events.utils import prune_event
 
-from synapse.util.async import ObservableDeferred
+from synapse.util.async import ObservableDeferred, run_on_reactor
 from synapse.util.logcontext import preserve_fn, PreserveLoggingContext
 from synapse.util.logutils import log_function
 from synapse.api.constants import EventTypes
@@ -89,12 +89,14 @@ class _EventPeristenceQueue(object):
 
         return deferred.observe()
 
-    def handle_queue(self, room_id, callback):
+    def handle_queue(self, room_id, per_item_callback):
         """Attempts to handle the queue for a room if not already being handled.
 
-        The given callback will be invoked with a 'queue' arg, which is a
-        generator over _EventPersistQueueItem's. The queue will finish if there
-        are no longer any items in the room queue.
+        The given callback will be invoked with for each item in the queue,1
+        of type _EventPersistQueueItem. The per_item_callback will continuously
+        be called with new items, unless the queue becomnes empty. The return
+        value of the function will be given to the deferreds waiting on the item,
+        exceptions will be passed to the deferres as well.
 
         This function should therefore be called whenever anything is added
         to the queue.
@@ -108,15 +110,26 @@ class _EventPeristenceQueue(object):
 
         self._currently_persisting_rooms.add(room_id)
 
-        try:
-            callback(self._get_drainining_queue(room_id))
-        finally:
-            self._currently_persisting_rooms.discard(room_id)
+        @defer.inlineCallbacks
+        def handle_queue_loop():
+            try:
+                queue = self._get_drainining_queue(room_id)
+                for item in queue:
+                    try:
+                        ret = yield per_item_callback(item)
+                        item.deferred.callback(ret)
+                    except Exception as e:
+                        item.deferred.errback(e)
+            finally:
+                queue = self._event_persist_queues.pop(room_id, None)
+                if queue:
+                    self._event_persist_queues[room_id] = queue
+                self._currently_persisting_rooms.discard(room_id)
+
+        preserve_fn(handle_queue_loop)()
 
     def _get_drainining_queue(self, room_id):
-        queue = self._event_persist_queues.pop(room_id, None)
-        if not queue:
-            return
+        queue = self._event_persist_queues.setdefault(room_id, deque())
 
         try:
             while True:
@@ -180,30 +193,22 @@ class EventsStore(SQLBaseStore):
 
     def _maybe_start_persisting(self, room_id):
         @defer.inlineCallbacks
-        def persisting_queue(queue):
-            for item in queue:
-                try:
-                    ret = None
-                    if item.current_state:
-                        for event, context in item.events_and_contexts:
-                            # There should only ever be one item in
-                            # events_and_contexts when current_state is
-                            # not None
-                            yield self._persist_event(
-                                event, context,
-                                current_state=item.current_state,
-                                backfilled=item.backfilled,
-                            )
-                    else:
-                        yield self._persist_events(
-                            item.events_and_contexts,
-                            backfilled=item.backfilled,
-                        )
-                    logger.info("Resolving with ret: %r", ret)
-                    item.deferred.callback(ret)
-                except Exception as e:
-                    logger.exception("Failed to persist events")
-                    item.deferred.errback(e)
+        def persisting_queue(item):
+            if item.current_state:
+                for event, context in item.events_and_contexts:
+                    # There should only ever be one item in
+                    # events_and_contexts when current_state is
+                    # not None
+                    yield self._persist_event(
+                        event, context,
+                        current_state=item.current_state,
+                        backfilled=item.backfilled,
+                    )
+            else:
+                yield self._persist_events(
+                    item.events_and_contexts,
+                    backfilled=item.backfilled,
+                )
 
         self._event_persist_queue.handle_queue(room_id, persisting_queue)
 
-- 
cgit 1.5.1


From fcb2c3f0db70593709c3bfbfc66772066c4aa061 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 May 2016 15:47:40 +0100
Subject: Remove unused import

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 9f8f0a0823..a27b7919cd 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -19,7 +19,7 @@ from twisted.internet import defer, reactor
 from synapse.events import FrozenEvent, USE_FROZEN_DICTS
 from synapse.events.utils import prune_event
 
-from synapse.util.async import ObservableDeferred, run_on_reactor
+from synapse.util.async import ObservableDeferred
 from synapse.util.logcontext import preserve_fn, PreserveLoggingContext
 from synapse.util.logutils import log_function
 from synapse.api.constants import EventTypes
-- 
cgit 1.5.1


From 4ea762c1a28edbf18d0a183ed35fb8a5a11847c5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 May 2016 10:08:21 +0100
Subject: Add cache to get_user_by_id

---
 synapse/storage/registration.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 7af0cae6a5..bda84a744a 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -101,6 +101,7 @@ class RegistrationStore(SQLBaseStore):
             make_guest,
             appservice_id
         )
+        self.get_user_by_id.invalidate((user_id,))
         self.is_guest.invalidate((user_id,))
 
     def _register(
@@ -156,6 +157,7 @@ class RegistrationStore(SQLBaseStore):
                 (next_id, user_id, token,)
             )
 
+    @cached()
     def get_user_by_id(self, user_id):
         return self._simple_select_one(
             table="users",
@@ -193,6 +195,7 @@ class RegistrationStore(SQLBaseStore):
         }, {
             'password_hash': password_hash
         })
+        self.get_user_by_id.invalidate((user_id,))
 
     @defer.inlineCallbacks
     def user_delete_access_tokens(self, user_id, except_token_ids=[]):
-- 
cgit 1.5.1


From 08dfa8eee258a43177b843e6d708c8c98357c6d7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 May 2016 10:36:03 +0100
Subject: Add and use get_domian_from_id

---
 synapse/api/auth.py            | 16 ++++++++--------
 synapse/handlers/_base.py      |  8 +++-----
 synapse/handlers/federation.py | 12 ++++--------
 synapse/handlers/presence.py   |  4 ++--
 synapse/storage/roommember.py  |  7 ++-----
 synapse/types.py               |  4 ++++
 6 files changed, 23 insertions(+), 28 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 9e912fdfbe..d3e9837c81 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -22,7 +22,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership, JoinRules
 from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError
-from synapse.types import Requester, RoomID, UserID, EventID
+from synapse.types import Requester, UserID, get_domian_from_id
 from synapse.util.logutils import log_function
 from synapse.util.logcontext import preserve_context_over_fn
 from synapse.util.metrics import Measure
@@ -91,8 +91,8 @@ class Auth(object):
                     "Room %r does not exist" % (event.room_id,)
                 )
 
-            creating_domain = RoomID.from_string(event.room_id).domain
-            originating_domain = UserID.from_string(event.sender).domain
+            creating_domain = get_domian_from_id(event.room_id)
+            originating_domain = get_domian_from_id(event.sender)
             if creating_domain != originating_domain:
                 if not self.can_federate(event, auth_events):
                     raise AuthError(
@@ -219,7 +219,7 @@ class Auth(object):
         for event in curr_state.values():
             if event.type == EventTypes.Member:
                 try:
-                    if UserID.from_string(event.state_key).domain != host:
+                    if get_domian_from_id(event.state_key) != host:
                         continue
                 except:
                     logger.warn("state_key not user_id: %s", event.state_key)
@@ -266,8 +266,8 @@ class Auth(object):
 
         target_user_id = event.state_key
 
-        creating_domain = RoomID.from_string(event.room_id).domain
-        target_domain = UserID.from_string(target_user_id).domain
+        creating_domain = get_domian_from_id(event.room_id)
+        target_domain = get_domian_from_id(target_user_id)
         if creating_domain != target_domain:
             if not self.can_federate(event, auth_events):
                 raise AuthError(
@@ -889,8 +889,8 @@ class Auth(object):
         if user_level >= redact_level:
             return False
 
-        redacter_domain = EventID.from_string(event.event_id).domain
-        redactee_domain = EventID.from_string(event.redacts).domain
+        redacter_domain = get_domian_from_id(event.event_id)
+        redactee_domain = get_domian_from_id(event.redacts)
         if redacter_domain == redactee_domain:
             return True
 
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 13a675b208..287024c1ca 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from synapse.api.errors import LimitExceededError, SynapseError, AuthError
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.api.constants import Membership, EventTypes
-from synapse.types import UserID, RoomAlias, Requester
+from synapse.types import UserID, RoomAlias, Requester, get_domian_from_id
 from synapse.push.action_generator import ActionGenerator
 
 from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
@@ -296,7 +296,7 @@ class BaseHandler(object):
                 return True
         for (state_key, membership) in room_members:
             if (
-                UserID.from_string(state_key).domain == self.hs.hostname
+                self.hs.is_mine_id(state_key)
                 and membership == Membership.JOIN
             ):
                 return True
@@ -421,9 +421,7 @@ class BaseHandler(object):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.JOIN:
-                        destinations.add(
-                            UserID.from_string(s.state_key).domain
-                        )
+                        destinations.add(get_domian_from_id(s.state_key))
             except SynapseError:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index d95e0b23b1..f38c6a8713 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -33,7 +33,7 @@ from synapse.util.frozenutils import unfreeze
 from synapse.crypto.event_signing import (
     compute_event_signature, add_hashes_and_signatures,
 )
-from synapse.types import UserID
+from synapse.types import UserID, get_domian_from_id
 
 from synapse.events.utils import prune_event
 
@@ -453,7 +453,7 @@ class FederationHandler(BaseHandler):
             joined_domains = {}
             for u, d in joined_users:
                 try:
-                    dom = UserID.from_string(u).domain
+                    dom = get_domian_from_id(u)
                     old_d = joined_domains.get(dom)
                     if old_d:
                         joined_domains[dom] = min(d, old_d)
@@ -743,9 +743,7 @@ class FederationHandler(BaseHandler):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.JOIN:
-                        destinations.add(
-                            UserID.from_string(s.state_key).domain
-                        )
+                        destinations.add(get_domian_from_id(s.state_key))
             except:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
@@ -970,9 +968,7 @@ class FederationHandler(BaseHandler):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.LEAVE:
-                        destinations.add(
-                            UserID.from_string(s.state_key).domain
-                        )
+                        destinations.add(get_domian_from_id(s.state_key))
             except:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 639567953a..a8529cce42 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -33,7 +33,7 @@ from synapse.util.logcontext import preserve_fn
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
-from synapse.types import UserID
+from synapse.types import UserID, get_domian_from_id
 import synapse.metrics
 
 from ._base import BaseHandler
@@ -440,7 +440,7 @@ class PresenceHandler(BaseHandler):
             if not local_states:
                 continue
 
-            host = UserID.from_string(user_id).domain
+            host = get_domian_from_id(user_id)
             hosts_to_states.setdefault(host, []).extend(local_states)
 
         # TODO: de-dup hosts_to_states, as a single host might have multiple
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 08a54cbdd1..9d6bfd5245 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -21,7 +21,7 @@ from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
 from synapse.api.constants import Membership
-from synapse.types import UserID
+from synapse.types import get_domian_from_id
 
 import logging
 
@@ -273,10 +273,7 @@ class RoomMemberStore(SQLBaseStore):
             room_id, membership=Membership.JOIN
         )
 
-        joined_domains = set(
-            UserID.from_string(r["user_id"]).domain
-            for r in rows
-        )
+        joined_domains = set(get_domian_from_id(r["user_id"]) for r in rows)
 
         return joined_domains
 
diff --git a/synapse/types.py b/synapse/types.py
index 5b166835bd..42fd9c7204 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -21,6 +21,10 @@ from collections import namedtuple
 Requester = namedtuple("Requester", ["user", "access_token_id", "is_guest"])
 
 
+def get_domian_from_id(string):
+    return string.split(":", 1)[1]
+
+
 class DomainSpecificString(
         namedtuple("DomainSpecificString", ("localpart", "domain"))
 ):
-- 
cgit 1.5.1


From 3b1930e8ecd571b6839ebb90c636dc63539d34a1 Mon Sep 17 00:00:00 2001
From: Matrix <matrix@matrix.org>
Date: Tue, 10 May 2016 16:42:37 +0100
Subject: unbreak schema

---
 synapse/storage/schema/delta/31/events.sql         | 16 ---------------
 .../storage/schema/delta/31/pusher_throttle.sql    | 23 ----------------------
 synapse/storage/schema/delta/32/events.sql         | 16 +++++++++++++++
 .../storage/schema/delta/32/pusher_throttle.sql    | 23 ++++++++++++++++++++++
 4 files changed, 39 insertions(+), 39 deletions(-)
 delete mode 100644 synapse/storage/schema/delta/31/events.sql
 delete mode 100644 synapse/storage/schema/delta/31/pusher_throttle.sql
 create mode 100644 synapse/storage/schema/delta/32/events.sql
 create mode 100644 synapse/storage/schema/delta/32/pusher_throttle.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/31/events.sql b/synapse/storage/schema/delta/31/events.sql
deleted file mode 100644
index 1dd0f9e170..0000000000
--- a/synapse/storage/schema/delta/31/events.sql
+++ /dev/null
@@ -1,16 +0,0 @@
-/* Copyright 2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-ALTER TABLE events ADD COLUMN received_ts BIGINT;
diff --git a/synapse/storage/schema/delta/31/pusher_throttle.sql b/synapse/storage/schema/delta/31/pusher_throttle.sql
deleted file mode 100644
index d86d30c13c..0000000000
--- a/synapse/storage/schema/delta/31/pusher_throttle.sql
+++ /dev/null
@@ -1,23 +0,0 @@
-/* Copyright 2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-CREATE TABLE pusher_throttle(
-    pusher BIGINT NOT NULL,
-    room_id TEXT NOT NULL,
-    last_sent_ts BIGINT,
-    throttle_ms BIGINT,
-    PRIMARY KEY (pusher, room_id)
-);
diff --git a/synapse/storage/schema/delta/32/events.sql b/synapse/storage/schema/delta/32/events.sql
new file mode 100644
index 0000000000..1dd0f9e170
--- /dev/null
+++ b/synapse/storage/schema/delta/32/events.sql
@@ -0,0 +1,16 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE events ADD COLUMN received_ts BIGINT;
diff --git a/synapse/storage/schema/delta/32/pusher_throttle.sql b/synapse/storage/schema/delta/32/pusher_throttle.sql
new file mode 100644
index 0000000000..d86d30c13c
--- /dev/null
+++ b/synapse/storage/schema/delta/32/pusher_throttle.sql
@@ -0,0 +1,23 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE TABLE pusher_throttle(
+    pusher BIGINT NOT NULL,
+    room_id TEXT NOT NULL,
+    last_sent_ts BIGINT,
+    throttle_ms BIGINT,
+    PRIMARY KEY (pusher, room_id)
+);
-- 
cgit 1.5.1


From 13d37c3c568ac7bfea2e03eff373189c57c0f3fd Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 13 May 2016 11:25:02 +0100
Subject: Fixup add_pusher

---
 synapse/storage/_base.py  |  8 +++++++-
 synapse/storage/pusher.py | 13 ++++++++++---
 2 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 1e27c2c0ce..258b251141 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -453,7 +453,9 @@ class SQLBaseStore(object):
             keyvalues (dict): The unique key tables and their new values
             values (dict): The nonunique columns and their new values
             insertion_values (dict): key/values to use when inserting
-        Returns: A deferred
+        Returns:
+            Deferred(bool): True if a new entry was created, False if an
+                exisitng one was updated.
         """
         return self.runInteraction(
             desc,
@@ -498,6 +500,10 @@ class SQLBaseStore(object):
             )
             txn.execute(sql, allvalues.values())
 
+            return True
+        else:
+            return False
+
     def _simple_select_one(self, table, keyvalues, retcols,
                            allow_none=False, desc="_simple_select_one"):
         """Executes a SELECT query on the named table, which is expected to
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index d9afd7ec87..9e8e2e2964 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -156,8 +156,7 @@ class PusherStore(SQLBaseStore):
                    profile_tag=""):
         with self._pushers_id_gen.get_next() as stream_id:
             def f(txn):
-                txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
-                return self._simple_upsert_txn(
+                newly_inserted = self._simple_upsert_txn(
                     txn,
                     "pushers",
                     {
@@ -178,11 +177,18 @@ class PusherStore(SQLBaseStore):
                         "id": stream_id,
                     },
                 )
-        defer.returnValue((yield self.runInteraction("add_pusher", f)))
+                if newly_inserted:
+                    # get_users_with_pushers_in_room only cares if the user has
+                    # at least *one* pusher.
+                    txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
+
+            yield self.runInteraction("add_pusher", f)
 
     @defer.inlineCallbacks
     def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id):
         def delete_pusher_txn(txn, stream_id):
+            txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
+
             self._simple_delete_one_txn(
                 txn,
                 "pushers",
@@ -194,6 +200,7 @@ class PusherStore(SQLBaseStore):
                 {"app_id": app_id, "pushkey": pushkey, "user_id": user_id},
                 {"stream_id": stream_id},
             )
+
         with self._pushers_id_gen.get_next() as stream_id:
             yield self.runInteraction(
                 "delete_pusher", delete_pusher_txn, stream_id
-- 
cgit 1.5.1


From 6da7f39d952f7c8bb805d8b863c737d98b240217 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 13 May 2016 11:41:23 +0100
Subject: Use tree cache for get_linearized_receipts_for_room

---
 synapse/storage/receipts.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 935fc503d9..669fc8ada2 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -100,7 +100,7 @@ class ReceiptsStore(SQLBaseStore):
 
         defer.returnValue([ev for res in results.values() for ev in res])
 
-    @cachedInlineCallbacks(num_args=3, max_entries=5000)
+    @cachedInlineCallbacks(num_args=3, max_entries=5000, lru=True, tree=True)
     def get_linearized_receipts_for_room(self, room_id, to_key, from_key=None):
         """Get receipts for a single room for sending to clients.
 
@@ -232,7 +232,7 @@ class ReceiptsStore(SQLBaseStore):
             self.get_receipts_for_user.invalidate, (user_id, receipt_type)
         )
         # FIXME: This shouldn't invalidate the whole cache
-        txn.call_after(self.get_linearized_receipts_for_room.invalidate_all)
+        txn.call_after(self.get_linearized_receipts_for_room.invalidate_many, (room_id,))
 
         txn.call_after(
             self._receipts_stream_cache.entity_has_changed,
@@ -367,7 +367,7 @@ class ReceiptsStore(SQLBaseStore):
             self.get_receipts_for_user.invalidate, (user_id, receipt_type)
         )
         # FIXME: This shouldn't invalidate the whole cache
-        txn.call_after(self.get_linearized_receipts_for_room.invalidate_all)
+        txn.call_after(self.get_linearized_receipts_for_room.invalidate_many, (room_id,))
 
         self._simple_delete_txn(
             txn,
-- 
cgit 1.5.1


From 5e500584738d4e9ce897e92056e20c7909052601 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 13 May 2016 13:28:07 +0100
Subject: Remove unused indices

This includes removing both unused indices and indices that are subsets
of other indices.
---
 synapse/storage/schema/delta/32/remove_indices.sql | 42 ++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 synapse/storage/schema/delta/32/remove_indices.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql
new file mode 100644
index 0000000000..314fa51287
--- /dev/null
+++ b/synapse/storage/schema/delta/32/remove_indices.sql
@@ -0,0 +1,42 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- The following indices are redundant, other indices are equivalent or
+-- supersets
+DROP INDEX IF EXISTS events_room_id;
+DROP INDEX IF EXISTS events_order;
+DROP INDEX IF EXISTS events_topological_ordering;
+DROP INDEX IF EXISTS events_stream_ordering;
+DROP INDEX IF EXISTS state_groups_id;
+DROP INDEX IF EXISTS event_to_state_groups_id;
+DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag;
+DROP INDEX IF EXISTS event_push_actions_room_id_user_id;
+
+DROP INDEX IF EXISTS event_destinations_id;
+DROP INDEX IF EXISTS st_extrem_id;
+DROP INDEX IF EXISTS event_content_hashes_id;
+DROP INDEX IF EXISTS event_signatures_id;
+DROP INDEX IF EXISTS event_edge_hashes_id;
+DROP INDEX IF EXISTS redactions_event_id;
+DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
+DROP INDEX IF EXISTS room_hosts_room_id;
+DROP INDEX IF EXISTS event_search_ev_ridx;
+
+
+-- The following indices were unused
+DROP INDEX IF EXISTS evauth_edges_auth_id;
+DROP INDEX IF EXISTS topics_room_id;
+DROP INDEX IF EXISTS presence_stream_state;
-- 
cgit 1.5.1


From 9295fa30a83c5ecdb6315b1cc39910af3284d6be Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 13 May 2016 14:16:57 +0100
Subject: Annotate the removed indicies with why they were removed.

---
 synapse/storage/schema/delta/32/remove_indices.sql | 33 ++++++++++------------
 1 file changed, 15 insertions(+), 18 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql
index 314fa51287..91eab6d63c 100644
--- a/synapse/storage/schema/delta/32/remove_indices.sql
+++ b/synapse/storage/schema/delta/32/remove_indices.sql
@@ -16,27 +16,24 @@
 
 -- The following indices are redundant, other indices are equivalent or
 -- supersets
-DROP INDEX IF EXISTS events_room_id;
-DROP INDEX IF EXISTS events_order;
-DROP INDEX IF EXISTS events_topological_ordering;
-DROP INDEX IF EXISTS events_stream_ordering;
-DROP INDEX IF EXISTS state_groups_id;
-DROP INDEX IF EXISTS event_to_state_groups_id;
-DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag;
-DROP INDEX IF EXISTS event_push_actions_room_id_user_id;
-
-DROP INDEX IF EXISTS event_destinations_id;
-DROP INDEX IF EXISTS st_extrem_id;
-DROP INDEX IF EXISTS event_content_hashes_id;
-DROP INDEX IF EXISTS event_signatures_id;
-DROP INDEX IF EXISTS event_edge_hashes_id;
-DROP INDEX IF EXISTS redactions_event_id;
-DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
-DROP INDEX IF EXISTS room_hosts_room_id;
-DROP INDEX IF EXISTS event_search_ev_ridx;
+DROP INDEX IF EXISTS events_room_id; -- Prefix of events_room_stream
+DROP INDEX IF EXISTS events_order; -- Prefix of events_order_topo_stream_room
+DROP INDEX IF EXISTS events_topological_ordering; -- Prefix of events_order_topo_stream_room
+DROP INDEX IF EXISTS events_stream_ordering; -- Duplicate of PRIMARY KEY
+DROP INDEX IF EXISTS state_groups_id; -- Duplicate of PRIMARY KEY
+DROP INDEX IF EXISTS event_to_state_groups_id; -- Duplicate of PRIMARY KEY
+DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -- Duplicate of UNIQUE CONSTRAINT
 
+DROP INDEX IF EXISTS event_destinations_id; -- Prefix of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS st_extrem_id; -- Prefix of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS event_content_hashes_id; -- Prefix of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS event_signatures_id; -- Prefix of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS event_edge_hashes_id; -- Prefix of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS redactions_event_id; -- Duplicate of UNIQUE CONSTRAINT
+DROP INDEX IF EXISTS room_hosts_room_id; -- Prefix of UNIQUE CONSTRAINT
 
 -- The following indices were unused
+DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
 DROP INDEX IF EXISTS evauth_edges_auth_id;
 DROP INDEX IF EXISTS topics_room_id;
 DROP INDEX IF EXISTS presence_stream_state;
-- 
cgit 1.5.1


From c9aff0736c75046b8923057309af91d1bd7a6985 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 13 May 2016 14:40:38 +0100
Subject: Remove topics table

---
 synapse/storage/schema/delta/32/remove_indices.sql | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql
index 91eab6d63c..f859be46a6 100644
--- a/synapse/storage/schema/delta/32/remove_indices.sql
+++ b/synapse/storage/schema/delta/32/remove_indices.sql
@@ -35,5 +35,4 @@ DROP INDEX IF EXISTS room_hosts_room_id; -- Prefix of UNIQUE CONSTRAINT
 -- The following indices were unused
 DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
 DROP INDEX IF EXISTS evauth_edges_auth_id;
-DROP INDEX IF EXISTS topics_room_id;
 DROP INDEX IF EXISTS presence_stream_state;
-- 
cgit 1.5.1


From 0c11c1be884d69bf0d45b6d1f55a71861e16ee8f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 13 May 2016 14:42:25 +0100
Subject: Spelling

---
 synapse/storage/_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 258b251141..e0d7098692 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -455,7 +455,7 @@ class SQLBaseStore(object):
             insertion_values (dict): key/values to use when inserting
         Returns:
             Deferred(bool): True if a new entry was created, False if an
-                exisitng one was updated.
+                existing one was updated.
         """
         return self.runInteraction(
             desc,
-- 
cgit 1.5.1


From b7381d5338e37b8c4c374f5458e578e05eb762d0 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 13 May 2016 15:46:41 +0100
Subject: Allow receipts for events we haven't seen in the db

---
 synapse/storage/receipts.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 94be820f86..fdcf28f3e1 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -249,9 +249,11 @@ class ReceiptsStore(SQLBaseStore):
             table="events",
             retcols=["topological_ordering", "stream_ordering"],
             keyvalues={"event_id": event_id},
+            allow_none=True
         )
-        topological_ordering = int(res["topological_ordering"])
-        stream_ordering = int(res["stream_ordering"])
+
+        topological_ordering = int(res["topological_ordering"]) if res else None
+        stream_ordering = int(res["stream_ordering"]) if res else None
 
         # We don't want to clobber receipts for more recent events, so we
         # have to compare orderings of existing receipts
@@ -264,7 +266,7 @@ class ReceiptsStore(SQLBaseStore):
         txn.execute(sql, (room_id, receipt_type, user_id))
         results = txn.fetchall()
 
-        if results:
+        if results and topological_ordering:
             for to, so, _ in results:
                 if int(to) > topological_ordering:
                     return False
@@ -294,7 +296,7 @@ class ReceiptsStore(SQLBaseStore):
             }
         )
 
-        if receipt_type == "m.read":
+        if receipt_type == "m.read" and topological_ordering:
             self._remove_push_actions_before_txn(
                 txn,
                 room_id=room_id,
-- 
cgit 1.5.1


From 1a3a2002ffc2569ec2afbbdfb110f94ce2728710 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 16 May 2016 19:17:03 +0100
Subject: Spell "domain" correctly

s/domian/domain/g
---
 synapse/api/auth.py            | 16 ++++++++--------
 synapse/handlers/federation.py |  8 ++++----
 synapse/handlers/message.py    |  4 ++--
 synapse/handlers/presence.py   |  4 ++--
 synapse/storage/roommember.py  |  4 ++--
 synapse/types.py               |  2 +-
 6 files changed, 19 insertions(+), 19 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 44e38b777a..2474a1453b 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -22,7 +22,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership, JoinRules
 from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError
-from synapse.types import Requester, UserID, get_domian_from_id
+from synapse.types import Requester, UserID, get_domain_from_id
 from synapse.util.logutils import log_function
 from synapse.util.logcontext import preserve_context_over_fn
 from synapse.util.metrics import Measure
@@ -91,8 +91,8 @@ class Auth(object):
                     "Room %r does not exist" % (event.room_id,)
                 )
 
-            creating_domain = get_domian_from_id(event.room_id)
-            originating_domain = get_domian_from_id(event.sender)
+            creating_domain = get_domain_from_id(event.room_id)
+            originating_domain = get_domain_from_id(event.sender)
             if creating_domain != originating_domain:
                 if not self.can_federate(event, auth_events):
                     raise AuthError(
@@ -219,7 +219,7 @@ class Auth(object):
         for event in curr_state.values():
             if event.type == EventTypes.Member:
                 try:
-                    if get_domian_from_id(event.state_key) != host:
+                    if get_domain_from_id(event.state_key) != host:
                         continue
                 except:
                     logger.warn("state_key not user_id: %s", event.state_key)
@@ -266,8 +266,8 @@ class Auth(object):
 
         target_user_id = event.state_key
 
-        creating_domain = get_domian_from_id(event.room_id)
-        target_domain = get_domian_from_id(target_user_id)
+        creating_domain = get_domain_from_id(event.room_id)
+        target_domain = get_domain_from_id(target_user_id)
         if creating_domain != target_domain:
             if not self.can_federate(event, auth_events):
                 raise AuthError(
@@ -890,8 +890,8 @@ class Auth(object):
         if user_level >= redact_level:
             return False
 
-        redacter_domain = get_domian_from_id(event.event_id)
-        redactee_domain = get_domian_from_id(event.redacts)
+        redacter_domain = get_domain_from_id(event.event_id)
+        redactee_domain = get_domain_from_id(event.redacts)
         if redacter_domain == redactee_domain:
             return True
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index c21d9d4d83..648a505e65 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -33,7 +33,7 @@ from synapse.util.frozenutils import unfreeze
 from synapse.crypto.event_signing import (
     compute_event_signature, add_hashes_and_signatures,
 )
-from synapse.types import UserID, get_domian_from_id
+from synapse.types import UserID, get_domain_from_id
 
 from synapse.events.utils import prune_event
 
@@ -453,7 +453,7 @@ class FederationHandler(BaseHandler):
             joined_domains = {}
             for u, d in joined_users:
                 try:
-                    dom = get_domian_from_id(u)
+                    dom = get_domain_from_id(u)
                     old_d = joined_domains.get(dom)
                     if old_d:
                         joined_domains[dom] = min(d, old_d)
@@ -744,7 +744,7 @@ class FederationHandler(BaseHandler):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.JOIN:
-                        destinations.add(get_domian_from_id(s.state_key))
+                        destinations.add(get_domain_from_id(s.state_key))
             except:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
@@ -970,7 +970,7 @@ class FederationHandler(BaseHandler):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.LEAVE:
-                        destinations.add(get_domian_from_id(s.state_key))
+                        destinations.add(get_domain_from_id(s.state_key))
             except:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 13154edb78..9386aeeb1c 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -23,7 +23,7 @@ from synapse.events.validator import EventValidator
 from synapse.push.action_generator import ActionGenerator
 from synapse.streams.config import PaginationConfig
 from synapse.types import (
-    UserID, RoomAlias, RoomStreamToken, StreamToken, get_domian_from_id
+    UserID, RoomAlias, RoomStreamToken, StreamToken, get_domain_from_id
 )
 from synapse.util import unwrapFirstError
 from synapse.util.async import concurrently_execute
@@ -902,7 +902,7 @@ class MessageHandler(BaseHandler):
             try:
                 if k[0] == EventTypes.Member:
                     if s.content["membership"] == Membership.JOIN:
-                        destinations.add(get_domian_from_id(s.state_key))
+                        destinations.add(get_domain_from_id(s.state_key))
             except SynapseError:
                 logger.warn(
                     "Failed to get destination from event %s", s.event_id
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index a8529cce42..72a0b272db 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -33,7 +33,7 @@ from synapse.util.logcontext import preserve_fn
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
-from synapse.types import UserID, get_domian_from_id
+from synapse.types import UserID, get_domain_from_id
 import synapse.metrics
 
 from ._base import BaseHandler
@@ -440,7 +440,7 @@ class PresenceHandler(BaseHandler):
             if not local_states:
                 continue
 
-            host = get_domian_from_id(user_id)
+            host = get_domain_from_id(user_id)
             hosts_to_states.setdefault(host, []).extend(local_states)
 
         # TODO: de-dup hosts_to_states, as a single host might have multiple
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 9d6bfd5245..223258785d 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -21,7 +21,7 @@ from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
 from synapse.api.constants import Membership
-from synapse.types import get_domian_from_id
+from synapse.types import get_domain_from_id
 
 import logging
 
@@ -273,7 +273,7 @@ class RoomMemberStore(SQLBaseStore):
             room_id, membership=Membership.JOIN
         )
 
-        joined_domains = set(get_domian_from_id(r["user_id"]) for r in rows)
+        joined_domains = set(get_domain_from_id(r["user_id"]) for r in rows)
 
         return joined_domains
 
diff --git a/synapse/types.py b/synapse/types.py
index 42fd9c7204..7b6ae44bdd 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -21,7 +21,7 @@ from collections import namedtuple
 Requester = namedtuple("Requester", ["user", "access_token_id", "is_guest"])
 
 
-def get_domian_from_id(string):
+def get_domain_from_id(string):
     return string.split(":", 1)[1]
 
 
-- 
cgit 1.5.1


From 816df9f2678d35c33922f4dcdc343a848b16f082 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 16 May 2016 19:51:43 +0100
Subject: get_room_members is unused now

---
 synapse/storage/roommember.py    | 18 ------------------
 tests/storage/test_roommember.py | 18 ------------------
 2 files changed, 36 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 223258785d..face685ed2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -137,24 +137,6 @@ class RoomMemberStore(SQLBaseStore):
             return [r["user_id"] for r in rows]
         return self.runInteraction("get_users_in_room", f)
 
-    def get_room_members(self, room_id, membership=None):
-        """Retrieve the current room member list for a room.
-
-        Args:
-            room_id (str): The room to get the list of members.
-            membership (synapse.api.constants.Membership): The filter to apply
-            to this list, or None to return all members with some state
-            associated with this room.
-        Returns:
-            list of namedtuples representing the members in this room.
-        """
-        return self.runInteraction(
-            "get_room_members",
-            self._get_members_events_txn,
-            room_id,
-            membership=membership,
-        ).addCallback(self._get_events)
-
     @cached()
     def get_invited_rooms_for_user(self, user_id):
         """ Get all the rooms the user is invited to
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 997090fe35..27b2b3d123 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -70,12 +70,6 @@ class RoomMemberStoreTestCase(unittest.TestCase):
     def test_one_member(self):
         yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)
 
-        self.assertEquals(
-            [self.u_alice.to_string()],
-            [m.user_id for m in (
-                yield self.store.get_room_members(self.room.to_string())
-            )]
-        )
         self.assertEquals(
             [self.room.to_string()],
             [m.room_id for m in (
@@ -85,18 +79,6 @@ class RoomMemberStoreTestCase(unittest.TestCase):
             )]
         )
 
-    @defer.inlineCallbacks
-    def test_two_members(self):
-        yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)
-        yield self.inject_room_member(self.room, self.u_bob, Membership.JOIN)
-
-        self.assertEquals(
-            {self.u_alice.to_string(), self.u_bob.to_string()},
-            {m.user_id for m in (
-                yield self.store.get_room_members(self.room.to_string())
-            )}
-        )
-
     @defer.inlineCallbacks
     def test_room_hosts(self):
         yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)
-- 
cgit 1.5.1


From 6a30a0bfd384f6ea99af8412508ba9fb375ae941 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 17 May 2016 11:28:58 +0100
Subject: Move the functions for parsing app service config

---
 synapse/config/appservice.py  | 107 +++++++++++++++++++++++++++++++++++++++++-
 synapse/storage/appservice.py | 105 ++---------------------------------------
 2 files changed, 109 insertions(+), 103 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 3bed542c4f..eade803909 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -12,7 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
+from ._base import Config, ConfigError
+
+from synapse.appservice import ApplicationService
+from synapse.types import UserID
+
+import urllib
+import yaml
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class AppServiceConfig(Config):
@@ -25,3 +34,99 @@ class AppServiceConfig(Config):
         # A list of application service config file to use
         app_service_config_files: []
         """
+
+
+def load_appservices(hostname, config_files):
+    """Returns a list of Application Services from the config files."""
+    if not isinstance(config_files, list):
+        logger.warning(
+            "Expected %s to be a list of AS config files.", config_files
+        )
+        return []
+
+    # Dicts of value -> filename
+    seen_as_tokens = {}
+    seen_ids = {}
+
+    appservices = []
+
+    for config_file in config_files:
+        try:
+            with open(config_file, 'r') as f:
+                appservice = _load_appservice(
+                    hostname, yaml.load(f), config_file
+                )
+                if appservice.id in seen_ids:
+                    raise ConfigError(
+                        "Cannot reuse ID across application services: "
+                        "%s (files: %s, %s)" % (
+                            appservice.id, config_file, seen_ids[appservice.id],
+                        )
+                    )
+                seen_ids[appservice.id] = config_file
+                if appservice.token in seen_as_tokens:
+                    raise ConfigError(
+                        "Cannot reuse as_token across application services: "
+                        "%s (files: %s, %s)" % (
+                            appservice.token,
+                            config_file,
+                            seen_as_tokens[appservice.token],
+                        )
+                    )
+                seen_as_tokens[appservice.token] = config_file
+                logger.info("Loaded application service: %s", appservice)
+                appservices.append(appservice)
+        except Exception as e:
+            logger.error("Failed to load appservice from '%s'", config_file)
+            logger.exception(e)
+            raise
+    return appservices
+
+
+def _load_appservice(hostname, as_info, config_filename):
+    required_string_fields = [
+        "id", "url", "as_token", "hs_token", "sender_localpart"
+    ]
+    for field in required_string_fields:
+        if not isinstance(as_info.get(field), basestring):
+            raise KeyError("Required string field: '%s' (%s)" % (
+                field, config_filename,
+            ))
+
+    localpart = as_info["sender_localpart"]
+    if urllib.quote(localpart) != localpart:
+        raise ValueError(
+            "sender_localpart needs characters which are not URL encoded."
+        )
+    user = UserID(localpart, hostname)
+    user_id = user.to_string()
+
+    # namespace checks
+    if not isinstance(as_info.get("namespaces"), dict):
+        raise KeyError("Requires 'namespaces' object.")
+    for ns in ApplicationService.NS_LIST:
+        # specific namespaces are optional
+        if ns in as_info["namespaces"]:
+            # expect a list of dicts with exclusive and regex keys
+            for regex_obj in as_info["namespaces"][ns]:
+                if not isinstance(regex_obj, dict):
+                    raise ValueError(
+                        "Expected namespace entry in %s to be an object,"
+                        " but got %s", ns, regex_obj
+                    )
+                if not isinstance(regex_obj.get("regex"), basestring):
+                    raise ValueError(
+                        "Missing/bad type 'regex' key in %s", regex_obj
+                    )
+                if not isinstance(regex_obj.get("exclusive"), bool):
+                    raise ValueError(
+                        "Missing/bad type 'exclusive' key in %s", regex_obj
+                    )
+    return ApplicationService(
+        token=as_info["as_token"],
+        url=as_info["url"],
+        namespaces=as_info["namespaces"],
+        hs_token=as_info["hs_token"],
+        sender=user_id,
+        id=as_info["id"],
+    )
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 371600eebb..feb9d228ae 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -13,16 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import urllib
-import yaml
 import simplejson as json
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
-from synapse.appservice import ApplicationService, AppServiceTransaction
-from synapse.config._base import ConfigError
+from synapse.appservice import AppServiceTransaction
+from synapse.config.appservice import load_appservices
 from synapse.storage.roommember import RoomsForUser
-from synapse.types import UserID
 from ._base import SQLBaseStore
 
 
@@ -34,7 +31,7 @@ class ApplicationServiceStore(SQLBaseStore):
     def __init__(self, hs):
         super(ApplicationServiceStore, self).__init__(hs)
         self.hostname = hs.hostname
-        self.services_cache = ApplicationServiceStore.load_appservices(
+        self.services_cache = load_appservices(
             hs.hostname,
             hs.config.app_service_config_files
         )
@@ -144,102 +141,6 @@ class ApplicationServiceStore(SQLBaseStore):
 
         return rooms_for_user_matching_user_id
 
-    @classmethod
-    def _load_appservice(cls, hostname, as_info, config_filename):
-        required_string_fields = [
-            "id", "url", "as_token", "hs_token", "sender_localpart"
-        ]
-        for field in required_string_fields:
-            if not isinstance(as_info.get(field), basestring):
-                raise KeyError("Required string field: '%s' (%s)" % (
-                    field, config_filename,
-                ))
-
-        localpart = as_info["sender_localpart"]
-        if urllib.quote(localpart) != localpart:
-            raise ValueError(
-                "sender_localpart needs characters which are not URL encoded."
-            )
-        user = UserID(localpart, hostname)
-        user_id = user.to_string()
-
-        # namespace checks
-        if not isinstance(as_info.get("namespaces"), dict):
-            raise KeyError("Requires 'namespaces' object.")
-        for ns in ApplicationService.NS_LIST:
-            # specific namespaces are optional
-            if ns in as_info["namespaces"]:
-                # expect a list of dicts with exclusive and regex keys
-                for regex_obj in as_info["namespaces"][ns]:
-                    if not isinstance(regex_obj, dict):
-                        raise ValueError(
-                            "Expected namespace entry in %s to be an object,"
-                            " but got %s", ns, regex_obj
-                        )
-                    if not isinstance(regex_obj.get("regex"), basestring):
-                        raise ValueError(
-                            "Missing/bad type 'regex' key in %s", regex_obj
-                        )
-                    if not isinstance(regex_obj.get("exclusive"), bool):
-                        raise ValueError(
-                            "Missing/bad type 'exclusive' key in %s", regex_obj
-                        )
-        return ApplicationService(
-            token=as_info["as_token"],
-            url=as_info["url"],
-            namespaces=as_info["namespaces"],
-            hs_token=as_info["hs_token"],
-            sender=user_id,
-            id=as_info["id"],
-        )
-
-    @classmethod
-    def load_appservices(cls, hostname, config_files):
-        """Returns a list of Application Services from the config files."""
-        if not isinstance(config_files, list):
-            logger.warning(
-                "Expected %s to be a list of AS config files.", config_files
-            )
-            return []
-
-        # Dicts of value -> filename
-        seen_as_tokens = {}
-        seen_ids = {}
-
-        appservices = []
-
-        for config_file in config_files:
-            try:
-                with open(config_file, 'r') as f:
-                    appservice = ApplicationServiceStore._load_appservice(
-                        hostname, yaml.load(f), config_file
-                    )
-                    if appservice.id in seen_ids:
-                        raise ConfigError(
-                            "Cannot reuse ID across application services: "
-                            "%s (files: %s, %s)" % (
-                                appservice.id, config_file, seen_ids[appservice.id],
-                            )
-                        )
-                    seen_ids[appservice.id] = config_file
-                    if appservice.token in seen_as_tokens:
-                        raise ConfigError(
-                            "Cannot reuse as_token across application services: "
-                            "%s (files: %s, %s)" % (
-                                appservice.token,
-                                config_file,
-                                seen_as_tokens[appservice.token],
-                            )
-                        )
-                    seen_as_tokens[appservice.token] = config_file
-                    logger.info("Loaded application service: %s", appservice)
-                    appservices.append(appservice)
-            except Exception as e:
-                logger.error("Failed to load appservice from '%s'", config_file)
-                logger.exception(e)
-                raise
-        return appservices
-
 
 class ApplicationServiceTransactionStore(SQLBaseStore):
 
-- 
cgit 1.5.1


From 42368ea8db5a2e6e15679cfc17b3c90075ea66ab Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 May 2016 11:38:00 +0100
Subject: Add desc to get_presence_for_users

---
 synapse/storage/presence.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py
index 07f5fae8dd..3fab57a7e8 100644
--- a/synapse/storage/presence.py
+++ b/synapse/storage/presence.py
@@ -149,6 +149,7 @@ class PresenceStore(SQLBaseStore):
                 "status_msg",
                 "currently_active",
             ),
+            desc="get_presence_for_users",
         )
 
         for row in rows:
-- 
cgit 1.5.1


From 149fa411e24a30b738094e40f02f18c2b9230cbe Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 20 May 2016 15:25:12 +0100
Subject: Only delete push actions after 30 days

---
 synapse/storage/event_push_actions.py | 42 ++++++++++++++++++++++++++++++-----
 synapse/storage/receipts.py           |  2 +-
 2 files changed, 38 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 9705db5c47..336c03c68a 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -22,6 +22,8 @@ import ujson as json
 
 logger = logging.getLogger(__name__)
 
+KEEP_PUSH_ACTIONS_FOR_MS = 30 * 24 * 60 * 60 * 1000
+
 
 class EventPushActionsStore(SQLBaseStore):
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):
@@ -224,16 +226,46 @@ class EventPushActionsStore(SQLBaseStore):
             (room_id, event_id)
         )
 
-    def _remove_push_actions_before_txn(self, txn, room_id, user_id,
-                                        topological_ordering):
+    def _remove_old_push_actions_before_txn(self, txn, room_id, user_id,
+                                            topological_ordering):
+        """
+        Purges old, stale push actions for a user and room before a given
+        topological_ordering
+        Args:
+            txn: The transcation
+            room_id: Room ID to delete from
+            user_id: user ID to delete for
+            topological_ordering: The lowest topological ordering which will
+                                  not be deleted.
+
+        Returns:
+
+        """
         txn.call_after(
             self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
             (room_id, user_id, )
         )
+
+        threshold = self._clock.time_msec() - KEEP_PUSH_ACTIONS_FOR_MS
+
+        # We need to join on the events table to get the received_ts for
+        # event_push_actions and sqlite won't let us use a join in a delete so
+        # we can't just delete where received_ts < x. Furthermore we can
+        # only identify event_push_actions by a tuple of room_id, event_id
+        # we we can't use a subquery.
+        # Instead, we look up the stream ordering for the last event in that
+        # room received before the threshold time and delete event_push_actions
+        # in the room with a stream_odering before that.
         txn.execute(
-            "DELETE FROM event_push_actions"
-            " WHERE room_id = ? AND user_id = ? AND topological_ordering < ?",
-            (room_id, user_id, topological_ordering,)
+            "DELETE FROM event_push_actions "
+            " WHERE user_id = ? AND room_id = ? AND "
+            " topological_ordering < ? AND stream_ordering < ("
+            "  SELECT stream_ordering FROM events"
+            "  WHERE room_id = ? AND received_ts < ?"
+            "  ORDER BY stream_ordering DESC"
+            "  LIMIT 1"
+            ")",
+            (user_id, room_id, topological_ordering, room_id, threshold)
         )
 
 
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index fdcf28f3e1..f1774f0e44 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -297,7 +297,7 @@ class ReceiptsStore(SQLBaseStore):
         )
 
         if receipt_type == "m.read" and topological_ordering:
-            self._remove_push_actions_before_txn(
+            self._remove_old_push_actions_before_txn(
                 txn,
                 room_id=room_id,
                 user_id=user_id,
-- 
cgit 1.5.1


From d4503e25ed01b6053bd5bb503f858a2ab934e350 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 20 May 2016 17:56:10 +0100
Subject: Make deleting push actions more efficient

There's no index on received_ts, so manually binary search using the stream_ordering index, and only update it once an hour.
---
 synapse/storage/__init__.py           |  9 +++++
 synapse/storage/_base.py              |  1 -
 synapse/storage/event_push_actions.py | 71 +++++++++++++++++++++++++++++------
 3 files changed, 68 insertions(+), 13 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index d970fde9e8..49feb77779 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -88,6 +88,7 @@ class DataStore(RoomMemberStore, RoomStore,
 
     def __init__(self, db_conn, hs):
         self.hs = hs
+        self._clock = hs.get_clock()
         self.database_engine = hs.database_engine
 
         self.client_ip_last_seen = Cache(
@@ -173,6 +174,14 @@ class DataStore(RoomMemberStore, RoomStore,
             prefilled_cache=push_rules_prefill,
         )
 
+        cur = db_conn.cursor()
+        self._find_stream_orderings_for_times_txn(cur)
+        cur.close()
+
+        self.find_stream_orderings_looping_call = self._clock.looping_call(
+            self._find_stream_orderings_for_times, 60 * 60 * 1000
+        )
+
         super(DataStore, self).__init__(hs)
 
     def take_presence_startup_info(self):
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index e0d7098692..56a0dd80f3 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -153,7 +153,6 @@ class SQLBaseStore(object):
     def __init__(self, hs):
         self.hs = hs
         self._db_pool = hs.get_db_pool()
-        self._clock = hs.get_clock()
 
         self._previous_txn_total_time = 0
         self._current_txn_total_time = 0
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 336c03c68a..4425d4bce5 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -22,10 +22,12 @@ import ujson as json
 
 logger = logging.getLogger(__name__)
 
-KEEP_PUSH_ACTIONS_FOR_MS = 30 * 24 * 60 * 60 * 1000
-
 
 class EventPushActionsStore(SQLBaseStore):
+    def __init__(self, hs):
+        self.stream_ordering_month_ago = None
+        super(EventPushActionsStore, self).__init__(hs)
+
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):
         """
         Args:
@@ -237,9 +239,6 @@ class EventPushActionsStore(SQLBaseStore):
             user_id: user ID to delete for
             topological_ordering: The lowest topological ordering which will
                                   not be deleted.
-
-        Returns:
-
         """
         txn.call_after(
             self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
@@ -259,15 +258,63 @@ class EventPushActionsStore(SQLBaseStore):
         txn.execute(
             "DELETE FROM event_push_actions "
             " WHERE user_id = ? AND room_id = ? AND "
-            " topological_ordering < ? AND stream_ordering < ("
-            "  SELECT stream_ordering FROM events"
-            "  WHERE room_id = ? AND received_ts < ?"
-            "  ORDER BY stream_ordering DESC"
-            "  LIMIT 1"
-            ")",
-            (user_id, room_id, topological_ordering, room_id, threshold)
+            " topological_ordering < ? AND stream_ordering < ?"
+            (user_id, room_id, topological_ordering, self.stream_ordering_month_ago)
+        )
+
+    @defer.inlineCallbacks
+    def _find_stream_orderings_for_times(self):
+        yield self.runInteraction(
+            "_find_stream_orderings_for_times",
+            self._find_stream_orderings_for_times_txn
+        )
+
+    def _find_stream_orderings_for_times_txn(self, txn):
+        logger.info("Searching for stream ordering 1 month ago")
+        self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn(
+            txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000
+        )
+        logger.info(
+            "Found stream ordering 1 month ago: it's %d",
+            self.stream_ordering_month_ago
         )
 
+    def _find_first_stream_ordering_after_ts_txn(self, txn, ts):
+        """
+        Find the stream_ordering of the first event that was received after
+        a given timestamp. This is relatively slow as there is no index on
+        received_ts but we can then use this to delete push actions before
+        this.
+
+        received_ts must necessarily be in the same order as stream_ordering
+        and stream_ordering is indexed, so we manually binary search using
+        stream_ordering
+        """
+        txn.execute("SELECT MAX(stream_ordering) FROM events")
+        max_stream_ordering = txn.fetchone()[0]
+
+        range_start = 0
+        range_end = max_stream_ordering
+
+        sql = (
+            "SELECT received_ts FROM events"
+            " WHERE stream_ordering > ?"
+            " ORDER BY stream_ordering"
+            " LIMIT 1"
+        )
+
+        while range_end - range_start > 1:
+            middle = int((range_end + range_start) / 2)
+            txn.execute(sql, (middle,))
+            middle_ts = txn.fetchone()[0]
+            if ts > middle_ts:
+                range_start = middle
+            else:
+                range_end = middle
+        logger.info("done: picking %d from %d and %d", range_end, range_start, range_end)
+
+        return range_end
+
 
 def _action_has_highlight(actions):
     for action in actions:
-- 
cgit 1.5.1


From 18d68bfee40078e4195adc43c93872d4a6facaf8 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 20 May 2016 17:58:09 +0100
Subject: Handle empty events table

---
 synapse/storage/event_push_actions.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 4425d4bce5..d630a846c9 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -293,6 +293,9 @@ class EventPushActionsStore(SQLBaseStore):
         txn.execute("SELECT MAX(stream_ordering) FROM events")
         max_stream_ordering = txn.fetchone()[0]
 
+        if max_stream_ordering is None:
+            return 0
+
         range_start = 0
         range_end = max_stream_ordering
 
-- 
cgit 1.5.1


From ccffb0965de8a567ba56eeae5c16945921268802 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 20 May 2016 17:59:10 +0100
Subject: Remove stale line

---
 synapse/storage/event_push_actions.py | 2 --
 1 file changed, 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index d630a846c9..cb38d5f525 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -245,8 +245,6 @@ class EventPushActionsStore(SQLBaseStore):
             (room_id, user_id, )
         )
 
-        threshold = self._clock.time_msec() - KEEP_PUSH_ACTIONS_FOR_MS
-
         # We need to join on the events table to get the received_ts for
         # event_push_actions and sqlite won't let us use a join in a delete so
         # we can't just delete where received_ts < x. Furthermore we can
-- 
cgit 1.5.1


From c2da3406fcadefa3efbd302ae22a21c7ffbac9ce Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 20 May 2016 18:03:31 +0100
Subject: Oops, missing comma

---
 synapse/storage/event_push_actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index cb38d5f525..88f9b4b642 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -256,7 +256,7 @@ class EventPushActionsStore(SQLBaseStore):
         txn.execute(
             "DELETE FROM event_push_actions "
             " WHERE user_id = ? AND room_id = ? AND "
-            " topological_ordering < ? AND stream_ordering < ?"
+            " topological_ordering < ? AND stream_ordering < ?",
             (user_id, room_id, topological_ordering, self.stream_ordering_month_ago)
         )
 
-- 
cgit 1.5.1


From 31b5395ab6f1bec6a7198f2302ec43a891c95d03 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 23 May 2016 16:32:01 +0100
Subject: Remove debug logging

---
 synapse/storage/event_push_actions.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 88f9b4b642..4dae51a172 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -312,7 +312,6 @@ class EventPushActionsStore(SQLBaseStore):
                 range_start = middle
             else:
                 range_end = middle
-        logger.info("done: picking %d from %d and %d", range_end, range_start, range_end)
 
         return range_end
 
-- 
cgit 1.5.1


From 85b992f621743adc45945f67b1b1bbf4aed198fb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 27 May 2016 10:44:44 +0100
Subject: Fix to allow start with postgres

---
 synapse/storage/__init__.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 49feb77779..8581796b7e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 from .appservice import (
     ApplicationServiceStore, ApplicationServiceTransactionStore
 )
-from ._base import Cache
+from ._base import Cache, LoggingTransaction
 from .directory import DirectoryStore
 from .events import EventsStore
 from .presence import PresenceStore, UserPresenceState
@@ -174,7 +174,12 @@ class DataStore(RoomMemberStore, RoomStore,
             prefilled_cache=push_rules_prefill,
         )
 
-        cur = db_conn.cursor()
+        cur = LoggingTransaction(
+            db_conn.cursor(),
+            name="_find_stream_orderings_for_times_txn",
+            database_engine=self.database_engine,
+            after_callbacks=[]
+        )
         self._find_stream_orderings_for_times_txn(cur)
         cur.close()
 
-- 
cgit 1.5.1


From e5b0bbcd3381e581ecf9876760bbe36c66fcb4fe Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 May 2016 13:46:58 +0100
Subject: Add caches to bulk_get_push_rules*

---
 synapse/push/bulk_push_rule_evaluator.py |  8 +++++---
 synapse/storage/push_rule.py             | 16 ++++++++++------
 2 files changed, 15 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 25e13b3423..25f2fb9da4 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
 
 
 def decode_rule_json(rule):
+    rule = dict(rule)
     rule['conditions'] = json.loads(rule['conditions'])
     rule['actions'] = json.loads(rule['actions'])
     return rule
@@ -39,6 +40,8 @@ def _get_rules(room_id, user_ids, store):
     rules_by_user = yield store.bulk_get_push_rules(user_ids)
     rules_enabled_by_user = yield store.bulk_get_push_rules_enabled(user_ids)
 
+    rules_by_user = {k: v for k, v in rules_by_user.items() if v is not None}
+
     rules_by_user = {
         uid: list_with_base_rules([
             decode_rule_json(rule_list)
@@ -51,11 +54,10 @@ def _get_rules(room_id, user_ids, store):
     # fetch disabled rules, but this won't account for any server default
     # rules the user has disabled, so we need to do this too.
     for uid in user_ids:
-        if uid not in rules_enabled_by_user:
+        user_enabled_map = rules_enabled_by_user.get(uid)
+        if not user_enabled_map:
             continue
 
-        user_enabled_map = rules_enabled_by_user[uid]
-
         for i, rule in enumerate(rules_by_user[uid]):
             rule_id = rule['rule_id']
 
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index d2bf7f2aec..f285f59afd 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from ._base import SQLBaseStore
-from synapse.util.caches.descriptors import cachedInlineCallbacks
+from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
 from twisted.internet import defer
 
 import logging
@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 class PushRuleStore(SQLBaseStore):
-    @cachedInlineCallbacks()
+    @cachedInlineCallbacks(lru=True)
     def get_push_rules_for_user(self, user_id):
         rows = yield self._simple_select_list(
             table="push_rules",
@@ -44,7 +44,7 @@ class PushRuleStore(SQLBaseStore):
 
         defer.returnValue(rows)
 
-    @cachedInlineCallbacks()
+    @cachedInlineCallbacks(lru=True)
     def get_push_rules_enabled_for_user(self, user_id):
         results = yield self._simple_select_list(
             table="push_rules_enable",
@@ -60,7 +60,8 @@ class PushRuleStore(SQLBaseStore):
             r['rule_id']: False if r['enabled'] == 0 else True for r in results
         })
 
-    @defer.inlineCallbacks
+    @cachedList(cached_method_name="get_push_rules_for_user",
+                list_name="user_ids", num_args=1, inlineCallbacks=True)
     def bulk_get_push_rules(self, user_ids):
         if not user_ids:
             defer.returnValue({})
@@ -75,13 +76,16 @@ class PushRuleStore(SQLBaseStore):
             desc="bulk_get_push_rules",
         )
 
-        rows.sort(key=lambda e: (-e["priority_class"], -e["priority"]))
+        rows.sort(
+            key=lambda row: (-int(row["priority_class"]), -int(row["priority"]))
+        )
 
         for row in rows:
             results.setdefault(row['user_name'], []).append(row)
         defer.returnValue(results)
 
-    @defer.inlineCallbacks
+    @cachedList(cached_method_name="get_push_rules_enabled_for_user",
+                list_name="user_ids", num_args=1, inlineCallbacks=True)
     def bulk_get_push_rules_enabled(self, user_ids):
         if not user_ids:
             defer.returnValue({})
-- 
cgit 1.5.1


From aefd2d1cbc83f2cdf5d45d4e9cd2a176747ac38a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 May 2016 15:32:32 +0100
Subject: Cache get_event_reference_hashes

---
 synapse/storage/signatures.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index b10f2a5787..ea6823f18d 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -19,17 +19,24 @@ from ._base import SQLBaseStore
 
 from unpaddedbase64 import encode_base64
 from synapse.crypto.event_signing import compute_event_reference_hash
+from synapse.util.caches.descriptors import cached, cachedList
 
 
 class SignatureStore(SQLBaseStore):
     """Persistence for event signatures and hashes"""
 
+    @cached(lru=True)
+    def get_event_reference_hash(self, event_id):
+        return self._get_event_reference_hashes_txn(event_id)
+
+    @cachedList(cached_method_name="get_event_reference_hash",
+                list_name="event_ids", num_args=1)
     def get_event_reference_hashes(self, event_ids):
         def f(txn):
-            return [
-                self._get_event_reference_hashes_txn(txn, ev)
-                for ev in event_ids
-            ]
+            return {
+                event_id: self._get_event_reference_hashes_txn(txn, event_id)
+                for event_id in event_ids
+            }
 
         return self.runInteraction(
             "get_event_reference_hashes",
@@ -41,15 +48,15 @@ class SignatureStore(SQLBaseStore):
         hashes = yield self.get_event_reference_hashes(
             event_ids
         )
-        hashes = [
-            {
+        hashes = {
+            e_id: {
                 k: encode_base64(v) for k, v in h.items()
                 if k == "sha256"
             }
-            for h in hashes
-        ]
+            for e_id, h in hashes.items()
+        }
 
-        defer.returnValue(zip(event_ids, hashes))
+        defer.returnValue(hashes.items())
 
     def _get_event_reference_hashes_txn(self, txn, event_id):
         """Get all the hashes for a given PDU.
-- 
cgit 1.5.1


From cca0093fa96cfec0566ff790ef990fad8b6763bb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 May 2016 15:44:08 +0100
Subject: Change fix

---
 synapse/rest/client/v1/push_rule.py |  3 ---
 synapse/storage/push_rule.py        | 10 ++++++++--
 2 files changed, 8 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index c135353aab..02d837ee6a 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -132,9 +132,6 @@ class PushRuleRestServlet(ClientV1RestServlet):
 
         enabled_map = yield self.store.get_push_rules_enabled_for_user(user_id)
 
-        rawrules = {k: v for k, v in rawrules.item() if k is not None}
-        enabled_map = {k: v for k, v in enabled_map.item() if k is not None}
-
         rules = format_push_rules_for_user(requester.user, rawrules, enabled_map)
 
         path = request.postpath[1:]
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index f285f59afd..65c20e8900 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -66,7 +66,10 @@ class PushRuleStore(SQLBaseStore):
         if not user_ids:
             defer.returnValue({})
 
-        results = {}
+        results = {
+            user_id: []
+            for user_id in user_ids
+        }
 
         rows = yield self._simple_select_many_batch(
             table="push_rules",
@@ -90,7 +93,10 @@ class PushRuleStore(SQLBaseStore):
         if not user_ids:
             defer.returnValue({})
 
-        results = {}
+        results = {
+            user_id: []
+            for user_id in user_ids
+        }
 
         rows = yield self._simple_select_many_batch(
             table="push_rules_enable",
-- 
cgit 1.5.1


From 1d4ee854e21626f64c945610e467f7e761534424 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 May 2016 15:45:53 +0100
Subject: Fix typo

---
 synapse/storage/push_rule.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 65c20e8900..216a8bf69c 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -94,7 +94,7 @@ class PushRuleStore(SQLBaseStore):
             defer.returnValue({})
 
         results = {
-            user_id: []
+            user_id: {}
             for user_id in user_ids
         }
 
-- 
cgit 1.5.1


From c8c5bf950a27e00e3e9ae57b98f38cab03cdc3c9 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 31 May 2016 17:10:40 +0100
Subject: Fix synapse/storage/schema/delta/30/as_users.py

---
 synapse/storage/schema/delta/30/as_users.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py
index b417e3ac08..5b7d8d1ab5 100644
--- a/synapse/storage/schema/delta/30/as_users.py
+++ b/synapse/storage/schema/delta/30/as_users.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from synapse.storage.appservice import ApplicationServiceStore
+from synapse.config.appservice import load_appservices
 
 
 logger = logging.getLogger(__name__)
@@ -38,7 +38,7 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs):
         logger.warning("Could not get app_service_config_files from config")
         pass
 
-    appservices = ApplicationServiceStore.load_appservices(
+    appservices = load_appservices(
         config.server_name, config_files
     )
 
-- 
cgit 1.5.1


From dea9f20f8ce105624933a819d7949d31b69aa141 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 May 2016 17:24:30 +0100
Subject: Force boolean

---
 synapse/storage/push_rule.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 216a8bf69c..ebb97c8474 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -106,7 +106,8 @@ class PushRuleStore(SQLBaseStore):
             desc="bulk_get_push_rules_enabled",
         )
         for row in rows:
-            results.setdefault(row['user_name'], {})[row['rule_id']] = row['enabled']
+            enabled = bool(row['enabled'])
+            results.setdefault(row['user_name'], {})[row['rule_id']] = enabled
         defer.returnValue(results)
 
     @defer.inlineCallbacks
-- 
cgit 1.5.1


From 43db0d9f6a314679bd25b82354e5c469e7a010b9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Jun 2016 10:31:09 +0100
Subject: Add get_users_with_read_receipts_in_room cache

---
 synapse/push/bulk_push_rule_evaluator.py |  8 ++++----
 synapse/storage/receipts.py              | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 25f2fb9da4..1e5c4b073c 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -87,13 +87,13 @@ def evaluator_for_event(event, hs, store):
     all_in_room = yield store.get_users_in_room(room_id)
     all_in_room = set(all_in_room)
 
-    receipts = yield store.get_receipts_for_room(room_id, "m.read")
+    users_with_receipts = yield store.get_users_with_read_receipts_in_room(room_id)
 
     # any users with pushers must be ours: they have pushers
     user_ids = set(users_with_pushers)
-    for r in receipts:
-        if hs.is_mine_id(r['user_id']) and r['user_id'] in all_in_room:
-            user_ids.add(r['user_id'])
+    for uid in users_with_receipts:
+        if hs.is_mine_id(uid) and uid in all_in_room:
+            user_ids.add(uid)
 
     # if this event is an invite event, we may need to run rules for the user
     # who's been invited, otherwise they won't get told they've been invited
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index fdcf28f3e1..964f30dff7 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -34,6 +34,26 @@ class ReceiptsStore(SQLBaseStore):
             "ReceiptsRoomChangeCache", self._receipts_id_gen.get_current_token()
         )
 
+    @cachedInlineCallbacks()
+    def get_users_with_read_receipts_in_room(self, room_id):
+        receipts = yield self.get_receipts_for_room(room_id, "m.read")
+        defer.returnValue(set(r['user_id'] for r in receipts))
+
+    def _invalidate_get_users_with_receipts_in_room(self, room_id, receipt_type,
+                                                    user_id):
+        if receipt_type != "m.read":
+            return
+
+        # Returns an ObservableDeferred
+        res = self.get_users_with_read_receipts_in_room.cache.get((room_id,), None)
+
+        if res and res.called and user_id in res.result:
+            # We'd only be adding to the set, so no point invalidating if the
+            # user is already there
+            return
+
+        self.get_users_with_read_receipts_in_room.invalidate((room_id,))
+
     @cached(num_args=2)
     def get_receipts_for_room(self, room_id, receipt_type):
         return self._simple_select_list(
@@ -228,6 +248,10 @@ class ReceiptsStore(SQLBaseStore):
         txn.call_after(
             self.get_receipts_for_room.invalidate, (room_id, receipt_type)
         )
+        txn.call_after(
+            self._invalidate_get_users_with_receipts_in_room,
+            room_id, receipt_type, user_id,
+        )
         txn.call_after(
             self.get_receipts_for_user.invalidate, (user_id, receipt_type)
         )
@@ -373,6 +397,10 @@ class ReceiptsStore(SQLBaseStore):
         txn.call_after(
             self.get_receipts_for_room.invalidate, (room_id, receipt_type)
         )
+        txn.call_after(
+            self._invalidate_get_users_with_receipts_in_room,
+            room_id, receipt_type, user_id,
+        )
         txn.call_after(
             self.get_receipts_for_user.invalidate, (user_id, receipt_type)
         )
-- 
cgit 1.5.1


From d60eed07109f61ebe2120e1eb566e5bb7095fbad Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 1 Jun 2016 11:45:43 +0100
Subject: Limit number of notifications in an email notification

---
 synapse/storage/event_push_actions.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 4dae51a172..940e11d7a2 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -119,7 +119,8 @@ class EventPushActionsStore(SQLBaseStore):
     @defer.inlineCallbacks
     def get_unread_push_actions_for_user_in_range(self, user_id,
                                                   min_stream_ordering,
-                                                  max_stream_ordering=None):
+                                                  max_stream_ordering=None,
+                                                  limit=20):
         def get_after_receipt(txn):
             sql = (
                 "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, "
@@ -151,7 +152,8 @@ class EventPushActionsStore(SQLBaseStore):
             if max_stream_ordering is not None:
                 sql += " AND ep.stream_ordering <= ?"
                 args.append(max_stream_ordering)
-            sql += " ORDER BY ep.stream_ordering ASC"
+            sql += " ORDER BY ep.stream_ordering ASC LIMIT ?"
+            args.append(limit)
             txn.execute(sql, args)
             return txn.fetchall()
         after_read_receipt = yield self.runInteraction(
-- 
cgit 1.5.1


From c8285564a3772db387fd5c94b1a82329dc320e36 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Jun 2016 11:08:45 +0100
Subject: Use state to calculate get_users_in_room

---
 synapse/push/action_generator.py         |  2 +-
 synapse/push/bulk_push_rule_evaluator.py | 27 ++++++++++++++---------
 synapse/storage/events.py                |  3 ---
 synapse/storage/pusher.py                | 38 +++++++++++++++++++++++---------
 synapse/storage/roommember.py            |  3 ---
 5 files changed, 45 insertions(+), 28 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py
index 9b208668b6..46e768e35c 100644
--- a/synapse/push/action_generator.py
+++ b/synapse/push/action_generator.py
@@ -40,7 +40,7 @@ class ActionGenerator:
     def handle_push_actions_for_event(self, event, context):
         with Measure(self.clock, "handle_push_actions_for_event"):
             bulk_evaluator = yield evaluator_for_event(
-                event, self.hs, self.store
+                event, self.hs, self.store, context.current_state
             )
 
             actions_by_user = yield bulk_evaluator.action_for_event_by_user(
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 1e5c4b073c..8c59e59e03 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -21,7 +21,7 @@ from twisted.internet import defer
 from .baserules import list_with_base_rules
 from .push_rule_evaluator import PushRuleEvaluatorForEvent
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, Membership
 from synapse.visibility import filter_events_for_clients
 
 
@@ -72,20 +72,24 @@ def _get_rules(room_id, user_ids, store):
 
 
 @defer.inlineCallbacks
-def evaluator_for_event(event, hs, store):
+def evaluator_for_event(event, hs, store, current_state):
     room_id = event.room_id
-
-    # users in the room who have pushers need to get push rules run because
-    # that's how their pushers work
-    users_with_pushers = yield store.get_users_with_pushers_in_room(room_id)
-
     # We also will want to generate notifs for other people in the room so
     # their unread countss are correct in the event stream, but to avoid
     # generating them for bot / AS users etc, we only do so for people who've
     # sent a read receipt into the room.
 
-    all_in_room = yield store.get_users_in_room(room_id)
-    all_in_room = set(all_in_room)
+    all_in_room = set(
+        e.state_key for e in current_state.values()
+        if e.type == EventTypes.Member and e.membership == Membership.JOIN
+    )
+
+    # users in the room who have pushers need to get push rules run because
+    # that's how their pushers work
+    if_users_with_pushers = yield store.get_if_users_have_pushers(all_in_room)
+    users_with_pushers = set(
+        uid for uid, have_pusher in if_users_with_pushers.items() if have_pusher
+    )
 
     users_with_receipts = yield store.get_users_with_read_receipts_in_room(room_id)
 
@@ -143,7 +147,10 @@ class BulkPushRuleEvaluator:
             self.store, user_tuples, [event], {event.event_id: current_state}
         )
 
-        room_members = yield self.store.get_users_in_room(self.room_id)
+        room_members = set(
+            e.state_key for e in current_state.values()
+            if e.type == EventTypes.Member and e.membership == Membership.JOIN
+        )
 
         evaluator = PushRuleEvaluatorForEvent(event, len(room_members))
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 4655669ba0..2b3f79577b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -342,9 +342,6 @@ class EventsStore(SQLBaseStore):
             txn.call_after(self._get_current_state_for_key.invalidate_all)
             txn.call_after(self.get_rooms_for_user.invalidate_all)
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
-            txn.call_after(
-                self.get_users_with_pushers_in_room.invalidate, (event.room_id,)
-            )
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_room_name_and_aliases.invalidate, (event.room_id,))
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 9e8e2e2964..39d5349eaa 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 
 from canonicaljson import encode_canonical_json
 
-from synapse.util.caches.descriptors import cachedInlineCallbacks
+from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
 
 import logging
 import simplejson as json
@@ -135,19 +135,35 @@ class PusherStore(SQLBaseStore):
             "get_all_updated_pushers", get_all_updated_pushers_txn
         )
 
-    @cachedInlineCallbacks(num_args=1)
-    def get_users_with_pushers_in_room(self, room_id):
-        users = yield self.get_users_in_room(room_id)
-
+    @cachedInlineCallbacks(lru=True, num_args=1)
+    def get_if_user_has_pusher(self, user_id):
         result = yield self._simple_select_many_batch(
+            table='pushers',
+            keyvalues={
+                'user_name': 'user_id',
+            },
+            retcol='user_name',
+            desc='get_if_user_has_pusher',
+            allow_none=True,
+        )
+
+        defer.returnValue(bool(result))
+
+    @cachedList(cached_method_name="get_if_user_has_pusher",
+                list_name="user_ids", num_args=1, inlineCallbacks=True)
+    def get_if_users_have_pushers(self, user_ids):
+        rows = yield self._simple_select_many_batch(
             table='pushers',
             column='user_name',
-            iterable=users,
+            iterable=user_ids,
             retcols=['user_name'],
-            desc='get_users_with_pushers_in_room'
+            desc='get_if_users_have_pushers'
         )
 
-        defer.returnValue([r['user_name'] for r in result])
+        result = {user_id: False for user_id in user_ids}
+        result.update({r['user_name']: True for r in rows})
+
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def add_pusher(self, user_id, access_token, kind, app_id,
@@ -178,16 +194,16 @@ class PusherStore(SQLBaseStore):
                     },
                 )
                 if newly_inserted:
-                    # get_users_with_pushers_in_room only cares if the user has
+                    # get_if_user_has_pusher only cares if the user has
                     # at least *one* pusher.
-                    txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
+                    txn.call_after(self.get_if_user_has_pusher.invalidate, (user_id,))
 
             yield self.runInteraction("add_pusher", f)
 
     @defer.inlineCallbacks
     def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id):
         def delete_pusher_txn(txn, stream_id):
-            txn.call_after(self.get_users_with_pushers_in_room.invalidate_all)
+            txn.call_after(self.get_if_user_has_pusher.invalidate, (user_id,))
 
             self._simple_delete_one_txn(
                 txn,
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index face685ed2..41b395e07c 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -58,9 +58,6 @@ class RoomMemberStore(SQLBaseStore):
             txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,))
             txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
-            txn.call_after(
-                self.get_users_with_pushers_in_room.invalidate, (event.room_id,)
-            )
             txn.call_after(
                 self._membership_stream_cache.entity_has_changed,
                 event.state_key, event.internal_metadata.stream_ordering
-- 
cgit 1.5.1


From 80f34d7b574e1a6b8bc922df41bd53b59260fcf2 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 2 Jun 2016 15:23:09 +0100
Subject: Fix setting the _clock in SQLBaseStore

---
 synapse/storage/_base.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 56a0dd80f3..32c6677d47 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -152,6 +152,7 @@ class SQLBaseStore(object):
 
     def __init__(self, hs):
         self.hs = hs
+        self._clock = hs.get_clock()
         self._db_pool = hs.get_db_pool()
 
         self._previous_txn_total_time = 0
-- 
cgit 1.5.1


From 9c26b390a2112590fe4252057dc1f081cb99a6b1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Jun 2016 11:34:06 +0100
Subject: Only get local users

---
 synapse/push/bulk_push_rule_evaluator.py | 7 +++++--
 synapse/storage/pusher.py                | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 8c59e59e03..d50db3b736 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -83,10 +83,13 @@ def evaluator_for_event(event, hs, store, current_state):
         e.state_key for e in current_state.values()
         if e.type == EventTypes.Member and e.membership == Membership.JOIN
     )
+    local_users_in_room = set(uid for uid in all_in_room if hs.is_mine_id(uid))
 
     # users in the room who have pushers need to get push rules run because
     # that's how their pushers work
-    if_users_with_pushers = yield store.get_if_users_have_pushers(all_in_room)
+    if_users_with_pushers = yield store.get_if_users_have_pushers(
+        local_users_in_room
+    )
     users_with_pushers = set(
         uid for uid, have_pusher in if_users_with_pushers.items() if have_pusher
     )
@@ -96,7 +99,7 @@ def evaluator_for_event(event, hs, store, current_state):
     # any users with pushers must be ours: they have pushers
     user_ids = set(users_with_pushers)
     for uid in users_with_receipts:
-        if hs.is_mine_id(uid) and uid in all_in_room:
+        if uid in local_users_in_room:
             user_ids.add(uid)
 
     # if this event is an invite event, we may need to run rules for the user
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 39d5349eaa..a7d7c54d7e 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -135,7 +135,7 @@ class PusherStore(SQLBaseStore):
             "get_all_updated_pushers", get_all_updated_pushers_txn
         )
 
-    @cachedInlineCallbacks(lru=True, num_args=1)
+    @cachedInlineCallbacks(lru=True, num_args=1, max_entries=15000)
     def get_if_user_has_pusher(self, user_id):
         result = yield self._simple_select_many_batch(
             table='pushers',
-- 
cgit 1.5.1


From 3ae915b27e4531031ee325931b3c62bc200ce798 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Fri, 3 Jun 2016 11:05:53 +0100
Subject: Add a slaved store for presence

---
 synapse/replication/slave/storage/presence.py | 59 +++++++++++++++++++++++++++
 synapse/storage/__init__.py                   |  6 +--
 2 files changed, 62 insertions(+), 3 deletions(-)
 create mode 100644 synapse/replication/slave/storage/presence.py

(limited to 'synapse/storage')

diff --git a/synapse/replication/slave/storage/presence.py b/synapse/replication/slave/storage/presence.py
new file mode 100644
index 0000000000..703f4a49bf
--- /dev/null
+++ b/synapse/replication/slave/storage/presence.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import BaseSlavedStore
+from ._slaved_id_tracker import SlavedIdTracker
+
+from synapse.util.caches.stream_change_cache import StreamChangeCache
+from synapse.storage import DataStore
+
+
+class SlavedPresenceStore(BaseSlavedStore):
+    def __init__(self, db_conn, hs):
+        super(SlavedPresenceStore, self).__init__(db_conn, hs)
+        self._presence_id_gen = SlavedIdTracker(
+            db_conn, "presence_stream", "stream_id",
+        )
+
+        self._presence_on_startup = self._get_active_presence(db_conn)
+
+        self.presence_stream_cache = self.presence_stream_cache = StreamChangeCache(
+            "PresenceStreamChangeCache", self._presence_id_gen.get_current_token()
+        )
+
+    _get_active_presence = DataStore._get_active_presence.__func__
+    take_presence_startup_info = DataStore.take_presence_startup_info.__func__
+    get_presence_for_users = DataStore.get_presence_for_users.__func__
+
+    def get_current_presence_token(self):
+        return self._presence_id_gen.get_current_token()
+
+    def stream_positions(self):
+        result = super(SlavedPresenceStore, self).stream_positions()
+        position = self._presence_id_gen.get_current_token()
+        result["presence"] = position
+        return result
+
+    def process_replication(self, result):
+        stream = result.get("presence")
+        if stream:
+            self._presence_id_gen.advance(int(stream["position"]))
+            for row in stream["rows"]:
+                position, user_id = row[:2]
+                self.presence_stream_cache.entity_has_changed(
+                    user_id, position
+                )
+
+        return super(SlavedPresenceStore, self).process_replication(result)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 8581796b7e..6928a213e8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -149,7 +149,7 @@ class DataStore(RoomMemberStore, RoomStore,
             "AccountDataAndTagsChangeCache", account_max,
         )
 
-        self.__presence_on_startup = self._get_active_presence(db_conn)
+        self._presence_on_startup = self._get_active_presence(db_conn)
 
         presence_cache_prefill, min_presence_val = self._get_cache_dict(
             db_conn, "presence_stream",
@@ -190,8 +190,8 @@ class DataStore(RoomMemberStore, RoomStore,
         super(DataStore, self).__init__(hs)
 
     def take_presence_startup_info(self):
-        active_on_startup = self.__presence_on_startup
-        self.__presence_on_startup = None
+        active_on_startup = self._presence_on_startup
+        self._presence_on_startup = None
         return active_on_startup
 
     def _get_active_presence(self, db_conn):
-- 
cgit 1.5.1


From 6a0afa582aa5bf816e082af31ac44e2a8fee28c0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Jun 2016 14:27:07 +0100
Subject: Load push rules in storage layer, so that they get cached

---
 synapse/handlers/sync.py                 |  5 ++---
 synapse/push/bulk_push_rule_evaluator.py | 28 -----------------------
 synapse/push/clientformat.py             | 30 ++++++++++++++++++-------
 synapse/rest/client/v1/push_rule.py      |  6 ++---
 synapse/storage/push_rule.py             | 38 +++++++++++++++++++++++++++++++-
 5 files changed, 63 insertions(+), 44 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 5307b62b85..be26a491ff 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -198,9 +198,8 @@ class SyncHandler(object):
     @defer.inlineCallbacks
     def push_rules_for_user(self, user):
         user_id = user.to_string()
-        rawrules = yield self.store.get_push_rules_for_user(user_id)
-        enabled_map = yield self.store.get_push_rules_enabled_for_user(user_id)
-        rules = format_push_rules_for_user(user, rawrules, enabled_map)
+        rules = yield self.store.get_push_rules_for_user(user_id)
+        rules = format_push_rules_for_user(user, rules)
         defer.returnValue(rules)
 
     @defer.inlineCallbacks
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index af5212a5d1..6e42121b1d 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -18,7 +18,6 @@ import ujson as json
 
 from twisted.internet import defer
 
-from .baserules import list_with_base_rules
 from .push_rule_evaluator import PushRuleEvaluatorForEvent
 
 from synapse.api.constants import EventTypes, Membership
@@ -38,36 +37,9 @@ def decode_rule_json(rule):
 @defer.inlineCallbacks
 def _get_rules(room_id, user_ids, store):
     rules_by_user = yield store.bulk_get_push_rules(user_ids)
-    rules_enabled_by_user = yield store.bulk_get_push_rules_enabled(user_ids)
 
     rules_by_user = {k: v for k, v in rules_by_user.items() if v is not None}
 
-    rules_by_user = {
-        uid: list_with_base_rules([
-            decode_rule_json(rule_list)
-            for rule_list in rules_by_user.get(uid, [])
-        ])
-        for uid in user_ids
-    }
-
-    # We apply the rules-enabled map here: bulk_get_push_rules doesn't
-    # fetch disabled rules, but this won't account for any server default
-    # rules the user has disabled, so we need to do this too.
-    for uid in user_ids:
-        user_enabled_map = rules_enabled_by_user.get(uid)
-        if not user_enabled_map:
-            continue
-
-        for i, rule in enumerate(rules_by_user[uid]):
-            rule_id = rule['rule_id']
-
-            if rule_id in user_enabled_map:
-                if rule.get('enabled', True) != bool(user_enabled_map[rule_id]):
-                    # Rules are cached across users.
-                    rule = dict(rule)
-                    rule['enabled'] = bool(user_enabled_map[rule_id])
-                    rules_by_user[uid][i] = rule
-
     defer.returnValue(rules_by_user)
 
 
diff --git a/synapse/push/clientformat.py b/synapse/push/clientformat.py
index ae9db9ec2f..b3983f7940 100644
--- a/synapse/push/clientformat.py
+++ b/synapse/push/clientformat.py
@@ -23,10 +23,7 @@ import copy
 import simplejson as json
 
 
-def format_push_rules_for_user(user, rawrules, enabled_map):
-    """Converts a list of rawrules and a enabled map into nested dictionaries
-    to match the Matrix client-server format for push rules"""
-
+def load_rules_for_user(user, rawrules, enabled_map):
     ruleslist = []
     for rawrule in rawrules:
         rule = dict(rawrule)
@@ -35,7 +32,26 @@ def format_push_rules_for_user(user, rawrules, enabled_map):
         ruleslist.append(rule)
 
     # We're going to be mutating this a lot, so do a deep copy
-    ruleslist = copy.deepcopy(list_with_base_rules(ruleslist))
+    rules = list(list_with_base_rules(ruleslist))
+
+    for i, rule in enumerate(rules):
+        rule_id = rule['rule_id']
+        if rule_id in enabled_map:
+            if rule.get('enabled', True) != bool(enabled_map[rule_id]):
+                # Rules are cached across users.
+                rule = dict(rule)
+                rule['enabled'] = bool(enabled_map[rule_id])
+                rules[i] = rule
+
+    return rules
+
+
+def format_push_rules_for_user(user, ruleslist):
+    """Converts a list of rawrules and a enabled map into nested dictionaries
+    to match the Matrix client-server format for push rules"""
+
+    # We're going to be mutating this a lot, so do a deep copy
+    ruleslist = copy.deepcopy(ruleslist)
 
     rules = {'global': {}, 'device': {}}
 
@@ -60,9 +76,7 @@ def format_push_rules_for_user(user, rawrules, enabled_map):
 
         template_rule = _rule_to_template(r)
         if template_rule:
-            if r['rule_id'] in enabled_map:
-                template_rule['enabled'] = enabled_map[r['rule_id']]
-            elif 'enabled' in r:
+            if 'enabled' in r:
                 template_rule['enabled'] = r['enabled']
             else:
                 template_rule['enabled'] = True
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 02d837ee6a..6bb4821ec6 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -128,11 +128,9 @@ class PushRuleRestServlet(ClientV1RestServlet):
         # we build up the full structure and then decide which bits of it
         # to send which means doing unnecessary work sometimes but is
         # is probably not going to make a whole lot of difference
-        rawrules = yield self.store.get_push_rules_for_user(user_id)
+        rules = yield self.store.get_push_rules_for_user(user_id)
 
-        enabled_map = yield self.store.get_push_rules_enabled_for_user(user_id)
-
-        rules = format_push_rules_for_user(requester.user, rawrules, enabled_map)
+        rules = format_push_rules_for_user(requester.user, rules)
 
         path = request.postpath[1:]
 
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index ebb97c8474..786d6f6d67 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -15,6 +15,7 @@
 
 from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
+from synapse.push.baserules import list_with_base_rules
 from twisted.internet import defer
 
 import logging
@@ -23,6 +24,29 @@ import simplejson as json
 logger = logging.getLogger(__name__)
 
 
+def _load_rules(rawrules, enabled_map):
+    ruleslist = []
+    for rawrule in rawrules:
+        rule = dict(rawrule)
+        rule["conditions"] = json.loads(rawrule["conditions"])
+        rule["actions"] = json.loads(rawrule["actions"])
+        ruleslist.append(rule)
+
+    # We're going to be mutating this a lot, so do a deep copy
+    rules = list(list_with_base_rules(ruleslist))
+
+    for i, rule in enumerate(rules):
+        rule_id = rule['rule_id']
+        if rule_id in enabled_map:
+            if rule.get('enabled', True) != bool(enabled_map[rule_id]):
+                # Rules are cached across users.
+                rule = dict(rule)
+                rule['enabled'] = bool(enabled_map[rule_id])
+                rules[i] = rule
+
+    return rules
+
+
 class PushRuleStore(SQLBaseStore):
     @cachedInlineCallbacks(lru=True)
     def get_push_rules_for_user(self, user_id):
@@ -42,7 +66,11 @@ class PushRuleStore(SQLBaseStore):
             key=lambda row: (-int(row["priority_class"]), -int(row["priority"]))
         )
 
-        defer.returnValue(rows)
+        enabled_map = yield self.get_push_rules_enabled_for_user(user_id)
+
+        rules = _load_rules(rows, enabled_map)
+
+        defer.returnValue(rules)
 
     @cachedInlineCallbacks(lru=True)
     def get_push_rules_enabled_for_user(self, user_id):
@@ -85,6 +113,14 @@ class PushRuleStore(SQLBaseStore):
 
         for row in rows:
             results.setdefault(row['user_name'], []).append(row)
+
+        enabled_map_by_user = yield self.bulk_get_push_rules_enabled(user_ids)
+
+        for user_id, rules in results.items():
+            results[user_id] = _load_rules(
+                rules, enabled_map_by_user.get(user_id, {})
+            )
+
         defer.returnValue(results)
 
     @cachedList(cached_method_name="get_push_rules_enabled_for_user",
-- 
cgit 1.5.1


From ccb56fc24bd7d2675dc21796b29333538ca0d5fa Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 2 Jun 2016 12:47:06 +0100
Subject: Make get_joined_hosts_for_room use get_users_in_room

---
 synapse/storage/roommember.py | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 41b395e07c..64b4bd371b 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -238,23 +238,10 @@ class RoomMemberStore(SQLBaseStore):
 
         return results
 
-    @cached(max_entries=5000)
+    @cachedInlineCallbacks(max_entries=5000)
     def get_joined_hosts_for_room(self, room_id):
-        return self.runInteraction(
-            "get_joined_hosts_for_room",
-            self._get_joined_hosts_for_room_txn,
-            room_id,
-        )
-
-    def _get_joined_hosts_for_room_txn(self, txn, room_id):
-        rows = self._get_members_rows_txn(
-            txn,
-            room_id, membership=Membership.JOIN
-        )
-
-        joined_domains = set(get_domain_from_id(r["user_id"]) for r in rows)
-
-        return joined_domains
+        user_ids = yield self.get_users_in_room(room_id)
+        defer.returnValue(set(get_domain_from_id(uid) for uid in user_ids))
 
     def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None):
         rows = self._get_members_rows_txn(
-- 
cgit 1.5.1


From eef541a2919649e6d756d45a29d47fe76cfe02e2 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Fri, 3 Jun 2016 14:42:35 +0100
Subject: Move insert_client_ip to a separate class

---
 synapse/storage/__init__.py   | 48 ++----------------------------
 synapse/storage/client_ips.py | 68 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 45 deletions(-)
 create mode 100644 synapse/storage/client_ips.py

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6928a213e8..e93c3de66c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 from .appservice import (
     ApplicationServiceStore, ApplicationServiceTransactionStore
 )
-from ._base import Cache, LoggingTransaction
+from ._base import LoggingTransaction
 from .directory import DirectoryStore
 from .events import EventsStore
 from .presence import PresenceStore, UserPresenceState
@@ -45,6 +45,7 @@ from .search import SearchStore
 from .tags import TagsStore
 from .account_data import AccountDataStore
 from .openid import OpenIdStore
+from .client_ips import ClientIpStore
 
 from .util.id_generators import IdGenerator, StreamIdGenerator, ChainedIdGenerator
 
@@ -58,12 +59,6 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-# Number of msec of granularity to store the user IP 'last seen' time. Smaller
-# times give more inserts into the database even for readonly API hits
-# 120 seconds == 2 minutes
-LAST_SEEN_GRANULARITY = 120 * 1000
-
-
 class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore,
                 PresenceStore, TransactionStore,
@@ -84,6 +79,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 AccountDataStore,
                 EventPushActionsStore,
                 OpenIdStore,
+                ClientIpStore,
                 ):
 
     def __init__(self, db_conn, hs):
@@ -91,11 +87,6 @@ class DataStore(RoomMemberStore, RoomStore,
         self._clock = hs.get_clock()
         self.database_engine = hs.database_engine
 
-        self.client_ip_last_seen = Cache(
-            name="client_ip_last_seen",
-            keylen=4,
-        )
-
         self._stream_id_gen = StreamIdGenerator(
             db_conn, "events", "stream_ordering",
             extra_tables=[("local_invites", "stream_id")]
@@ -216,39 +207,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return [UserPresenceState(**row) for row in rows]
 
-    @defer.inlineCallbacks
-    def insert_client_ip(self, user, access_token, ip, user_agent):
-        now = int(self._clock.time_msec())
-        key = (user.to_string(), access_token, ip)
-
-        try:
-            last_seen = self.client_ip_last_seen.get(key)
-        except KeyError:
-            last_seen = None
-
-        # Rate-limited inserts
-        if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
-            defer.returnValue(None)
-
-        self.client_ip_last_seen.prefill(key, now)
-
-        # It's safe not to lock here: a) no unique constraint,
-        # b) LAST_SEEN_GRANULARITY makes concurrent updates incredibly unlikely
-        yield self._simple_upsert(
-            "user_ips",
-            keyvalues={
-                "user_id": user.to_string(),
-                "access_token": access_token,
-                "ip": ip,
-                "user_agent": user_agent,
-            },
-            values={
-                "last_seen": now,
-            },
-            desc="insert_client_ip",
-            lock=False,
-        )
-
     @defer.inlineCallbacks
     def count_daily_users(self):
         """
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
new file mode 100644
index 0000000000..a90990e006
--- /dev/null
+++ b/synapse/storage/client_ips.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import SQLBaseStore, Cache
+
+from twisted.internet import defer
+
+
+# Number of msec of granularity to store the user IP 'last seen' time. Smaller
+# times give more inserts into the database even for readonly API hits
+# 120 seconds == 2 minutes
+LAST_SEEN_GRANULARITY = 120 * 1000
+
+
+class ClientIpStore(SQLBaseStore):
+
+    def __init__(self, hs):
+        self.client_ip_last_seen = Cache(
+            name="client_ip_last_seen",
+            keylen=4,
+        )
+
+        super(ClientIpStore, self).__init__(hs)
+
+    @defer.inlineCallbacks
+    def insert_client_ip(self, user, access_token, ip, user_agent):
+        now = int(self._clock.time_msec())
+        key = (user.to_string(), access_token, ip)
+
+        try:
+            last_seen = self.client_ip_last_seen.get(key)
+        except KeyError:
+            last_seen = None
+
+        # Rate-limited inserts
+        if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
+            defer.returnValue(None)
+
+        self.client_ip_last_seen.prefill(key, now)
+
+        # It's safe not to lock here: a) no unique constraint,
+        # b) LAST_SEEN_GRANULARITY makes concurrent updates incredibly unlikely
+        yield self._simple_upsert(
+            "user_ips",
+            keyvalues={
+                "user_id": user.to_string(),
+                "access_token": access_token,
+                "ip": ip,
+                "user_agent": user_agent,
+            },
+            values={
+                "last_seen": now,
+            },
+            desc="insert_client_ip",
+            lock=False,
+        )
-- 
cgit 1.5.1


From 05e01f21d7012c1853ff566c8a76aa66087bfbd7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 3 Jun 2016 17:12:48 +0100
Subject: Remove event fetching from DB threads

---
 synapse/replication/slave/storage/events.py |   5 -
 synapse/storage/appservice.py               |  21 +++--
 synapse/storage/events.py                   | 138 ----------------------------
 synapse/storage/room.py                     |  46 ++++++----
 synapse/storage/search.py                   |  29 +++---
 synapse/storage/stream.py                   |  34 +++----
 tests/storage/test_appservice.py            |   2 +-
 7 files changed, 75 insertions(+), 200 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index cbc1ae4190..877c68508c 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -131,15 +131,10 @@ class SlavedEventStore(BaseSlavedStore):
     _get_events_from_cache = DataStore._get_events_from_cache.__func__
 
     _invalidate_get_event_cache = DataStore._invalidate_get_event_cache.__func__
-    _parse_events_txn = DataStore._parse_events_txn.__func__
-    _get_events_txn = DataStore._get_events_txn.__func__
-    _get_event_txn = DataStore._get_event_txn.__func__
     _enqueue_events = DataStore._enqueue_events.__func__
     _do_fetch = DataStore._do_fetch.__func__
-    _fetch_events_txn = DataStore._fetch_events_txn.__func__
     _fetch_event_rows = DataStore._fetch_event_rows.__func__
     _get_event_from_row = DataStore._get_event_from_row.__func__
-    _get_event_from_row_txn = DataStore._get_event_from_row_txn.__func__
     _get_rooms_for_user_where_membership_is_txn = (
         DataStore._get_rooms_for_user_where_membership_is_txn.__func__
     )
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index feb9d228ae..ffb7d4a25b 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -298,6 +298,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
             dict(txn_id=txn_id, as_id=service.id)
         )
 
+    @defer.inlineCallbacks
     def get_oldest_unsent_txn(self, service):
         """Get the oldest transaction which has not been sent for this
         service.
@@ -308,12 +309,23 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
             A Deferred which resolves to an AppServiceTransaction or
             None.
         """
-        return self.runInteraction(
+        entry = yield self.runInteraction(
             "get_oldest_unsent_appservice_txn",
             self._get_oldest_unsent_txn,
             service
         )
 
+        if not entry:
+            defer.returnValue(None)
+
+        event_ids = json.loads(entry["event_ids"])
+
+        events = yield self.get_events(event_ids)
+
+        defer.returnValue(AppServiceTransaction(
+            service=service, id=entry["txn_id"], events=events
+        ))
+
     def _get_oldest_unsent_txn(self, txn, service):
         # Monotonically increasing txn ids, so just select the smallest
         # one in the txns table (we delete them when they are sent)
@@ -328,12 +340,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
 
         entry = rows[0]
 
-        event_ids = json.loads(entry["event_ids"])
-        events = self._get_events_txn(txn, event_ids)
-
-        return AppServiceTransaction(
-            service=service, id=entry["txn_id"], events=events
-        )
+        return entry
 
     def _get_last_txn(self, txn, service_id):
         txn.execute(
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 2b3f79577b..b710505a7e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -762,41 +762,6 @@ class EventsStore(SQLBaseStore):
             if e_id in event_map and event_map[e_id]
         ])
 
-    def _get_events_txn(self, txn, event_ids, check_redacted=True,
-                        get_prev_content=False, allow_rejected=False):
-        if not event_ids:
-            return []
-
-        event_map = self._get_events_from_cache(
-            event_ids,
-            check_redacted=check_redacted,
-            get_prev_content=get_prev_content,
-            allow_rejected=allow_rejected,
-        )
-
-        missing_events_ids = [e for e in event_ids if e not in event_map]
-
-        if not missing_events_ids:
-            return [
-                event_map[e_id] for e_id in event_ids
-                if e_id in event_map and event_map[e_id]
-            ]
-
-        missing_events = self._fetch_events_txn(
-            txn,
-            missing_events_ids,
-            check_redacted=check_redacted,
-            get_prev_content=get_prev_content,
-            allow_rejected=allow_rejected,
-        )
-
-        event_map.update(missing_events)
-
-        return [
-            event_map[e_id] for e_id in event_ids
-            if e_id in event_map and event_map[e_id]
-        ]
-
     def _invalidate_get_event_cache(self, event_id):
         for check_redacted in (False, True):
             for get_prev_content in (False, True):
@@ -804,18 +769,6 @@ class EventsStore(SQLBaseStore):
                     (event_id, check_redacted, get_prev_content)
                 )
 
-    def _get_event_txn(self, txn, event_id, check_redacted=True,
-                       get_prev_content=False, allow_rejected=False):
-
-        events = self._get_events_txn(
-            txn, [event_id],
-            check_redacted=check_redacted,
-            get_prev_content=get_prev_content,
-            allow_rejected=allow_rejected,
-        )
-
-        return events[0] if events else None
-
     def _get_events_from_cache(self, events, check_redacted, get_prev_content,
                                allow_rejected):
         event_map = {}
@@ -981,34 +934,6 @@ class EventsStore(SQLBaseStore):
 
         return rows
 
-    def _fetch_events_txn(self, txn, events, check_redacted=True,
-                          get_prev_content=False, allow_rejected=False):
-        if not events:
-            return {}
-
-        rows = self._fetch_event_rows(
-            txn, events,
-        )
-
-        if not allow_rejected:
-            rows[:] = [r for r in rows if not r["rejects"]]
-
-        res = [
-            self._get_event_from_row_txn(
-                txn,
-                row["internal_metadata"], row["json"], row["redacts"],
-                check_redacted=check_redacted,
-                get_prev_content=get_prev_content,
-                rejected_reason=row["rejects"],
-            )
-            for row in rows
-        ]
-
-        return {
-            r.event_id: r
-            for r in res
-        }
-
     @defer.inlineCallbacks
     def _get_event_from_row(self, internal_metadata, js, redacted,
                             check_redacted=True, get_prev_content=False,
@@ -1070,69 +995,6 @@ class EventsStore(SQLBaseStore):
 
         defer.returnValue(ev)
 
-    def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted,
-                                check_redacted=True, get_prev_content=False,
-                                rejected_reason=None):
-        d = json.loads(js)
-        internal_metadata = json.loads(internal_metadata)
-
-        if rejected_reason:
-            rejected_reason = self._simple_select_one_onecol_txn(
-                txn,
-                table="rejections",
-                keyvalues={"event_id": rejected_reason},
-                retcol="reason",
-            )
-
-        ev = FrozenEvent(
-            d,
-            internal_metadata_dict=internal_metadata,
-            rejected_reason=rejected_reason,
-        )
-
-        if check_redacted and redacted:
-            ev = prune_event(ev)
-
-            redaction_id = self._simple_select_one_onecol_txn(
-                txn,
-                table="redactions",
-                keyvalues={"redacts": ev.event_id},
-                retcol="event_id",
-            )
-
-            ev.unsigned["redacted_by"] = redaction_id
-            # Get the redaction event.
-
-            because = self._get_event_txn(
-                txn,
-                redaction_id,
-                check_redacted=False
-            )
-
-            if because:
-                ev.unsigned["redacted_because"] = because
-
-        if get_prev_content and "replaces_state" in ev.unsigned:
-            prev = self._get_event_txn(
-                txn,
-                ev.unsigned["replaces_state"],
-                get_prev_content=False,
-            )
-            if prev:
-                ev.unsigned["prev_content"] = prev.content
-                ev.unsigned["prev_sender"] = prev.sender
-
-        self._get_event_cache.prefill(
-            (ev.event_id, check_redacted, get_prev_content), ev
-        )
-
-        return ev
-
-    def _parse_events_txn(self, txn, rows):
-        event_ids = [r["event_id"] for r in rows]
-
-        return self._get_events_txn(txn, event_ids)
-
     @defer.inlineCallbacks
     def count_daily_messages(self):
         """
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 26933e593a..97f9f1929c 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -194,32 +194,44 @@ class RoomStore(SQLBaseStore):
 
     @cachedInlineCallbacks()
     def get_room_name_and_aliases(self, room_id):
-        def f(txn):
+        def get_room_name(txn):
             sql = (
-                "SELECT event_id FROM current_state_events "
-                "WHERE room_id = ? "
+                "SELECT name FROM room_names"
+                " INNER JOIN current_state_events USING (room_id, event_id)"
+                " WHERE room_id = ?"
+                " LIMIT 1"
             )
 
-            sql += " AND ((type = 'm.room.name' AND state_key = '')"
-            sql += " OR type = 'm.room.aliases')"
-
             txn.execute(sql, (room_id,))
-            results = self.cursor_to_dict(txn)
+            rows = txn.fetchall()
+            if rows:
+                return rows[0][0]
+            else:
+                return None
 
-            return self._parse_events_txn(txn, results)
+            return [row[0] for row in txn.fetchall()]
 
-        events = yield self.runInteraction("get_room_name_and_aliases", f)
+        def get_room_aliases(txn):
+            sql = (
+                "SELECT content FROM current_state_events"
+                " INNER JOIN events USING (room_id, event_id)"
+                " WHERE room_id = ?"
+            )
+            txn.execute(sql, (room_id,))
+            return [row[0] for row in txn.fetchall()]
+
+        name = yield self.runInteraction("get_room_name", get_room_name)
+        alias_contents = yield self.runInteraction("get_room_aliases", get_room_aliases)
 
-        name = None
         aliases = []
 
-        for e in events:
-            if e.type == 'm.room.name':
-                if 'name' in e.content:
-                    name = e.content['name']
-            elif e.type == 'm.room.aliases':
-                if 'aliases' in e.content:
-                    aliases.extend(e.content['aliases'])
+        for c in alias_contents:
+            try:
+                content = json.loads(c)
+            except:
+                continue
+
+            aliases.extend(content.get('aliases', []))
 
         defer.returnValue((name, aliases))
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 0224299625..12941d1775 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -21,6 +21,7 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
 import logging
 import re
+import ujson as json
 
 
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class SearchStore(BackgroundUpdateStore):
 
         def reindex_search_txn(txn):
             sql = (
-                "SELECT stream_ordering, event_id FROM events"
+                "SELECT stream_ordering, event_id, room_id, type, content FROM events"
                 " WHERE ? <= stream_ordering AND stream_ordering < ?"
                 " AND (%s)"
                 " ORDER BY stream_ordering DESC"
@@ -61,28 +62,30 @@ class SearchStore(BackgroundUpdateStore):
 
             txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
 
-            rows = txn.fetchall()
+            rows = self.cursor_to_dict(txn)
             if not rows:
                 return 0
 
-            min_stream_id = rows[-1][0]
-            event_ids = [row[1] for row in rows]
-
-            events = self._get_events_txn(txn, event_ids)
+            min_stream_id = rows[-1]["stream_ordering"]
 
             event_search_rows = []
-            for event in events:
+            for row in rows:
                 try:
-                    event_id = event.event_id
-                    room_id = event.room_id
-                    content = event.content
-                    if event.type == "m.room.message":
+                    event_id = row["event_id"]
+                    room_id = row["room_id"]
+                    etype = row["type"]
+                    try:
+                        content = json.loads(row["content"])
+                    except:
+                        continue
+
+                    if etype == "m.room.message":
                         key = "content.body"
                         value = content["body"]
-                    elif event.type == "m.room.topic":
+                    elif etype == "m.room.topic":
                         key = "content.topic"
                         value = content["topic"]
-                    elif event.type == "m.room.name":
+                    elif etype == "m.room.name":
                         key = "content.name"
                         value = content["name"]
                 except (KeyError, AttributeError):
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 95b12559a6..b9ad965fd6 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -132,29 +132,25 @@ class StreamStore(SQLBaseStore):
                         return True
                 return False
 
-            ret = self._get_events_txn(
-                txn,
-                # apply the filter on the room id list
-                [
-                    r["event_id"] for r in rows
-                    if app_service_interested(r)
-                ],
-                get_prev_content=True
-            )
+            return [r for r in rows if app_service_interested(r)]
 
-            self._set_before_and_after(ret, rows)
+        rows = yield self.runInteraction("get_appservice_room_stream", f)
 
-            if rows:
-                key = "s%d" % max(r["stream_ordering"] for r in rows)
-            else:
-                # Assume we didn't get anything because there was nothing to
-                # get.
-                key = to_key
+        ret = yield self._get_events(
+            [r["event_id"] for r in rows],
+            get_prev_content=True
+        )
 
-            return ret, key
+        self._set_before_and_after(ret, rows, topo_order=from_id is None)
 
-        results = yield self.runInteraction("get_appservice_room_stream", f)
-        defer.returnValue(results)
+        if rows:
+            key = "s%d" % max(r["stream_ordering"] for r in rows)
+        else:
+            # Assume we didn't get anything because there was nothing to
+            # get.
+            key = to_key
+
+        defer.returnValue((ret, key))
 
     @defer.inlineCallbacks
     def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0,
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 5734198121..f44c4870e3 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -357,7 +357,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
         other_events = [Mock(event_id="e5"), Mock(event_id="e6")]
 
         # we aren't testing store._base stuff here, so mock this out
-        self.store._get_events_txn = Mock(return_value=events)
+        self.store.get_events = Mock(return_value=events)
 
         yield self._insert_txn(self.as_list[1]["id"], 9, other_events)
         yield self._insert_txn(service.id, 10, events)
-- 
cgit 1.5.1


From 10ea3f46ba3eda2f7c220a5e5902b687feb3042c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 3 Jun 2016 17:55:32 +0100
Subject: Change the way we cache events

---
 synapse/storage/events.py | 80 ++++++++++++++++++++++++-----------------------
 1 file changed, 41 insertions(+), 39 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index b710505a7e..779743b8f9 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -139,6 +139,9 @@ class _EventPeristenceQueue(object):
             pass
 
 
+_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
+
+
 class EventsStore(SQLBaseStore):
     EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
 
@@ -741,7 +744,6 @@ class EventsStore(SQLBaseStore):
         event_map = self._get_events_from_cache(
             event_ids,
             check_redacted=check_redacted,
-            get_prev_content=get_prev_content,
             allow_rejected=allow_rejected,
         )
 
@@ -751,40 +753,49 @@ class EventsStore(SQLBaseStore):
             missing_events = yield self._enqueue_events(
                 missing_events_ids,
                 check_redacted=check_redacted,
-                get_prev_content=get_prev_content,
                 allow_rejected=allow_rejected,
             )
 
             event_map.update(missing_events)
 
-        defer.returnValue([
+        events = [
             event_map[e_id] for e_id in event_id_list
             if e_id in event_map and event_map[e_id]
-        ])
+        ]
+
+        if get_prev_content:
+            for event in events:
+                if "replaces_state" in event.unsigned:
+                    prev = yield self.get_event(
+                        event.unsigned["replaces_state"],
+                        get_prev_content=False,
+                        allow_none=True,
+                    )
+                    if prev:
+                        event.unsigned = dict(event.unsigned)
+                        event.unsigned["prev_content"] = prev.content
+                        event.unsigned["prev_sender"] = prev.sender
+
+        defer.returnValue(events)
 
     def _invalidate_get_event_cache(self, event_id):
-        for check_redacted in (False, True):
-            for get_prev_content in (False, True):
-                self._get_event_cache.invalidate(
-                    (event_id, check_redacted, get_prev_content)
-                )
+            self._get_event_cache.invalidate((event_id,))
 
-    def _get_events_from_cache(self, events, check_redacted, get_prev_content,
-                               allow_rejected):
+    def _get_events_from_cache(self, events, check_redacted, allow_rejected):
         event_map = {}
 
         for event_id in events:
-            try:
-                ret = self._get_event_cache.get(
-                    (event_id, check_redacted, get_prev_content,)
-                )
+            ret = self._get_event_cache.get((event_id,), None)
+            if not ret:
+                continue
 
-                if allow_rejected or not ret.rejected_reason:
-                    event_map[event_id] = ret
+            if allow_rejected or not ret.event.rejected_reason:
+                if check_redacted and ret.redacted_event:
+                    event_map[event_id] = ret.redacted_event
                 else:
-                    event_map[event_id] = None
-            except KeyError:
-                pass
+                    event_map[event_id] = ret.event
+            else:
+                event_map[event_id] = None
 
         return event_map
 
@@ -855,8 +866,7 @@ class EventsStore(SQLBaseStore):
                         reactor.callFromThread(fire, event_list)
 
     @defer.inlineCallbacks
-    def _enqueue_events(self, events, check_redacted=True,
-                        get_prev_content=False, allow_rejected=False):
+    def _enqueue_events(self, events, check_redacted=True, allow_rejected=False):
         """Fetches events from the database using the _event_fetch_list. This
         allows batch and bulk fetching of events - it allows us to fetch events
         without having to create a new transaction for each request for events.
@@ -895,7 +905,6 @@ class EventsStore(SQLBaseStore):
                 preserve_fn(self._get_event_from_row)(
                     row["internal_metadata"], row["json"], row["redacts"],
                     check_redacted=check_redacted,
-                    get_prev_content=get_prev_content,
                     rejected_reason=row["rejects"],
                 )
                 for row in rows
@@ -936,8 +945,7 @@ class EventsStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def _get_event_from_row(self, internal_metadata, js, redacted,
-                            check_redacted=True, get_prev_content=False,
-                            rejected_reason=None):
+                            check_redacted=True, rejected_reason=None):
         d = json.loads(js)
         internal_metadata = json.loads(internal_metadata)
 
@@ -949,14 +957,17 @@ class EventsStore(SQLBaseStore):
                 desc="_get_event_from_row",
             )
 
-        ev = FrozenEvent(
+        original_ev = FrozenEvent(
             d,
             internal_metadata_dict=internal_metadata,
             rejected_reason=rejected_reason,
         )
 
+        ev = original_ev
+        redacted_event = None
         if check_redacted and redacted:
             ev = prune_event(ev)
+            redacted_event = ev
 
             redaction_id = yield self._simple_select_one_onecol(
                 table="redactions",
@@ -979,19 +990,10 @@ class EventsStore(SQLBaseStore):
                 # will serialise this field correctly
                 ev.unsigned["redacted_because"] = because
 
-        if get_prev_content and "replaces_state" in ev.unsigned:
-            prev = yield self.get_event(
-                ev.unsigned["replaces_state"],
-                get_prev_content=False,
-                allow_none=True,
-            )
-            if prev:
-                ev.unsigned["prev_content"] = prev.content
-                ev.unsigned["prev_sender"] = prev.sender
-
-        self._get_event_cache.prefill(
-            (ev.event_id, check_redacted, get_prev_content), ev
-        )
+        self._get_event_cache.prefill((ev.event_id,), _EventCacheEntry(
+            event=original_ev,
+            redacted_event=redacted_event,
+        ))
 
         defer.returnValue(ev)
 
-- 
cgit 1.5.1


From cffe46408f40db082df76adc263cf5014031ae54 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 3 Jun 2016 18:25:21 +0100
Subject: Don't rely on options when inserting event into cache

---
 synapse/storage/events.py | 83 ++++++++++++++++++++++++-----------------------
 1 file changed, 43 insertions(+), 40 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 779743b8f9..5db24e86f9 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -741,13 +741,12 @@ class EventsStore(SQLBaseStore):
         event_id_list = event_ids
         event_ids = set(event_ids)
 
-        event_map = self._get_events_from_cache(
+        event_entry_map = self._get_events_from_cache(
             event_ids,
-            check_redacted=check_redacted,
             allow_rejected=allow_rejected,
         )
 
-        missing_events_ids = [e for e in event_ids if e not in event_map]
+        missing_events_ids = [e for e in event_ids if e not in event_entry_map]
 
         if missing_events_ids:
             missing_events = yield self._enqueue_events(
@@ -756,32 +755,40 @@ class EventsStore(SQLBaseStore):
                 allow_rejected=allow_rejected,
             )
 
-            event_map.update(missing_events)
+            event_entry_map.update(missing_events)
 
-        events = [
-            event_map[e_id] for e_id in event_id_list
-            if e_id in event_map and event_map[e_id]
-        ]
+        events = []
+        for event_id in event_id_list:
+            entry = event_entry_map.get(event_id, None)
+            if not entry:
+                continue
 
-        if get_prev_content:
-            for event in events:
-                if "replaces_state" in event.unsigned:
-                    prev = yield self.get_event(
-                        event.unsigned["replaces_state"],
-                        get_prev_content=False,
-                        allow_none=True,
-                    )
-                    if prev:
-                        event.unsigned = dict(event.unsigned)
-                        event.unsigned["prev_content"] = prev.content
-                        event.unsigned["prev_sender"] = prev.sender
+            if allow_rejected or not entry.event.rejected_reason:
+                if check_redacted and entry.redacted_event:
+                    event = entry.redacted_event
+                else:
+                    event = entry.event
+
+                events.append(event)
+
+                if get_prev_content:
+                    if "replaces_state" in event.unsigned:
+                        prev = yield self.get_event(
+                            event.unsigned["replaces_state"],
+                            get_prev_content=False,
+                            allow_none=True,
+                        )
+                        if prev:
+                            event.unsigned = dict(event.unsigned)
+                            event.unsigned["prev_content"] = prev.content
+                            event.unsigned["prev_sender"] = prev.sender
 
         defer.returnValue(events)
 
     def _invalidate_get_event_cache(self, event_id):
             self._get_event_cache.invalidate((event_id,))
 
-    def _get_events_from_cache(self, events, check_redacted, allow_rejected):
+    def _get_events_from_cache(self, events, allow_rejected):
         event_map = {}
 
         for event_id in events:
@@ -790,10 +797,7 @@ class EventsStore(SQLBaseStore):
                 continue
 
             if allow_rejected or not ret.event.rejected_reason:
-                if check_redacted and ret.redacted_event:
-                    event_map[event_id] = ret.redacted_event
-                else:
-                    event_map[event_id] = ret.event
+                event_map[event_id] = ret
             else:
                 event_map[event_id] = None
 
@@ -904,7 +908,6 @@ class EventsStore(SQLBaseStore):
             [
                 preserve_fn(self._get_event_from_row)(
                     row["internal_metadata"], row["json"], row["redacts"],
-                    check_redacted=check_redacted,
                     rejected_reason=row["rejects"],
                 )
                 for row in rows
@@ -913,7 +916,7 @@ class EventsStore(SQLBaseStore):
         )
 
         defer.returnValue({
-            e.event_id: e
+            e.event.event_id: e
             for e in res if e
         })
 
@@ -945,7 +948,7 @@ class EventsStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def _get_event_from_row(self, internal_metadata, js, redacted,
-                            check_redacted=True, rejected_reason=None):
+                            rejected_reason=None):
         d = json.loads(js)
         internal_metadata = json.loads(internal_metadata)
 
@@ -954,7 +957,7 @@ class EventsStore(SQLBaseStore):
                 table="rejections",
                 keyvalues={"event_id": rejected_reason},
                 retcol="reason",
-                desc="_get_event_from_row",
+                desc="_get_event_from_row_rejected_reason",
             )
 
         original_ev = FrozenEvent(
@@ -963,20 +966,18 @@ class EventsStore(SQLBaseStore):
             rejected_reason=rejected_reason,
         )
 
-        ev = original_ev
         redacted_event = None
-        if check_redacted and redacted:
-            ev = prune_event(ev)
-            redacted_event = ev
+        if redacted:
+            redacted_event = prune_event(original_ev)
 
             redaction_id = yield self._simple_select_one_onecol(
                 table="redactions",
-                keyvalues={"redacts": ev.event_id},
+                keyvalues={"redacts": redacted_event.event_id},
                 retcol="event_id",
-                desc="_get_event_from_row",
+                desc="_get_event_from_row_redactions",
             )
 
-            ev.unsigned["redacted_by"] = redaction_id
+            redacted_event.unsigned["redacted_by"] = redaction_id
             # Get the redaction event.
 
             because = yield self.get_event(
@@ -988,14 +989,16 @@ class EventsStore(SQLBaseStore):
             if because:
                 # It's fine to do add the event directly, since get_pdu_json
                 # will serialise this field correctly
-                ev.unsigned["redacted_because"] = because
+                redacted_event.unsigned["redacted_because"] = because
 
-        self._get_event_cache.prefill((ev.event_id,), _EventCacheEntry(
+        cache_entry = _EventCacheEntry(
             event=original_ev,
             redacted_event=redacted_event,
-        ))
+        )
+
+        self._get_event_cache.prefill((original_ev.event_id,), cache_entry)
 
-        defer.returnValue(ev)
+        defer.returnValue(cache_entry)
 
     @defer.inlineCallbacks
     def count_daily_messages(self):
-- 
cgit 1.5.1


From 70aee0717c22acf7eabb5f158cbaf527137bc90e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 6 Jun 2016 11:08:12 +0100
Subject: Add events to cache when we persist them

---
 synapse/storage/events.py | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5db24e86f9..16398dc0a8 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -635,6 +635,8 @@ class EventsStore(SQLBaseStore):
             ],
         )
 
+        self._add_to_cache(txn, events_and_contexts)
+
         if backfilled:
             # Backfilled events come before the current state so we don't need
             # to update the current state table
@@ -676,6 +678,45 @@ class EventsStore(SQLBaseStore):
 
         return
 
+    def _add_to_cache(self, txn, events_and_contexts):
+        to_prefill = []
+
+        rows = []
+        N = 200
+        for i in range(0, len(events_and_contexts), N):
+            ev_map = {
+                e[0].event_id: e[0]
+                for e in events_and_contexts[i:i + N]
+            }
+            if not ev_map:
+                break
+
+            sql = (
+                "SELECT "
+                " e.event_id as event_id, "
+                " r.redacts as redacts,"
+                " rej.event_id as rejects "
+                " FROM events as e"
+                " LEFT JOIN rejections as rej USING (event_id)"
+                " LEFT JOIN redactions as r ON e.event_id = r.redacts"
+                " WHERE e.event_id IN (%s)"
+            ) % (",".join(["?"] * len(ev_map)),)
+
+            txn.execute(sql, ev_map.keys())
+            rows = self.cursor_to_dict(txn)
+            for row in rows:
+                event = ev_map[row["event_id"]]
+                if not row["rejects"] and not row["redacts"]:
+                    to_prefill.append(_EventCacheEntry(
+                        event=event,
+                        redacted_event=None,
+                    ))
+
+        def prefill():
+            for cache_entry in to_prefill:
+                self._get_event_cache.prefill((cache_entry[0].event_id,), cache_entry)
+        txn.call_after(prefill)
+
     def _store_redaction(self, txn, event):
         # invalidate the cache for the redacted event
         txn.call_after(self._invalidate_get_event_cache, event.redacts)
-- 
cgit 1.5.1


From 7aa778fba9bb81087c3a1029e0a0d4ff55b1a065 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 6 Jun 2016 11:58:09 +0100
Subject: Add metric counter for number of persisted events

---
 synapse/storage/events.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5db24e86f9..ff4f742f6a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -27,6 +27,9 @@ from synapse.api.constants import EventTypes
 from canonicaljson import encode_canonical_json
 from collections import deque, namedtuple
 
+import synapse
+import synapse.metrics
+
 
 import logging
 import math
@@ -35,6 +38,10 @@ import ujson as json
 logger = logging.getLogger(__name__)
 
 
+metrics = synapse.metrics.get_metrics_for(__name__)
+persist_event_counter = metrics.register_counter("persisted_events")
+
+
 def encode_json(json_object):
     if USE_FROZEN_DICTS:
         # ujson doesn't like frozen_dicts
@@ -261,6 +268,7 @@ class EventsStore(SQLBaseStore):
                         events_and_contexts=chunk,
                         backfilled=backfilled,
                     )
+                    persist_event_counter.inc_by(len(chunk))
 
     @defer.inlineCallbacks
     @log_function
@@ -278,6 +286,7 @@ class EventsStore(SQLBaseStore):
                         current_state=current_state,
                         backfilled=backfilled,
                     )
+                    persist_event_counter.inc()
         except _RollbackButIsFineException:
             pass
 
-- 
cgit 1.5.1


From 310197bab5cf8ed2c26fae522f15f092dbcdff58 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Jun 2016 09:34:50 +0100
Subject: Fix AS retries

---
 synapse/storage/appservice.py    | 4 ++--
 tests/storage/test_appservice.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index ffb7d4a25b..a281571637 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -320,10 +320,10 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
 
         event_ids = json.loads(entry["event_ids"])
 
-        events = yield self.get_events(event_ids)
+        event_map = yield self.get_events(event_ids)
 
         defer.returnValue(AppServiceTransaction(
-            service=service, id=entry["txn_id"], events=events
+            service=service, id=entry["txn_id"], events=event_map.values()
         ))
 
     def _get_oldest_unsent_txn(self, txn, service):
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index f44c4870e3..6db4b966db 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -353,21 +353,21 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_get_oldest_unsent_txn(self):
         service = Mock(id=self.as_list[0]["id"])
-        events = [Mock(event_id="e1"), Mock(event_id="e2")]
+        events = {"e1": Mock(event_id="e1"), "e2": Mock(event_id="e2")}
         other_events = [Mock(event_id="e5"), Mock(event_id="e6")]
 
         # we aren't testing store._base stuff here, so mock this out
         self.store.get_events = Mock(return_value=events)
 
         yield self._insert_txn(self.as_list[1]["id"], 9, other_events)
-        yield self._insert_txn(service.id, 10, events)
+        yield self._insert_txn(service.id, 10, events.values())
         yield self._insert_txn(service.id, 11, other_events)
         yield self._insert_txn(service.id, 12, other_events)
 
         txn = yield self.store.get_oldest_unsent_txn(service)
         self.assertEquals(service, txn.service)
         self.assertEquals(10, txn.id)
-        self.assertEquals(events, txn.events)
+        self.assertEquals(events.values(), txn.events)
 
     @defer.inlineCallbacks
     def test_get_appservices_by_state_single(self):
-- 
cgit 1.5.1


From 84379062f9ec259abc302af321d4ed8f5a958c01 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Jun 2016 10:24:50 +0100
Subject: Fix AS retries, but with correct ordering

---
 synapse/storage/appservice.py    | 4 ++--
 tests/storage/test_appservice.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index a281571637..d1ee533fac 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -320,10 +320,10 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
 
         event_ids = json.loads(entry["event_ids"])
 
-        event_map = yield self.get_events(event_ids)
+        events = yield self._get_events(event_ids)
 
         defer.returnValue(AppServiceTransaction(
-            service=service, id=entry["txn_id"], events=event_map.values()
+            service=service, id=entry["txn_id"], events=events
         ))
 
     def _get_oldest_unsent_txn(self, txn, service):
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 6db4b966db..3e2862daae 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -353,21 +353,21 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_get_oldest_unsent_txn(self):
         service = Mock(id=self.as_list[0]["id"])
-        events = {"e1": Mock(event_id="e1"), "e2": Mock(event_id="e2")}
+        events = [Mock(event_id="e1"), Mock(event_id="e2")]
         other_events = [Mock(event_id="e5"), Mock(event_id="e6")]
 
         # we aren't testing store._base stuff here, so mock this out
-        self.store.get_events = Mock(return_value=events)
+        self.store._get_events = Mock(return_value=events)
 
         yield self._insert_txn(self.as_list[1]["id"], 9, other_events)
-        yield self._insert_txn(service.id, 10, events.values())
+        yield self._insert_txn(service.id, 10, events)
         yield self._insert_txn(service.id, 11, other_events)
         yield self._insert_txn(service.id, 12, other_events)
 
         txn = yield self.store.get_oldest_unsent_txn(service)
         self.assertEquals(service, txn.service)
         self.assertEquals(10, txn.id)
-        self.assertEquals(events.values(), txn.events)
+        self.assertEquals(events, txn.events)
 
     @defer.inlineCallbacks
     def test_get_appservices_by_state_single(self):
-- 
cgit 1.5.1


From 0b2158719c43eab87ab7a9448ae1d85008b92b92 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 7 Jun 2016 15:07:11 +0100
Subject: Remove dead code.

Loading push rules now happens in the datastore, so we can remove
the methods that loaded them outside the datastore.

The ``waiting_for_join_list`` in federation handler is populated by
anything, so can be removed.

The ``_get_members_events_txn`` method isn't called from anywhere
so can be removed.
---
 synapse/handlers/federation.py           | 13 -------------
 synapse/push/bulk_push_rule_evaluator.py |  8 --------
 synapse/push/clientformat.py             | 26 --------------------------
 synapse/storage/roommember.py            |  7 -------
 4 files changed, 54 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 648a505e65..ff83c608e7 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -66,10 +66,6 @@ class FederationHandler(BaseHandler):
 
         self.hs = hs
 
-        self.distributor.observe("user_joined_room", self.user_joined_room)
-
-        self.waiting_for_join_list = {}
-
         self.store = hs.get_datastore()
         self.replication_layer = hs.get_replication_layer()
         self.state_handler = hs.get_state_handler()
@@ -1091,15 +1087,6 @@ class FederationHandler(BaseHandler):
     def get_min_depth_for_context(self, context):
         return self.store.get_min_depth(context)
 
-    @log_function
-    def user_joined_room(self, user, room_id):
-        waiters = self.waiting_for_join_list.get(
-            (user.to_string(), room_id),
-            []
-        )
-        while waiters:
-            waiters.pop().callback(None)
-
     @defer.inlineCallbacks
     @log_function
     def _handle_new_event(self, origin, event, state=None, auth_events=None,
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 6e42121b1d..756e5da513 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import logging
-import ujson as json
 
 from twisted.internet import defer
 
@@ -27,13 +26,6 @@ from synapse.visibility import filter_events_for_clients
 logger = logging.getLogger(__name__)
 
 
-def decode_rule_json(rule):
-    rule = dict(rule)
-    rule['conditions'] = json.loads(rule['conditions'])
-    rule['actions'] = json.loads(rule['actions'])
-    return rule
-
-
 @defer.inlineCallbacks
 def _get_rules(room_id, user_ids, store):
     rules_by_user = yield store.bulk_get_push_rules(user_ids)
diff --git a/synapse/push/clientformat.py b/synapse/push/clientformat.py
index b3983f7940..e0331b2d2d 100644
--- a/synapse/push/clientformat.py
+++ b/synapse/push/clientformat.py
@@ -13,37 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.push.baserules import list_with_base_rules
-
 from synapse.push.rulekinds import (
     PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP
 )
 
 import copy
-import simplejson as json
-
-
-def load_rules_for_user(user, rawrules, enabled_map):
-    ruleslist = []
-    for rawrule in rawrules:
-        rule = dict(rawrule)
-        rule["conditions"] = json.loads(rawrule["conditions"])
-        rule["actions"] = json.loads(rawrule["actions"])
-        ruleslist.append(rule)
-
-    # We're going to be mutating this a lot, so do a deep copy
-    rules = list(list_with_base_rules(ruleslist))
-
-    for i, rule in enumerate(rules):
-        rule_id = rule['rule_id']
-        if rule_id in enabled_map:
-            if rule.get('enabled', True) != bool(enabled_map[rule_id]):
-                # Rules are cached across users.
-                rule = dict(rule)
-                rule['enabled'] = bool(enabled_map[rule_id])
-                rules[i] = rule
-
-    return rules
 
 
 def format_push_rules_for_user(user, ruleslist):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 64b4bd371b..8bd693be72 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -243,13 +243,6 @@ class RoomMemberStore(SQLBaseStore):
         user_ids = yield self.get_users_in_room(room_id)
         defer.returnValue(set(get_domain_from_id(uid) for uid in user_ids))
 
-    def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None):
-        rows = self._get_members_rows_txn(
-            txn,
-            room_id, membership, user_id,
-        )
-        return [r["event_id"] for r in rows]
-
     def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None):
         where_clause = "c.room_id = ?"
         where_values = [room_id]
-- 
cgit 1.5.1