Merge pull request #456 from matrix-org/store_event_actions

Send unread notification counts

16 files changed, 515 insertions, 35 deletions

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index b474042e84..66e35de6e4 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -19,6 +19,7 @@ from synapse.api.errors import LimitExceededError, SynapseError, AuthError
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.api.constants import Membership, EventTypes
 from synapse.types import UserID, RoomAlias
+from synapse.push.action_generator import ActionGenerator
 
 from synapse.util.logcontext import PreserveLoggingContext
 
@@ -252,6 +253,11 @@ class BaseHandler(object):
             event, context=context
         )
 
+        action_generator = ActionGenerator(self.store)
+        yield action_generator.handle_push_actions_for_event(
+            event, self
+        )
+
         destinations = set(extra_destinations)
         for k, s in context.current_state.items():
             try:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index c1704afc53..5dee24e21d 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -36,6 +36,8 @@ from synapse.events.utils import prune_event
 
 from synapse.util.retryutils import NotRetryingDestination
 
+from synapse.push.action_generator import ActionGenerator
+
 from twisted.internet import defer
 
 import itertools
@@ -242,6 +244,12 @@ class FederationHandler(BaseHandler):
                     user = UserID.from_string(event.state_key)
                     yield user_joined_room(self.distributor, user, event.room_id)
 
+        if not backfilled and not event.internal_metadata.is_outlier():
+            action_generator = ActionGenerator(self.store)
+            yield action_generator.handle_push_actions_for_event(
+                event, self
+            )
+
     @defer.inlineCallbacks
     def _filter_events_for_server(self, server_name, room_id, events):
         event_to_state = yield self.store.get_state_for_events(
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 6f111ff63e..ba26d13d49 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -84,7 +84,8 @@ class RegistrationHandler(BaseHandler):
         localpart=None,
         password=None,
         generate_token=True,
-        guest_access_token=None
+        guest_access_token=None,
+        make_guest=False
     ):
         """Registers a new client on the server.
 
@@ -118,6 +119,7 @@ class RegistrationHandler(BaseHandler):
                 token=token,
                 password_hash=password_hash,
                 was_guest=guest_access_token is not None,
+                make_guest=make_guest,
             )
 
             yield registered_user(self.distributor, user)
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 3bc18b4338..27fdbe28ee 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -54,6 +54,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
     "state",             # dict[(str, str), FrozenEvent]
     "ephemeral",
     "account_data",
+    "unread_notification_count",
 ])):
     __slots__ = []
 
@@ -66,6 +67,8 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
             or self.state
             or self.ephemeral
             or self.account_data
+            # nb the notification count does not, er, count: if there's nothing
+            # else in the result, we don't need to send it.
         )
 
 
@@ -163,6 +166,18 @@ class SyncHandler(BaseHandler):
         else:
             return self.incremental_sync_with_gap(sync_config, since_token)
 
+    def last_read_event_id_for_room_and_user(self, room_id, user_id, ephemeral_by_room):
+        if room_id not in ephemeral_by_room:
+            return None
+        for e in ephemeral_by_room[room_id]:
+            if e['type'] != 'm.receipt':
+                continue
+            for receipt_event_id, val in e['content'].items():
+                if 'm.read' in val:
+                    if user_id in val['m.read']:
+                        return receipt_event_id
+        return None
+
     @defer.inlineCallbacks
     def full_state_sync(self, sync_config, timeline_since_token):
         """Get a sync for a client which is starting without any state.
@@ -274,6 +289,13 @@ class SyncHandler(BaseHandler):
             room_id, sync_config, now_token, since_token=timeline_since_token
         )
 
+        notifs = yield self.unread_notifs_for_room_id(
+            room_id, sync_config, ephemeral_by_room
+        )
+        notif_count = None
+        if notifs is not None:
+            notif_count = len(notifs)
+
         current_state = yield self.get_state_at(room_id, now_token)
 
         defer.returnValue(JoinedSyncResult(
@@ -284,6 +306,7 @@ class SyncHandler(BaseHandler):
             account_data=self.account_data_for_room(
                 room_id, tags_by_room, account_data_by_room
             ),
+            unread_notification_count=notif_count,
         ))
 
     def account_data_for_user(self, account_data):
@@ -423,6 +446,13 @@ class SyncHandler(BaseHandler):
         )
         now_token = now_token.copy_and_replace("presence_key", presence_key)
 
+        # We now fetch all ephemeral events for this room in order to get
+        # this users current read receipt. This could almost certainly be
+        # optimised.
+        _, all_ephemeral_by_room = yield self.ephemeral_by_room(
+            sync_config, now_token
+        )
+
         now_token, ephemeral_by_room = yield self.ephemeral_by_room(
             sync_config, now_token, since_token
         )
@@ -496,6 +526,13 @@ class SyncHandler(BaseHandler):
                 else:
                     prev_batch = now_token
 
+                notifs = yield self.unread_notifs_for_room_id(
+                    room_id, sync_config, all_ephemeral_by_room
+                )
+                notif_count = None
+                if notifs is not None:
+                    notif_count = len(notifs)
+
                 just_joined = yield self.check_joined_room(sync_config, state)
                 if just_joined:
                     logger.debug("User has just joined %s: needs full state",
@@ -516,6 +553,7 @@ class SyncHandler(BaseHandler):
                     account_data=self.account_data_for_room(
                         room_id, tags_by_room, account_data_by_room
                     ),
+                    unread_notification_count=notif_count
                 )
                 logger.debug("Result for room %s: %r", room_id, room_sync)
 
@@ -650,6 +688,13 @@ class SyncHandler(BaseHandler):
         if just_joined:
             state = yield self.get_state_at(room_id, now_token)
 
+        notifs = yield self.unread_notifs_for_room_id(
+            room_id, sync_config, ephemeral_by_room
+        )
+        notif_count = None
+        if notifs is not None:
+            notif_count = len(notifs)
+
         room_sync = JoinedSyncResult(
             room_id=room_id,
             timeline=batch,
@@ -658,6 +703,7 @@ class SyncHandler(BaseHandler):
             account_data=self.account_data_for_room(
                 room_id, tags_by_room, account_data_by_room
             ),
+            unread_notification_count=notif_count,
         )
 
         logger.debug("Room sync: %r", room_sync)
@@ -788,3 +834,20 @@ class SyncHandler(BaseHandler):
             if join_event.content["membership"] == Membership.JOIN:
                 return True
         return False
+
+    @defer.inlineCallbacks
+    def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room):
+        last_unread_event_id = self.last_read_event_id_for_room_and_user(
+            room_id, sync_config.user.to_string(), ephemeral_by_room
+        )
+
+        notifs = []
+        if last_unread_event_id:
+            notifs = yield self.store.get_unread_event_push_actions_by_room_for_user(
+                room_id, sync_config.user.to_string(), last_unread_event_id
+            )
+        else:
+            # There is no new information in this period, so your notification
+            # count is whatever it was last time.
+            defer.returnValue(None)
+        defer.returnValue(notifs)
diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 7dc656b7cb..c5ddfb564c 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -27,6 +27,9 @@ import random
 logger = logging.getLogger(__name__)
 
 
+# Pushers could now be moved to pull out of the event_push_actions table instead
+# of listening on the event stream: this would avoid them having to run the
+# rules again.
 class Pusher(object):
     INITIAL_BACKOFF = 1000
     MAX_BACKOFF = 60 * 60 * 1000
@@ -157,21 +160,7 @@ class Pusher(object):
         actions = yield rule_evaluator.actions_for_event(single_event)
         tweaks = rule_evaluator.tweaks_for_actions(actions)
 
-        if len(actions) == 0:
-            logger.warn("Empty actions! Using default action.")
-            actions = Pusher.DEFAULT_ACTIONS
-
-        if 'notify' not in actions and 'dont_notify' not in actions:
-            logger.warn("Neither notify nor dont_notify in actions: adding default")
-            actions.extend(Pusher.DEFAULT_ACTIONS)
-
-        if 'dont_notify' in actions:
-            logger.debug(
-                "%s for %s: dont_notify",
-                single_event['event_id'], self.user_name
-            )
-            processed = True
-        else:
+        if 'notify' in actions:
             rejected = yield self.dispatch_push(single_event, tweaks)
             self.has_unread = True
             if isinstance(rejected, list) or isinstance(rejected, tuple):
@@ -192,6 +181,8 @@ class Pusher(object):
                         yield self.hs.get_pusherpool().remove_pusher(
                             self.app_id, pk, self.user_name
                         )
+        else:
+            processed = True
 
         if not self.alive:
             return
diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py
new file mode 100644
index 0000000000..4cf94f6c61
--- /dev/null
+++ b/synapse/push/action_generator.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import bulk_push_rule_evaluator
+
+import logging
+
+from synapse.api.constants import EventTypes
+
+logger = logging.getLogger(__name__)
+
+
+class ActionGenerator:
+    def __init__(self, store):
+        self.store = store
+        # really we want to get all user ids and all profile tags too,
+        # since we want the actions for each profile tag for every user and
+        # also actions for a client with no profile tag for each user.
+        # Currently the event stream doesn't support profile tags on an
+        # event stream, so we just run the rules for a client with no profile
+        # tag (ie. we just need all the users).
+
+    @defer.inlineCallbacks
+    def handle_push_actions_for_event(self, event, handler):
+        if event.type == EventTypes.Redaction and event.redacts is not None:
+            yield self.store.remove_push_actions_for_event_id(
+                event.room_id, event.redacts
+            )
+
+        bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id(
+            event.room_id, self.store
+        )
+
+        actions_by_user = yield bulk_evaluator.action_for_event_by_user(event, handler)
+
+        yield self.store.set_push_actions_for_event_and_users(
+            event,
+            [
+                (uid, None, actions) for uid, actions in actions_by_user.items()
+            ]
+        )
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
new file mode 100644
index 0000000000..ce244fa959
--- /dev/null
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import simplejson as json
+
+from twisted.internet import defer
+
+from synapse.types import UserID
+
+import baserules
+from push_rule_evaluator import PushRuleEvaluator
+
+from synapse.events.utils import serialize_event
+
+logger = logging.getLogger(__name__)
+
+
+def decode_rule_json(rule):
+    rule['conditions'] = json.loads(rule['conditions'])
+    rule['actions'] = json.loads(rule['actions'])
+    return rule
+
+
+@defer.inlineCallbacks
+def evaluator_for_room_id(room_id, store):
+    users = yield store.get_users_in_room(room_id)
+    rules_by_user = yield store.bulk_get_push_rules(users)
+    rules_by_user = {
+        uid: baserules.list_with_base_rules(
+            [decode_rule_json(rule_list) for rule_list in rules_by_user[uid]]
+            if uid in rules_by_user else [],
+            UserID.from_string(uid),
+        )
+        for uid in users
+    }
+    member_events = yield store.get_current_state(
+        room_id=room_id,
+        event_type='m.room.member',
+    )
+    display_names = {}
+    for ev in member_events:
+        if ev.content.get("displayname"):
+            display_names[ev.state_key] = ev.content.get("displayname")
+
+    defer.returnValue(BulkPushRuleEvaluator(
+        room_id, rules_by_user, display_names, users, store
+    ))
+
+
+class BulkPushRuleEvaluator:
+    """
+    Runs push rules for all users in a room.
+    This is faster than running PushRuleEvaluator for each user because it
+    fetches all the rules for all the users in one (batched) db query
+    rather than doing multiple queries per-user. It currently uses
+    the same logic to run the actual rules, but could be optimised further
+    (see https://matrix.org/jira/browse/SYN-562)
+    """
+    def __init__(self, room_id, rules_by_user, display_names, users_in_room, store):
+        self.room_id = room_id
+        self.rules_by_user = rules_by_user
+        self.display_names = display_names
+        self.users_in_room = users_in_room
+        self.store = store
+
+    @defer.inlineCallbacks
+    def action_for_event_by_user(self, event, handler):
+        actions_by_user = {}
+
+        for uid, rules in self.rules_by_user.items():
+            display_name = None
+            if uid in self.display_names:
+                display_name = self.display_names[uid]
+
+            is_guest = yield self.store.is_guest(UserID.from_string(uid))
+            filtered = yield handler._filter_events_for_client(
+                uid, [event], is_guest=is_guest
+            )
+            if len(filtered) == 0:
+                continue
+
+            for rule in rules:
+                if 'enabled' in rule and not rule['enabled']:
+                    continue
+
+                # XXX: profile tags
+                if BulkPushRuleEvaluator.event_matches_rule(
+                    event, rule,
+                    display_name, len(self.users_in_room), None
+                ):
+                    actions = [x for x in rule['actions'] if x != 'dont_notify']
+                    if len(actions) > 0:
+                        actions_by_user[uid] = actions
+                    break
+        defer.returnValue(actions_by_user)
+
+    @staticmethod
+    def event_matches_rule(event, rule,
+                           display_name, room_member_count, profile_tag):
+        matches = True
+
+        # passing the clock all the way into here is extremely awkward and push
+        # rules do not care about any of the relative timestamps, so we just
+        # pass 0 for the current time.
+        client_event = serialize_event(event, 0)
+
+        for cond in rule['conditions']:
+            matches &= PushRuleEvaluator._event_fulfills_condition(
+                client_event, cond, display_name, room_member_count, profile_tag
+            )
+        return matches
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index dec81566ba..705ab8c967 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -43,7 +43,7 @@ def evaluator_for_user_name_and_profile_tag(user_name, profile_tag, room_id, sto
 
 
 class PushRuleEvaluator:
-    DEFAULT_ACTIONS = ['dont_notify']
+    DEFAULT_ACTIONS = []
     INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$")
 
     def __init__(self, user_name, profile_tag, raw_rules, enabled_map, room_id,
@@ -85,7 +85,7 @@ class PushRuleEvaluator:
         """
         if ev['user_id'] == self.user_name:
             # let's assume you probably know about messages you sent yourself
-            defer.returnValue(['dont_notify'])
+            defer.returnValue([])
 
         room_id = ev['room_id']
 
@@ -113,7 +113,8 @@ class PushRuleEvaluator:
             for c in conditions:
                 matches &= self._event_fulfills_condition(
                     ev, c, display_name=my_display_name,
-                    room_member_count=room_member_count
+                    room_member_count=room_member_count,
+                    profile_tag=self.profile_tag
                 )
             logger.debug(
                 "Rule %s %s",
@@ -131,6 +132,11 @@ class PushRuleEvaluator:
                     "%s matches for user %s, event %s",
                     r['rule_id'], self.user_name, ev['event_id']
                 )
+
+                # filter out dont_notify as we treat an empty actions list
+                # as dont_notify, and this doesn't take up a row in our database
+                actions = [x for x in actions if x != 'dont_notify']
+
                 defer.returnValue(actions)
 
         logger.info(
@@ -151,16 +157,18 @@ class PushRuleEvaluator:
                                           re.sub(r'\\\-', '-', x.group(2)))), r)
         return r
 
-    def _event_fulfills_condition(self, ev, condition, display_name, room_member_count):
+    @staticmethod
+    def _event_fulfills_condition(ev, condition,
+                                  display_name, room_member_count, profile_tag):
         if condition['kind'] == 'event_match':
             if 'pattern' not in condition:
                 logger.warn("event_match condition with no pattern")
                 return False
             # XXX: optimisation: cache our pattern regexps
             if condition['key'] == 'content.body':
-                r = r'\b%s\b' % self._glob_to_regexp(condition['pattern'])
+                r = r'\b%s\b' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
             else:
-                r = r'^%s$' % self._glob_to_regexp(condition['pattern'])
+                r = r'^%s$' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
             val = _value_for_dotted_key(condition['key'], ev)
             if val is None:
                 return False
@@ -169,7 +177,7 @@ class PushRuleEvaluator:
         elif condition['kind'] == 'device':
             if 'profile_tag' not in condition:
                 return True
-            return condition['profile_tag'] == self.profile_tag
+            return condition['profile_tag'] == profile_tag
 
         elif condition['kind'] == 'contains_display_name':
             # This is special because display names can be different
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 25389ceded..c4d025b465 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -259,7 +259,10 @@ class RegisterRestServlet(RestServlet):
     def _do_guest_registration(self):
         if not self.hs.config.allow_guest_access:
             defer.returnValue((403, "Guest access is disabled"))
-        user_id, _ = yield self.registration_handler.register(generate_token=False)
+        user_id, _ = yield self.registration_handler.register(
+            generate_token=False,
+            make_guest=True
+        )
         access_token = self.auth_handler.generate_access_token(user_id, ["guest = true"])
         defer.returnValue((200, {
             "user_id": user_id,
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index c05e7d50c8..2558dc415c 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -311,6 +311,7 @@ class SyncRestServlet(RestServlet):
         if joined:
             ephemeral_events = filter.filter_room_ephemeral(room.ephemeral)
             result["ephemeral"] = {"events": ephemeral_events}
+            result["unread_notification_count"] = room.unread_notification_count
 
         return result
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 2b650f9fa3..7a3f6c4662 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -33,6 +33,7 @@ from .pusher import PusherStore
 from .push_rule import PushRuleStore
 from .media_repository import MediaRepositoryStore
 from .rejections import RejectionsStore
+from .event_push_actions import EventPushActionsStore
 
 from .state import StateStore
 from .signatures import SignatureStore
@@ -75,6 +76,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 SearchStore,
                 TagsStore,
                 AccountDataStore,
+                EventPushActionsStore
                 ):
 
     def __init__(self, hs):
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
new file mode 100644
index 0000000000..5b44431ab9
--- /dev/null
+++ b/synapse/storage/event_push_actions.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import SQLBaseStore
+from twisted.internet import defer
+
+import logging
+import simplejson as json
+
+logger = logging.getLogger(__name__)
+
+
+class EventPushActionsStore(SQLBaseStore):
+    @defer.inlineCallbacks
+    def set_push_actions_for_event_and_users(self, event, tuples):
+        """
+        :param event: the event set actions for
+        :param tuples: list of tuples of (user_id, profile_tag, actions)
+        """
+        values = []
+        for uid, profile_tag, actions in tuples:
+            values.append({
+                'room_id': event.room_id,
+                'event_id': event.event_id,
+                'user_id': uid,
+                'profile_tag': profile_tag,
+                'actions': json.dumps(actions)
+            })
+
+        yield self.runInteraction(
+            "set_actions_for_event_and_users",
+            self._simple_insert_many_txn,
+            EventPushActionsTable.table_name,
+            values
+        )
+
+    @defer.inlineCallbacks
+    def get_unread_event_push_actions_by_room_for_user(
+            self, room_id, user_id, last_read_event_id
+    ):
+        def _get_unread_event_push_actions_by_room(txn):
+            sql = (
+                "SELECT stream_ordering, topological_ordering"
+                " FROM events"
+                " WHERE room_id = ? AND event_id = ?"
+            )
+            txn.execute(
+                sql, (room_id, last_read_event_id)
+            )
+            results = txn.fetchall()
+            if len(results) == 0:
+                return []
+
+            stream_ordering = results[0][0]
+            topological_ordering = results[0][1]
+
+            sql = (
+                "SELECT ea.event_id, ea.actions"
+                " FROM event_push_actions ea, events e"
+                " WHERE ea.room_id = e.room_id"
+                " AND ea.event_id = e.event_id"
+                " AND ea.user_id = ?"
+                " AND ea.room_id = ?"
+                " AND ("
+                "       e.topological_ordering > ?"
+                "       OR (e.topological_ordering = ? AND e.stream_ordering > ?)"
+                ")"
+            )
+            txn.execute(sql, (
+                user_id, room_id,
+                topological_ordering, topological_ordering, stream_ordering
+            )
+            )
+            return [
+                {"event_id": row[0], "actions": row[1]} for row in txn.fetchall()
+            ]
+
+        ret = yield self.runInteraction(
+            "get_unread_event_push_actions_by_room",
+            _get_unread_event_push_actions_by_room
+        )
+        defer.returnValue(ret)
+
+    @defer.inlineCallbacks
+    def remove_push_actions_for_event_id(self, room_id, event_id):
+        def f(txn):
+            txn.execute(
+                "DELETE FROM event_push_actions WHERE room_id = ? AND event_id = ?",
+                (room_id, event_id)
+            )
+        yield self.runInteraction(
+            "remove_push_actions_for_event_id",
+            f
+        )
+
+
+class EventPushActionsTable(object):
+    table_name = "event_push_actions"
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index a913ea7c50..0829262f42 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -56,6 +56,47 @@ class PushRuleStore(SQLBaseStore):
         })
 
     @defer.inlineCallbacks
+    def bulk_get_push_rules(self, user_ids):
+        batch_size = 100
+
+        def f(txn, user_ids_to_fetch):
+            sql = (
+                "SELECT " +
+                ",".join("pr."+x for x in PushRuleTable.fields) +
+                " FROM " + PushRuleTable.table_name + " pr " +
+                " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " +
+                " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " +
+                " WHERE pr.user_name " +
+                " IN (" + ",".join("?" for _ in user_ids_to_fetch) + ")"
+                " AND (pre.enabled is null or pre.enabled = 1)"
+                " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC"
+            )
+            txn.execute(sql, user_ids_to_fetch)
+            return txn.fetchall()
+
+        results = {}
+
+        batch_start = 0
+        while batch_start < len(user_ids):
+            batch_end = min(len(user_ids), batch_size)
+            batch_user_ids = user_ids[batch_start:batch_end]
+            batch_start = batch_end
+
+            rows = yield self.runInteraction(
+                "bulk_get_push_rules", f, batch_user_ids
+            )
+
+            for r in rows:
+                rawdict = {
+                    PushRuleTable.fields[i]: r[i] for i in range(len(r))
+                }
+
+                if rawdict['user_name'] not in results:
+                    results[rawdict['user_name']] = []
+                results[rawdict['user_name']].append(rawdict)
+        defer.returnValue(results)
+
+    @defer.inlineCallbacks
     def add_push_rule(self, before, after, **kwargs):
         vals = kwargs
         if 'conditions' in vals:
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index f0fa0bd33c..ece71f2ee8 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from synapse.api.errors import StoreError, Codes
 
 from ._base import SQLBaseStore
-from synapse.util.caches.descriptors import cached
+from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
 
 class RegistrationStore(SQLBaseStore):
@@ -73,7 +73,8 @@ class RegistrationStore(SQLBaseStore):
         )
 
     @defer.inlineCallbacks
-    def register(self, user_id, token, password_hash, was_guest=False):
+    def register(self, user_id, token, password_hash,
+                 was_guest=False, make_guest=False):
         """Attempts to register an account.
 
         Args:
@@ -82,15 +83,18 @@ class RegistrationStore(SQLBaseStore):
             password_hash (str): Optional. The password hash for this user.
             was_guest (bool): Optional. Whether this is a guest account being
                 upgraded to a non-guest account.
+            make_guest (boolean): True if the the new user should be guest,
+                false to add a regular user account.
         Raises:
             StoreError if the user_id could not be registered.
         """
         yield self.runInteraction(
             "register",
-            self._register, user_id, token, password_hash, was_guest
+            self._register, user_id, token, password_hash, was_guest, make_guest
         )
+        self.is_guest.invalidate((user_id,))
 
-    def _register(self, txn, user_id, token, password_hash, was_guest):
+    def _register(self, txn, user_id, token, password_hash, was_guest, make_guest):
         now = int(self.clock.time())
 
         next_id = self._access_tokens_id_gen.get_next_txn(txn)
@@ -99,13 +103,15 @@ class RegistrationStore(SQLBaseStore):
             if was_guest:
                 txn.execute("UPDATE users SET"
                             " password_hash = ?,"
-                            " upgrade_ts = ?"
+                            " upgrade_ts = ?,"
+                            " is_guest = ?"
                             " WHERE name = ?",
-                            [password_hash, now, user_id])
+                            [password_hash, now, make_guest, user_id])
             else:
-                txn.execute("INSERT INTO users(name, password_hash, creation_ts) "
-                            "VALUES (?,?,?)",
-                            [user_id, password_hash, now])
+                txn.execute("INSERT INTO users "
+                            "(name, password_hash, creation_ts, is_guest) "
+                            "VALUES (?,?,?,?)",
+                            [user_id, password_hash, now, make_guest])
         except self.database_engine.module.IntegrityError:
             raise StoreError(
                 400, "User ID already taken.", errcode=Codes.USER_IN_USE
@@ -126,7 +132,7 @@ class RegistrationStore(SQLBaseStore):
             keyvalues={
                 "name": user_id,
             },
-            retcols=["name", "password_hash"],
+            retcols=["name", "password_hash", "is_guest"],
             allow_none=True,
         )
 
@@ -249,9 +255,21 @@ class RegistrationStore(SQLBaseStore):
 
         defer.returnValue(res if res else False)
 
+    @cachedInlineCallbacks()
+    def is_guest(self, user):
+        res = yield self._simple_select_one_onecol(
+            table="users",
+            keyvalues={"name": user.to_string()},
+            retcol="is_guest",
+            allow_none=True,
+            desc="is_guest",
+        )
+
+        defer.returnValue(res if res else False)
+
     def _query_for_auth(self, txn, token):
         sql = (
-            "SELECT users.name, access_tokens.id as token_id"
+            "SELECT users.name, users.is_guest, access_tokens.id as token_id"
             " FROM users"
             " INNER JOIN access_tokens on users.name = access_tokens.user_id"
             " WHERE token = ?"
diff --git a/synapse/storage/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/delta/28/event_push_actions.sql
new file mode 100644
index 0000000000..bdf6ae3f24
--- /dev/null
+++ b/synapse/storage/schema/delta/28/event_push_actions.sql
@@ -0,0 +1,26 @@
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS event_push_actions(
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    profile_tag VARCHAR(32),
+    actions TEXT NOT NULL,
+    CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag)
+);
+
+
+CREATE INDEX event_push_actions_room_id_event_id_user_id_profile_tag on event_push_actions(room_id, event_id, user_id, profile_tag);
diff --git a/synapse/storage/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/delta/28/users_is_guest.sql
new file mode 100644
index 0000000000..80792e85de
--- /dev/null
+++ b/synapse/storage/schema/delta/28/users_is_guest.sql
@@ -0,0 +1,22 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE users ADD is_guest admin SMALLINT DEFAULT 0 NOT NULL;
+/*
+ * NB: any guest users created between 27 and 28 will be incorrectly
+ * marked as not guests: we don't bother to fill these in correctly
+ * because guest access is not really complete in 27 anyway so it's
+ * very unlikley there will be any guest users created.
+ */