Merge branch 'develop' into markjh/edu_frequency

Conflicts:
	synapse/handlers/federation.py
	synapse/handlers/room.py

11 files changed, 109 insertions, 69 deletions

diff --git a/README.rst b/README.rst
index 9149f2feec..80e1b26e60 100644
--- a/README.rst
+++ b/README.rst
@@ -155,7 +155,7 @@ To set up your homeserver, run (in your virtualenv, as before)::
         --generate-config \
         --report-stats=[yes|no]
 
-Substituting your host and domain name as appropriate.
+...substituting your host and domain name as appropriate.
 
 This will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your Home Server to
@@ -168,10 +168,11 @@ key in the <server name>.signing.key file (the second word, which by default is
 
 By default, registration of new users is disabled. You can either enable
 registration in the config by specifying ``enable_registration: true``
-(it is then recommended to also set up CAPTCHA), or
+(it is then recommended to also set up CAPTCHA - see docs/CAPTCHA_SETUP), or
 you can use the command line to register new users::
 
     $ source ~/.synapse/bin/activate
+    $ synctl start # if not already running
     $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
     New user localpart: erikj
     Password:
@@ -181,6 +182,16 @@ you can use the command line to register new users::
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
 a TURN server.  See docs/turn-howto.rst for details.
 
+Running Synapse
+===============
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. ``~/.synapse``), and::
+
+    cd ~/.synapse
+    source ./bin/activate
+    synctl start
+
 Using PostgreSQL
 ================
 
@@ -203,16 +214,6 @@ may have a few regressions relative to SQLite.
 For information on how to install and use PostgreSQL, please see
 `docs/postgres.rst <docs/postgres.rst>`_.
 
-Running Synapse
-===============
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. ``~/.synapse``), and::
-
-    cd ~/.synapse
-    source ./bin/activate
-    synctl start
-
 Platform Specific Instructions
 ==============================
 
@@ -439,6 +440,10 @@ SRV record, as that is the name other machines will expect it to have::
     python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
+If you've already generated the config file, you need to edit the "server_name"
+in you  ```homeserver.yaml``` file. If you've already started Synapse and a
+database has been created, you will have to recreate the database.
+
 You may additionally want to pass one or more "-v" options, in order to
 increase the verbosity of logging output; at least for initial testing.
 
diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index f5d4cc37fd..8340c72618 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -152,7 +152,7 @@ if __name__ == '__main__':
     while referrer_depth:
         referrer_depth -= 1
         for entry in result.values():
-            for used_by in entry["used"]:
+            for used_by in entry.get("used", ()):
                 referrers.add(used_by)
         for name, definition in names.items():
             if not name in referrers:
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 5c2d6bfeab..187edd516b 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -133,6 +133,7 @@ class ServerConfig(Config):
         # The domain name of the server, with optional explicit port.
         # This is used by remote servers to connect to this server,
         # e.g. matrix.org, localhost:8080, etc.
+        # This is also the last part of your UserID.
         server_name: "%(server_name)s"
 
         # When running as a daemon, the file to store the pid in
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index 0e4c0d4d06..fe300433e6 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -28,6 +28,18 @@ import random
 logger = logging.getLogger(__name__)
 
 
+def started_user_eventstream(distributor, user):
+    return distributor.fire("started_user_eventstream", user)
+
+
+def stopped_user_eventstream(distributor, user):
+    return distributor.fire("stopped_user_eventstream", user)
+
+
+def user_joined_room(distributor, user, room_id):
+    return distributor.fire("user_joined_room", user, room_id)
+
+
 class EventStreamHandler(BaseHandler):
 
     def __init__(self, hs):
@@ -66,7 +78,7 @@ class EventStreamHandler(BaseHandler):
                 except:
                     logger.exception("Failed to cancel event timer")
             else:
-                yield self.distributor.fire("started_user_eventstream", user)
+                yield started_user_eventstream(self.distributor, user)
 
         self._streams_per_user[user] += 1
 
@@ -89,7 +101,7 @@ class EventStreamHandler(BaseHandler):
 
                 self._stop_timer_per_user.pop(user, None)
 
-                return self.distributor.fire("stopped_user_eventstream", user)
+                return stopped_user_eventstream(self.distributor, user)
 
             logger.debug("Scheduling _later: for %s", user)
             self._stop_timer_per_user[user] = (
@@ -120,9 +132,7 @@ class EventStreamHandler(BaseHandler):
                 timeout = random.randint(int(timeout*0.9), int(timeout*1.1))
 
             if is_guest:
-                yield self.distributor.fire(
-                    "user_joined_room", user=auth_user, room_id=room_id
-                )
+                yield user_joined_room(self.distributor, auth_user, room_id)
 
             events, tokens = yield self.notifier.get_events_for(
                 auth_user, pagin_config, timeout,
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e5fb1dd3c9..a03a5f494c 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -44,6 +44,10 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+def user_joined_room(distributor, user, room_id):
+    return distributor.fire("user_joined_room", user, room_id)
+
+
 class FederationHandler(BaseHandler):
     """Handles events that originated from federation.
         Responsible for:
@@ -60,10 +64,7 @@ class FederationHandler(BaseHandler):
 
         self.hs = hs
 
-        self.distributor.observe(
-            "user_joined_room",
-            self._on_user_joined
-        )
+        self.distributor.observe("user_joined_room", self.user_joined_room)
 
         self.waiting_for_join_list = {}
 
@@ -239,9 +240,7 @@ class FederationHandler(BaseHandler):
                 prev_state = context.current_state.get((event.type, event.state_key))
                 if not prev_state or prev_state.membership != Membership.JOIN:
                     user = UserID.from_string(event.state_key)
-                    yield self.distributor.fire(
-                        "user_joined_room", user=user, room_id=event.room_id
-                    )
+                    yield user_joined_room(self.distributor, user, event.room_id)
 
     @defer.inlineCallbacks
     def _filter_events_for_server(self, server_name, room_id, events):
@@ -738,9 +737,7 @@ class FederationHandler(BaseHandler):
         if event.type == EventTypes.Member:
             if event.content["membership"] == Membership.JOIN:
                 user = UserID.from_string(event.state_key)
-                yield self.distributor.fire(
-                    "user_joined_room", user=user, room_id=event.room_id
-                )
+                yield user_joined_room(self.distributor, user, event.room_id)
 
         new_pdu = event
 
@@ -1087,7 +1084,7 @@ class FederationHandler(BaseHandler):
         return self.store.get_min_depth(context)
 
     @log_function
-    def _on_user_joined(self, user, room_id):
+    def user_joined_room(self, user, room_id):
         waiters = self.waiting_for_join_list.get(
             (user.to_string(), room_id),
             []
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index e959ce50be..cb0361ac49 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -31,6 +31,10 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+def collect_presencelike_data(distributor, user, content):
+    return distributor.fire("changed_presencelike_data", user, content)
+
+
 class MessageHandler(BaseHandler):
 
     def __init__(self, hs):
@@ -195,10 +199,8 @@ class MessageHandler(BaseHandler):
             if membership == Membership.JOIN:
                 joinee = UserID.from_string(builder.state_key)
                 # If event doesn't include a display name, add one.
-                yield self.distributor.fire(
-                    "collect_presencelike_data",
-                    joinee,
-                    builder.content
+                yield collect_presencelike_data(
+                    self.distributor, joinee, builder.content
                 )
 
         if token_id is not None:
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index e95e821c9a..63d6f30a7b 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -62,6 +62,14 @@ def partitionbool(l, func):
     return ret.get(True, []), ret.get(False, [])
 
 
+def user_presence_changed(distributor, user, statuscache):
+    return distributor.fire("user_presence_changed", user, statuscache)
+
+
+def collect_presencelike_data(distributor, user, content):
+    return distributor.fire("collect_presencelike_data", user, content)
+
+
 class PresenceHandler(BaseHandler):
 
     STATE_LEVELS = {
@@ -361,9 +369,7 @@ class PresenceHandler(BaseHandler):
         yield self.store.set_presence_state(
             target_user.localpart, state_to_store
         )
-        yield self.distributor.fire(
-            "collect_presencelike_data", target_user, state
-        )
+        yield collect_presencelike_data(self.distributor, target_user, state)
 
         if now_level > was_level:
             state["last_active"] = self.clock.time_msec()
@@ -878,7 +884,7 @@ class PresenceHandler(BaseHandler):
             room_ids=room_ids,
             statuscache=statuscache,
         )
-        yield self.distributor.fire("user_presence_changed", user, statuscache)
+        yield user_presence_changed(self.distributor, user, statuscache)
 
     @defer.inlineCallbacks
     def incoming_presence(self, origin, content):
@@ -1116,9 +1122,7 @@ class PresenceHandler(BaseHandler):
                     self._user_cachemap[user].get_state()["last_active"]
                 )
 
-            yield self.distributor.fire(
-                "collect_presencelike_data", user, state
-            )
+            yield collect_presencelike_data(self.distributor, user, state)
 
         if "last_active" in state:
             state = dict(state)
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 799faffe53..576c6f09b4 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -28,6 +28,14 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+def changed_presencelike_data(distributor, user, state):
+    return distributor.fire("changed_presencelike_data", user, state)
+
+
+def collect_presencelike_data(distributor, user, content):
+    return distributor.fire("collect_presencelike_data", user, content)
+
+
 class ProfileHandler(BaseHandler):
 
     def __init__(self, hs):
@@ -95,11 +103,9 @@ class ProfileHandler(BaseHandler):
             target_user.localpart, new_displayname
         )
 
-        yield self.distributor.fire(
-            "changed_presencelike_data", target_user, {
-                "displayname": new_displayname,
-            }
-        )
+        yield changed_presencelike_data(self.distributor, target_user, {
+            "displayname": new_displayname,
+        })
 
         yield self._update_join_states(target_user)
 
@@ -144,11 +150,9 @@ class ProfileHandler(BaseHandler):
             target_user.localpart, new_avatar_url
         )
 
-        yield self.distributor.fire(
-            "changed_presencelike_data", target_user, {
-                "avatar_url": new_avatar_url,
-            }
-        )
+        yield changed_presencelike_data(self.distributor, target_user, {
+            "avatar_url": new_avatar_url,
+        })
 
         yield self._update_join_states(target_user)
 
@@ -208,9 +212,7 @@ class ProfileHandler(BaseHandler):
                 "membership": Membership.JOIN,
             }
 
-            yield self.distributor.fire(
-                "collect_presencelike_data", user, content
-            )
+            yield collect_presencelike_data(self.distributor, user, content)
 
             msg_handler = self.hs.get_handlers().message_handler
             try:
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 493a087031..5166bc7b62 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -31,6 +31,10 @@ import urllib
 logger = logging.getLogger(__name__)
 
 
+def registered_user(distributor, user):
+    return distributor.fire("registered_user", user)
+
+
 class RegistrationHandler(BaseHandler):
 
     def __init__(self, hs):
@@ -98,7 +102,7 @@ class RegistrationHandler(BaseHandler):
                 password_hash=password_hash
             )
 
-            yield self.distributor.fire("registered_user", user)
+            yield registered_user(self.distributor, user)
         else:
             # autogen a random user ID
             attempts = 0
@@ -117,7 +121,7 @@ class RegistrationHandler(BaseHandler):
                         token=token,
                         password_hash=password_hash)
 
-                    self.distributor.fire("registered_user", user)
+                    yield registered_user(self.distributor, user)
                 except SynapseError:
                     # if user id is taken, just generate another
                     user_id = None
@@ -167,7 +171,7 @@ class RegistrationHandler(BaseHandler):
             token=token,
             password_hash=""
         )
-        self.distributor.fire("registered_user", user)
+        registered_user(self.distributor, user)
         defer.returnValue((user_id, token))
 
     @defer.inlineCallbacks
@@ -215,7 +219,7 @@ class RegistrationHandler(BaseHandler):
                 token=token,
                 password_hash=None
             )
-            yield self.distributor.fire("registered_user", user)
+            yield registered_user(self.distributor, user)
         except Exception, e:
             yield self.store.add_access_token_to_user(user_id, token)
             # Ignore Registration errors
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index cb6ac37758..1e18038e15 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -41,6 +41,18 @@ logger = logging.getLogger(__name__)
 id_server_scheme = "https://"
 
 
+def collect_presencelike_data(distributor, user, content):
+    return distributor.fire("collect_presencelike_data", user, content)
+
+
+def user_left_room(distributor, user, room_id):
+    return distributor.fire("user_left_room", user=user, room_id=room_id)
+
+
+def user_joined_room(distributor, user, room_id):
+    return distributor.fire("user_joined_room", user=user, room_id=room_id)
+
+
 class RoomCreationHandler(BaseHandler):
 
     PRESETS_DICT = {
@@ -438,9 +450,7 @@ class RoomMemberHandler(BaseHandler):
 
             if prev_state and prev_state.membership == Membership.JOIN:
                 user = UserID.from_string(event.user_id)
-                self.distributor.fire(
-                    "user_left_room", user=user, room_id=event.room_id
-                )
+                user_left_room(self.distributor, user, event.room_id)
 
         defer.returnValue({"room_id": room_id})
 
@@ -458,9 +468,7 @@ class RoomMemberHandler(BaseHandler):
             raise SynapseError(404, "No known servers")
 
         # If event doesn't include a display name, add one.
-        yield self.distributor.fire(
-            "collect_presencelike_data", joinee, content
-        )
+        yield collect_presencelike_data(self.distributor, joinee, content)
 
         content.update({"membership": Membership.JOIN})
         builder = self.event_builder_factory.new({
@@ -520,9 +528,7 @@ class RoomMemberHandler(BaseHandler):
         prev_state = context.current_state.get((event.type, event.state_key))
         if not prev_state or prev_state.membership != Membership.JOIN:
             user = UserID.from_string(event.user_id)
-            yield self.distributor.fire(
-                "user_joined_room", user=user, room_id=room_id
-            )
+            yield user_joined_room(self.distributor, user, room_id)
 
     @defer.inlineCallbacks
     def get_inviter(self, event):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index df6390cf05..65ef2f85bf 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -240,9 +240,18 @@ class SearchHandler(BaseHandler):
                 last_event_id = room_events[-1].event_id
                 pagination_token = results_map[last_event_id]["pagination_token"]
 
-                global_next_batch = encode_base64("%s\n%s\n%s" % (
-                    "all", "", pagination_token
-                ))
+                # We want to respect the given batch group and group keys so
+                # that if people blindly use the top level `next_batch` token
+                # it returns more from the same group (if applicable) rather
+                # than reverting to searching all results again.
+                if batch_group and batch_group_key:
+                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                        batch_group, batch_group_key, pagination_token
+                    ))
+                else:
+                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                        "all", "", pagination_token
+                    ))
 
                 for room_id, group in room_groups.items():
                     group["next_batch"] = encode_base64("%s\n%s\n%s" % (