From 2f9eafdd369796d8b7731b24ab8cf6a98ad19e29 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Jul 2017 14:52:27 +0100
Subject: Add local group server support

---
 synapse/handlers/groups_local.py | 278 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 278 insertions(+)
 create mode 100644 synapse/handlers/groups_local.py

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
new file mode 100644
index 0000000000..3df255b05a
--- /dev/null
+++ b/synapse/handlers/groups_local.py
@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: Validate attestations
+# TODO: Allow users to "knock" or simpkly join depending on rules
+# TODO: is_priveged flag to users and is_public to users and rooms
+# TODO: Roles
+# TODO: Audit log for admins (profile updates, membership changes, users who tried
+#       to join but were rejected, etc)
+# TODO: Flairs
+# TODO: Add group memebership  /sync
+
+
+def _create_rerouter(name):
+    def f(self, group_id, *args, **kwargs):
+        if self.is_mine_id(group_id):
+            return getattr(self.groups_server_handler, name)(
+                group_id, *args, **kwargs
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return getattr(repl_layer, name)(group_id, *args, **kwargs)
+    return f
+
+
+class GroupsLocalHandler(object):
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.room_list_handler = hs.get_room_list_handler()
+        self.groups_server_handler = hs.get_groups_server_handler()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.keyring = hs.get_keyring()
+        self.is_mine_id = hs.is_mine_id
+        self.signing_key = hs.config.signing_key[0]
+        self.server_name = hs.hostname
+        self.attestations = hs.get_groups_attestation_signing()
+
+        # Ensure attestations get renewed
+        hs.get_groups_attestation_renewer()
+
+    get_group_profile = _create_rerouter("get_group_profile")
+    get_rooms_in_group = _create_rerouter("get_rooms_in_group")
+
+    update_group_summary_room = _create_rerouter("update_group_summary_room")
+    delete_group_summary_room = _create_rerouter("delete_group_summary_room")
+
+    update_group_category = _create_rerouter("update_group_category")
+    delete_group_category = _create_rerouter("delete_group_category")
+    get_group_category = _create_rerouter("get_group_category")
+    get_group_categories = _create_rerouter("get_group_categories")
+
+    update_group_summary_user = _create_rerouter("update_group_summary_user")
+    delete_group_summary_user = _create_rerouter("delete_group_summary_user")
+
+    update_group_role = _create_rerouter("update_group_role")
+    delete_group_role = _create_rerouter("delete_group_role")
+    get_group_role = _create_rerouter("get_group_role")
+    get_group_roles = _create_rerouter("get_group_roles")
+
+    @defer.inlineCallbacks
+    def get_group_summary(self, group_id, requester_user_id):
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.get_group_summary(
+                group_id, requester_user_id
+            )
+            defer.returnValue(res)
+
+        repl_layer = self.hs.get_replication_layer()
+        res = yield repl_layer.get_group_summary(group_id, requester_user_id)
+
+        chunk = res["users_section"]["users"]
+        valid_users = []
+        for entry in chunk:
+            g_user_id = entry["user_id"]
+            attestation = entry.pop("attestation")
+            try:
+                yield self.attestations.verify_attestation(
+                    attestation,
+                    group_id=group_id,
+                    user_id=g_user_id,
+                )
+                valid_users.append(entry)
+            except Exception as e:
+                logger.info("Failed to verify user is in group: %s", e)
+
+        res["users_section"]["users"] = valid_users
+
+        res["users_section"]["users"].sort(key=lambda e: e.get("order", 0))
+        res["rooms_section"]["rooms"].sort(key=lambda e: e.get("order", 0))
+
+        defer.returnValue(res)
+
+    def create_group(self, group_id, user_id, content):
+        logger.info("Asking to create group with ID: %r", group_id)
+
+        if self.is_mine_id(group_id):
+            return self.groups_server_handler.create_group(
+                group_id, user_id, content
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return repl_layer.create_group(group_id, user_id, content)  # TODO
+
+    def add_room(self, group_id, user_id, room_id, content):
+        if self.is_mine_id(group_id):
+            return self.groups_server_handler.add_room(
+                group_id, user_id, room_id, content
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return repl_layer.add_room_to_group(group_id, user_id, room_id, content)  # TODO
+
+    @defer.inlineCallbacks
+    def get_users_in_group(self, group_id, requester_user_id):
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.get_users_in_group(
+                group_id, requester_user_id
+            )
+            defer.returnValue(res)
+
+        repl_layer = self.hs.get_replication_layer()
+        res = yield repl_layer.get_users_in_group(group_id, requester_user_id)  # TODO
+
+        chunk = res["chunk"]
+        valid_entries = []
+        for entry in chunk:
+            g_user_id = entry["user_id"]
+            attestation = entry.pop("attestation")
+            try:
+                yield self.attestations.verify_attestation(
+                    attestation,
+                    group_id=group_id,
+                    user_id=g_user_id,
+                )
+                valid_entries.append(entry)
+            except Exception as e:
+                logger.info("Failed to verify user is in group: %s", e)
+
+        res["chunk"] = valid_entries
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def join_group(self, group_id, user_id, content):
+        raise NotImplementedError()  # TODO
+
+    @defer.inlineCallbacks
+    def accept_invite(self, group_id, user_id, content):
+        if self.is_mine_id(group_id):
+            yield self.groups_server_handler.accept_invite(
+                group_id, user_id, content
+            )
+            local_attestation = None
+            remote_attestation = None
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content["attestation"] = local_attestation
+
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.accept_group_invite(group_id, user_id, content)
+
+            remote_attestation = res["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                group_id=group_id,
+                user_id=user_id,
+            )
+
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="join",
+            is_admin=False,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def invite(self, group_id, user_id, requester_user_id, config):
+        content = {
+            "requester_user_id": requester_user_id,
+            "config": config,
+        }
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.invite_to_group(
+                group_id, user_id, requester_user_id, content,
+            )
+        else:
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.invite_to_group(
+                group_id, user_id, content,
+            )
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def on_invite(self, group_id, user_id, content):
+        # TODO: Support auto join and rejection
+
+        if not self.is_mine_id(user_id):
+            raise SynapseError(400, "User not on this server")
+
+        local_profile = {}
+        if "profile" in content:
+            if "name" in content["profile"]:
+                local_profile["name"] = content["profile"]["name"]
+            if "avatar_url" in content["profile"]:
+                local_profile["avatar_url"] = content["profile"]["avatar_url"]
+
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="invite",
+            content={"profile": local_profile, "inviter": content["inviter"]},
+        )
+
+        defer.returnValue({"state": "invite"})
+
+    @defer.inlineCallbacks
+    def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        if user_id == requester_user_id:
+            yield self.store.register_user_group_membership(
+                group_id, user_id,
+                membership="leave",
+            )
+
+            # TODO: Should probably remember that we tried to leave so that we can
+            # retry if the group server is currently down.
+
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.remove_user_from_group(
+                group_id, user_id, requester_user_id, content,
+            )
+        else:
+            content["requester_user_id"] = requester_user_id
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.remove_user_from_group(
+                group_id, user_id, content
+            )  # TODO
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def user_removed_from_group(self, group_id, user_id, content):
+        # TODO: Check if user in group
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="leave",
+        )
+
+    @defer.inlineCallbacks
+    def get_joined_groups(self, user_id):
+        group_ids = yield self.store.get_joined_groups(user_id)
+        defer.returnValue({"groups": group_ids})
-- 
cgit 1.5.1


From 68f34e85cebcacef428d1f38942990c43c3cdd01 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 10:29:57 +0100
Subject: Use transport client directly

---
 synapse/handlers/groups_local.py | 43 +++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 3df255b05a..e0f53120be 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -32,15 +32,17 @@ logger = logging.getLogger(__name__)
 # TODO: Add group memebership  /sync
 
 
-def _create_rerouter(name):
+def _create_rerouter(func_name):
+    """Returns a function that looks at the group id and calls the function
+    on federation or the local group server if the group is local
+    """
     def f(self, group_id, *args, **kwargs):
         if self.is_mine_id(group_id):
-            return getattr(self.groups_server_handler, name)(
+            return getattr(self.groups_server_handler, func_name)(
                 group_id, *args, **kwargs
             )
 
-        repl_layer = self.hs.get_replication_layer()
-        return getattr(repl_layer, name)(group_id, *args, **kwargs)
+        return getattr(self.transport_client, func_name)(group_id, *args, **kwargs)
     return f
 
 
@@ -50,6 +52,7 @@ class GroupsLocalHandler(object):
         self.store = hs.get_datastore()
         self.room_list_handler = hs.get_room_list_handler()
         self.groups_server_handler = hs.get_groups_server_handler()
+        self.transport_client = hs.get_federation_transport_client()
         self.auth = hs.get_auth()
         self.clock = hs.get_clock()
         self.keyring = hs.get_keyring()
@@ -82,15 +85,19 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
+        """Get the group summary for a group.
+
+        If the group is remote we check that the users have valid attestations.
+        """
         if self.is_mine_id(group_id):
             res = yield self.groups_server_handler.get_group_summary(
                 group_id, requester_user_id
             )
             defer.returnValue(res)
 
-        repl_layer = self.hs.get_replication_layer()
-        res = yield repl_layer.get_group_summary(group_id, requester_user_id)
+        res = yield self.transport_client.get_group_summary(group_id, requester_user_id)
 
+        # Loop through the users and validate the attestations.
         chunk = res["users_section"]["users"]
         valid_users = []
         for entry in chunk:
@@ -121,8 +128,7 @@ class GroupsLocalHandler(object):
                 group_id, user_id, content
             )
 
-        repl_layer = self.hs.get_replication_layer()
-        return repl_layer.create_group(group_id, user_id, content)  # TODO
+        return self.transport_client.create_group(group_id, user_id, content)  # TODO
 
     def add_room(self, group_id, user_id, room_id, content):
         if self.is_mine_id(group_id):
@@ -130,8 +136,9 @@ class GroupsLocalHandler(object):
                 group_id, user_id, room_id, content
             )
 
-        repl_layer = self.hs.get_replication_layer()
-        return repl_layer.add_room_to_group(group_id, user_id, room_id, content)  # TODO
+        return self.transport_client.add_room_to_group(
+            group_id, user_id, room_id, content,
+        )  # TODO
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -141,8 +148,9 @@ class GroupsLocalHandler(object):
             )
             defer.returnValue(res)
 
-        repl_layer = self.hs.get_replication_layer()
-        res = yield repl_layer.get_users_in_group(group_id, requester_user_id)  # TODO
+        res = yield self.transport_client.get_users_in_group(
+            group_id, requester_user_id,
+        )  # TODO
 
         chunk = res["chunk"]
         valid_entries = []
@@ -179,8 +187,9 @@ class GroupsLocalHandler(object):
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content["attestation"] = local_attestation
 
-            repl_layer = self.hs.get_replication_layer()
-            res = yield repl_layer.accept_group_invite(group_id, user_id, content)
+            res = yield self.transport_client.accept_group_invite(
+                group_id, user_id, content,
+            )
 
             remote_attestation = res["attestation"]
 
@@ -211,8 +220,7 @@ class GroupsLocalHandler(object):
                 group_id, user_id, requester_user_id, content,
             )
         else:
-            repl_layer = self.hs.get_replication_layer()
-            res = yield repl_layer.invite_to_group(
+            res = yield self.transport_client.invite_to_group(
                 group_id, user_id, content,
             )
 
@@ -257,8 +265,7 @@ class GroupsLocalHandler(object):
             )
         else:
             content["requester_user_id"] = requester_user_id
-            repl_layer = self.hs.get_replication_layer()
-            res = yield repl_layer.remove_user_from_group(
+            res = yield self.transport_client.remove_user_from_group(
                 group_id, user_id, content
             )  # TODO
 
-- 
cgit 1.5.1


From cccfcfa7b9e2af28bb56d6f970754fe5aa07ad56 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 10:31:59 +0100
Subject: Comments

---
 synapse/federation/transport/server.py | 34 +++++++++++++++++++---------------
 synapse/handlers/groups_local.py       |  3 +++
 2 files changed, 22 insertions(+), 15 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 0f08334f33..d68e90d2f7 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -715,21 +715,6 @@ class FederationGroupsInviteServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
-class FederationGroupsLocalInviteServlet(BaseFederationServlet):
-    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite$"
-
-    @defer.inlineCallbacks
-    def on_POST(self, origin, content, query, group_id, user_id):
-        if get_domain_from_id(group_id) != origin:
-            raise SynapseError(403, "group_id doesn't match origin")
-
-        new_content = yield self.handler.on_invite(
-            group_id, user_id, content,
-        )
-
-        defer.returnValue((200, new_content))
-
-
 class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
     """Accept an invitation from the group server
     """
@@ -765,7 +750,26 @@ class FederationGroupsRemoveUserServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsLocalInviteServlet(BaseFederationServlet):
+    """A group server has invited a local user
+    """
+    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id, user_id):
+        if get_domain_from_id(group_id) != origin:
+            raise SynapseError(403, "group_id doesn't match origin")
+
+        new_content = yield self.handler.on_invite(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
+    """A group server has removed a local user
+    """
     PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove$"
 
     @defer.inlineCallbacks
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index e0f53120be..0857b14c7a 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -64,6 +64,9 @@ class GroupsLocalHandler(object):
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
 
+    # The following functions merely route the query to the local groups server
+    # or federation depending on if the group is local or remote
+
     get_group_profile = _create_rerouter("get_group_profile")
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
-- 
cgit 1.5.1


From e5ea6dd021ea71f3b5bc9a37fb896c351ee550b1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 14:37:06 +0100
Subject: Add client apis

---
 synapse/federation/transport/client.py | 196 +++++++++++++++++++++++++++++++--
 synapse/handlers/groups_local.py       |   2 +-
 2 files changed, 188 insertions(+), 10 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ea340e345c..500f3622a2 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -476,10 +476,10 @@ class TransportLayerClient(object):
     def get_group_profile(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/profile" % (group_id,)
 
-        return self.client.post_json(
+        return self.client.get_json(
             destination=destination,
             path=path,
-            data={"requester_user_id": requester_user_id},
+            args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
 
@@ -487,10 +487,10 @@ class TransportLayerClient(object):
     def get_group_summary(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/summary" % (group_id,)
 
-        return self.client.post_json(
+        return self.client.get_json(
             destination=destination,
             path=path,
-            data={"requester_user_id": requester_user_id},
+            args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
 
@@ -498,10 +498,22 @@ class TransportLayerClient(object):
     def get_group_rooms(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/rooms" % (group_id,)
 
+        return self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    def add_room_to_group(self, destination, group_id, requester_user_id, room_id,
+                          content):
+        path = PREFIX + "/groups/%s/room/%s" % (group_id, room_id,)
+
         return self.client.post_json(
             destination=destination,
             path=path,
-            data={"requester_user_id": requester_user_id},
+            args={"requester_user_id": requester_user_id},
+            data=content,
             ignore_backoff=True,
         )
 
@@ -509,10 +521,10 @@ class TransportLayerClient(object):
     def get_group_users(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/users" % (group_id,)
 
-        return self.client.post_json(
+        return self.client.get_json(
             destination=destination,
             path=path,
-            data={"requester_user_id": requester_user_id},
+            args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
 
@@ -528,12 +540,13 @@ class TransportLayerClient(object):
         )
 
     @log_function
-    def invite_to_group(self, destination, group_id, user_id, content):
+    def invite_to_group(self, destination, group_id, user_id, requester_user_id, content):
         path = PREFIX + "/groups/%s/users/%s/invite" % (group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
             path=path,
+            args=requester_user_id,
             data=content,
             ignore_backoff=True,
         )
@@ -554,12 +567,14 @@ class TransportLayerClient(object):
         )
 
     @log_function
-    def remove_user_from_group(self, destination, group_id, user_id, content):
+    def remove_user_from_group(self, destination, group_id, requester_user_id,
+                               user_id, content):
         path = PREFIX + "/groups/%s/users/%s/remove" % (group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
             path=path,
+            args={"requester_user_id": requester_user_id},
             data=content,
             ignore_backoff=True,
         )
@@ -594,3 +609,166 @@ class TransportLayerClient(object):
             data=content,
             ignore_backoff=True,
         )
+
+    @log_function
+    def update_group_summary_room(self, destination, group_id, user_id, room_id,
+                                  category_id, content):
+        if category_id:
+            path = PREFIX + "/groups/%s/summary/categories/%s/rooms/%s" % (
+                group_id, category_id, room_id,
+            )
+        else:
+            path = PREFIX + "/groups/%s/summary/rooms/%s" % (group_id, room_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def delete_group_summary_room(self, destination, group_id, user_id, room_id,
+                                  category_id):
+        if category_id:
+            path = PREFIX + "/groups/%s/summary/categories/%s/rooms/%s" % (
+                group_id, category_id, room_id,
+            )
+        else:
+            path = PREFIX + "/groups/%s/summary/rooms/%s" % (group_id, room_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_categories(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/categories" % (group_id,)
+
+        return self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_category(self, destination, group_id, requester_user_id, category_id):
+        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+
+        return self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def update_group_category(self, destination, group_id, requester_user_id, category_id,
+                              content):
+        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def delete_group_category(self, destination, group_id, requester_user_id,
+                              category_id):
+        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_roles(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/roles" % (group_id,)
+
+        return self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_role(self, destination, group_id, requester_user_id, role_id):
+        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+
+        return self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def update_group_role(self, destination, group_id, requester_user_id, role_id,
+                          content):
+        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def delete_group_role(self, destination, group_id, requester_user_id, role_id):
+        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def update_group_summary_user(self, destination, group_id, requester_user_id,
+                                  user_id, role_id, content):
+        if role_id:
+            path = PREFIX + "/groups/%s/summary/roles/%s/users/%s" % (
+                group_id, role_id, user_id,
+            )
+        else:
+            path = PREFIX + "/groups/%s/summary/users/%s" % (group_id, user_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def delete_group_summary_user(self, destination, group_id, requester_user_id,
+                                  user_id, role_id):
+        if role_id:
+            path = PREFIX + "/groups/%s/summary/roles/%s/users/%s" % (
+                group_id, role_id, user_id,
+            )
+        else:
+            path = PREFIX + "/groups/%s/summary/users/%s" % (group_id, user_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 0857b14c7a..6962210526 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -224,7 +224,7 @@ class GroupsLocalHandler(object):
             )
         else:
             res = yield self.transport_client.invite_to_group(
-                group_id, user_id, content,
+                group_id, user_id, requester_user_id, content,
             )
 
         defer.returnValue(res)
-- 
cgit 1.5.1


From 332839f6ea4f4ae6e89c383bfb334b6ddecd3e53 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 14:45:37 +0100
Subject: Update federation client pokes

---
 synapse/handlers/groups_local.py | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 6962210526..7d7fc5d976 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -16,6 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
+from synapse.types import get_domain_from_id
 
 import logging
 
@@ -41,8 +42,11 @@ def _create_rerouter(func_name):
             return getattr(self.groups_server_handler, func_name)(
                 group_id, *args, **kwargs
             )
-
-        return getattr(self.transport_client, func_name)(group_id, *args, **kwargs)
+        else:
+            destination = get_domain_from_id(group_id)
+            return getattr(self.transport_client, func_name)(
+                destination, group_id, *args, **kwargs
+            )
     return f
 
 
@@ -98,7 +102,9 @@ class GroupsLocalHandler(object):
             )
             defer.returnValue(res)
 
-        res = yield self.transport_client.get_group_summary(group_id, requester_user_id)
+        res = yield self.transport_client.get_group_summary(
+            get_domain_from_id(group_id), group_id, requester_user_id,
+        )
 
         # Loop through the users and validate the attestations.
         chunk = res["users_section"]["users"]
@@ -131,7 +137,9 @@ class GroupsLocalHandler(object):
                 group_id, user_id, content
             )
 
-        return self.transport_client.create_group(group_id, user_id, content)  # TODO
+        return self.transport_client.create_group(
+            get_domain_from_id(group_id), group_id, user_id, content,
+        )  # TODO
 
     def add_room(self, group_id, user_id, room_id, content):
         if self.is_mine_id(group_id):
@@ -140,8 +148,8 @@ class GroupsLocalHandler(object):
             )
 
         return self.transport_client.add_room_to_group(
-            group_id, user_id, room_id, content,
-        )  # TODO
+            get_domain_from_id(group_id), group_id, user_id, room_id, content,
+        )
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -151,9 +159,9 @@ class GroupsLocalHandler(object):
             )
             defer.returnValue(res)
 
-        res = yield self.transport_client.get_users_in_group(
-            group_id, requester_user_id,
-        )  # TODO
+        res = yield self.transport_client.get_group_users(
+            get_domain_from_id(group_id), group_id, requester_user_id,
+        )
 
         chunk = res["chunk"]
         valid_entries = []
@@ -191,7 +199,7 @@ class GroupsLocalHandler(object):
             content["attestation"] = local_attestation
 
             res = yield self.transport_client.accept_group_invite(
-                group_id, user_id, content,
+                get_domain_from_id(group_id), group_id, user_id, content,
             )
 
             remote_attestation = res["attestation"]
@@ -224,7 +232,8 @@ class GroupsLocalHandler(object):
             )
         else:
             res = yield self.transport_client.invite_to_group(
-                group_id, user_id, requester_user_id, content,
+                get_domain_from_id(group_id), group_id, user_id, requester_user_id,
+                content,
             )
 
         defer.returnValue(res)
@@ -269,8 +278,8 @@ class GroupsLocalHandler(object):
         else:
             content["requester_user_id"] = requester_user_id
             res = yield self.transport_client.remove_user_from_group(
-                group_id, user_id, content
-            )  # TODO
+                get_domain_from_id(group_id), group_id, user_id, content
+            )
 
         defer.returnValue(res)
 
-- 
cgit 1.5.1


From 94ecd871a047707da5998f83440c039d064de8aa Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 16:38:54 +0100
Subject: Fix typos

---
 synapse/federation/transport/client.py |  4 ++--
 synapse/handlers/groups_local.py       |  5 +++--
 synapse/storage/group_server.py        | 25 +++++++++++++++++--------
 3 files changed, 22 insertions(+), 12 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 500f3622a2..e4d84c06c1 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -495,7 +495,7 @@ class TransportLayerClient(object):
         )
 
     @log_function
-    def get_group_rooms(self, destination, group_id, requester_user_id):
+    def get_rooms_in_group(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/rooms" % (group_id,)
 
         return self.client.get_json(
@@ -518,7 +518,7 @@ class TransportLayerClient(object):
         )
 
     @log_function
-    def get_group_users(self, destination, group_id, requester_user_id):
+    def get_users_in_group(self, destination, group_id, requester_user_id):
         path = PREFIX + "/groups/%s/users" % (group_id,)
 
         return self.client.get_json(
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 7d7fc5d976..50f7fce885 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -159,7 +159,7 @@ class GroupsLocalHandler(object):
             )
             defer.returnValue(res)
 
-        res = yield self.transport_client.get_group_users(
+        res = yield self.transport_client.get_users_in_group(
             get_domain_from_id(group_id), group_id, requester_user_id,
         )
 
@@ -278,7 +278,8 @@ class GroupsLocalHandler(object):
         else:
             content["requester_user_id"] = requester_user_id
             res = yield self.transport_client.remove_user_from_group(
-                get_domain_from_id(group_id), group_id, user_id, content
+                get_domain_from_id(group_id), group_id, requester_user_id,
+                user_id, content,
             )
 
         defer.returnValue(res)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 3c6ee7df68..0a69e0f501 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -776,7 +776,7 @@ class GroupServerStore(SQLBaseStore):
             remote_attestation (dict): If remote group then store the remote
                 attestation from the group, else None.
         """
-        def _register_user_group_membership_txn(txn, next_id):
+        def _register_user_group_membership_txn(txn):
             # TODO: Upsert?
             self._simple_delete_txn(
                 txn,
@@ -797,7 +797,6 @@ class GroupServerStore(SQLBaseStore):
                     "content": json.dumps(content),
                 },
             )
-            self._group_updates_stream_cache.entity_has_changed(user_id, next_id)
 
             # TODO: Insert profile to ensuer it comes down stream if its a join.
 
@@ -820,7 +819,7 @@ class GroupServerStore(SQLBaseStore):
                             "group_id": group_id,
                             "user_id": user_id,
                             "valid_until_ms": remote_attestation["valid_until_ms"],
-                            "attestation": json.dumps(remote_attestation),
+                            "attestation_json": json.dumps(remote_attestation),
                         }
                     )
             else:
@@ -841,11 +840,10 @@ class GroupServerStore(SQLBaseStore):
                     },
                 )
 
-        with self._group_updates_id_gen.get_next() as next_id:
-            yield self.runInteraction(
-                "register_user_group_membership",
-                _register_user_group_membership_txn, next_id,
-            )
+        yield self.runInteraction(
+            "register_user_group_membership",
+            _register_user_group_membership_txn,
+        )
 
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, name, avatar_url, short_description,
@@ -928,3 +926,14 @@ class GroupServerStore(SQLBaseStore):
             defer.returnValue(json.loads(row["attestation_json"]))
 
         defer.returnValue(None)
+
+    def get_joined_groups(self, user_id):
+        return self._simple_select_onecol(
+            table="local_group_membership",
+            keyvalues={
+                "user_id": user_id,
+                "membership": "join",
+            },
+            retcol="group_id",
+            desc="get_joined_groups",
+        )
-- 
cgit 1.5.1


From 14a34f12d755e7516dc81348d811d47dc51f026d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 17:28:42 +0100
Subject: Comments

---
 synapse/federation/transport/server.py |  2 +-
 synapse/groups/groups_server.py        |  2 +-
 synapse/handlers/groups_local.py       | 29 +++++++++++++++++++----------
 synapse/rest/client/v2_alpha/groups.py |  4 +++-
 4 files changed, 24 insertions(+), 13 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 29e966ac29..1332b49f35 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -672,7 +672,7 @@ class FederationGroupsAddRoomsServlet(BaseFederationServlet):
         if get_domain_from_id(requester_user_id) != origin:
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
-        new_content = yield self.handler.add_room(
+        new_content = yield self.handler.add_room_to_group(
             group_id, requester_user_id, room_id, content
         )
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index b9ad9507f4..1b6e354ca3 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -430,7 +430,7 @@ class GroupsServerHandler(object):
         })
 
     @defer.inlineCallbacks
-    def add_room(self, group_id, requester_user_id, room_id, content):
+    def add_room_to_group(self, group_id, requester_user_id, room_id, content):
         """Add room to group
         """
         yield self.check_group_is_ours(
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 50f7fce885..0b80348c82 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -74,6 +74,8 @@ class GroupsLocalHandler(object):
     get_group_profile = _create_rerouter("get_group_profile")
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
+    add_room_to_group = _create_rerouter("add_room_to_group")
+
     update_group_summary_room = _create_rerouter("update_group_summary_room")
     delete_group_summary_room = _create_rerouter("delete_group_summary_room")
 
@@ -130,6 +132,9 @@ class GroupsLocalHandler(object):
         defer.returnValue(res)
 
     def create_group(self, group_id, user_id, content):
+        """Create a group
+        """
+
         logger.info("Asking to create group with ID: %r", group_id)
 
         if self.is_mine_id(group_id):
@@ -141,18 +146,10 @@ class GroupsLocalHandler(object):
             get_domain_from_id(group_id), group_id, user_id, content,
         )  # TODO
 
-    def add_room(self, group_id, user_id, room_id, content):
-        if self.is_mine_id(group_id):
-            return self.groups_server_handler.add_room(
-                group_id, user_id, room_id, content
-            )
-
-        return self.transport_client.add_room_to_group(
-            get_domain_from_id(group_id), group_id, user_id, room_id, content,
-        )
-
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
+        """Get users in a group
+        """
         if self.is_mine_id(group_id):
             res = yield self.groups_server_handler.get_users_in_group(
                 group_id, requester_user_id
@@ -184,10 +181,14 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def join_group(self, group_id, user_id, content):
+        """Request to join a group
+        """
         raise NotImplementedError()  # TODO
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
+        """Accept an invite to a group
+        """
         if self.is_mine_id(group_id):
             yield self.groups_server_handler.accept_invite(
                 group_id, user_id, content
@@ -222,6 +223,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def invite(self, group_id, user_id, requester_user_id, config):
+        """Invite a user to a group
+        """
         content = {
             "requester_user_id": requester_user_id,
             "config": config,
@@ -240,6 +243,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def on_invite(self, group_id, user_id, content):
+        """One of our users were invited to a group
+        """
         # TODO: Support auto join and rejection
 
         if not self.is_mine_id(user_id):
@@ -262,6 +267,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        """Remove a user from a group
+        """
         if user_id == requester_user_id:
             yield self.store.register_user_group_membership(
                 group_id, user_id,
@@ -286,6 +293,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def user_removed_from_group(self, group_id, user_id, content):
+        """One of our users was removed/kicked from a group
+        """
         # TODO: Check if user in group
         yield self.store.register_user_group_membership(
             group_id, user_id,
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index 787967c3a2..f937d856fd 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -405,7 +405,9 @@ class GroupAdminRoomsServlet(RestServlet):
         user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        result = yield self.groups_handler.add_room(group_id, user_id, room_id, content)
+        result = yield self.groups_handler.add_room_to_group(
+            group_id, user_id, room_id, content,
+        )
 
         defer.returnValue((200, result))
 
-- 
cgit 1.5.1


From 6f443a74cf6ab0bfe452289f9888580725987765 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Jul 2017 09:46:33 +0100
Subject: Add update group profile API

---
 synapse/federation/transport/server.py | 12 ++++++++++++
 synapse/groups/groups_server.py        | 16 ++++++++++++++++
 synapse/handlers/groups_local.py       |  1 +
 synapse/rest/client/v2_alpha/groups.py | 12 ++++++++++++
 synapse/storage/group_server.py        | 11 +++++++++++
 5 files changed, 52 insertions(+)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 1332b49f35..e04750fd2a 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -642,6 +642,18 @@ class FederationGroupsSummaryServlet(BaseFederationServlet):
 
         defer.returnValue((200, new_content))
 
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.update_group_profile(
+            group_id, requester_user_id, content
+        )
+
+        defer.returnValue((200, new_content))
+
 
 class FederationGroupsRoomsServlet(BaseFederationServlet):
     """Get the rooms in a group on behalf of a user
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 1b6e354ca3..322aad2a6f 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -341,6 +341,22 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(404, "Unknown group")
 
+    @defer.inlineCallbacks
+    def update_group_profile(self, group_id, requester_user_id, content):
+        """Update the group profile
+        """
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id,
+        )
+
+        profile = {}
+        for keyname in ("name", "avatar_url", "short_description",
+                        "long_description"):
+            if keyname in content:
+                profile[keyname] = content[keyname]
+
+        yield self.store.update_group_profile(group_id, profile)
+
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
         """Get the users in group as seen by requester_user_id.
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 0b80348c82..b2c920da38 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -72,6 +72,7 @@ class GroupsLocalHandler(object):
     # or federation depending on if the group is local or remote
 
     get_group_profile = _create_rerouter("get_group_profile")
+    update_group_profile = _create_rerouter("update_group_profile")
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
     add_room_to_group = _create_rerouter("add_room_to_group")
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index f937d856fd..64d803d489 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -45,6 +45,18 @@ class GroupServlet(RestServlet):
 
         defer.returnValue((200, group_description))
 
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id, content):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        yield self.groups_handler.update_group_profile(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, {}))
+
 
 class GroupSummaryServlet(RestServlet):
     """Get the full group summary
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 0a69e0f501..4197d22d88 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -860,6 +860,17 @@ class GroupServerStore(SQLBaseStore):
             desc="create_group",
         )
 
+    @defer.inlineCallbacks
+    def update_group_profile(self, group_id, profile,):
+        yield self._simple_update_one(
+            table="groups",
+            keyvalues={
+                "group_id": group_id,
+            },
+            updatevalues=profile,
+            desc="create_group",
+        )
+
     def get_attestations_need_renewals(self, valid_until_ms):
         """Get all attestations that need to be renewed until givent time
         """
-- 
cgit 1.5.1


From 2cc998fed879357376edb35d5088d88a078dd576 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Jul 2017 17:13:18 +0100
Subject: Fix replication. And notify

---
 synapse/app/synchrotron.py                  |  6 ++++
 synapse/handlers/groups_local.py            | 20 ++++++++---
 synapse/replication/slave/storage/groups.py | 54 +++++++++++++++++++++++++++++
 synapse/replication/tcp/streams.py          | 20 +++++++++++
 synapse/storage/group_server.py             | 23 ++++++++++++
 5 files changed, 119 insertions(+), 4 deletions(-)
 create mode 100644 synapse/replication/slave/storage/groups.py

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 4bdd99a966..d06a05acd9 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -41,6 +41,7 @@ from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.room import RoomStore
+from synapse.replication.slave.storage.groups import SlavedGroupServerStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -75,6 +76,7 @@ class SynchrotronSlavedStore(
     SlavedRegistrationStore,
     SlavedFilteringStore,
     SlavedPresenceStore,
+    SlavedGroupServerStore,
     SlavedDeviceInboxStore,
     SlavedDeviceStore,
     SlavedClientIpStore,
@@ -409,6 +411,10 @@ class SyncReplicationHandler(ReplicationClientHandler):
             )
         elif stream_name == "presence":
             yield self.presence_handler.process_replication_rows(token, rows)
+        elif stream_name == "receipts":
+            self.notifier.on_new_event(
+                "groups_key", token, users=[row.user_id for row in rows],
+            )
 
 
 def start(config_options):
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 0b80348c82..4182ea5afa 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -211,13 +211,16 @@ class GroupsLocalHandler(object):
                 user_id=user_id,
             )
 
-        yield self.store.register_user_group_membership(
+        token = yield self.store.register_user_group_membership(
             group_id, user_id,
             membership="join",
             is_admin=False,
             local_attestation=local_attestation,
             remote_attestation=remote_attestation,
         )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
 
         defer.returnValue({})
 
@@ -257,11 +260,14 @@ class GroupsLocalHandler(object):
             if "avatar_url" in content["profile"]:
                 local_profile["avatar_url"] = content["profile"]["avatar_url"]
 
-        yield self.store.register_user_group_membership(
+        token = yield self.store.register_user_group_membership(
             group_id, user_id,
             membership="invite",
             content={"profile": local_profile, "inviter": content["inviter"]},
         )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
 
         defer.returnValue({"state": "invite"})
 
@@ -270,10 +276,13 @@ class GroupsLocalHandler(object):
         """Remove a user from a group
         """
         if user_id == requester_user_id:
-            yield self.store.register_user_group_membership(
+            token = yield self.store.register_user_group_membership(
                 group_id, user_id,
                 membership="leave",
             )
+            self.notifier.on_new_event(
+                "groups_key", token, users=[user_id],
+            )
 
             # TODO: Should probably remember that we tried to leave so that we can
             # retry if the group server is currently down.
@@ -296,10 +305,13 @@ class GroupsLocalHandler(object):
         """One of our users was removed/kicked from a group
         """
         # TODO: Check if user in group
-        yield self.store.register_user_group_membership(
+        token = yield self.store.register_user_group_membership(
             group_id, user_id,
             membership="leave",
         )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
 
     @defer.inlineCallbacks
     def get_joined_groups(self, user_id):
diff --git a/synapse/replication/slave/storage/groups.py b/synapse/replication/slave/storage/groups.py
new file mode 100644
index 0000000000..0bc4bce5b0
--- /dev/null
+++ b/synapse/replication/slave/storage/groups.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import BaseSlavedStore
+from ._slaved_id_tracker import SlavedIdTracker
+from synapse.storage import DataStore
+from synapse.util.caches.stream_change_cache import StreamChangeCache
+
+
+class SlavedGroupServerStore(BaseSlavedStore):
+    def __init__(self, db_conn, hs):
+        super(SlavedGroupServerStore, self).__init__(db_conn, hs)
+
+        self.hs = hs
+
+        self._group_updates_id_gen = SlavedIdTracker(
+            db_conn, "local_group_updates", "stream_id",
+        )
+        self._group_updates_stream_cache = StreamChangeCache(
+            "_group_updates_stream_cache", self._group_updates_id_gen.get_current_token(),
+        )
+
+    get_groups_changes_for_user = DataStore.get_groups_changes_for_user.__func__
+    get_group_stream_token = DataStore.get_group_stream_token.__func__
+    get_all_groups_for_user = DataStore.get_all_groups_for_user.__func__
+
+    def stream_positions(self):
+        result = super(SlavedGroupServerStore, self).stream_positions()
+        result["groups"] = self._group_updates_id_gen.get_current_token()
+        return result
+
+    def process_replication_rows(self, stream_name, token, rows):
+        if stream_name == "groups":
+            self._group_updates_id_gen.advance(token)
+            for row in rows:
+                self._group_updates_stream_cache.entity_has_changed(
+                    row.user_id, token
+                )
+
+        return super(SlavedGroupServerStore, self).process_replication_rows(
+            stream_name, token, rows
+        )
diff --git a/synapse/replication/tcp/streams.py b/synapse/replication/tcp/streams.py
index fbafe12cc2..4c60bf79f9 100644
--- a/synapse/replication/tcp/streams.py
+++ b/synapse/replication/tcp/streams.py
@@ -118,6 +118,12 @@ CurrentStateDeltaStreamRow = namedtuple("CurrentStateDeltaStream", (
     "state_key",  # str
     "event_id",  # str, optional
 ))
+GroupsStreamRow = namedtuple("GroupsStreamRow", (
+    "group_id",  # str
+    "user_id",  # str
+    "type",  # str
+    "content",  # dict
+))
 
 
 class Stream(object):
@@ -464,6 +470,19 @@ class CurrentStateDeltaStream(Stream):
         super(CurrentStateDeltaStream, self).__init__(hs)
 
 
+class GroupServerStream(Stream):
+    NAME = "groups"
+    ROW_TYPE = GroupsStreamRow
+
+    def __init__(self, hs):
+        store = hs.get_datastore()
+
+        self.current_token = store.get_group_stream_token
+        self.update_function = store.get_all_groups_changes
+
+        super(GroupServerStream, self).__init__(hs)
+
+
 STREAMS_MAP = {
     stream.NAME: stream
     for stream in (
@@ -482,5 +501,6 @@ STREAMS_MAP = {
         TagAccountDataStream,
         AccountDataStream,
         CurrentStateDeltaStream,
+        GroupServerStream,
     )
 }
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 45f0a4c599..5006ac863f 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -853,6 +853,8 @@ class GroupServerStore(SQLBaseStore):
                     },
                 )
 
+            return next_id
+
         with self._group_updates_id_gen.get_next() as next_id:
             yield self.runInteraction(
                 "register_user_group_membership",
@@ -993,5 +995,26 @@ class GroupServerStore(SQLBaseStore):
             "get_groups_changes_for_user", _get_groups_changes_for_user_txn,
         )
 
+    def get_all_groups_changes(self, from_token, to_token, limit):
+        from_token = int(from_token)
+        has_changed = self._group_updates_stream_cache.has_any_entity_changed(
+            from_token,
+        )
+        if not has_changed:
+            return []
+
+        def _get_all_groups_changes_txn(txn):
+            sql = """
+                SELECT stream_id, group_id, user_id, type, content
+                FROM local_group_updates
+                WHERE ? < stream_id AND stream_id <= ?
+                LIMIT ?
+            """
+            txn.execute(sql, (from_token, to_token, limit,))
+            return txn.fetchall()
+        return self.runInteraction(
+            "get_all_groups_changes", _get_all_groups_changes_txn,
+        )
+
     def get_group_stream_token(self):
         return self._group_updates_id_gen.get_current_token()
-- 
cgit 1.5.1


From 960dae3340221168e1e019f250cbc2dd430a1aee Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Jul 2017 17:14:44 +0100
Subject: Add notifier

---
 synapse/handlers/groups_local.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 4182ea5afa..0c329e633d 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -63,6 +63,7 @@ class GroupsLocalHandler(object):
         self.is_mine_id = hs.is_mine_id
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
+        self.notifier = hs.get_notifier()
         self.attestations = hs.get_groups_attestation_signing()
 
         # Ensure attestations get renewed
-- 
cgit 1.5.1


From a1e67bcb974e098cbbe2fbe6072bc7d7658936f9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 4 Aug 2017 10:07:10 +0100
Subject: Remove stale TODO comments

---
 synapse/handlers/groups_local.py | 10 ----------
 1 file changed, 10 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index d0ed988224..b8b1e754c7 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -23,16 +23,6 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-# TODO: Validate attestations
-# TODO: Allow users to "knock" or simpkly join depending on rules
-# TODO: is_priveged flag to users and is_public to users and rooms
-# TODO: Roles
-# TODO: Audit log for admins (profile updates, membership changes, users who tried
-#       to join but were rejected, etc)
-# TODO: Flairs
-# TODO: Add group memebership  /sync
-
-
 def _create_rerouter(func_name):
     """Returns a function that looks at the group id and calls the function
     on federation or the local group server if the group is local
-- 
cgit 1.5.1


From 05e21285aae4a0411a9ec1151ce006297fa3ca91 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 8 Aug 2017 11:50:09 +0100
Subject: Store whether the user wants to publicise their membership of a group

---
 synapse/handlers/groups_local.py                 | 4 ++++
 synapse/storage/group_server.py                  | 2 ++
 synapse/storage/schema/delta/43/group_server.sql | 1 +
 3 files changed, 7 insertions(+)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index b8b1e754c7..3a738ef36f 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -203,12 +203,16 @@ class GroupsLocalHandler(object):
                 user_id=user_id,
             )
 
+        # TODO: Check that the group is public and we're being added publically
+        is_publicised = content.get("publicise", False)
+
         token = yield self.store.register_user_group_membership(
             group_id, user_id,
             membership="join",
             is_admin=False,
             local_attestation=local_attestation,
             remote_attestation=remote_attestation,
+            is_publicised=is_publicised,
         )
         self.notifier.on_new_event(
             "groups_key", token, users=[user_id],
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index f44e80b514..31514f3cdb 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -840,6 +840,7 @@ class GroupServerStore(SQLBaseStore):
                                        is_admin=False, content={},
                                        local_attestation=None,
                                        remote_attestation=None,
+                                       is_publicised=False,
                                        ):
         """Registers that a local user is a member of a (local or remote) group.
 
@@ -873,6 +874,7 @@ class GroupServerStore(SQLBaseStore):
                     "user_id": user_id,
                     "is_admin": is_admin,
                     "membership": membership,
+                    "is_publicised": is_publicised,
                     "content": json.dumps(content),
                 },
             )
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index 92f3339c94..01ac0edc35 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -150,6 +150,7 @@ CREATE TABLE local_group_membership (
     user_id TEXT NOT NULL,
     is_admin BOOLEAN NOT NULL,
     membership TEXT NOT NULL,
+    is_publicised TEXT NOT NULL,  -- if the user is publicising their membership
     content TEXT NOT NULL
 );
 
-- 
cgit 1.5.1


From ef8e5786770ff285ebdf1fce420b5aa86437673c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 Aug 2017 13:36:22 +0100
Subject: Add bulk group publicised lookup API

---
 synapse/federation/transport/client.py | 15 ++++++++++
 synapse/federation/transport/server.py | 17 +++++++++++
 synapse/handlers/groups_local.py       | 42 ++++++++++++++++++++++++++
 synapse/rest/client/v2_alpha/groups.py | 54 ++++++++++++++++++++++++++++++++++
 synapse/storage/group_server.py        | 14 +++++++++
 5 files changed, 142 insertions(+)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 073d3abb2a..ce68cc4937 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -812,3 +812,18 @@ class TransportLayerClient(object):
             args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
+
+    def bulk_get_publicised_groups(self, destination, user_ids):
+        """Get the groups a list of users are publicising
+        """
+
+        path = PREFIX + "/get_groups_publicised"
+
+        content = {"user_ids": user_ids}
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index e04750fd2a..b5f07c50bf 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1050,6 +1050,22 @@ class FederationGroupsSummaryUsersServlet(BaseFederationServlet):
         defer.returnValue((200, resp))
 
 
+class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
+    """Get roles in a group
+    """
+    PATH = (
+        "/get_groups_publicised$"
+    )
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query):
+        resp = yield self.handler.bulk_get_publicised_groups(
+            content["user_ids"], proxy=False,
+        )
+
+        defer.returnValue((200, resp))
+
+
 FEDERATION_SERVLET_CLASSES = (
     FederationSendServlet,
     FederationPullServlet,
@@ -1102,6 +1118,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
 GROUP_LOCAL_SERVLET_CLASSES = (
     FederationGroupsLocalInviteServlet,
     FederationGroupsRemoveLocalUserServlet,
+    FederationGroupsBulkPublicisedServlet,
 )
 
 
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 3a738ef36f..c980623bbc 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -313,3 +313,45 @@ class GroupsLocalHandler(object):
     def get_joined_groups(self, user_id):
         group_ids = yield self.store.get_joined_groups(user_id)
         defer.returnValue({"groups": group_ids})
+
+    @defer.inlineCallbacks
+    def get_publicised_groups_for_user(self, user_id):
+        if self.hs.is_mine_id(user_id):
+            result = yield self.store.get_publicised_groups_for_user(user_id)
+            defer.returnValue({"groups": result})
+        else:
+            result = yield self.transport_client.get_publicised_groups_for_user(
+                get_domain_from_id(user_id), user_id
+            )
+            # TODO: Verify attestations
+            defer.returnValue(result)
+
+    @defer.inlineCallbacks
+    def bulk_get_publicised_groups(self, user_ids, proxy=True):
+        destinations = {}
+        locals = []
+
+        for user_id in user_ids:
+            if self.hs.is_mine_id(user_id):
+                locals.append(user_id)
+            else:
+                destinations.setdefault(
+                    get_domain_from_id(user_id), []
+                ).append(user_id)
+
+        if not proxy and destinations:
+            raise SynapseError(400, "Some user_ids are not local")
+
+        results = {}
+        for destination, dest_user_ids in destinations.iteritems():
+            r = yield self.transport_client.bulk_get_publicised_groups(
+                destination, dest_user_ids,
+            )
+            results.update(r)
+
+        for uid in locals:
+            results[uid] = yield self.store.get_publicised_groups_for_user(
+                uid
+            )
+
+        defer.returnValue({"users": results})
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index 9b1116acee..97d7948bb9 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -584,6 +584,59 @@ class GroupSelfUpdatePublicityServlet(RestServlet):
         defer.returnValue((200, {}))
 
 
+class PublicisedGroupsForUserServlet(RestServlet):
+    """Get the list of groups a user is advertising
+    """
+    PATTERNS = client_v2_patterns(
+        "/publicised_groups/(?P<user_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(PublicisedGroupsForUserServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.store = hs.get_datastore()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        yield self.auth.get_user_by_req(request)
+
+        result = yield self.groups_handler.get_publicised_groups_for_user(
+            user_id
+        )
+
+        defer.returnValue((200, result))
+
+
+class PublicisedGroupsForUsersServlet(RestServlet):
+    """Get the list of groups a user is advertising
+    """
+    PATTERNS = client_v2_patterns(
+        "/publicised_groups$"
+    )
+
+    def __init__(self, hs):
+        super(PublicisedGroupsForUsersServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.store = hs.get_datastore()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        yield self.auth.get_user_by_req(request)
+
+        content = parse_json_object_from_request(request)
+        user_ids = content["user_ids"]
+
+        result = yield self.groups_handler.bulk_get_publicised_groups(
+            user_ids
+        )
+
+        defer.returnValue((200, result))
+
+
 class GroupsForUserServlet(RestServlet):
     """Get all groups the logged in user is joined to
     """
@@ -627,3 +680,4 @@ def register_servlets(hs, http_server):
     GroupRolesServlet(hs).register(http_server)
     GroupSelfUpdatePublicityServlet(hs).register(http_server)
     GroupSummaryUsersRoleServlet(hs).register(http_server)
+    PublicisedGroupsForUserServlet(hs).register(http_server)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 10e757e975..0c35b03d2a 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -835,6 +835,20 @@ class GroupServerStore(SQLBaseStore):
             desc="add_room_to_group",
         )
 
+    def get_publicised_groups_for_user(self, user_id):
+        """Get all groups a user is publicising
+        """
+        return self._simple_select_onecol(
+            table="local_group_membership",
+            keyvalues={
+                "user_id": user_id,
+                "membership": "join",
+                "is_publicised": True,
+            },
+            retcol="group_id",
+            desc="get_publicised_groups_for_user",
+        )
+
     def update_group_publicity(self, group_id, user_id, publicise):
         """Update whether the user is publicising their membership of the group
         """
-- 
cgit 1.5.1


From ba3ff7918b54ae431aaaedb3d12650c93d366c04 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 11 Aug 2017 13:42:42 +0100
Subject: Fixup

---
 synapse/handlers/groups_local.py       | 22 +++++++++++++---------
 synapse/rest/client/v2_alpha/groups.py |  1 +
 2 files changed, 14 insertions(+), 9 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index c980623bbc..274fed9278 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -329,27 +329,31 @@ class GroupsLocalHandler(object):
     @defer.inlineCallbacks
     def bulk_get_publicised_groups(self, user_ids, proxy=True):
         destinations = {}
-        locals = []
+        local_users = set()
 
         for user_id in user_ids:
             if self.hs.is_mine_id(user_id):
-                locals.append(user_id)
+                local_users.add(user_id)
             else:
                 destinations.setdefault(
-                    get_domain_from_id(user_id), []
-                ).append(user_id)
+                    get_domain_from_id(user_id), set()
+                ).add(user_id)
 
         if not proxy and destinations:
             raise SynapseError(400, "Some user_ids are not local")
 
         results = {}
+        failed_results = []
         for destination, dest_user_ids in destinations.iteritems():
-            r = yield self.transport_client.bulk_get_publicised_groups(
-                destination, dest_user_ids,
-            )
-            results.update(r)
+            try:
+                r = yield self.transport_client.bulk_get_publicised_groups(
+                    destination, list(dest_user_ids),
+                )
+                results.update(r["users"])
+            except Exception:
+                failed_results.extend(dest_user_ids)
 
-        for uid in locals:
+        for uid in local_users:
             results[uid] = yield self.store.get_publicised_groups_for_user(
                 uid
             )
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index 97d7948bb9..b469058e9d 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -681,3 +681,4 @@ def register_servlets(hs, http_server):
     GroupSelfUpdatePublicityServlet(hs).register(http_server)
     GroupSummaryUsersRoleServlet(hs).register(http_server)
     PublicisedGroupsForUserServlet(hs).register(http_server)
+    PublicisedGroupsForUsersServlet(hs).register(http_server)
-- 
cgit 1.5.1


From 27ebc5c8f299488ccc0a6f100ec3b248cd81a058 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Aug 2017 11:21:34 +0100
Subject: Add remote profile cache

---
 synapse/groups/groups_server.py                   | 18 +++++
 synapse/handlers/groups_local.py                  | 17 +++-
 synapse/handlers/profile.py                       | 81 ++++++++++++++++++-
 synapse/storage/profile.py                        | 98 +++++++++++++++++++++++
 synapse/storage/schema/delta/43/profile_cache.sql | 28 +++++++
 5 files changed, 237 insertions(+), 5 deletions(-)
 create mode 100644 synapse/storage/schema/delta/43/profile_cache.sql

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index f25f327eb9..6bccae4bfb 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -503,6 +503,13 @@ class GroupsServerHandler(object):
                 get_domain_from_id(user_id), group_id, user_id, content
             )
 
+            user_profile = res.get("user_profile", {})
+            yield self.store.add_remote_profile_cache(
+                user_id,
+                displayname=user_profile.get("displayname"),
+                avatar_url=user_profile.get("avatar_url"),
+            )
+
         if res["state"] == "join":
             if not self.hs.is_mine_id(user_id):
                 remote_attestation = res["attestation"]
@@ -627,6 +634,9 @@ class GroupsServerHandler(object):
                     get_domain_from_id(user_id), group_id, user_id, {}
                 )
 
+        if not self.hs.is_mine_id(user_id):
+            yield self.store.maybe_delete_remote_profile_cache(user_id)
+
         defer.returnValue({})
 
     @defer.inlineCallbacks
@@ -647,6 +657,7 @@ class GroupsServerHandler(object):
         avatar_url = profile.get("avatar_url")
         short_description = profile.get("short_description")
         long_description = profile.get("long_description")
+        user_profile = content.get("user_profile", {})
 
         yield self.store.create_group(
             group_id,
@@ -679,6 +690,13 @@ class GroupsServerHandler(object):
             remote_attestation=remote_attestation,
         )
 
+        if not self.hs.is_mine_id(user_id):
+            yield self.store.add_remote_profile_cache(
+                user_id,
+                displayname=user_profile.get("displayname"),
+                avatar_url=user_profile.get("avatar_url"),
+            )
+
         defer.returnValue({
             "group_id": group_id,
         })
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 274fed9278..bfa10bde5a 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -56,6 +56,9 @@ class GroupsLocalHandler(object):
         self.notifier = hs.get_notifier()
         self.attestations = hs.get_groups_attestation_signing()
 
+        handlers = hs.get_handlers()
+        self.profile_handler = handlers.profile_handler
+
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
 
@@ -123,6 +126,7 @@ class GroupsLocalHandler(object):
 
         defer.returnValue(res)
 
+    @defer.inlineCallbacks
     def create_group(self, group_id, user_id, content):
         """Create a group
         """
@@ -130,13 +134,16 @@ class GroupsLocalHandler(object):
         logger.info("Asking to create group with ID: %r", group_id)
 
         if self.is_mine_id(group_id):
-            return self.groups_server_handler.create_group(
+            res = yield self.groups_server_handler.create_group(
                 group_id, user_id, content
             )
+            defer.returnValue(res)
 
-        return self.transport_client.create_group(
+        content["user_profile"] = yield self.profile_handler.get_profile(user_id)
+        res = yield self.transport_client.create_group(
             get_domain_from_id(group_id), group_id, user_id, content,
-        )  # TODO
+        )
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -265,7 +272,9 @@ class GroupsLocalHandler(object):
             "groups_key", token, users=[user_id],
         )
 
-        defer.returnValue({"state": "invite"})
+        user_profile = yield self.profile_handler.get_profile(user_id)
+
+        defer.returnValue({"state": "invite", "user_profile": user_profile})
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 7abee98dea..57e22edb0d 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -19,7 +19,7 @@ from twisted.internet import defer
 
 import synapse.types
 from synapse.api.errors import SynapseError, AuthError, CodeMessageException
-from synapse.types import UserID
+from synapse.types import UserID, get_domain_from_id
 from ._base import BaseHandler
 
 
@@ -27,15 +27,53 @@ logger = logging.getLogger(__name__)
 
 
 class ProfileHandler(BaseHandler):
+    PROFILE_UPDATE_MS = 60 * 1000
+    PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
 
     def __init__(self, hs):
         super(ProfileHandler, self).__init__(hs)
 
+        self.clock = hs.get_clock()
+
         self.federation = hs.get_replication_layer()
         self.federation.register_query_handler(
             "profile", self.on_profile_query
         )
 
+        self.clock.looping_call(self._update_remote_profile_cache, self.PROFILE_UPDATE_MS)
+
+    @defer.inlineCallbacks
+    def get_profile(self, user_id):
+        target_user = UserID.from_string(user_id)
+        if self.hs.is_mine(target_user):
+            displayname = yield self.store.get_profile_displayname(
+                target_user.localpart
+            )
+            avatar_url = yield self.store.get_profile_avatar_url(
+                target_user.localpart
+            )
+
+            defer.returnValue({
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+            })
+        else:
+            try:
+                result = yield self.federation.make_query(
+                    destination=target_user.domain,
+                    query_type="profile",
+                    args={
+                        "user_id": user_id,
+                    },
+                    ignore_backoff=True,
+                )
+                defer.returnValue(result)
+            except CodeMessageException as e:
+                if e.code != 404:
+                    logger.exception("Failed to get displayname")
+
+                raise
+
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
         if self.hs.is_mine(target_user):
@@ -182,3 +220,44 @@ class ProfileHandler(BaseHandler):
                     "Failed to update join event for room %s - %s",
                     room_id, str(e.message)
                 )
+
+    def _update_remote_profile_cache(self):
+        """Called periodically to check profiles of remote users we havent'
+        checked in a while.
+        """
+        entries = yield self.store.get_remote_profile_cache_entries_that_expire(
+            last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS
+        )
+
+        for user_id, displayname, avatar_url in entries:
+            is_subcscribed = yield self.store.is_subscribed_remote_profile_for_user(
+                user_id,
+            )
+            if not is_subcscribed:
+                yield self.store.maybe_delete_remote_profile_cache(user_id)
+                continue
+
+            try:
+                profile = yield self.federation.make_query(
+                    destination=get_domain_from_id(user_id),
+                    query_type="profile",
+                    args={
+                        "user_id": user_id,
+                    },
+                    ignore_backoff=True,
+                )
+            except:
+                logger.exception("Failed to get avatar_url")
+
+                yield self.store.update_remote_profile_cache(
+                    user_id, displayname, avatar_url
+                )
+                continue
+
+            new_name = profile.get("displayname")
+            new_avatar = profile.get("avatar_url")
+
+            # We always hit update to update the last_check timestamp
+            yield self.store.update_remote_profile_cache(
+                user_id, new_name, new_avatar
+            )
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index 26a40905ae..dca6af8a77 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from twisted.internet import defer
+
 from ._base import SQLBaseStore
 
 
@@ -55,3 +57,99 @@ class ProfileStore(SQLBaseStore):
             updatevalues={"avatar_url": new_avatar_url},
             desc="set_profile_avatar_url",
         )
+
+    def get_from_remote_profile_cache(self, user_id):
+        return self._simple_select_one(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            retcols=("displayname", "avatar_url", "last_check"),
+            allow_none=True,
+            desc="get_from_remote_profile_cache",
+        )
+
+    def add_remote_profile_cache(self, user_id, displayname, avatar_url):
+        """Ensure we are caching the remote user's profiles.
+
+        This should only be called when `is_subscribed_remote_profile_for_user`
+        would return true for the user.
+        """
+        return self._simple_upsert(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            values={
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+                "last_check": self._clock.time_msec(),
+            },
+            desc="add_remote_profile_cache",
+        )
+
+    def update_remote_profile_cache(self, user_id, displayname, avatar_url):
+        return self._simple_update(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            values={
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+                "last_check": self._clock.time_msec(),
+            },
+            desc="update_remote_profile_cache",
+        )
+
+    @defer.inlineCallbacks
+    def maybe_delete_remote_profile_cache(self, user_id):
+        """Check if we still care about the remote user's profile, and if we
+        don't then remove their profile from the cache
+        """
+        subscribed = yield self.is_subscribed_remote_profile_for_user(user_id)
+        if not subscribed:
+            yield self._simple_delete(
+                table="remote_profile_cache",
+                keyvalues={"user_id": user_id},
+                desc="delete_remote_profile_cache",
+            )
+
+    def get_remote_profile_cache_entries_that_expire(self, last_checked):
+        """Get all users who haven't been checked since `last_checked`
+        """
+        def _get_remote_profile_cache_entries_that_expire_txn(txn):
+            sql = """
+                SELECT user_id, displayname, avatar_url
+                FROM remote_profile_cache
+                WHERE last_check < ?
+            """
+
+            txn.execute(sql, (last_checked,))
+
+            return self.cursor_to_dict(txn)
+
+        return self.runInteraction(
+            "get_remote_profile_cache_entries_that_expire",
+            _get_remote_profile_cache_entries_that_expire_txn,
+        )
+
+    @defer.inlineCallbacks
+    def is_subscribed_remote_profile_for_user(self, user_id):
+        """Check whether we are interested in a remote user's profile.
+        """
+        res = yield self._simple_select_one_onecol(
+            table="group_users",
+            keyvalues={"user_id": user_id},
+            retcol="user_id",
+            allow_none=True,
+            desc="should_update_remote_profile_cache_for_user",
+        )
+
+        if res:
+            defer.returnValue(True)
+
+        res = yield self._simple_select_one_onecol(
+            table="group_invites",
+            keyvalues={"user_id": user_id},
+            retcol="user_id",
+            allow_none=True,
+            desc="should_update_remote_profile_cache_for_user",
+        )
+
+        if res:
+            defer.returnValue(True)
diff --git a/synapse/storage/schema/delta/43/profile_cache.sql b/synapse/storage/schema/delta/43/profile_cache.sql
new file mode 100644
index 0000000000..e5ddc84df0
--- /dev/null
+++ b/synapse/storage/schema/delta/43/profile_cache.sql
@@ -0,0 +1,28 @@
+/* Copyright 2017 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- A subset of remote users whose profiles we have cached.
+-- Whether a user is in this table or not is defined by the storage function
+-- `is_subscribed_remote_profile_for_user`
+CREATE TABLE remote_profile_cache (
+    user_id TEXT NOT NULL,
+    displayname TEXT,
+    avatar_url TEXT,
+    last_check BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
+CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);
-- 
cgit 1.5.1


From bf81f3cf2c3e5b1d96953f3116c22aee05fb79b3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Aug 2017 14:34:56 +0100
Subject: Split out profile handler to fix tests

---
 synapse/handlers/__init__.py         |  2 --
 synapse/handlers/groups_local.py     |  3 +--
 synapse/handlers/message.py          |  3 ++-
 synapse/handlers/profile.py          | 13 ++++++++-----
 synapse/handlers/register.py         |  4 ++--
 synapse/handlers/room_member.py      |  4 +++-
 synapse/rest/client/v1/profile.py    | 18 +++++++++---------
 synapse/server.py                    |  5 +++++
 tests/handlers/test_profile.py       |  4 +---
 tests/handlers/test_register.py      |  5 +++--
 tests/rest/client/v1/test_profile.py |  3 +--
 11 files changed, 35 insertions(+), 29 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 5ad408f549..53213cdccf 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -20,7 +20,6 @@ from .room import (
 from .room_member import RoomMemberHandler
 from .message import MessageHandler
 from .federation import FederationHandler
-from .profile import ProfileHandler
 from .directory import DirectoryHandler
 from .admin import AdminHandler
 from .identity import IdentityHandler
@@ -52,7 +51,6 @@ class Handlers(object):
         self.room_creation_handler = RoomCreationHandler(hs)
         self.room_member_handler = RoomMemberHandler(hs)
         self.federation_handler = FederationHandler(hs)
-        self.profile_handler = ProfileHandler(hs)
         self.directory_handler = DirectoryHandler(hs)
         self.admin_handler = AdminHandler(hs)
         self.identity_handler = IdentityHandler(hs)
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index bfa10bde5a..1950c12f1f 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -56,8 +56,7 @@ class GroupsLocalHandler(object):
         self.notifier = hs.get_notifier()
         self.attestations = hs.get_groups_attestation_signing()
 
-        handlers = hs.get_handlers()
-        self.profile_handler = handlers.profile_handler
+        self.profile_handler = hs.get_profile_handler()
 
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index be4f123c54..5b8f20b73c 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -47,6 +47,7 @@ class MessageHandler(BaseHandler):
         self.state = hs.get_state_handler()
         self.clock = hs.get_clock()
         self.validator = EventValidator()
+        self.profile_handler = hs.get_profile_handler()
 
         self.pagination_lock = ReadWriteLock()
 
@@ -210,7 +211,7 @@ class MessageHandler(BaseHandler):
 
                 if membership in {Membership.JOIN, Membership.INVITE}:
                     # If event doesn't include a display name, add one.
-                    profile = self.hs.get_handlers().profile_handler
+                    profile = self.profile_handler
                     content = builder.content
 
                     try:
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 57e22edb0d..5e34501c7a 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -22,18 +22,21 @@ from synapse.api.errors import SynapseError, AuthError, CodeMessageException
 from synapse.types import UserID, get_domain_from_id
 from ._base import BaseHandler
 
-
 logger = logging.getLogger(__name__)
 
 
-class ProfileHandler(BaseHandler):
+class ProfileHandler(object):
     PROFILE_UPDATE_MS = 60 * 1000
     PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
 
     def __init__(self, hs):
-        super(ProfileHandler, self).__init__(hs)
-
+        self.hs = hs
+        self.store = hs.get_datastore()
         self.clock = hs.get_clock()
+        self.ratelimiter = hs.get_ratelimiter()
+
+        # AWFUL hack to get at BaseHandler.ratelimit
+        self.base_handler = BaseHandler(hs)
 
         self.federation = hs.get_replication_layer()
         self.federation.register_query_handler(
@@ -194,7 +197,7 @@ class ProfileHandler(BaseHandler):
         if not self.hs.is_mine(user):
             return
 
-        yield self.ratelimit(requester)
+        yield self.base_handler.ratelimit(requester)
 
         room_ids = yield self.store.get_rooms_for_user(
             user.to_string(),
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ee3a2269a8..560fb36254 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -36,6 +36,7 @@ class RegistrationHandler(BaseHandler):
         super(RegistrationHandler, self).__init__(hs)
 
         self.auth = hs.get_auth()
+        self.profile_handler = hs.get_profile_handler()
         self.captcha_client = CaptchaServerHttpClient(hs)
 
         self._next_generated_user_id = None
@@ -423,8 +424,7 @@ class RegistrationHandler(BaseHandler):
 
         if displayname is not None:
             logger.info("setting user display name: %s -> %s", user_id, displayname)
-            profile_handler = self.hs.get_handlers().profile_handler
-            yield profile_handler.set_displayname(
+            yield self.profile_handler.set_displayname(
                 user, requester, displayname, by_admin=True,
             )
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index b3f979b246..dadc19d45b 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -45,6 +45,8 @@ class RoomMemberHandler(BaseHandler):
     def __init__(self, hs):
         super(RoomMemberHandler, self).__init__(hs)
 
+        self.profile_handler = hs.get_profile_handler()
+
         self.member_linearizer = Linearizer(name="member")
 
         self.clock = hs.get_clock()
@@ -255,7 +257,7 @@ class RoomMemberHandler(BaseHandler):
 
                 content["membership"] = Membership.JOIN
 
-                profile = self.hs.get_handlers().profile_handler
+                profile = self.profile_handler
                 if not content_specified:
                     content["displayname"] = yield profile.get_displayname(target)
                     content["avatar_url"] = yield profile.get_avatar_url(target)
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index 1a5045c9ec..d7edc34245 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -26,13 +26,13 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(ProfileDisplaynameRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.profile_handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         user = UserID.from_string(user_id)
 
-        displayname = yield self.handlers.profile_handler.get_displayname(
+        displayname = yield self.profile_handler.get_displayname(
             user,
         )
 
@@ -55,7 +55,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
         except:
             defer.returnValue((400, "Unable to parse name"))
 
-        yield self.handlers.profile_handler.set_displayname(
+        yield self.profile_handler.set_displayname(
             user, requester, new_name, is_admin)
 
         defer.returnValue((200, {}))
@@ -69,13 +69,13 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(ProfileAvatarURLRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.profile_handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         user = UserID.from_string(user_id)
 
-        avatar_url = yield self.handlers.profile_handler.get_avatar_url(
+        avatar_url = yield self.profile_handler.get_avatar_url(
             user,
         )
 
@@ -97,7 +97,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
         except:
             defer.returnValue((400, "Unable to parse name"))
 
-        yield self.handlers.profile_handler.set_avatar_url(
+        yield self.profile_handler.set_avatar_url(
             user, requester, new_name, is_admin)
 
         defer.returnValue((200, {}))
@@ -111,16 +111,16 @@ class ProfileRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(ProfileRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.profile_handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         user = UserID.from_string(user_id)
 
-        displayname = yield self.handlers.profile_handler.get_displayname(
+        displayname = yield self.profile_handler.get_displayname(
             user,
         )
-        avatar_url = yield self.handlers.profile_handler.get_avatar_url(
+        avatar_url = yield self.profile_handler.get_avatar_url(
             user,
         )
 
diff --git a/synapse/server.py b/synapse/server.py
index d0a6272766..5b892cc390 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -51,6 +51,7 @@ from synapse.handlers.receipts import ReceiptsHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.user_directory import UserDirectoyHandler
 from synapse.handlers.groups_local import GroupsLocalHandler
+from synapse.handlers.profile import ProfileHandler
 from synapse.groups.groups_server import GroupsServerHandler
 from synapse.groups.attestations import GroupAttestionRenewer, GroupAttestationSigning
 from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory
@@ -114,6 +115,7 @@ class HomeServer(object):
         'application_service_scheduler',
         'application_service_handler',
         'device_message_handler',
+        'profile_handler',
         'notifier',
         'distributor',
         'client_resource',
@@ -258,6 +260,9 @@ class HomeServer(object):
     def build_initial_sync_handler(self):
         return InitialSyncHandler(self)
 
+    def build_profile_handler(self):
+        return ProfileHandler(self)
+
     def build_event_sources(self):
         return EventSources(self)
 
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index 2a203129ca..a5f47181d7 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -62,8 +62,6 @@ class ProfileTestCase(unittest.TestCase):
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
 
-        hs.handlers = ProfileHandlers(hs)
-
         self.store = hs.get_datastore()
 
         self.frank = UserID.from_string("@1234ABCD:test")
@@ -72,7 +70,7 @@ class ProfileTestCase(unittest.TestCase):
 
         yield self.store.create_profile(self.frank.localpart)
 
-        self.handler = hs.get_handlers().profile_handler
+        self.handler = hs.get_profile_handler()
 
     @defer.inlineCallbacks
     def test_get_my_name(self):
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index c8cf9a63ec..e990e45220 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -40,13 +40,14 @@ class RegistrationTestCase(unittest.TestCase):
         self.hs = yield setup_test_homeserver(
             handlers=None,
             http_client=None,
-            expire_access_token=True)
+            expire_access_token=True,
+            profile_handler=Mock(),
+        )
         self.macaroon_generator = Mock(
             generate_access_token=Mock(return_value='secret'))
         self.hs.get_macaroon_generator = Mock(return_value=self.macaroon_generator)
         self.hs.handlers = RegistrationHandlers(self.hs)
         self.handler = self.hs.get_handlers().registration_handler
-        self.hs.get_handlers().profile_handler = Mock()
 
     @defer.inlineCallbacks
     def test_user_is_created_and_logged_in_if_doesnt_exist(self):
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 1e95e97538..dddcf51b69 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -46,6 +46,7 @@ class ProfileTestCase(unittest.TestCase):
             resource_for_client=self.mock_resource,
             federation=Mock(),
             replication_layer=Mock(),
+            profile_handler=self.mock_handler
         )
 
         def _get_user_by_req(request=None, allow_guest=False):
@@ -53,8 +54,6 @@ class ProfileTestCase(unittest.TestCase):
 
         hs.get_v1auth().get_user_by_req = _get_user_by_req
 
-        hs.get_handlers().profile_handler = self.mock_handler
-
         profile.register_servlets(hs, self.mock_resource)
 
     @defer.inlineCallbacks
-- 
cgit 1.5.1


From 93e504d04e5e92ec1d54b3c7c860adc2cfb0e9f4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 19 Sep 2017 10:35:35 +0100
Subject: Ensure that creator of group sees group down /sync

---
 synapse/handlers/groups_local.py | 34 ++++++++++++++++++++++++++++++----
 synapse/handlers/sync.py         |  1 +
 2 files changed, 31 insertions(+), 4 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 1950c12f1f..b4833f8ef8 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -136,12 +136,38 @@ class GroupsLocalHandler(object):
             res = yield self.groups_server_handler.create_group(
                 group_id, user_id, content
             )
-            defer.returnValue(res)
+            local_attestation = None
+            remote_attestation = None
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content["attestation"] = local_attestation
+
+            content["user_profile"] = yield self.profile_handler.get_profile(user_id)
+
+            res = yield self.transport_client.create_group(
+                get_domain_from_id(group_id), group_id, user_id, content,
+            )
+
+            remote_attestation = res["attestation"]
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                group_id=group_id,
+                user_id=user_id,
+            )
 
-        content["user_profile"] = yield self.profile_handler.get_profile(user_id)
-        res = yield self.transport_client.create_group(
-            get_domain_from_id(group_id), group_id, user_id, content,
+        is_publicised = content.get("publicise", False)
+        token = yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="join",
+            is_admin=True,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+            is_publicised=is_publicised,
         )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
+
         defer.returnValue(res)
 
     @defer.inlineCallbacks
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index f2e4ffcec6..69c1bc189e 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -637,6 +637,7 @@ class SyncHandler(object):
 
             if membership == "join":
                 if gtype == "membership":
+                    # TODO: Add profile
                     content.pop("membership", None)
                     joined[group_id] = content["content"]
                 else:
-- 
cgit 1.5.1


From 95298783bb86e13da56e07fa3f73b6e2de053c08 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 26 Sep 2017 11:04:37 +0100
Subject: Add is_publicised to group summary

---
 synapse/handlers/groups_local.py | 56 ++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 25 deletions(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index b4833f8ef8..14fdf06b58 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -96,32 +96,38 @@ class GroupsLocalHandler(object):
             res = yield self.groups_server_handler.get_group_summary(
                 group_id, requester_user_id
             )
-            defer.returnValue(res)
-
-        res = yield self.transport_client.get_group_summary(
-            get_domain_from_id(group_id), group_id, requester_user_id,
-        )
-
-        # Loop through the users and validate the attestations.
-        chunk = res["users_section"]["users"]
-        valid_users = []
-        for entry in chunk:
-            g_user_id = entry["user_id"]
-            attestation = entry.pop("attestation")
-            try:
-                yield self.attestations.verify_attestation(
-                    attestation,
-                    group_id=group_id,
-                    user_id=g_user_id,
-                )
-                valid_users.append(entry)
-            except Exception as e:
-                logger.info("Failed to verify user is in group: %s", e)
-
-        res["users_section"]["users"] = valid_users
+        else:
+            res = yield self.transport_client.get_group_summary(
+                get_domain_from_id(group_id), group_id, requester_user_id,
+            )
 
-        res["users_section"]["users"].sort(key=lambda e: e.get("order", 0))
-        res["rooms_section"]["rooms"].sort(key=lambda e: e.get("order", 0))
+            # Loop through the users and validate the attestations.
+            chunk = res["users_section"]["users"]
+            valid_users = []
+            for entry in chunk:
+                g_user_id = entry["user_id"]
+                attestation = entry.pop("attestation")
+                try:
+                    yield self.attestations.verify_attestation(
+                        attestation,
+                        group_id=group_id,
+                        user_id=g_user_id,
+                    )
+                    valid_users.append(entry)
+                except Exception as e:
+                    logger.info("Failed to verify user is in group: %s", e)
+
+            res["users_section"]["users"] = valid_users
+
+            res["users_section"]["users"].sort(key=lambda e: e.get("order", 0))
+            res["rooms_section"]["rooms"].sort(key=lambda e: e.get("order", 0))
+
+        # Add `is_publicised` flag to indicate whether the user has publicised their
+        # membership of the group on their profile
+        result = yield self.store.get_publicised_groups_for_user(requester_user_id)
+        is_publicised = group_id in result
+
+        res.setdefault("user", {})["is_publicised"] = is_publicised
 
         defer.returnValue(res)
 
-- 
cgit 1.5.1


From 17b8e2bd02ad0abbd25103b637eb8490f3a53507 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 26 Sep 2017 15:52:41 +0100
Subject: Add remove room API

---
 synapse/federation/transport/client.py | 12 ++++++++++++
 synapse/federation/transport/server.py | 14 +++++++++++++-
 synapse/groups/groups_server.py        | 12 ++++++++++++
 synapse/handlers/groups_local.py       |  1 +
 synapse/rest/client/v2_alpha/groups.py | 11 +++++++++++
 synapse/storage/group_server.py        | 23 +++++++++++++++++++++++
 6 files changed, 72 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers/groups_local.py')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ce68cc4937..36f6eb75e9 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -525,6 +525,18 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
+        """Remove a room from a group
+        """
+        path = PREFIX + "/groups/%s/room/%s" % (group_id, room_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
     @log_function
     def get_users_in_group(self, destination, group_id, requester_user_id):
         """Get users in a group
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index b5f07c50bf..c7565e0737 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -674,7 +674,7 @@ class FederationGroupsRoomsServlet(BaseFederationServlet):
 
 
 class FederationGroupsAddRoomsServlet(BaseFederationServlet):
-    """Add room to group
+    """Add/remove room from group
     """
     PATH = "/groups/(?P<group_id>[^/]*)/room/(?<room_id>)$"
 
@@ -690,6 +690,18 @@ class FederationGroupsAddRoomsServlet(BaseFederationServlet):
 
         defer.returnValue((200, new_content))
 
+    @defer.inlineCallbacks
+    def on_DELETE(self, origin, content, query, group_id, room_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.remove_room_from_group(
+            group_id, requester_user_id, room_id,
+        )
+
+        defer.returnValue((200, new_content))
+
 
 class FederationGroupsUsersServlet(BaseFederationServlet):
     """Get the users in a group on behalf of a user
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 699d8a5265..10bf61d178 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -472,6 +472,18 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
+    @defer.inlineCallbacks
+    def remove_room_from_group(self, group_id, requester_user_id, room_id):
+        """Remove room from group
+        """
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id
+        )
+
+        yield self.store.remove_room_from_group(group_id, room_id)
+
+        defer.returnValue({})
+
     @defer.inlineCallbacks
     def invite_to_group(self, group_id, user_id, requester_user_id, content):
         """Invite user to group
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 14fdf06b58..a2bacbfc38 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -69,6 +69,7 @@ class GroupsLocalHandler(object):
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
     add_room_to_group = _create_rerouter("add_room_to_group")
+    remove_room_from_group = _create_rerouter("remove_room_from_group")
 
     update_group_summary_room = _create_rerouter("update_group_summary_room")
     delete_group_summary_room = _create_rerouter("delete_group_summary_room")
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index b469058e9d..8f3ce15b02 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -423,6 +423,17 @@ class GroupAdminRoomsServlet(RestServlet):
 
         defer.returnValue((200, result))
 
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        result = yield self.groups_handler.remove_room_from_group(
+            group_id, user_id, room_id,
+        )
+
+        defer.returnValue((200, result))
+
 
 class GroupAdminUsersInviteServlet(RestServlet):
     """Invite a user to the group
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index d0b5ad231a..4fe9172adc 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -843,6 +843,29 @@ class GroupServerStore(SQLBaseStore):
             desc="add_room_to_group",
         )
 
+    def remove_room_from_group(self, group_id, room_id):
+        def _remove_room_from_group_txn(txn):
+            self._simple_delete_txn(
+                txn,
+                table="group_rooms",
+                keyvalues={
+                    "group_id": group_id,
+                    "room_id": room_id,
+                },
+            )
+
+            self._simple_delete_txn(
+                txn,
+                table="group_summary_rooms",
+                keyvalues={
+                    "group_id": group_id,
+                    "room_id": room_id,
+                },
+            )
+        return self.runInteraction(
+            "remove_room_from_group", _remove_room_from_group_txn,
+        )
+
     def get_publicised_groups_for_user(self, user_id):
         """Get all groups a user is publicising
         """
-- 
cgit 1.5.1