From b8ca494ee9e42e5b1aca8958088bd35cc5707437 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Jul 2017 15:44:15 +0100
Subject: Initial group server implementation

---
 synapse/groups/__init__.py      |   0
 synapse/groups/attestations.py  | 120 +++++++++++++
 synapse/groups/groups_server.py | 382 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 502 insertions(+)
 create mode 100644 synapse/groups/__init__.py
 create mode 100644 synapse/groups/attestations.py
 create mode 100644 synapse/groups/groups_server.py

(limited to 'synapse/groups')

diff --git a/synapse/groups/__init__.py b/synapse/groups/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
new file mode 100644
index 0000000000..d83076a9b3
--- /dev/null
+++ b/synapse/groups/attestations.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.types import get_domain_from_id
+from synapse.util.logcontext import preserve_fn
+
+from signedjson.sign import sign_json
+
+
+DEFAULT_ATTESTATION_LENGTH_MS = 3 * 24 * 60 * 60 * 1000
+MIN_ATTESTATION_LENGTH_MS = 1 * 60 * 60 * 1000
+UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
+
+
+class GroupAttestationSigning(object):
+    def __init__(self, hs):
+        self.keyring = hs.get_keyring()
+        self.clock = hs.get_clock()
+        self.server_name = hs.hostname
+        self.signing_key = hs.config.signing_key[0]
+
+    @defer.inlineCallbacks
+    def verify_attestation(self, attestation, group_id, user_id, server_name=None):
+        if not server_name:
+            if get_domain_from_id(group_id) == self.server_name:
+                server_name = get_domain_from_id(user_id)
+            else:
+                server_name = get_domain_from_id(group_id)
+
+        if user_id != attestation["user_id"]:
+            raise SynapseError(400, "Attestation has incorrect user_id")
+
+        if group_id != attestation["group_id"]:
+            raise SynapseError(400, "Attestation has incorrect group_id")
+
+        valid_until_ms = attestation["valid_until_ms"]
+        if valid_until_ms - self.clock.time_msec() < MIN_ATTESTATION_LENGTH_MS:
+            raise SynapseError(400, "Attestation not valid for long enough")
+
+        yield self.keyring.verify_json_for_server(server_name, attestation)
+
+    def create_attestation(self, group_id, user_id):
+        return sign_json({
+            "group_id": group_id,
+            "user_id": user_id,
+            "valid_until_ms": self.clock.time_msec() + DEFAULT_ATTESTATION_LENGTH_MS,
+        }, self.server_name, self.signing_key)
+
+
+class GroupAttestionRenewer(object):
+    def __init__(self, hs):
+        self.clock = hs.get_clock()
+        self.store = hs.get_datastore()
+        self.assestations = hs.get_groups_attestation_signing()
+        self.transport_client = hs.get_federation_transport_client()
+
+        self._renew_attestations_loop = self.clock.looping_call(
+            self._renew_attestations, 30 * 60 * 1000,
+        )
+
+    @defer.inlineCallbacks
+    def on_renew_attestation(self, group_id, user_id, content):
+        attestation = content["attestation"]
+
+        yield self.attestations.verify_attestation(
+            attestation,
+            user_id=user_id,
+            group_id=group_id,
+        )
+
+        yield self.store.update_remote_attestion(group_id, user_id, attestation)
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def _renew_attestations(self):
+        now = self.clock.time_msec()
+
+        rows = yield self.store.get_attestations_need_renewals(
+            now + UPDATE_ATTESTATION_TIME_MS
+        )
+
+        @defer.inlineCallbacks
+        def _renew_attestation(self, group_id, user_id):
+            attestation = self.attestations.create_attestation(group_id, user_id)
+
+            if self.hs.is_mine_id(group_id):
+                destination = get_domain_from_id(user_id)
+            else:
+                destination = get_domain_from_id(group_id)
+
+            yield self.transport_client.renew_group_attestation(
+                destination, group_id, user_id,
+                content={"attestation": attestation},
+            )
+
+            yield self.store.update_attestation_renewal(
+                group_id, user_id, attestation
+            )
+
+        for row in rows:
+            group_id = row["group_id"]
+            user_id = row["user_id"]
+
+            preserve_fn(_renew_attestation)(group_id, user_id)
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
new file mode 100644
index 0000000000..195f1eae54
--- /dev/null
+++ b/synapse/groups/groups_server.py
@@ -0,0 +1,382 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.types import UserID, get_domain_from_id
+
+
+import functools
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: Allow users to "knock" or simpkly join depending on rules
+# TODO: Federation admin APIs
+# TODO: is_priveged flag to users and is_public to users and rooms
+# TODO: Audit log for admins (profile updates, membership changes, users who tried
+#       to join but were rejected, etc)
+# TODO: Flairs
+
+
+UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
+
+
+def check_group_is_ours(and_exists=False):
+    def g(func):
+        @functools.wraps(func)
+        @defer.inlineCallbacks
+        def h(self, group_id, *args, **kwargs):
+            if not self.is_mine_id(group_id):
+                raise SynapseError(400, "Group not on this server")
+            if and_exists:
+                group = yield self.store.get_group(group_id)
+                if not group:
+                    raise SynapseError(404, "Unknown group")
+
+            res = yield func(self, group_id, *args, **kwargs)
+            defer.returnValue(res)
+
+        return h
+    return g
+
+
+class GroupsServerHandler(object):
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.room_list_handler = hs.get_room_list_handler()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.keyring = hs.get_keyring()
+        self.is_mine_id = hs.is_mine_id
+        self.signing_key = hs.config.signing_key[0]
+        self.server_name = hs.hostname
+        self.attestations = hs.get_groups_attestation_signing()
+        self.transport_client = hs.get_federation_transport_client()
+
+        # Ensure attestations get renewed
+        hs.get_groups_attestation_renewer()
+
+    @check_group_is_ours()
+    @defer.inlineCallbacks
+    def get_group_profile(self, group_id, requester_user_id):
+        group_description = yield self.store.get_group(group_id)
+
+        if group_description:
+            defer.returnValue(group_description)
+        else:
+            raise SynapseError(404, "Unknown group")
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def get_users_in_group(self, group_id, requester_user_id):
+        is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
+
+        user_results = yield self.store.get_users_in_group(
+            group_id, include_private=is_user_in_group,
+        )
+
+        chunk = []
+        for user_result in user_results:
+            g_user_id = user_result["user_id"]
+            is_public = user_result["is_public"]
+
+            entry = {"user_id": g_user_id}
+
+            # TODO: Get profile information
+
+            if not is_public:
+                entry["is_public"] = False
+
+            if not self.is_mine_id(requester_user_id):
+                attestation = yield self.store.get_remote_attestation(group_id, g_user_id)
+                if not attestation:
+                    continue
+
+                entry["attestation"] = attestation
+            else:
+                entry["attestation"] = self.attestations.create_attestation(
+                    group_id, g_user_id,
+                )
+
+            chunk.append(entry)
+
+        # TODO: If admin add lists of users whose attestations have timed out
+
+        defer.returnValue({
+            "chunk": chunk,
+            "total_user_count_estimate": len(user_results),
+        })
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def get_rooms_in_group(self, group_id, requester_user_id):
+        is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
+
+        room_results = yield self.store.get_rooms_in_group(
+            group_id, include_private=is_user_in_group,
+        )
+
+        chunk = []
+        for room_result in room_results:
+            room_id = room_result["room_id"]
+            is_public = room_result["is_public"]
+
+            joined_users = yield self.store.get_users_in_room(room_id)
+            entry = yield self.room_list_handler.generate_room_entry(
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
+            )
+
+            if not entry:
+                continue
+
+            if not is_public:
+                entry["is_public"] = False
+
+            chunk.append(entry)
+
+        chunk.sort(key=lambda e: -e["num_joined_members"])
+
+        defer.returnValue({
+            "chunk": chunk,
+            "total_room_count_estimate": len(room_results),
+        })
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def add_room(self, group_id, requester_user_id, room_id, content):
+        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        # TODO: Check if room has already been added
+
+        visibility = content.get("visibility")
+        if visibility:
+            vis_type = visibility["type"]
+            if vis_type not in ("public", "private"):
+                raise SynapseError(
+                    400, "Synapse only supports 'public'/'private' visibility"
+                )
+            is_public = vis_type == "public"
+        else:
+            is_public = True
+
+        yield self.store.add_room_to_group(group_id, room_id, is_public=is_public)
+
+        defer.returnValue({})
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def invite_to_group(self, group_id, user_id, requester_user_id, content):
+        is_admin = yield self.store.is_user_admin_in_group(
+            group_id, requester_user_id
+        )
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        # TODO: Check if user knocked
+        # TODO: Check if user is already invited
+
+        group = yield self.store.get_group(group_id)
+        content = {
+            "profile": {
+                "name": group["name"],
+                "avatar_url": group["avatar_url"],
+            },
+            "inviter": requester_user_id,
+        }
+
+        if self.hs.is_mine_id(user_id):
+            raise NotImplementedError()
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content.update({
+                "attestation": local_attestation,
+            })
+
+            res = yield self.transport_client.invite_to_group_notification(
+                get_domain_from_id(user_id), group_id, user_id, content
+            )
+
+        if res["state"] == "join":
+            if not self.hs.is_mine_id(user_id):
+                remote_attestation = res["attestation"]
+
+                yield self.attestations.verify_attestation(
+                    remote_attestation,
+                    user_id=user_id,
+                    group_id=group_id,
+                )
+            else:
+                remote_attestation = None
+
+            yield self.store.add_user_to_group(
+                group_id, user_id,
+                is_admin=False,
+                is_public=False,  # TODO
+                local_attestation=local_attestation,
+                remote_attestation=remote_attestation,
+            )
+        elif res["state"] == "invite":
+            yield self.store.add_group_invite(
+                group_id, user_id,
+            )
+            defer.returnValue({
+                "state": "invite"
+            })
+        elif res["state"] == "reject":
+            defer.returnValue({
+                "state": "reject"
+            })
+        else:
+            raise SynapseError(502, "Unknown state returned by HS")
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def accept_invite(self, group_id, user_id, content):
+        if not self.store.is_user_invited_to_local_group(group_id, user_id):
+            raise SynapseError(403, "User not invited to group")
+
+        if not self.hs.is_mine_id(user_id):
+            remote_attestation = content["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                user_id=user_id,
+                group_id=group_id,
+            )
+        else:
+            remote_attestation = None
+
+        local_attestation = self.attestations.create_attestation(group_id, user_id)
+
+        visibility = content.get("visibility")
+        if visibility:
+            vis_type = visibility["type"]
+            if vis_type not in ("public", "private"):
+                raise SynapseError(
+                    400, "Synapse only supports 'public'/'private' visibility"
+                )
+            is_public = vis_type == "public"
+        else:
+            is_public = True
+
+        yield self.store.add_user_to_group(
+            group_id, user_id,
+            is_admin=False,
+            is_public=is_public,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({
+            "state": "join",
+            "attestation": local_attestation,
+        })
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def knock(self, group_id, user_id, content):
+        pass
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def accept_knock(self, group_id, user_id, content):
+        pass
+
+    @check_group_is_ours(and_exists=True)
+    @defer.inlineCallbacks
+    def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        is_kick = False
+        if requester_user_id != user_id:
+            is_admin = yield self.store.is_user_admin_in_group(
+                group_id, requester_user_id
+            )
+            if not is_admin:
+                raise SynapseError(403, "User is not admin in group")
+
+            is_kick = True
+
+        yield self.store.remove_user_to_group(
+            group_id, user_id,
+        )
+
+        if is_kick:
+            if self.hs.is_mine_id(user_id):
+                raise NotImplementedError()
+            else:
+                yield self.transport_client.remove_user_from_group_notification(
+                    get_domain_from_id(user_id), group_id, user_id, {}
+                )
+
+        defer.returnValue({})
+
+    @check_group_is_ours()
+    @defer.inlineCallbacks
+    def create_group(self, group_id, user_id, content):
+        logger.info("Attempting to create group with ID: %r", group_id)
+        group = yield self.store.get_group(group_id)
+        if group:
+            raise SynapseError(400, "Group already exists")
+
+        is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
+        if not is_admin and not group_id.startswith("+u/"):
+            raise SynapseError(403, "Group ID must start with '+u/' or be a server admin")
+
+        profile = content.get("profile", {})
+        name = profile.get("name")
+        avatar_url = profile.get("avatar_url")
+        short_description = profile.get("short_description")
+        long_description = profile.get("long_description")
+
+        yield self.store.create_group(
+            group_id,
+            user_id,
+            name=name,
+            avatar_url=avatar_url,
+            short_description=short_description,
+            long_description=long_description,
+        )
+
+        if not self.hs.is_mine_id(user_id):
+            remote_attestation = content["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                user_id=user_id,
+                group_id=group_id,
+            )
+
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+        else:
+            local_attestation = None
+            remote_attestation = None
+
+        yield self.store.add_user_to_group(
+            group_id, user_id,
+            is_admin=True,
+            is_public=True,  # TODO
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({
+            "group_id": group_id,
+        })
-- 
cgit 1.4.1


From 83936293eb3ddb8998191b537153eaeec5e7afb0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 09:58:59 +0100
Subject: Comments

---
 synapse/groups/attestations.py                   |  29 +++++-
 synapse/groups/groups_server.py                  | 108 +++++++++++++++--------
 synapse/storage/group_server.py                  |  32 ++++++-
 synapse/storage/schema/delta/43/group_server.sql |   6 +-
 4 files changed, 132 insertions(+), 43 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index d83076a9b3..6937fa44cf 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -28,6 +28,8 @@ UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
 
 
 class GroupAttestationSigning(object):
+    """Creates and verifies group attestations.
+    """
     def __init__(self, hs):
         self.keyring = hs.get_keyring()
         self.clock = hs.get_clock()
@@ -36,11 +38,20 @@ class GroupAttestationSigning(object):
 
     @defer.inlineCallbacks
     def verify_attestation(self, attestation, group_id, user_id, server_name=None):
+        """Verifies that the given attestation matches the given paramaters.
+
+        An optional server_name can be supplied to explicitly set which server's
+        signature is expected. Otherwise assumes that either the group_id or user_id
+        is local and uses the other's server as the one to check.
+        """
+
         if not server_name:
             if get_domain_from_id(group_id) == self.server_name:
                 server_name = get_domain_from_id(user_id)
-            else:
+            elif get_domain_from_id(user_id) == self.server_name:
                 server_name = get_domain_from_id(group_id)
+            else:
+                raise Exception("Expected eitehr group_id or user_id to be local")
 
         if user_id != attestation["user_id"]:
             raise SynapseError(400, "Attestation has incorrect user_id")
@@ -48,6 +59,7 @@ class GroupAttestationSigning(object):
         if group_id != attestation["group_id"]:
             raise SynapseError(400, "Attestation has incorrect group_id")
 
+        # TODO:
         valid_until_ms = attestation["valid_until_ms"]
         if valid_until_ms - self.clock.time_msec() < MIN_ATTESTATION_LENGTH_MS:
             raise SynapseError(400, "Attestation not valid for long enough")
@@ -55,6 +67,9 @@ class GroupAttestationSigning(object):
         yield self.keyring.verify_json_for_server(server_name, attestation)
 
     def create_attestation(self, group_id, user_id):
+        """Create an attestation for the group_id and user_id with default
+        validity length.
+        """
         return sign_json({
             "group_id": group_id,
             "user_id": user_id,
@@ -63,11 +78,15 @@ class GroupAttestationSigning(object):
 
 
 class GroupAttestionRenewer(object):
+    """Responsible for sending and receiving attestation updates.
+    """
+
     def __init__(self, hs):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
         self.assestations = hs.get_groups_attestation_signing()
         self.transport_client = hs.get_federation_transport_client()
+        self.is_mine_id = hs.is_mind_id
 
         self._renew_attestations_loop = self.clock.looping_call(
             self._renew_attestations, 30 * 60 * 1000,
@@ -75,8 +94,13 @@ class GroupAttestionRenewer(object):
 
     @defer.inlineCallbacks
     def on_renew_attestation(self, group_id, user_id, content):
+        """When a remote updates an attestation
+        """
         attestation = content["attestation"]
 
+        if not self.is_mine_id(group_id) and not self.is_mine_id(user_id):
+            raise SynapseError(400, "Neither user not group are on this server")
+
         yield self.attestations.verify_attestation(
             attestation,
             user_id=user_id,
@@ -89,6 +113,9 @@ class GroupAttestionRenewer(object):
 
     @defer.inlineCallbacks
     def _renew_attestations(self):
+        """Called periodically to check if we need to update any of our attestations
+        """
+
         now = self.clock.time_msec()
 
         rows = yield self.store.get_attestations_need_renewals(
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 195f1eae54..44083100f7 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -19,7 +19,6 @@ from synapse.api.errors import SynapseError
 from synapse.types import UserID, get_domain_from_id
 
 
-import functools
 import logging
 
 logger = logging.getLogger(__name__)
@@ -33,28 +32,6 @@ logger = logging.getLogger(__name__)
 # TODO: Flairs
 
 
-UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
-
-
-def check_group_is_ours(and_exists=False):
-    def g(func):
-        @functools.wraps(func)
-        @defer.inlineCallbacks
-        def h(self, group_id, *args, **kwargs):
-            if not self.is_mine_id(group_id):
-                raise SynapseError(400, "Group not on this server")
-            if and_exists:
-                group = yield self.store.get_group(group_id)
-                if not group:
-                    raise SynapseError(404, "Unknown group")
-
-            res = yield func(self, group_id, *args, **kwargs)
-            defer.returnValue(res)
-
-        return h
-    return g
-
-
 class GroupsServerHandler(object):
     def __init__(self, hs):
         self.hs = hs
@@ -72,9 +49,28 @@ class GroupsServerHandler(object):
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
 
-    @check_group_is_ours()
+    @defer.inlineCallbacks
+    def check_group_is_ours(self, group_id, and_exists=False):
+        """Check that the group is ours, and optionally if it exists.
+
+        If group does exist then return group.
+        """
+        if not self.is_mine_id(group_id):
+            raise SynapseError(400, "Group not on this server")
+
+        group = yield self.store.get_group(group_id)
+        if and_exists and not group:
+            raise SynapseError(404, "Unknown group")
+
+        defer.returnValue(group)
+
     @defer.inlineCallbacks
     def get_group_profile(self, group_id, requester_user_id):
+        """Get the group profile as seen by requester_user_id
+        """
+
+        yield self.check_group_is_ours(group_id)
+
         group_description = yield self.store.get_group(group_id)
 
         if group_description:
@@ -82,9 +78,13 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(404, "Unknown group")
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
+        """Get the users in group as seen by requester_user_id
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
         is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
 
         user_results = yield self.store.get_users_in_group(
@@ -123,9 +123,13 @@ class GroupsServerHandler(object):
             "total_user_count_estimate": len(user_results),
         })
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def get_rooms_in_group(self, group_id, requester_user_id):
+        """Get the rooms in group as seen by requester_user_id
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
         is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
 
         room_results = yield self.store.get_rooms_in_group(
@@ -158,9 +162,13 @@ class GroupsServerHandler(object):
             "total_room_count_estimate": len(room_results),
         })
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def add_room(self, group_id, requester_user_id, room_id, content):
+        """Add room to group
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
         is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
         if not is_admin:
             raise SynapseError(403, "User is not admin in group")
@@ -182,9 +190,13 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def invite_to_group(self, group_id, user_id, requester_user_id, content):
+        """Invite user to group
+        """
+
+        group = yield self.check_group_is_ours(group_id, and_exists=True)
+
         is_admin = yield self.store.is_user_admin_in_group(
             group_id, requester_user_id
         )
@@ -194,7 +206,6 @@ class GroupsServerHandler(object):
         # TODO: Check if user knocked
         # TODO: Check if user is already invited
 
-        group = yield self.store.get_group(group_id)
         content = {
             "profile": {
                 "name": group["name"],
@@ -248,9 +259,16 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(502, "Unknown state returned by HS")
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
+        """User tries to accept an invite to the group.
+
+        This is different from them asking to join, and so should error if no
+        invite exists (and they're not a member of the group)
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
         if not self.store.is_user_invited_to_local_group(group_id, user_id):
             raise SynapseError(403, "User not invited to group")
 
@@ -291,19 +309,33 @@ class GroupsServerHandler(object):
             "attestation": local_attestation,
         })
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def knock(self, group_id, user_id, content):
-        pass
+        """A user requests becoming a member of the group
+        """
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        raise NotImplementedError()
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def accept_knock(self, group_id, user_id, content):
-        pass
+        """Accept a users knock to the room.
+
+        Errors if the user hasn't knocked, rather than inviting them.
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        raise NotImplementedError()
 
-    @check_group_is_ours(and_exists=True)
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        """Remove a user from the group; either a user is leaving or and admin
+        kicked htem.
+        """
+
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
         is_kick = False
         if requester_user_id != user_id:
             is_admin = yield self.store.is_user_admin_in_group(
@@ -314,7 +346,7 @@ class GroupsServerHandler(object):
 
             is_kick = True
 
-        yield self.store.remove_user_to_group(
+        yield self.store.remove_user_from_group(
             group_id, user_id,
         )
 
@@ -328,11 +360,11 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
-    @check_group_is_ours()
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, content):
+        group = yield self.check_group_is_ours(group_id)
+
         logger.info("Attempting to create group with ID: %r", group_id)
-        group = yield self.store.get_group(group_id)
         if group:
             raise SynapseError(400, "Group already exists")
 
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 01d9a982c8..327d770862 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -89,6 +89,8 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def add_group_invite(self, group_id, user_id):
+        """Record that the group server has invited a user
+        """
         return self._simple_insert(
             table="group_invites",
             values={
@@ -99,6 +101,8 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def is_user_invited_to_local_group(self, group_id, user_id):
+        """Has the group server invited a user?
+        """
         return self._simple_select_one_onecol(
             table="group_invites",
             keyvalues={
@@ -112,6 +116,19 @@ class GroupServerStore(SQLBaseStore):
 
     def add_user_to_group(self, group_id, user_id, is_admin=False, is_public=True,
                           local_attestation=None, remote_attestation=None):
+        """Add a user to the group server.
+
+        Args:
+            group_id (str)
+            user_id (str)
+            is_admin (bool)
+            is_public (bool)
+            local_attestation (dict): The attestation the GS created to give
+                to the remote server. Optional if the user and group are on the
+                same server
+            remote_attestation (dict): The attestation given to GS by remote
+                server. Optional if the user and group are on the same server
+        """
         def _add_user_to_group_txn(txn):
             self._simple_insert_txn(
                 txn,
@@ -159,8 +176,8 @@ class GroupServerStore(SQLBaseStore):
             "add_user_to_group", _add_user_to_group_txn
         )
 
-    def remove_user_to_group(self, group_id, user_id):
-        def _remove_user_to_group_txn(txn):
+    def remove_user_from_group(self, group_id, user_id):
+        def _remove_user_from_group_txn(txn):
             self._simple_delete_txn(
                 txn,
                 table="group_users",
@@ -193,7 +210,7 @@ class GroupServerStore(SQLBaseStore):
                     "user_id": user_id,
                 },
             )
-        return self.runInteraction("remove_user_to_group", _remove_user_to_group_txn)
+        return self.runInteraction("remove_user_from_group", _remove_user_from_group_txn)
 
     def add_room_to_group(self, group_id, room_id, is_public):
         return self._simple_insert(
@@ -222,6 +239,8 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def get_attestations_need_renewals(self, valid_until_ms):
+        """Get all attestations that need to be renewed until givent time
+        """
         def _get_attestations_need_renewals_txn(txn):
             sql = """
                 SELECT group_id, user_id FROM group_attestations_renewals
@@ -234,6 +253,8 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def update_attestation_renewal(self, group_id, user_id, attestation):
+        """Update an attestation that we have renewed
+        """
         return self._simple_update_one(
             table="group_attestations_renewals",
             keyvalues={
@@ -247,6 +268,8 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def update_remote_attestion(self, group_id, user_id, attestation):
+        """Update an attestation that a remote has renewed
+        """
         return self._simple_update_one(
             table="group_attestations_remote",
             keyvalues={
@@ -262,6 +285,9 @@ class GroupServerStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def get_remote_attestation(self, group_id, user_id):
+        """Get the attestation that proves the remote agrees that the user is
+        in the group.
+        """
         row = yield self._simple_select_one(
             table="group_attestations_remote",
             keyvalues={
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index 6f1a941990..5dc7a497e2 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -24,6 +24,7 @@ CREATE TABLE groups (
 CREATE UNIQUE INDEX groups_idx ON groups(group_id);
 
 
+-- list of users the group server thinks are joined
 CREATE TABLE group_users (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
@@ -35,7 +36,7 @@ CREATE TABLE group_users (
 CREATE INDEX groups_users_g_idx ON group_users(group_id, user_id);
 CREATE INDEX groups_users_u_idx ON group_users(user_id);
 
-
+-- list of users the group server thinks are invited
 CREATE TABLE group_invites (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL
@@ -55,6 +56,7 @@ CREATE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
 CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
 
 
+-- List of  attestations we've given out and need to renew
 CREATE TABLE group_attestations_renewals (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
@@ -65,6 +67,8 @@ CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals(gr
 CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals(user_id);
 CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals(valid_until_ms);
 
+
+-- List of attestations we've received from remotes and are interested in.
 CREATE TABLE group_attestations_remote (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
-- 
cgit 1.4.1


From 429925a5e9d24bef0533d936d2bca8a149b2ad1c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 11:44:08 +0100
Subject: Lift out visibility parsing

---
 synapse/groups/groups_server.py | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 44083100f7..1ac946abc3 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -175,16 +175,7 @@ class GroupsServerHandler(object):
 
         # TODO: Check if room has already been added
 
-        visibility = content.get("visibility")
-        if visibility:
-            vis_type = visibility["type"]
-            if vis_type not in ("public", "private"):
-                raise SynapseError(
-                    400, "Synapse only supports 'public'/'private' visibility"
-                )
-            is_public = vis_type == "public"
-        else:
-            is_public = True
+        is_public = _parse_visibility_from_contents(content)
 
         yield self.store.add_room_to_group(group_id, room_id, is_public=is_public)
 
@@ -285,16 +276,7 @@ class GroupsServerHandler(object):
 
         local_attestation = self.attestations.create_attestation(group_id, user_id)
 
-        visibility = content.get("visibility")
-        if visibility:
-            vis_type = visibility["type"]
-            if vis_type not in ("public", "private"):
-                raise SynapseError(
-                    400, "Synapse only supports 'public'/'private' visibility"
-                )
-            is_public = vis_type == "public"
-        else:
-            is_public = True
+        is_public = _parse_visibility_from_contents(content)
 
         yield self.store.add_user_to_group(
             group_id, user_id,
@@ -412,3 +394,22 @@ class GroupsServerHandler(object):
         defer.returnValue({
             "group_id": group_id,
         })
+
+
+def _parse_visibility_from_contents(content):
+    """Given a content for a request parse out whether the entity should be
+    public or not
+    """
+
+    visibility = content.get("visibility")
+    if visibility:
+        vis_type = visibility["type"]
+        if vis_type not in ("public", "private"):
+            raise SynapseError(
+                400, "Synapse only supports 'public'/'private' visibility"
+            )
+        is_public = vis_type == "public"
+    else:
+        is_public = True
+
+    return is_public
-- 
cgit 1.4.1


From 8ba89f1050c523506b99be376b9b052fd68e5bd5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 11:45:32 +0100
Subject: Remove u/ requirement

---
 synapse/groups/groups_server.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 1ac946abc3..61fe0d49d9 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -350,9 +350,10 @@ class GroupsServerHandler(object):
         if group:
             raise SynapseError(400, "Group already exists")
 
+        # TODO: Add config to enforce that only server admins can create rooms
         is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
-        if not is_admin and not group_id.startswith("+u/"):
-            raise SynapseError(403, "Group ID must start with '+u/' or be a server admin")
+        if not is_admin:
+            raise SynapseError(403, "Only server admin can create group on this server")
 
         profile = content.get("profile", {})
         name = profile.get("name")
-- 
cgit 1.4.1


From 0aac30d53b1dba2f399cad0044a905286d8c79d2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 14:23:50 +0100
Subject: Comments

---
 synapse/groups/attestations.py                   | 4 ++--
 synapse/groups/groups_server.py                  | 6 +++++-
 synapse/storage/schema/delta/43/group_server.sql | 4 ++--
 3 files changed, 9 insertions(+), 5 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 6937fa44cf..0741b55c1c 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -38,7 +38,7 @@ class GroupAttestationSigning(object):
 
     @defer.inlineCallbacks
     def verify_attestation(self, attestation, group_id, user_id, server_name=None):
-        """Verifies that the given attestation matches the given paramaters.
+        """Verifies that the given attestation matches the given parameters.
 
         An optional server_name can be supplied to explicitly set which server's
         signature is expected. Otherwise assumes that either the group_id or user_id
@@ -51,7 +51,7 @@ class GroupAttestationSigning(object):
             elif get_domain_from_id(user_id) == self.server_name:
                 server_name = get_domain_from_id(group_id)
             else:
-                raise Exception("Expected eitehr group_id or user_id to be local")
+                raise Exception("Expected either group_id or user_id to be local")
 
         if user_id != attestation["user_id"]:
             raise SynapseError(400, "Attestation has incorrect user_id")
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 61fe0d49d9..414c95e3fe 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -80,7 +80,9 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
-        """Get the users in group as seen by requester_user_id
+        """Get the users in group as seen by requester_user_id.
+
+        The ordering is arbitrary at the moment
         """
 
         yield self.check_group_is_ours(group_id, and_exists=True)
@@ -126,6 +128,8 @@ class GroupsServerHandler(object):
     @defer.inlineCallbacks
     def get_rooms_in_group(self, group_id, requester_user_id):
         """Get the rooms in group as seen by requester_user_id
+
+        This returns rooms in order of decreasing number of joined users
         """
 
         yield self.check_group_is_ours(group_id, and_exists=True)
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index 5dc7a497e2..bfe8c2ca4a 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -28,7 +28,7 @@ CREATE UNIQUE INDEX groups_idx ON groups(group_id);
 CREATE TABLE group_users (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
-    is_admin BOOLEAN NOT NULL,
+    is_admin BOOLEAN NOT NULL,  -- whether the users membership can be seen by everyone
     is_public BOOLEAN NOT NULL
 );
 
@@ -49,7 +49,7 @@ CREATE INDEX groups_invites_u_idx ON group_invites(user_id);
 CREATE TABLE group_rooms (
     group_id TEXT NOT NULL,
     room_id TEXT NOT NULL,
-    is_public BOOLEAN NOT NULL
+    is_public BOOLEAN NOT NULL  -- whether the room can be seen by everyone
 );
 
 CREATE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
-- 
cgit 1.4.1


From 26752df503880b565a49350e0dd8881f9b2285e9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 14:29:03 +0100
Subject: Typo

---
 synapse/groups/attestations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 0741b55c1c..9ac09366d3 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -86,7 +86,7 @@ class GroupAttestionRenewer(object):
         self.store = hs.get_datastore()
         self.assestations = hs.get_groups_attestation_signing()
         self.transport_client = hs.get_federation_transport_client()
-        self.is_mine_id = hs.is_mind_id
+        self.is_mine_id = hs.is_mine_id
 
         self._renew_attestations_loop = self.clock.looping_call(
             self._renew_attestations, 30 * 60 * 1000,
-- 
cgit 1.4.1


From bbb739d24a448c500dbc56c9cedf262d42c7f2f4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Jul 2017 14:31:36 +0100
Subject: Comment

---
 synapse/groups/attestations.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 9ac09366d3..5ef7a12cb7 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -22,8 +22,10 @@ from synapse.util.logcontext import preserve_fn
 from signedjson.sign import sign_json
 
 
+# Default validity duration for new attestations we create
 DEFAULT_ATTESTATION_LENGTH_MS = 3 * 24 * 60 * 60 * 1000
-MIN_ATTESTATION_LENGTH_MS = 1 * 60 * 60 * 1000
+
+# Start trying to update our attestations when they come this close to expiring
 UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
 
 
@@ -58,11 +60,12 @@ class GroupAttestationSigning(object):
 
         if group_id != attestation["group_id"]:
             raise SynapseError(400, "Attestation has incorrect group_id")
-
-        # TODO:
         valid_until_ms = attestation["valid_until_ms"]
-        if valid_until_ms - self.clock.time_msec() < MIN_ATTESTATION_LENGTH_MS:
-            raise SynapseError(400, "Attestation not valid for long enough")
+
+        # TODO: We also want to check that *new* attestations that people give
+        # us to store are valid for at least a little while.
+        if valid_until_ms < self.clock.time_msec():
+            raise SynapseError(400, "Attestation expired")
 
         yield self.keyring.verify_json_for_server(server_name, attestation)
 
-- 
cgit 1.4.1


From a62406aaa5c4ef3780e42c9de443a2cc1e82cd9a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Jul 2017 15:44:40 +0100
Subject: Add group summary APIs

---
 synapse/federation/transport/server.py           |  17 +
 synapse/groups/groups_server.py                  | 256 ++++++++-
 synapse/storage/group_server.py                  | 643 +++++++++++++++++++++++
 synapse/storage/schema/delta/43/group_server.sql |  56 ++
 4 files changed, 970 insertions(+), 2 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 5d6ff79235..bbb66190e0 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -627,6 +627,22 @@ class FederationGroupsProfileServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsSummaryServlet(BaseFederationServlet):
+    PATH = "/groups/(?P<group_id>[^/]*)/summary$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id):
+        requester_user_id = content["requester_user_id"]
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.get_group_summary(
+            group_id, requester_user_id
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsRoomsServlet(BaseFederationServlet):
     """Get the rooms in a group on behalf of a user
     """
@@ -784,6 +800,7 @@ ROOM_LIST_CLASSES = (
 
 GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsProfileServlet,
+    FederationGroupsSummaryServlet,
     FederationGroupsRoomsServlet,
     FederationGroupsUsersServlet,
     FederationGroupsInviteServlet,
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 414c95e3fe..29a911e18e 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -64,6 +64,255 @@ class GroupsServerHandler(object):
 
         defer.returnValue(group)
 
+    @defer.inlineCallbacks
+    def get_group_summary(self, group_id, requester_user_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
+
+        profile = yield self.get_group_profile(group_id, requester_user_id)
+
+        users, roles = yield self.store.get_users_for_summary_by_role(
+            group_id, include_private=is_user_in_group,
+        )
+
+        # TODO: Add profiles to users
+        # TODO: Add assestations to users
+
+        rooms, categories = yield self.store.get_rooms_for_summary_by_category(
+            group_id, include_private=is_user_in_group,
+        )
+
+        for room_entry in rooms:
+            room_id = room_entry["room_id"]
+            joined_users = yield self.store.get_users_in_room(room_id)
+            entry = yield self.room_list_handler.generate_room_entry(
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
+            )
+            entry.pop("room_id", None)
+
+            room_entry["profile"] = entry
+
+        rooms.sort(key=lambda e: e.get("order", 0))
+
+        for entry in users:
+            user_id = entry["user_id"]
+
+            if not self.is_mine_id(requester_user_id):
+                attestation = yield self.store.get_remote_attestation(group_id, user_id)
+                if not attestation:
+                    continue
+
+                entry["attestation"] = attestation
+            else:
+                entry["attestation"] = self.attestations.create_attestation(
+                    group_id, user_id,
+                )
+
+        users.sort(key=lambda e: e.get("order", 0))
+
+        defer.returnValue({
+            "profile": profile,
+            "users_section": {
+                "users": users,
+                "roles": roles,
+                "total_user_count_estimate": 0,  # TODO
+            },
+            "rooms_section": {
+                "rooms": rooms,
+                "categories": categories,
+                "total_room_count_estimate": 0,  # TODO
+            },
+        })
+
+    @defer.inlineCallbacks
+    def update_group_summary_room(self, group_id, user_id, room_id, category_id, content):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        order = content.get("order", None)
+
+        is_public = _parse_visibility_from_contents(content)
+
+        yield self.store.add_room_to_summary(
+            group_id=group_id,
+            room_id=room_id,
+            category_id=category_id,
+            order=order,
+            is_public=is_public,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def delete_group_summary_room(self, group_id, user_id, room_id, category_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        yield self.store.remove_room_from_summary(
+            group_id=group_id,
+            room_id=room_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def get_group_categories(self, group_id, user_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        categories = yield self.store.get_group_categories(
+            group_id=group_id,
+        )
+        defer.returnValue({"categories": categories})
+
+    @defer.inlineCallbacks
+    def get_group_category(self, group_id, user_id, category_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        res = yield self.store.get_group_category(
+            group_id=group_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def update_group_category(self, group_id, user_id, category_id, content):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        is_public = _parse_visibility_from_contents(content)
+        profile = content.get("profile")
+
+        yield self.store.upsert_group_category(
+            group_id=group_id,
+            category_id=category_id,
+            is_public=is_public,
+            profile=profile,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def delete_group_category(self, group_id, user_id, category_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        yield self.store.remove_group_category(
+            group_id=group_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def get_group_roles(self, group_id, user_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        roles = yield self.store.get_group_roles(
+            group_id=group_id,
+        )
+        defer.returnValue({"roles": roles})
+
+    @defer.inlineCallbacks
+    def get_group_role(self, group_id, user_id, role_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        res = yield self.store.get_group_role(
+            group_id=group_id,
+            role_id=role_id,
+        )
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def update_group_role(self, group_id, user_id, role_id, content):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        is_public = _parse_visibility_from_contents(content)
+
+        profile = content.get("profile")
+
+        yield self.store.upsert_group_role(
+            group_id=group_id,
+            role_id=role_id,
+            is_public=is_public,
+            profile=profile,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def delete_group_role(self, group_id, user_id, role_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        yield self.store.remove_group_role(
+            group_id=group_id,
+            role_id=role_id,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def update_group_summary_user(self, group_id, requester_user_id, user_id, role_id,
+                                  content):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        order = content.get("order", None)
+
+        is_public = _parse_visibility_from_contents(content)
+
+        yield self.store.add_user_to_summary(
+            group_id=group_id,
+            user_id=user_id,
+            role_id=role_id,
+            order=order,
+            is_public=is_public,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
+        yield self.check_group_is_ours(group_id, and_exists=True)
+
+        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
+        if not is_admin:
+            raise SynapseError(403, "User is not admin in group")
+
+        yield self.store.remove_user_from_summary(
+            group_id=group_id,
+            user_id=user_id,
+            role_id=role_id,
+        )
+
+        defer.returnValue({})
+
     @defer.inlineCallbacks
     def get_group_profile(self, group_id, requester_user_id):
         """Get the group profile as seen by requester_user_id
@@ -210,7 +459,9 @@ class GroupsServerHandler(object):
         }
 
         if self.hs.is_mine_id(user_id):
-            raise NotImplementedError()
+            groups_local = self.hs.get_groups_local_handler()
+            res = yield groups_local.on_invite(group_id, user_id, content)
+            local_attestation = None
         else:
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content.update({
@@ -338,7 +589,8 @@ class GroupsServerHandler(object):
 
         if is_kick:
             if self.hs.is_mine_id(user_id):
-                raise NotImplementedError()
+                groups_local = self.hs.get_groups_local_handler()
+                yield groups_local.user_removed_from_group(group_id, user_id, {})
             else:
                 yield self.transport_client.remove_user_from_group_notification(
                     get_domain_from_id(user_id), group_id, user_id, {}
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 105ab9920e..f4818ff174 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -15,11 +15,16 @@
 
 from twisted.internet import defer
 
+from synapse.api.errors import SynapseError
+
 from ._base import SQLBaseStore
 
 import ujson as json
 
 
+_DEFAULT_CATEGORY_ID = "default"
+
+
 class GroupServerStore(SQLBaseStore):
     def get_group(self, group_id):
         return self._simple_select_one(
@@ -64,6 +69,492 @@ class GroupServerStore(SQLBaseStore):
             desc="get_rooms_in_group",
         )
 
+    def get_rooms_for_summary_by_category(self, group_id, include_private=False):
+        def _get_rooms_for_summary_txn(txn):
+            keyvalues = {
+                "group_id": group_id,
+            }
+            if not include_private:
+                keyvalues["is_public"] = True
+
+            sql = """
+                SELECT room_id, is_public, category_id, room_order
+                FROM group_summary_rooms
+                WHERE group_id = ?
+            """
+
+            if not include_private:
+                sql += " AND is_public = ?"
+                txn.execute(sql, (group_id, True))
+            else:
+                txn.execute(sql, (group_id,))
+
+            rooms = [
+                {
+                    "room_id": row[0],
+                    "is_public": row[1],
+                    "category_id": row[2] if row[2] != _DEFAULT_CATEGORY_ID else None,
+                    "order": row[3],
+                }
+                for row in txn
+            ]
+
+            sql = """
+                SELECT category_id, is_public, profile, cat_order
+                FROM group_summary_room_categories
+                INNER JOIN group_room_categories USING (group_id, category_id)
+                WHERE group_id = ?
+            """
+
+            if not include_private:
+                sql += " AND is_public = ?"
+                txn.execute(sql, (group_id, True))
+            else:
+                txn.execute(sql, (group_id,))
+
+            categories = {
+                row[0]: {
+                    "is_public": row[1],
+                    "profile": json.loads(row[2]),
+                    "order": row[3],
+                }
+                for row in txn
+            }
+
+            return rooms, categories
+        return self.runInteraction(
+            "get_rooms_for_summary", _get_rooms_for_summary_txn
+        )
+
+    def add_room_to_summary(self, group_id, room_id, category_id, order, is_public):
+        return self.runInteraction(
+            "add_room_to_summary", self._add_room_to_summary_txn,
+            group_id, room_id, category_id, order, is_public,
+        )
+
+    def _add_room_to_summary_txn(self, txn, group_id, room_id, category_id, order,
+                                 is_public):
+        if category_id is None:
+            category_id = _DEFAULT_CATEGORY_ID
+        else:
+            cat_exists = self._simple_select_one_onecol_txn(
+                txn,
+                table="group_room_categories",
+                keyvalues={
+                    "group_id": group_id,
+                    "category_id": category_id,
+                },
+                retcol="group_id",
+                allow_none=True,
+            )
+            if not cat_exists:
+                raise SynapseError(400, "Category doesn't exist")
+
+            # TODO: Check room is part of group already
+            cat_exists = self._simple_select_one_onecol_txn(
+                txn,
+                table="group_summary_room_categories",
+                keyvalues={
+                    "group_id": group_id,
+                    "category_id": category_id,
+                },
+                retcol="group_id",
+                allow_none=True,
+            )
+            if not cat_exists:
+                txn.execute("""
+                    INSERT INTO group_summary_room_categories
+                    (group_id, category_id, cat_order)
+                    SELECT ?, ?, COALESCE(MAX(cat_order), 1)
+                    FROM group_summary_room_categories
+                    WHERE group_id = ? AND category_id = ?
+                """, (group_id, category_id, group_id, category_id))
+
+        existing = self._simple_select_one_txn(
+            txn,
+            table="group_summary_rooms",
+            keyvalues={
+                "group_id": group_id,
+                "room_id": room_id,
+                "category_id": category_id,
+            },
+            retcols=("room_order", "is_public",),
+            allow_none=True,
+        )
+
+        if order is not None:
+            sql = """
+                UPDATE group_summary_rooms SET room_order = room_order + 1
+                WHERE group_id = ? AND category_id = ? AND room_order >= ?
+            """
+            txn.execute(sql, (group_id, category_id, order,))
+        elif not existing:
+            sql = """
+                SELECT COALESCE(MAX(room_order), 0) + 1 FROM group_summary_rooms
+                WHERE group_id = ? AND category_id = ?
+            """
+            txn.execute(sql, (group_id, category_id,))
+            order, = txn.fetchone()
+
+        if existing:
+            to_update = {}
+            if order is not None:
+                to_update["room_order"] = order
+            if is_public is not None:
+                to_update["is_public"] = is_public
+            self._simple_update_txn(
+                txn,
+                table="group_summary_rooms",
+                keyvalues={
+                    "group_id": group_id,
+                    "category_id": category_id,
+                    "room_id": room_id,
+                },
+                values=to_update,
+            )
+        else:
+            if is_public is None:
+                is_public = True
+
+            self._simple_insert_txn(
+                txn,
+                table="group_summary_rooms",
+                values={
+                    "group_id": group_id,
+                    "category_id": category_id,
+                    "room_id": room_id,
+                    "room_order": order,
+                    "is_public": is_public,
+                },
+            )
+
+    def remove_room_from_summary(self, group_id, room_id, category_id):
+        if category_id is None:
+            category_id = _DEFAULT_CATEGORY_ID
+
+        return self._simple_delete(
+            table="group_summary_rooms",
+            keyvalues={
+                "group_id": group_id,
+                "category_id": category_id,
+                "room_id": room_id,
+            },
+            desc="remove_room_from_summary",
+        )
+
+    @defer.inlineCallbacks
+    def get_group_categories(self, group_id):
+        rows = yield self._simple_select_list(
+            table="group_room_categories",
+            keyvalues={
+                "group_id": group_id,
+            },
+            retcols=("category_id", "is_public", "profile"),
+            desc="get_group_categories",
+        )
+
+        defer.returnValue({
+            row["category_id"]: {
+                "is_public": row["is_public"],
+                "profile": json.loads(row["profile"]),
+            }
+            for row in rows
+        })
+
+    @defer.inlineCallbacks
+    def get_group_category(self, group_id, category_id):
+        category = yield self._simple_select_one(
+            table="group_room_categories",
+            keyvalues={
+                "group_id": group_id,
+                "category_id": category_id,
+            },
+            retcols=("is_public", "profile"),
+            desc="get_group_category",
+        )
+
+        category["profile"] = json.loads(category["profile"])
+
+        defer.returnValue(category)
+
+    def upsert_group_category(self, group_id, category_id, profile, is_public):
+        insertion_values = {}
+        update_values = {"category_id": category_id}  # This cannot be empty
+
+        if profile is None:
+            insertion_values["profile"] = "{}"
+        else:
+            update_values["profile"] = json.dumps(profile)
+
+        if is_public is None:
+            insertion_values["is_public"] = True
+        else:
+            update_values["is_public"] = is_public
+
+        return self._simple_upsert(
+            table="group_room_categories",
+            keyvalues={
+                "group_id": group_id,
+                "category_id": category_id,
+            },
+            values=update_values,
+            insertion_values=insertion_values,
+            desc="upsert_group_category",
+        )
+
+    def remove_group_category(self, group_id, category_id):
+        return self._simple_delete(
+            table="group_room_categories",
+            keyvalues={
+                "group_id": group_id,
+                "category_id": category_id,
+            },
+            desc="remove_group_category",
+        )
+
+    @defer.inlineCallbacks
+    def get_group_roles(self, group_id):
+        rows = yield self._simple_select_list(
+            table="group_roles",
+            keyvalues={
+                "group_id": group_id,
+            },
+            retcols=("role_id", "is_public", "profile"),
+            desc="get_group_roles",
+        )
+
+        defer.returnValue({
+            row["role_id"]: {
+                "is_public": row["is_public"],
+                "profile": json.loads(row["profile"]),
+            }
+            for row in rows
+        })
+
+    @defer.inlineCallbacks
+    def get_group_role(self, group_id, role_id):
+        role = yield self._simple_select_one(
+            table="group_roles",
+            keyvalues={
+                "group_id": group_id,
+                "role_id": role_id,
+            },
+            retcols=("is_public", "profile"),
+            desc="get_group_role",
+        )
+
+        role["profile"] = json.loads(role["profile"])
+
+        defer.returnValue(role)
+
+    def upsert_group_role(self, group_id, role_id, profile, is_public):
+        insertion_values = {}
+        update_values = {"role_id": role_id}  # This cannot be empty
+
+        if profile is None:
+            insertion_values["profile"] = "{}"
+        else:
+            update_values["profile"] = json.dumps(profile)
+
+        if is_public is None:
+            insertion_values["is_public"] = True
+        else:
+            update_values["is_public"] = is_public
+
+        return self._simple_upsert(
+            table="group_roles",
+            keyvalues={
+                "group_id": group_id,
+                "role_id": role_id,
+            },
+            values=update_values,
+            insertion_values=insertion_values,
+            desc="upsert_group_role",
+        )
+
+    def remove_group_role(self, group_id, role_id):
+        return self._simple_delete(
+            table="group_roles",
+            keyvalues={
+                "group_id": group_id,
+                "role_id": role_id,
+            },
+            desc="remove_group_role",
+        )
+
+    def add_user_to_summary(self, group_id, user_id, role_id, order, is_public):
+        return self.runInteraction(
+            "add_user_to_summary", self._add_user_to_summary_txn,
+            group_id, user_id, role_id, order, is_public,
+        )
+
+    def _add_user_to_summary_txn(self, txn, group_id, user_id, role_id, order,
+                                 is_public):
+        if role_id is None:
+            role_id = _DEFAULT_CATEGORY_ID
+        else:
+            role_exists = self._simple_select_one_onecol_txn(
+                txn,
+                table="group_roles",
+                keyvalues={
+                    "group_id": group_id,
+                    "role_id": role_id,
+                },
+                retcol="group_id",
+                allow_none=True,
+            )
+            if not role_exists:
+                raise SynapseError(400, "Role doesn't exist")
+
+            # TODO: Check room is part of group already
+            role_exists = self._simple_select_one_onecol_txn(
+                txn,
+                table="group_summary_roles",
+                keyvalues={
+                    "group_id": group_id,
+                    "role_id": role_id,
+                },
+                retcol="group_id",
+                allow_none=True,
+            )
+            if not role_exists:
+                txn.execute("""
+                    INSERT INTO group_summary_roles
+                    (group_id, role_id, role_order)
+                    SELECT ?, ?, COALESCE(MAX(role_order), 1)
+                    FROM group_summary_roles
+                    WHERE group_id = ? AND role_id = ?
+                """, (group_id, role_id, group_id, role_id))
+
+        existing = self._simple_select_one_txn(
+            txn,
+            table="group_summary_users",
+            keyvalues={
+                "group_id": group_id,
+                "user_id": user_id,
+                "role_id": role_id,
+            },
+            retcols=("user_order", "is_public",),
+            allow_none=True,
+        )
+
+        if order is not None:
+            sql = """
+                UPDATE group_summary_users SET user_order = user_order + 1
+                WHERE group_id = ? AND role_id = ? AND user_order >= ?
+            """
+            txn.execute(sql, (group_id, role_id, order,))
+        elif not existing:
+            sql = """
+                SELECT COALESCE(MAX(user_order), 0) + 1 FROM group_summary_users
+                WHERE group_id = ? AND role_id = ?
+            """
+            txn.execute(sql, (group_id, role_id,))
+            order, = txn.fetchone()
+
+        if existing:
+            to_update = {}
+            if order is not None:
+                to_update["user_order"] = order
+            if is_public is not None:
+                to_update["is_public"] = is_public
+            self._simple_update_txn(
+                txn,
+                table="group_summary_users",
+                keyvalues={
+                    "group_id": group_id,
+                    "role_id": role_id,
+                    "user_id": user_id,
+                },
+                values=to_update,
+            )
+        else:
+            if is_public is None:
+                is_public = True
+
+            self._simple_insert_txn(
+                txn,
+                table="group_summary_users",
+                values={
+                    "group_id": group_id,
+                    "role_id": role_id,
+                    "user_id": user_id,
+                    "user_order": order,
+                    "is_public": is_public,
+                },
+            )
+
+    def remove_user_from_summary(self, group_id, user_id, role_id):
+        if role_id is None:
+            role_id = _DEFAULT_CATEGORY_ID
+
+        return self._simple_delete(
+            table="group_summary_users",
+            keyvalues={
+                "group_id": group_id,
+                "role_id": role_id,
+                "user_id": user_id,
+            },
+            desc="remove_user_from_summary",
+        )
+
+    def get_users_for_summary_by_role(self, group_id, include_private=False):
+        def _get_users_for_summary_txn(txn):
+            keyvalues = {
+                "group_id": group_id,
+            }
+            if not include_private:
+                keyvalues["is_public"] = True
+
+            sql = """
+                SELECT user_id, is_public, role_id, user_order
+                FROM group_summary_users
+                WHERE group_id = ?
+            """
+
+            if not include_private:
+                sql += " AND is_public = ?"
+                txn.execute(sql, (group_id, True))
+            else:
+                txn.execute(sql, (group_id,))
+
+            users = [
+                {
+                    "user_id": row[0],
+                    "is_public": row[1],
+                    "role_id": row[2] if row[2] != _DEFAULT_CATEGORY_ID else None,
+                    "order": row[3],
+                }
+                for row in txn
+            ]
+
+            sql = """
+                SELECT role_id, is_public, profile, role_order
+                FROM group_summary_roles
+                INNER JOIN group_roles USING (group_id, role_id)
+                WHERE group_id = ?
+            """
+
+            if not include_private:
+                sql += " AND is_public = ?"
+                txn.execute(sql, (group_id, True))
+            else:
+                txn.execute(sql, (group_id,))
+
+            roles = {
+                row[0]: {
+                    "is_public": row[1],
+                    "profile": json.loads(row[2]),
+                    "order": row[3],
+                }
+                for row in txn
+            }
+
+            return users, roles
+        return self.runInteraction(
+            "get_users_for_summary_by_role", _get_users_for_summary_txn
+        )
+
     def is_user_in_group(self, user_id, group_id):
         return self._simple_select_one_onecol(
             table="group_users",
@@ -223,6 +714,103 @@ class GroupServerStore(SQLBaseStore):
             desc="add_room_to_group",
         )
 
+    @defer.inlineCallbacks
+    def register_user_group_membership(self, group_id, user_id, membership,
+                                       is_admin=False, content={},
+                                       local_attestation=None,
+                                       remote_attestation=None,
+                                       ):
+        def _register_user_group_membership_txn(txn, next_id):
+            # TODO: Upsert?
+            self._simple_delete_txn(
+                txn,
+                table="local_group_membership",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                },
+            )
+            self._simple_insert_txn(
+                txn,
+                table="local_group_membership",
+                values={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "is_admin": is_admin,
+                    "membership": membership,
+                    "content": json.dumps(content),
+                },
+            )
+            self._simple_delete_txn(
+                txn,
+                table="local_group_updates",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "type": "membership",
+                },
+            )
+            self._simple_insert_txn(
+                txn,
+                table="local_group_updates",
+                values={
+                    "stream_id": next_id,
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "type": "membership",
+                    "content": json.dumps({"membership": membership, "content": content}),
+                }
+            )
+            self._group_updates_stream_cache.entity_has_changed(user_id, next_id)
+
+            # TODO: Insert profile to ensuer it comes down stream if its a join.
+
+            if membership == "join":
+                if local_attestation:
+                    self._simple_insert_txn(
+                        txn,
+                        table="group_attestations_renewals",
+                        values={
+                            "group_id": group_id,
+                            "user_id": user_id,
+                            "valid_until_ms": local_attestation["valid_until_ms"],
+                        }
+                    )
+                if remote_attestation:
+                    self._simple_insert_txn(
+                        txn,
+                        table="group_attestations_remote",
+                        values={
+                            "group_id": group_id,
+                            "user_id": user_id,
+                            "valid_until_ms": remote_attestation["valid_until_ms"],
+                            "attestation": json.dumps(remote_attestation),
+                        }
+                    )
+            else:
+                self._simple_delete_txn(
+                    txn,
+                    table="group_attestations_renewals",
+                    keyvalues={
+                        "group_id": group_id,
+                        "user_id": user_id,
+                    },
+                )
+                self._simple_delete_txn(
+                    txn,
+                    table="group_attestations_remote",
+                    keyvalues={
+                        "group_id": group_id,
+                        "user_id": user_id,
+                    },
+                )
+
+        with self._group_updates_id_gen.get_next() as next_id:
+            yield self.runInteraction(
+                "register_user_group_membership",
+                _register_user_group_membership_txn, next_id,
+            )
+
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, name, avatar_url, short_description,
                      long_description,):
@@ -238,6 +826,61 @@ class GroupServerStore(SQLBaseStore):
             desc="create_group",
         )
 
+    def get_joined_groups(self, user_id):
+        return self._simple_select_onecol(
+            table="local_group_membership",
+            keyvalues={
+                "user_id": user_id,
+                "membership": "join",
+            },
+            retcol="group_id",
+            desc="get_joined_groups",
+        )
+
+    def get_all_groups_for_user(self, user_id, now_token):
+        def _get_all_groups_for_user_txn(txn):
+            sql = """
+                SELECT group_id, type, membership, u.content
+                FROM local_group_updates AS u
+                INNER JOIN local_group_membership USING (group_id, user_id)
+                WHERE user_id = ? AND membership != 'leave'
+                    AND stream_id <= ?
+            """
+            txn.execute(sql, (user_id, now_token,))
+            return self.cursor_to_dict(txn)
+        return self.runInteraction(
+            "get_all_groups_for_user", _get_all_groups_for_user_txn,
+        )
+
+    def get_groups_changes_for_user(self, user_id, from_token, to_token):
+        from_token = int(from_token)
+        has_changed = self._group_updates_stream_cache.has_entity_changed(
+            user_id, from_token,
+        )
+        if not has_changed:
+            return []
+
+        def _get_groups_changes_for_user_txn(txn):
+            sql = """
+                SELECT group_id, membership, type, u.content
+                FROM local_group_updates AS u
+                INNER JOIN local_group_membership USING (group_id, user_id)
+                WHERE user_id = ? AND ? < stream_id AND stream_id <= ?
+            """
+            txn.execute(sql, (user_id, from_token, to_token,))
+            return [{
+                "group_id": group_id,
+                "membership": membership,
+                "type": gtype,
+                "content": json.loads(content_json),
+            } for group_id, membership, gtype, content_json in txn]
+        return self.runInteraction(
+            "get_groups_changes_for_user", _get_groups_changes_for_user_txn,
+        )
+
+    def get_group_stream_token(self):
+        return self._group_updates_id_gen.get_current_token()
+
     def get_attestations_need_renewals(self, valid_until_ms):
         """Get all attestations that need to be renewed until givent time
         """
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index c223ee275a..3013b89b7e 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -56,6 +56,62 @@ CREATE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
 CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
 
 
+CREATE TABLE group_summary_rooms (
+    group_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    category_id TEXT NOT NULL,
+    room_order BIGINT NOT NULL,
+    is_public BOOLEAN NOT NULL,
+    UNIQUE (group_id, category_id, room_id, room_order),
+    CHECK (room_order > 0)
+);
+
+CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
+
+CREATE TABLE group_summary_room_categories (
+    group_id TEXT NOT NULL,
+    category_id TEXT NOT NULL,
+    cat_order BIGINT NOT NULL,
+    UNIQUE (group_id, category_id, cat_order),
+    CHECK (cat_order > 0)
+);
+
+CREATE TABLE group_room_categories (
+    group_id TEXT NOT NULL,
+    category_id TEXT NOT NULL,
+    profile TEXT NOT NULL,
+    is_public BOOLEAN NOT NULL,
+    UNIQUE (group_id, category_id)
+);
+
+
+CREATE TABLE group_summary_users (
+    group_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    role_id TEXT NOT NULL,
+    user_order BIGINT NOT NULL,
+    is_public BOOLEAN NOT NULL
+);
+
+CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
+
+CREATE TABLE group_summary_roles (
+    group_id TEXT NOT NULL,
+    role_id TEXT NOT NULL,
+    role_order BIGINT NOT NULL,
+    UNIQUE (group_id, role_id, role_order),
+    CHECK (role_order > 0)
+);
+
+CREATE TABLE group_roles (
+    group_id TEXT NOT NULL,
+    role_id TEXT NOT NULL,
+    profile TEXT NOT NULL,
+    is_public BOOLEAN NOT NULL,
+    UNIQUE (group_id, role_id)
+);
+
+
 -- List of  attestations we've given out and need to renew
 CREATE TABLE group_attestations_renewals (
     group_id TEXT NOT NULL,
-- 
cgit 1.4.1


From 8d55877c9eaffc8dcaf26cdc7a032c774c9a2f5b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Jul 2017 11:43:39 +0100
Subject: Simplify checking if admin

---
 synapse/groups/groups_server.py | 72 +++++++++++------------------------------
 1 file changed, 19 insertions(+), 53 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 29a911e18e..ec45da2d7a 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -50,7 +50,7 @@ class GroupsServerHandler(object):
         hs.get_groups_attestation_renewer()
 
     @defer.inlineCallbacks
-    def check_group_is_ours(self, group_id, and_exists=False):
+    def check_group_is_ours(self, group_id, and_exists=False, and_is_admin=None):
         """Check that the group is ours, and optionally if it exists.
 
         If group does exist then return group.
@@ -62,6 +62,11 @@ class GroupsServerHandler(object):
         if and_exists and not group:
             raise SynapseError(404, "Unknown group")
 
+        if and_is_admin:
+            is_admin = yield self.store.is_user_admin_in_group(group_id, and_is_admin)
+            if not is_admin:
+                raise SynapseError(403, "User is not admin in group")
+
         defer.returnValue(group)
 
     @defer.inlineCallbacks
@@ -128,11 +133,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_summary_room(self, group_id, user_id, room_id, category_id, content):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         order = content.get("order", None)
 
@@ -150,11 +151,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_summary_room(self, group_id, user_id, room_id, category_id):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_room_from_summary(
             group_id=group_id,
@@ -186,11 +183,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_category(self, group_id, user_id, category_id, content):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         is_public = _parse_visibility_from_contents(content)
         profile = content.get("profile")
@@ -206,11 +199,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_category(self, group_id, user_id, category_id):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_group_category(
             group_id=group_id,
@@ -240,11 +229,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_role(self, group_id, user_id, role_id, content):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         is_public = _parse_visibility_from_contents(content)
 
@@ -261,11 +246,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_role(self, group_id, user_id, role_id):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_group_role(
             group_id=group_id,
@@ -277,11 +258,7 @@ class GroupsServerHandler(object):
     @defer.inlineCallbacks
     def update_group_summary_user(self, group_id, requester_user_id, user_id, role_id,
                                   content):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         order = content.get("order", None)
 
@@ -299,11 +276,7 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_user_from_summary(
             group_id=group_id,
@@ -419,12 +392,9 @@ class GroupsServerHandler(object):
     def add_room(self, group_id, requester_user_id, room_id, content):
         """Add room to group
         """
-
-        yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(group_id, requester_user_id)
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id
+        )
 
         # TODO: Check if room has already been added
 
@@ -439,13 +409,9 @@ class GroupsServerHandler(object):
         """Invite user to group
         """
 
-        group = yield self.check_group_is_ours(group_id, and_exists=True)
-
-        is_admin = yield self.store.is_user_admin_in_group(
-            group_id, requester_user_id
+        group = yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id
         )
-        if not is_admin:
-            raise SynapseError(403, "User is not admin in group")
 
         # TODO: Check if user knocked
         # TODO: Check if user is already invited
-- 
cgit 1.4.1


From 26451a09eb938e6a72be3d77ff8c9e3fd2b33539 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Jul 2017 14:11:59 +0100
Subject: Comments

---
 synapse/groups/groups_server.py                  | 38 ++++++++++++++++++++++++
 synapse/storage/group_server.py                  | 29 ++++++++++++++++++
 synapse/storage/schema/delta/43/group_server.sql | 17 +++++++----
 3 files changed, 79 insertions(+), 5 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index ec45da2d7a..83dfcd0fd4 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -54,6 +54,12 @@ class GroupsServerHandler(object):
         """Check that the group is ours, and optionally if it exists.
 
         If group does exist then return group.
+
+        Args:
+            group_id (str)
+            and_exists (bool): whether to also check if group exists
+            and_is_admin (str): whether to also check if given str is a user_id
+                that is an admin
         """
         if not self.is_mine_id(group_id):
             raise SynapseError(400, "Group not on this server")
@@ -71,6 +77,14 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
+        """Get the summary for a group as seen by requester_user_id.
+
+        The group summary consists of the profile of the room, and a curated
+        list of users and rooms. These list *may* be organised by role/category.
+        The roles/categories are ordered, and so are the users/rooms within them.
+
+        A user/room may appear in multiple roles/categories.
+        """
         yield self.check_group_is_ours(group_id, and_exists=True)
 
         is_user_in_group = yield self.store.is_user_in_group(requester_user_id, group_id)
@@ -133,6 +147,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_summary_room(self, group_id, user_id, room_id, category_id, content):
+        """Add/update a room to the group summary
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         order = content.get("order", None)
@@ -151,6 +167,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_summary_room(self, group_id, user_id, room_id, category_id):
+        """Remove a room from the summary
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_room_from_summary(
@@ -163,6 +181,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_group_categories(self, group_id, user_id):
+        """Get all categories in a group (as seen by user)
+        """
         yield self.check_group_is_ours(group_id, and_exists=True)
 
         categories = yield self.store.get_group_categories(
@@ -172,6 +192,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_group_category(self, group_id, user_id, category_id):
+        """Get a specific category in a group (as seen by user)
+        """
         yield self.check_group_is_ours(group_id, and_exists=True)
 
         res = yield self.store.get_group_category(
@@ -183,6 +205,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_category(self, group_id, user_id, category_id, content):
+        """Add/Update a group category
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         is_public = _parse_visibility_from_contents(content)
@@ -199,6 +223,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_category(self, group_id, user_id, category_id):
+        """Delete a group category
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_group_category(
@@ -210,6 +236,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_group_roles(self, group_id, user_id):
+        """Get all roles in a group (as seen by user)
+        """
         yield self.check_group_is_ours(group_id, and_exists=True)
 
         roles = yield self.store.get_group_roles(
@@ -219,6 +247,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def get_group_role(self, group_id, user_id, role_id):
+        """Get a specific role in a group (as seen by user)
+        """
         yield self.check_group_is_ours(group_id, and_exists=True)
 
         res = yield self.store.get_group_role(
@@ -229,6 +259,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def update_group_role(self, group_id, user_id, role_id, content):
+        """Add/update a role in a group
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         is_public = _parse_visibility_from_contents(content)
@@ -246,6 +278,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_role(self, group_id, user_id, role_id):
+        """Remove role from group
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_group_role(
@@ -258,6 +292,8 @@ class GroupsServerHandler(object):
     @defer.inlineCallbacks
     def update_group_summary_user(self, group_id, requester_user_id, user_id, role_id,
                                   content):
+        """Add/update a users entry in the group summary
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         order = content.get("order", None)
@@ -276,6 +312,8 @@ class GroupsServerHandler(object):
 
     @defer.inlineCallbacks
     def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
+        """Remove a user from the group summary
+        """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
         yield self.store.remove_user_from_summary(
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index f4818ff174..18bfaeda6e 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -22,6 +22,8 @@ from ._base import SQLBaseStore
 import ujson as json
 
 
+# The category ID for the "default" category. We don't store as null in the
+# database to avoid the fun of null != null
 _DEFAULT_CATEGORY_ID = "default"
 
 
@@ -70,6 +72,10 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def get_rooms_for_summary_by_category(self, group_id, include_private=False):
+        """Get the rooms and categories that should be included in a summary request
+
+        Returns ([rooms], [categories])
+        """
         def _get_rooms_for_summary_txn(txn):
             keyvalues = {
                 "group_id": group_id,
@@ -134,6 +140,14 @@ class GroupServerStore(SQLBaseStore):
 
     def _add_room_to_summary_txn(self, txn, group_id, room_id, category_id, order,
                                  is_public):
+        """Add room to summary.
+
+        This automatically adds the room to the end of the list of rooms to be
+        included in the summary response. If a role is given then user will
+        be added under that category (the category will automatically be added tothe
+        the summary if a user is listed under that role in the summary).
+        """
+
         if category_id is None:
             category_id = _DEFAULT_CATEGORY_ID
         else:
@@ -278,6 +292,8 @@ class GroupServerStore(SQLBaseStore):
         defer.returnValue(category)
 
     def upsert_group_category(self, group_id, category_id, profile, is_public):
+        """Add/update room category for group
+        """
         insertion_values = {}
         update_values = {"category_id": category_id}  # This cannot be empty
 
@@ -348,6 +364,8 @@ class GroupServerStore(SQLBaseStore):
         defer.returnValue(role)
 
     def upsert_group_role(self, group_id, role_id, profile, is_public):
+        """Add/remove user role
+        """
         insertion_values = {}
         update_values = {"role_id": role_id}  # This cannot be empty
 
@@ -390,6 +408,13 @@ class GroupServerStore(SQLBaseStore):
 
     def _add_user_to_summary_txn(self, txn, group_id, user_id, role_id, order,
                                  is_public):
+        """Add user to summary.
+
+        This automatically adds the user to the end of the list of users to be
+        included in the summary response. If a role is given then user will
+        be added under that role (the role will automatically be added to the
+        summary if a user is listed under that role in the summary).
+        """
         if role_id is None:
             role_id = _DEFAULT_CATEGORY_ID
         else:
@@ -499,6 +524,10 @@ class GroupServerStore(SQLBaseStore):
         )
 
     def get_users_for_summary_by_role(self, group_id, include_private=False):
+        """Get the users and roles that should be included in a summary request
+
+        Returns ([users], [roles])
+        """
         def _get_users_for_summary_txn(txn):
             keyvalues = {
                 "group_id": group_id,
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index 3013b89b7e..472aab0a78 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -56,18 +56,21 @@ CREATE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
 CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
 
 
+-- Rooms to include in the summary
 CREATE TABLE group_summary_rooms (
     group_id TEXT NOT NULL,
     room_id TEXT NOT NULL,
     category_id TEXT NOT NULL,
     room_order BIGINT NOT NULL,
-    is_public BOOLEAN NOT NULL,
+    is_public BOOLEAN NOT NULL, -- whether the room should be show to everyone
     UNIQUE (group_id, category_id, room_id, room_order),
     CHECK (room_order > 0)
 );
 
 CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms(group_id, room_id, category_id);
 
+
+-- Categories to include in the summary
 CREATE TABLE group_summary_room_categories (
     group_id TEXT NOT NULL,
     category_id TEXT NOT NULL,
@@ -76,25 +79,27 @@ CREATE TABLE group_summary_room_categories (
     CHECK (cat_order > 0)
 );
 
+-- The categories in the group
 CREATE TABLE group_room_categories (
     group_id TEXT NOT NULL,
     category_id TEXT NOT NULL,
     profile TEXT NOT NULL,
-    is_public BOOLEAN NOT NULL,
+    is_public BOOLEAN NOT NULL, -- whether the category should be show to everyone
     UNIQUE (group_id, category_id)
 );
 
-
+-- The users to include in the group summary
 CREATE TABLE group_summary_users (
     group_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
     role_id TEXT NOT NULL,
     user_order BIGINT NOT NULL,
-    is_public BOOLEAN NOT NULL
+    is_public BOOLEAN NOT NULL  -- whether the user should be show to everyone
 );
 
 CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id);
 
+-- The roles to include in the group summary
 CREATE TABLE group_summary_roles (
     group_id TEXT NOT NULL,
     role_id TEXT NOT NULL,
@@ -103,11 +108,13 @@ CREATE TABLE group_summary_roles (
     CHECK (role_order > 0)
 );
 
+
+-- The roles in a groups
 CREATE TABLE group_roles (
     group_id TEXT NOT NULL,
     role_id TEXT NOT NULL,
     profile TEXT NOT NULL,
-    is_public BOOLEAN NOT NULL,
+    is_public BOOLEAN NOT NULL,  -- whether the role should be show to everyone
     UNIQUE (group_id, role_id)
 );
 
-- 
cgit 1.4.1


From 5bbb53580a3fc732f0e4aab49f7893bd4f7e2a43 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Jul 2017 10:25:29 +0100
Subject: raise NotImplementedError

---
 synapse/groups/groups_server.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 83dfcd0fd4..dc9d361f57 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -463,9 +463,7 @@ class GroupsServerHandler(object):
         }
 
         if self.hs.is_mine_id(user_id):
-            groups_local = self.hs.get_groups_local_handler()
-            res = yield groups_local.on_invite(group_id, user_id, content)
-            local_attestation = None
+            raise NotImplementedError()
         else:
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content.update({
@@ -593,8 +591,7 @@ class GroupsServerHandler(object):
 
         if is_kick:
             if self.hs.is_mine_id(user_id):
-                groups_local = self.hs.get_groups_local_handler()
-                yield groups_local.user_removed_from_group(group_id, user_id, {})
+                raise NotImplementedError()
             else:
                 yield self.transport_client.remove_user_from_group_notification(
                     get_domain_from_id(user_id), group_id, user_id, {}
-- 
cgit 1.4.1


From a78cda4bafd1eb33a40e8d841de311ea2dbbc086 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Jul 2017 11:17:07 +0100
Subject: Remove TODO

---
 synapse/groups/groups_server.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index dc9d361f57..a00bafe3af 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -96,7 +96,6 @@ class GroupsServerHandler(object):
         )
 
         # TODO: Add profiles to users
-        # TODO: Add assestations to users
 
         rooms, categories = yield self.store.get_rooms_for_summary_by_category(
             group_id, include_private=is_user_in_group,
-- 
cgit 1.4.1


From 2f9eafdd369796d8b7731b24ab8cf6a98ad19e29 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Jul 2017 14:52:27 +0100
Subject: Add local group server support

---
 synapse/federation/transport/client.py           |  77 +++
 synapse/federation/transport/server.py           |  44 ++
 synapse/groups/groups_server.py                  |   7 +-
 synapse/handlers/groups_local.py                 | 278 ++++++++++
 synapse/rest/__init__.py                         |   2 +
 synapse/rest/client/v2_alpha/groups.py           | 642 +++++++++++++++++++++++
 synapse/server.py                                |   5 +
 synapse/storage/__init__.py                      |  15 +
 synapse/storage/group_server.py                  | 152 ++++++
 synapse/storage/schema/delta/43/group_server.sql |  28 +
 10 files changed, 1248 insertions(+), 2 deletions(-)
 create mode 100644 synapse/handlers/groups_local.py
 create mode 100644 synapse/rest/client/v2_alpha/groups.py

(limited to 'synapse/groups')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index d0f8da7516..ea340e345c 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -472,6 +472,72 @@ class TransportLayerClient(object):
 
         defer.returnValue(content)
 
+    @log_function
+    def get_group_profile(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/profile" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_summary(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/summary" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_rooms(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/rooms" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def get_group_users(self, destination, group_id, requester_user_id):
+        path = PREFIX + "/groups/%s/users" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def accept_group_invite(self, destination, group_id, user_id, content):
+        path = PREFIX + "/groups/%s/users/%s/accept_invite" % (group_id, user_id)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
+
+    @log_function
+    def invite_to_group(self, destination, group_id, user_id, content):
+        path = PREFIX + "/groups/%s/users/%s/invite" % (group_id, user_id)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
+
     @log_function
     def invite_to_group_notification(self, destination, group_id, user_id, content):
         """Sent by group server to inform a user's server that they have been
@@ -487,6 +553,17 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    @log_function
+    def remove_user_from_group(self, destination, group_id, user_id, content):
+        path = PREFIX + "/groups/%s/users/%s/remove" % (group_id, user_id)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
+
     @log_function
     def remove_user_from_group_notification(self, destination, group_id, user_id,
                                             content):
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 4f7d2546cf..0f08334f33 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -715,6 +715,21 @@ class FederationGroupsInviteServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsLocalInviteServlet(BaseFederationServlet):
+    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id, user_id):
+        if get_domain_from_id(group_id) != origin:
+            raise SynapseError(403, "group_id doesn't match origin")
+
+        new_content = yield self.handler.on_invite(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
     """Accept an invitation from the group server
     """
@@ -750,6 +765,21 @@ class FederationGroupsRemoveUserServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
+    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id, user_id):
+        if get_domain_from_id(group_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        new_content = yield self.handler.user_removed_from_group(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
     """A group or user's server renews their attestation
     """
@@ -1053,6 +1083,12 @@ GROUP_SERVER_SERVLET_CLASSES = (
 )
 
 
+GROUP_LOCAL_SERVLET_CLASSES = (
+    FederationGroupsLocalInviteServlet,
+    FederationGroupsRemoveLocalUserServlet,
+)
+
+
 GROUP_ATTESTATION_SERVLET_CLASSES = (
     FederationGroupsRenewAttestaionServlet,
 )
@@ -1083,6 +1119,14 @@ def register_servlets(hs, resource, authenticator, ratelimiter):
             server_name=hs.hostname,
         ).register(resource)
 
+    for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
+        servletclass(
+            handler=hs.get_groups_local_handler(),
+            authenticator=authenticator,
+            ratelimiter=ratelimiter,
+            server_name=hs.hostname,
+        ).register(resource)
+
     for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
         servletclass(
             handler=hs.get_groups_attestation_renewer(),
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index a00bafe3af..c8559577f7 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -462,7 +462,9 @@ class GroupsServerHandler(object):
         }
 
         if self.hs.is_mine_id(user_id):
-            raise NotImplementedError()
+            groups_local = self.hs.get_groups_local_handler()
+            res = yield groups_local.on_invite(group_id, user_id, content)
+            local_attestation = None
         else:
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content.update({
@@ -590,7 +592,8 @@ class GroupsServerHandler(object):
 
         if is_kick:
             if self.hs.is_mine_id(user_id):
-                raise NotImplementedError()
+                groups_local = self.hs.get_groups_local_handler()
+                yield groups_local.user_removed_from_group(group_id, user_id, {})
             else:
                 yield self.transport_client.remove_user_from_group_notification(
                     get_domain_from_id(user_id), group_id, user_id, {}
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
new file mode 100644
index 0000000000..3df255b05a
--- /dev/null
+++ b/synapse/handlers/groups_local.py
@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: Validate attestations
+# TODO: Allow users to "knock" or simpkly join depending on rules
+# TODO: is_priveged flag to users and is_public to users and rooms
+# TODO: Roles
+# TODO: Audit log for admins (profile updates, membership changes, users who tried
+#       to join but were rejected, etc)
+# TODO: Flairs
+# TODO: Add group memebership  /sync
+
+
+def _create_rerouter(name):
+    def f(self, group_id, *args, **kwargs):
+        if self.is_mine_id(group_id):
+            return getattr(self.groups_server_handler, name)(
+                group_id, *args, **kwargs
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return getattr(repl_layer, name)(group_id, *args, **kwargs)
+    return f
+
+
+class GroupsLocalHandler(object):
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.room_list_handler = hs.get_room_list_handler()
+        self.groups_server_handler = hs.get_groups_server_handler()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.keyring = hs.get_keyring()
+        self.is_mine_id = hs.is_mine_id
+        self.signing_key = hs.config.signing_key[0]
+        self.server_name = hs.hostname
+        self.attestations = hs.get_groups_attestation_signing()
+
+        # Ensure attestations get renewed
+        hs.get_groups_attestation_renewer()
+
+    get_group_profile = _create_rerouter("get_group_profile")
+    get_rooms_in_group = _create_rerouter("get_rooms_in_group")
+
+    update_group_summary_room = _create_rerouter("update_group_summary_room")
+    delete_group_summary_room = _create_rerouter("delete_group_summary_room")
+
+    update_group_category = _create_rerouter("update_group_category")
+    delete_group_category = _create_rerouter("delete_group_category")
+    get_group_category = _create_rerouter("get_group_category")
+    get_group_categories = _create_rerouter("get_group_categories")
+
+    update_group_summary_user = _create_rerouter("update_group_summary_user")
+    delete_group_summary_user = _create_rerouter("delete_group_summary_user")
+
+    update_group_role = _create_rerouter("update_group_role")
+    delete_group_role = _create_rerouter("delete_group_role")
+    get_group_role = _create_rerouter("get_group_role")
+    get_group_roles = _create_rerouter("get_group_roles")
+
+    @defer.inlineCallbacks
+    def get_group_summary(self, group_id, requester_user_id):
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.get_group_summary(
+                group_id, requester_user_id
+            )
+            defer.returnValue(res)
+
+        repl_layer = self.hs.get_replication_layer()
+        res = yield repl_layer.get_group_summary(group_id, requester_user_id)
+
+        chunk = res["users_section"]["users"]
+        valid_users = []
+        for entry in chunk:
+            g_user_id = entry["user_id"]
+            attestation = entry.pop("attestation")
+            try:
+                yield self.attestations.verify_attestation(
+                    attestation,
+                    group_id=group_id,
+                    user_id=g_user_id,
+                )
+                valid_users.append(entry)
+            except Exception as e:
+                logger.info("Failed to verify user is in group: %s", e)
+
+        res["users_section"]["users"] = valid_users
+
+        res["users_section"]["users"].sort(key=lambda e: e.get("order", 0))
+        res["rooms_section"]["rooms"].sort(key=lambda e: e.get("order", 0))
+
+        defer.returnValue(res)
+
+    def create_group(self, group_id, user_id, content):
+        logger.info("Asking to create group with ID: %r", group_id)
+
+        if self.is_mine_id(group_id):
+            return self.groups_server_handler.create_group(
+                group_id, user_id, content
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return repl_layer.create_group(group_id, user_id, content)  # TODO
+
+    def add_room(self, group_id, user_id, room_id, content):
+        if self.is_mine_id(group_id):
+            return self.groups_server_handler.add_room(
+                group_id, user_id, room_id, content
+            )
+
+        repl_layer = self.hs.get_replication_layer()
+        return repl_layer.add_room_to_group(group_id, user_id, room_id, content)  # TODO
+
+    @defer.inlineCallbacks
+    def get_users_in_group(self, group_id, requester_user_id):
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.get_users_in_group(
+                group_id, requester_user_id
+            )
+            defer.returnValue(res)
+
+        repl_layer = self.hs.get_replication_layer()
+        res = yield repl_layer.get_users_in_group(group_id, requester_user_id)  # TODO
+
+        chunk = res["chunk"]
+        valid_entries = []
+        for entry in chunk:
+            g_user_id = entry["user_id"]
+            attestation = entry.pop("attestation")
+            try:
+                yield self.attestations.verify_attestation(
+                    attestation,
+                    group_id=group_id,
+                    user_id=g_user_id,
+                )
+                valid_entries.append(entry)
+            except Exception as e:
+                logger.info("Failed to verify user is in group: %s", e)
+
+        res["chunk"] = valid_entries
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def join_group(self, group_id, user_id, content):
+        raise NotImplementedError()  # TODO
+
+    @defer.inlineCallbacks
+    def accept_invite(self, group_id, user_id, content):
+        if self.is_mine_id(group_id):
+            yield self.groups_server_handler.accept_invite(
+                group_id, user_id, content
+            )
+            local_attestation = None
+            remote_attestation = None
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content["attestation"] = local_attestation
+
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.accept_group_invite(group_id, user_id, content)
+
+            remote_attestation = res["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                group_id=group_id,
+                user_id=user_id,
+            )
+
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="join",
+            is_admin=False,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({})
+
+    @defer.inlineCallbacks
+    def invite(self, group_id, user_id, requester_user_id, config):
+        content = {
+            "requester_user_id": requester_user_id,
+            "config": config,
+        }
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.invite_to_group(
+                group_id, user_id, requester_user_id, content,
+            )
+        else:
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.invite_to_group(
+                group_id, user_id, content,
+            )
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def on_invite(self, group_id, user_id, content):
+        # TODO: Support auto join and rejection
+
+        if not self.is_mine_id(user_id):
+            raise SynapseError(400, "User not on this server")
+
+        local_profile = {}
+        if "profile" in content:
+            if "name" in content["profile"]:
+                local_profile["name"] = content["profile"]["name"]
+            if "avatar_url" in content["profile"]:
+                local_profile["avatar_url"] = content["profile"]["avatar_url"]
+
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="invite",
+            content={"profile": local_profile, "inviter": content["inviter"]},
+        )
+
+        defer.returnValue({"state": "invite"})
+
+    @defer.inlineCallbacks
+    def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        if user_id == requester_user_id:
+            yield self.store.register_user_group_membership(
+                group_id, user_id,
+                membership="leave",
+            )
+
+            # TODO: Should probably remember that we tried to leave so that we can
+            # retry if the group server is currently down.
+
+        if self.is_mine_id(group_id):
+            res = yield self.groups_server_handler.remove_user_from_group(
+                group_id, user_id, requester_user_id, content,
+            )
+        else:
+            content["requester_user_id"] = requester_user_id
+            repl_layer = self.hs.get_replication_layer()
+            res = yield repl_layer.remove_user_from_group(
+                group_id, user_id, content
+            )  # TODO
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def user_removed_from_group(self, group_id, user_id, content):
+        # TODO: Check if user in group
+        yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="leave",
+        )
+
+    @defer.inlineCallbacks
+    def get_joined_groups(self, user_id):
+        group_ids = yield self.store.get_joined_groups(user_id)
+        defer.returnValue({"groups": group_ids})
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 3d809d181b..16f5a73b95 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -52,6 +52,7 @@ from synapse.rest.client.v2_alpha import (
     thirdparty,
     sendtodevice,
     user_directory,
+    groups,
 )
 
 from synapse.http.server import JsonResource
@@ -102,3 +103,4 @@ class ClientRestResource(JsonResource):
         thirdparty.register_servlets(hs, client_resource)
         sendtodevice.register_servlets(hs, client_resource)
         user_directory.register_servlets(hs, client_resource)
+        groups.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
new file mode 100644
index 0000000000..255552c365
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -0,0 +1,642 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.types import GroupID
+
+from ._base import client_v2_patterns
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class GroupServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/profile$")
+
+    def __init__(self, hs):
+        super(GroupServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        group_description = yield self.groups_handler.get_group_profile(group_id, user_id)
+
+        defer.returnValue((200, group_description))
+
+
+class GroupSummaryServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/summary$")
+
+    def __init__(self, hs):
+        super(GroupSummaryServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        get_group_summary = yield self.groups_handler.get_group_summary(group_id, user_id)
+
+        defer.returnValue((200, get_group_summary))
+
+
+class GroupSummaryRoomsServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/summary/rooms$")
+
+    def __init__(self, hs):
+        super(GroupSummaryServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        get_group_summary = yield self.groups_handler.get_group_summary(group_id, user_id)
+
+        defer.returnValue((200, get_group_summary))
+
+
+class GroupSummaryRoomsDefaultCatServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/summary/rooms/(?P<room_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSummaryRoomsDefaultCatServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_summary_room(
+            group_id, user_id,
+            room_id=room_id,
+            category_id=None,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_summary_room(
+            group_id, user_id,
+            room_id=room_id,
+            category_id=None,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupSummaryRoomsCatServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/summary"
+        "/categories/(?P<category_id>[^/]+)/rooms/(?P<room_id>[^/]+)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSummaryRoomsCatServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, category_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_summary_room(
+            group_id, user_id,
+            room_id=room_id,
+            category_id=category_id,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, category_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_summary_room(
+            group_id, user_id,
+            room_id=room_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupCategoryServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupCategoryServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id, category_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        category = yield self.groups_handler.get_group_category(
+            group_id, user_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue((200, category))
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, category_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_category(
+            group_id, user_id,
+            category_id=category_id,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, category_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_category(
+            group_id, user_id,
+            category_id=category_id,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupCategoriesServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/categories/$"
+    )
+
+    def __init__(self, hs):
+        super(GroupCategoriesServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        category = yield self.groups_handler.get_group_categories(
+            group_id, user_id,
+        )
+
+        defer.returnValue((200, category))
+
+
+class GroupRoleServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupRoleServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id, role_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        category = yield self.groups_handler.get_group_role(
+            group_id, user_id,
+            role_id=role_id,
+        )
+
+        defer.returnValue((200, category))
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, role_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_role(
+            group_id, user_id,
+            role_id=role_id,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, role_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_role(
+            group_id, user_id,
+            role_id=role_id,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupRolesServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/roles/$"
+    )
+
+    def __init__(self, hs):
+        super(GroupRolesServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        category = yield self.groups_handler.get_group_roles(
+            group_id, user_id,
+        )
+
+        defer.returnValue((200, category))
+
+
+class GroupSummaryUsersDefaultRoleServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/summary/users/(?P<user_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSummaryUsersDefaultRoleServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_summary_user(
+            group_id, requester_user_id,
+            user_id=user_id,
+            role_id=None,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_summary_user(
+            group_id, requester_user_id,
+            user_id=user_id,
+            role_id=None,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupSummaryUsersRoleServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/summary"
+        "/roles/(?P<role_id>[^/]+)/users/(?P<user_id>[^/]+)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSummaryUsersRoleServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, role_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        resp = yield self.groups_handler.update_group_summary_user(
+            group_id, requester_user_id,
+            user_id=user_id,
+            role_id=role_id,
+            content=content,
+        )
+
+        defer.returnValue((200, resp))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, role_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        resp = yield self.groups_handler.delete_group_summary_user(
+            group_id, requester_user_id,
+            user_id=user_id,
+            role_id=role_id,
+        )
+
+        defer.returnValue((200, resp))
+
+
+class GroupRoomServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/rooms$")
+
+    def __init__(self, hs):
+        super(GroupRoomServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        result = yield self.groups_handler.get_rooms_in_group(group_id, user_id)
+
+        defer.returnValue((200, result))
+
+
+class GroupUsersServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/users$")
+
+    def __init__(self, hs):
+        super(GroupUsersServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        result = yield self.groups_handler.get_users_in_group(group_id, user_id)
+
+        defer.returnValue((200, result))
+
+
+class GroupCreateServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/create_group$")
+
+    def __init__(self, hs):
+        super(GroupCreateServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+        self.server_name = hs.hostname
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        # TODO: Create group on remote server
+        content = parse_json_object_from_request(request)
+        localpart = content.pop("localpart")
+        group_id = GroupID.create(localpart, self.server_name).to_string()
+
+        result = yield self.groups_handler.create_group(group_id, user_id, content)
+
+        defer.returnValue((200, result))
+
+
+class GroupAdminRoomsServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/admin/rooms/(?P<room_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupAdminRoomsServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        result = yield self.groups_handler.add_room(group_id, user_id, room_id, content)
+
+        defer.returnValue((200, result))
+
+
+class GroupAdminUsersInviteServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/admin/users/invite/(?P<user_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupAdminUsersInviteServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+        self.store = hs.get_datastore()
+        self.is_mine_id = hs.is_mine_id
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        config = content.get("config", {})
+        result = yield self.groups_handler.invite(
+            group_id, user_id, requester_user_id, config,
+        )
+
+        defer.returnValue((200, result))
+
+
+class GroupAdminUsersKickServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/admin/users/remove/(?P<user_id>[^/]*)$"
+    )
+
+    def __init__(self, hs):
+        super(GroupAdminUsersKickServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id, user_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        result = yield self.groups_handler.remove_user_from_group(
+            group_id, user_id, requester_user_id, content,
+        )
+
+        defer.returnValue((200, result))
+
+
+class GroupSelfLeaveServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/self/leave$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSelfLeaveServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        result = yield self.groups_handler.remove_user_from_group(
+            group_id, requester_user_id, requester_user_id, content,
+        )
+
+        defer.returnValue((200, result))
+
+
+class GroupSelfJoinServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/self/join$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSelfJoinServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        result = yield self.groups_handler.join_group(
+            group_id, requester_user_id, content,
+        )
+
+        defer.returnValue((200, result))
+
+
+class GroupSelfAcceptInviteServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/groups/(?P<group_id>[^/]*)/self/accept_invite$"
+    )
+
+    def __init__(self, hs):
+        super(GroupSelfAcceptInviteServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        result = yield self.groups_handler.accept_invite(
+            group_id, requester_user_id, content,
+        )
+
+        defer.returnValue((200, result))
+
+
+class GroupsForUserServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/joined_groups$"
+    )
+
+    def __init__(self, hs):
+        super(GroupsForUserServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        result = yield self.groups_handler.get_joined_groups(user_id)
+
+        defer.returnValue((200, result))
+
+
+def register_servlets(hs, http_server):
+    GroupServlet(hs).register(http_server)
+    GroupSummaryServlet(hs).register(http_server)
+    GroupUsersServlet(hs).register(http_server)
+    GroupRoomServlet(hs).register(http_server)
+    GroupCreateServlet(hs).register(http_server)
+    GroupAdminRoomsServlet(hs).register(http_server)
+    GroupAdminUsersInviteServlet(hs).register(http_server)
+    GroupAdminUsersKickServlet(hs).register(http_server)
+    GroupSelfLeaveServlet(hs).register(http_server)
+    GroupSelfJoinServlet(hs).register(http_server)
+    GroupSelfAcceptInviteServlet(hs).register(http_server)
+    GroupsForUserServlet(hs).register(http_server)
+    GroupSummaryRoomsDefaultCatServlet(hs).register(http_server)
+    GroupCategoryServlet(hs).register(http_server)
+    GroupCategoriesServlet(hs).register(http_server)
+    GroupSummaryRoomsCatServlet(hs).register(http_server)
+    GroupRoleServlet(hs).register(http_server)
+    GroupRolesServlet(hs).register(http_server)
+    GroupSummaryUsersDefaultRoleServlet(hs).register(http_server)
+    GroupSummaryUsersRoleServlet(hs).register(http_server)
diff --git a/synapse/server.py b/synapse/server.py
index d857cca848..d0a6272766 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -50,6 +50,7 @@ from synapse.handlers.initial_sync import InitialSyncHandler
 from synapse.handlers.receipts import ReceiptsHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.user_directory import UserDirectoyHandler
+from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.groups.groups_server import GroupsServerHandler
 from synapse.groups.attestations import GroupAttestionRenewer, GroupAttestationSigning
 from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory
@@ -141,6 +142,7 @@ class HomeServer(object):
         'read_marker_handler',
         'action_generator',
         'user_directory_handler',
+        'groups_local_handler',
         'groups_server_handler',
         'groups_attestation_signing',
         'groups_attestation_renewer',
@@ -314,6 +316,9 @@ class HomeServer(object):
     def build_user_directory_handler(self):
         return UserDirectoyHandler(self)
 
+    def build_groups_local_handler(self):
+        return GroupsLocalHandler(self)
+
     def build_groups_server_handler(self):
         return GroupsServerHandler(self)
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index fdee9f1ad5..594566eb38 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -136,6 +136,9 @@ class DataStore(RoomMemberStore, RoomStore,
             db_conn, "pushers", "id",
             extra_tables=[("deleted_pushers", "stream_id")],
         )
+        self._group_updates_id_gen = StreamIdGenerator(
+            db_conn, "local_group_updates", "stream_id",
+        )
 
         if isinstance(self.database_engine, PostgresEngine):
             self._cache_id_gen = StreamIdGenerator(
@@ -236,6 +239,18 @@ class DataStore(RoomMemberStore, RoomStore,
             prefilled_cache=curr_state_delta_prefill,
         )
 
+        _group_updates_prefill, min_group_updates_id = self._get_cache_dict(
+            db_conn, "local_group_updates",
+            entity_column="user_id",
+            stream_column="stream_id",
+            max_value=self._group_updates_id_gen.get_current_token(),
+            limit=1000,
+        )
+        self._group_updates_stream_cache = StreamChangeCache(
+            "_group_updates_stream_cache", min_group_updates_id,
+            prefilled_cache=_group_updates_prefill,
+        )
+
         cur = LoggingTransaction(
             db_conn.cursor(),
             name="_find_stream_orderings_for_times_txn",
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index e8a799d8c7..036549d437 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -756,6 +756,103 @@ class GroupServerStore(SQLBaseStore):
             desc="add_room_to_group",
         )
 
+    @defer.inlineCallbacks
+    def register_user_group_membership(self, group_id, user_id, membership,
+                                       is_admin=False, content={},
+                                       local_attestation=None,
+                                       remote_attestation=None,
+                                       ):
+        def _register_user_group_membership_txn(txn, next_id):
+            # TODO: Upsert?
+            self._simple_delete_txn(
+                txn,
+                table="local_group_membership",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                },
+            )
+            self._simple_insert_txn(
+                txn,
+                table="local_group_membership",
+                values={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "is_admin": is_admin,
+                    "membership": membership,
+                    "content": json.dumps(content),
+                },
+            )
+            self._simple_delete_txn(
+                txn,
+                table="local_group_updates",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "type": "membership",
+                },
+            )
+            self._simple_insert_txn(
+                txn,
+                table="local_group_updates",
+                values={
+                    "stream_id": next_id,
+                    "group_id": group_id,
+                    "user_id": user_id,
+                    "type": "membership",
+                    "content": json.dumps({"membership": membership, "content": content}),
+                }
+            )
+            self._group_updates_stream_cache.entity_has_changed(user_id, next_id)
+
+            # TODO: Insert profile to ensuer it comes down stream if its a join.
+
+            if membership == "join":
+                if local_attestation:
+                    self._simple_insert_txn(
+                        txn,
+                        table="group_attestations_renewals",
+                        values={
+                            "group_id": group_id,
+                            "user_id": user_id,
+                            "valid_until_ms": local_attestation["valid_until_ms"],
+                        }
+                    )
+                if remote_attestation:
+                    self._simple_insert_txn(
+                        txn,
+                        table="group_attestations_remote",
+                        values={
+                            "group_id": group_id,
+                            "user_id": user_id,
+                            "valid_until_ms": remote_attestation["valid_until_ms"],
+                            "attestation": json.dumps(remote_attestation),
+                        }
+                    )
+            else:
+                self._simple_delete_txn(
+                    txn,
+                    table="group_attestations_renewals",
+                    keyvalues={
+                        "group_id": group_id,
+                        "user_id": user_id,
+                    },
+                )
+                self._simple_delete_txn(
+                    txn,
+                    table="group_attestations_remote",
+                    keyvalues={
+                        "group_id": group_id,
+                        "user_id": user_id,
+                    },
+                )
+
+        with self._group_updates_id_gen.get_next() as next_id:
+            yield self.runInteraction(
+                "register_user_group_membership",
+                _register_user_group_membership_txn, next_id,
+            )
+
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, name, avatar_url, short_description,
                      long_description,):
@@ -771,6 +868,61 @@ class GroupServerStore(SQLBaseStore):
             desc="create_group",
         )
 
+    def get_joined_groups(self, user_id):
+        return self._simple_select_onecol(
+            table="local_group_membership",
+            keyvalues={
+                "user_id": user_id,
+                "membership": "join",
+            },
+            retcol="group_id",
+            desc="get_joined_groups",
+        )
+
+    def get_all_groups_for_user(self, user_id, now_token):
+        def _get_all_groups_for_user_txn(txn):
+            sql = """
+                SELECT group_id, type, membership, u.content
+                FROM local_group_updates AS u
+                INNER JOIN local_group_membership USING (group_id, user_id)
+                WHERE user_id = ? AND membership != 'leave'
+                    AND stream_id <= ?
+            """
+            txn.execute(sql, (user_id, now_token,))
+            return self.cursor_to_dict(txn)
+        return self.runInteraction(
+            "get_all_groups_for_user", _get_all_groups_for_user_txn,
+        )
+
+    def get_groups_changes_for_user(self, user_id, from_token, to_token):
+        from_token = int(from_token)
+        has_changed = self._group_updates_stream_cache.has_entity_changed(
+            user_id, from_token,
+        )
+        if not has_changed:
+            return []
+
+        def _get_groups_changes_for_user_txn(txn):
+            sql = """
+                SELECT group_id, membership, type, u.content
+                FROM local_group_updates AS u
+                INNER JOIN local_group_membership USING (group_id, user_id)
+                WHERE user_id = ? AND ? < stream_id AND stream_id <= ?
+            """
+            txn.execute(sql, (user_id, from_token, to_token,))
+            return [{
+                "group_id": group_id,
+                "membership": membership,
+                "type": gtype,
+                "content": json.loads(content_json),
+            } for group_id, membership, gtype, content_json in txn]
+        return self.runInteraction(
+            "get_groups_changes_for_user", _get_groups_changes_for_user_txn,
+        )
+
+    def get_group_stream_token(self):
+        return self._group_updates_id_gen.get_current_token()
+
     def get_attestations_need_renewals(self, valid_until_ms):
         """Get all attestations that need to be renewed until givent time
         """
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index 472aab0a78..e32db8b313 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -142,3 +142,31 @@ CREATE TABLE group_attestations_remote (
 CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote(group_id, user_id);
 CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote(user_id);
 CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote(valid_until_ms);
+
+
+CREATE TABLE local_group_membership (
+    group_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    is_admin BOOLEAN NOT NULL,
+    membership TEXT NOT NULL,
+    content TEXT NOT NULL
+);
+
+CREATE INDEX local_group_membership_u_idx ON local_group_membership(user_id, group_id);
+CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id);
+
+
+CREATE TABLE local_group_updates (
+    stream_id BIGINT NOT NULL,
+    group_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    content TEXT NOT NULL
+);
+
+
+CREATE TABLE local_group_profiles (
+    group_id TEXT NOT NULL,
+    name TEXT,
+    avatar_url TEXT
+);
-- 
cgit 1.4.1


From 6027b1992f507cb1a11836797827d9e69e2a6020 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 16:51:25 +0100
Subject: Fix permissions

---
 synapse/groups/groups_server.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index c8559577f7..b9ad9507f4 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -293,7 +293,9 @@ class GroupsServerHandler(object):
                                   content):
         """Add/update a users entry in the group summary
         """
-        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id,
+        )
 
         order = content.get("order", None)
 
@@ -313,7 +315,9 @@ class GroupsServerHandler(object):
     def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
         """Remove a user from the group summary
         """
-        yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id,
+        )
 
         yield self.store.remove_user_from_summary(
             group_id=group_id,
-- 
cgit 1.4.1


From 14a34f12d755e7516dc81348d811d47dc51f026d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 18 Jul 2017 17:28:42 +0100
Subject: Comments

---
 synapse/federation/transport/server.py |  2 +-
 synapse/groups/groups_server.py        |  2 +-
 synapse/handlers/groups_local.py       | 29 +++++++++++++++++++----------
 synapse/rest/client/v2_alpha/groups.py |  4 +++-
 4 files changed, 24 insertions(+), 13 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 29e966ac29..1332b49f35 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -672,7 +672,7 @@ class FederationGroupsAddRoomsServlet(BaseFederationServlet):
         if get_domain_from_id(requester_user_id) != origin:
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
-        new_content = yield self.handler.add_room(
+        new_content = yield self.handler.add_room_to_group(
             group_id, requester_user_id, room_id, content
         )
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index b9ad9507f4..1b6e354ca3 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -430,7 +430,7 @@ class GroupsServerHandler(object):
         })
 
     @defer.inlineCallbacks
-    def add_room(self, group_id, requester_user_id, room_id, content):
+    def add_room_to_group(self, group_id, requester_user_id, room_id, content):
         """Add room to group
         """
         yield self.check_group_is_ours(
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 50f7fce885..0b80348c82 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -74,6 +74,8 @@ class GroupsLocalHandler(object):
     get_group_profile = _create_rerouter("get_group_profile")
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
+    add_room_to_group = _create_rerouter("add_room_to_group")
+
     update_group_summary_room = _create_rerouter("update_group_summary_room")
     delete_group_summary_room = _create_rerouter("delete_group_summary_room")
 
@@ -130,6 +132,9 @@ class GroupsLocalHandler(object):
         defer.returnValue(res)
 
     def create_group(self, group_id, user_id, content):
+        """Create a group
+        """
+
         logger.info("Asking to create group with ID: %r", group_id)
 
         if self.is_mine_id(group_id):
@@ -141,18 +146,10 @@ class GroupsLocalHandler(object):
             get_domain_from_id(group_id), group_id, user_id, content,
         )  # TODO
 
-    def add_room(self, group_id, user_id, room_id, content):
-        if self.is_mine_id(group_id):
-            return self.groups_server_handler.add_room(
-                group_id, user_id, room_id, content
-            )
-
-        return self.transport_client.add_room_to_group(
-            get_domain_from_id(group_id), group_id, user_id, room_id, content,
-        )
-
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
+        """Get users in a group
+        """
         if self.is_mine_id(group_id):
             res = yield self.groups_server_handler.get_users_in_group(
                 group_id, requester_user_id
@@ -184,10 +181,14 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def join_group(self, group_id, user_id, content):
+        """Request to join a group
+        """
         raise NotImplementedError()  # TODO
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
+        """Accept an invite to a group
+        """
         if self.is_mine_id(group_id):
             yield self.groups_server_handler.accept_invite(
                 group_id, user_id, content
@@ -222,6 +223,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def invite(self, group_id, user_id, requester_user_id, config):
+        """Invite a user to a group
+        """
         content = {
             "requester_user_id": requester_user_id,
             "config": config,
@@ -240,6 +243,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def on_invite(self, group_id, user_id, content):
+        """One of our users were invited to a group
+        """
         # TODO: Support auto join and rejection
 
         if not self.is_mine_id(user_id):
@@ -262,6 +267,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
+        """Remove a user from a group
+        """
         if user_id == requester_user_id:
             yield self.store.register_user_group_membership(
                 group_id, user_id,
@@ -286,6 +293,8 @@ class GroupsLocalHandler(object):
 
     @defer.inlineCallbacks
     def user_removed_from_group(self, group_id, user_id, content):
+        """One of our users was removed/kicked from a group
+        """
         # TODO: Check if user in group
         yield self.store.register_user_group_membership(
             group_id, user_id,
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index 787967c3a2..f937d856fd 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -405,7 +405,9 @@ class GroupAdminRoomsServlet(RestServlet):
         user_id = requester.user.to_string()
 
         content = parse_json_object_from_request(request)
-        result = yield self.groups_handler.add_room(group_id, user_id, room_id, content)
+        result = yield self.groups_handler.add_room_to_group(
+            group_id, user_id, room_id, content,
+        )
 
         defer.returnValue((200, result))
 
-- 
cgit 1.4.1


From 6f443a74cf6ab0bfe452289f9888580725987765 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Jul 2017 09:46:33 +0100
Subject: Add update group profile API

---
 synapse/federation/transport/server.py | 12 ++++++++++++
 synapse/groups/groups_server.py        | 16 ++++++++++++++++
 synapse/handlers/groups_local.py       |  1 +
 synapse/rest/client/v2_alpha/groups.py | 12 ++++++++++++
 synapse/storage/group_server.py        | 11 +++++++++++
 5 files changed, 52 insertions(+)

(limited to 'synapse/groups')

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 1332b49f35..e04750fd2a 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -642,6 +642,18 @@ class FederationGroupsSummaryServlet(BaseFederationServlet):
 
         defer.returnValue((200, new_content))
 
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.update_group_profile(
+            group_id, requester_user_id, content
+        )
+
+        defer.returnValue((200, new_content))
+
 
 class FederationGroupsRoomsServlet(BaseFederationServlet):
     """Get the rooms in a group on behalf of a user
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 1b6e354ca3..322aad2a6f 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -341,6 +341,22 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(404, "Unknown group")
 
+    @defer.inlineCallbacks
+    def update_group_profile(self, group_id, requester_user_id, content):
+        """Update the group profile
+        """
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id,
+        )
+
+        profile = {}
+        for keyname in ("name", "avatar_url", "short_description",
+                        "long_description"):
+            if keyname in content:
+                profile[keyname] = content[keyname]
+
+        yield self.store.update_group_profile(group_id, profile)
+
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
         """Get the users in group as seen by requester_user_id.
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 0b80348c82..b2c920da38 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -72,6 +72,7 @@ class GroupsLocalHandler(object):
     # or federation depending on if the group is local or remote
 
     get_group_profile = _create_rerouter("get_group_profile")
+    update_group_profile = _create_rerouter("update_group_profile")
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
     add_room_to_group = _create_rerouter("add_room_to_group")
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index f937d856fd..64d803d489 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -45,6 +45,18 @@ class GroupServlet(RestServlet):
 
         defer.returnValue((200, group_description))
 
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id, content):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+        yield self.groups_handler.update_group_profile(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, {}))
+
 
 class GroupSummaryServlet(RestServlet):
     """Get the full group summary
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 0a69e0f501..4197d22d88 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -860,6 +860,17 @@ class GroupServerStore(SQLBaseStore):
             desc="create_group",
         )
 
+    @defer.inlineCallbacks
+    def update_group_profile(self, group_id, profile,):
+        yield self._simple_update_one(
+            table="groups",
+            keyvalues={
+                "group_id": group_id,
+            },
+            updatevalues=profile,
+            desc="create_group",
+        )
+
     def get_attestations_need_renewals(self, valid_until_ms):
         """Get all attestations that need to be renewed until givent time
         """
-- 
cgit 1.4.1


From 0ab153d2014d871c13b02dbd1c6bf7c0cc0bcedc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Jul 2017 16:24:18 +0100
Subject: Check values are strings

---
 synapse/groups/groups_server.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 322aad2a6f..b1ee43ef90 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -353,7 +353,10 @@ class GroupsServerHandler(object):
         for keyname in ("name", "avatar_url", "short_description",
                         "long_description"):
             if keyname in content:
-                profile[keyname] = content[keyname]
+                value = content[keyname]
+                if not isinstance(value, basestring):
+                    raise SynapseError(400, "%r value is not a string" % (keyname,))
+                profile[keyname] = value
 
         yield self.store.update_group_profile(group_id, profile)
 
-- 
cgit 1.4.1


From b76ef6ccb8e00610c791a78b940d396da82fb1ce Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 24 Jul 2017 13:55:39 +0100
Subject: Include users membership in group in summary API

---
 synapse/groups/groups_server.py |  5 ++++
 synapse/storage/group_server.py | 55 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index b1ee43ef90..f25f327eb9 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -130,6 +130,10 @@ class GroupsServerHandler(object):
 
         users.sort(key=lambda e: e.get("order", 0))
 
+        membership_info = yield self.store.get_users_membership_info_in_group(
+            group_id, requester_user_id,
+        )
+
         defer.returnValue({
             "profile": profile,
             "users_section": {
@@ -142,6 +146,7 @@ class GroupsServerHandler(object):
                 "categories": categories,
                 "total_room_count_estimate": 0,  # TODO
             },
+            "user": membership_info,
         })
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 258c3168aa..989a10eea6 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -672,6 +672,61 @@ class GroupServerStore(SQLBaseStore):
             allow_none=True,
         )
 
+    @defer.inlineCallbacks
+    def get_users_membership_info_in_group(self, group_id, user_id):
+        """Get a dict describing the memebrship of a user in a group.
+
+        Example if joined:
+
+            {
+                "memebrship": "joined",
+                "is_public": True,
+                "is_privileged": False,
+            }
+
+        Returns an empty dict if the user is not joined/invited/etc
+        """
+        def _get_users_membership_in_group_txn(txn):
+            row = self._simple_select_one_txn(
+                table="group_users",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                },
+                retcols=("is_admin", "is_public"),
+                allow_none=True,
+                desc="is_user_adim_in_group",
+            )
+
+            if row:
+                return {
+                    "memebrship": "joined",
+                    "is_public": row["is_public"],
+                    "is_privileged": row["is_admin"],
+                }
+
+            row = self._simple_select_one_onecol_txn(
+                table="group_invites",
+                keyvalues={
+                    "group_id": group_id,
+                    "user_id": user_id,
+                },
+                retcol="user_id",
+                desc="is_user_invited_to_local_group",
+                allow_none=True,
+            )
+
+            if row:
+                return {
+                    "memebrship": "invited",
+                }
+
+            return {}
+
+        return self.runInteraction(
+            "get_users_membership_info_in_group", _get_users_membership_in_group_txn,
+        )
+
     def add_user_to_group(self, group_id, user_id, is_admin=False, is_public=True,
                           local_attestation=None, remote_attestation=None):
         """Add a user to the group server.
-- 
cgit 1.4.1


From 27ebc5c8f299488ccc0a6f100ec3b248cd81a058 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Aug 2017 11:21:34 +0100
Subject: Add remote profile cache

---
 synapse/groups/groups_server.py                   | 18 +++++
 synapse/handlers/groups_local.py                  | 17 +++-
 synapse/handlers/profile.py                       | 81 ++++++++++++++++++-
 synapse/storage/profile.py                        | 98 +++++++++++++++++++++++
 synapse/storage/schema/delta/43/profile_cache.sql | 28 +++++++
 5 files changed, 237 insertions(+), 5 deletions(-)
 create mode 100644 synapse/storage/schema/delta/43/profile_cache.sql

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index f25f327eb9..6bccae4bfb 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -503,6 +503,13 @@ class GroupsServerHandler(object):
                 get_domain_from_id(user_id), group_id, user_id, content
             )
 
+            user_profile = res.get("user_profile", {})
+            yield self.store.add_remote_profile_cache(
+                user_id,
+                displayname=user_profile.get("displayname"),
+                avatar_url=user_profile.get("avatar_url"),
+            )
+
         if res["state"] == "join":
             if not self.hs.is_mine_id(user_id):
                 remote_attestation = res["attestation"]
@@ -627,6 +634,9 @@ class GroupsServerHandler(object):
                     get_domain_from_id(user_id), group_id, user_id, {}
                 )
 
+        if not self.hs.is_mine_id(user_id):
+            yield self.store.maybe_delete_remote_profile_cache(user_id)
+
         defer.returnValue({})
 
     @defer.inlineCallbacks
@@ -647,6 +657,7 @@ class GroupsServerHandler(object):
         avatar_url = profile.get("avatar_url")
         short_description = profile.get("short_description")
         long_description = profile.get("long_description")
+        user_profile = content.get("user_profile", {})
 
         yield self.store.create_group(
             group_id,
@@ -679,6 +690,13 @@ class GroupsServerHandler(object):
             remote_attestation=remote_attestation,
         )
 
+        if not self.hs.is_mine_id(user_id):
+            yield self.store.add_remote_profile_cache(
+                user_id,
+                displayname=user_profile.get("displayname"),
+                avatar_url=user_profile.get("avatar_url"),
+            )
+
         defer.returnValue({
             "group_id": group_id,
         })
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 274fed9278..bfa10bde5a 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -56,6 +56,9 @@ class GroupsLocalHandler(object):
         self.notifier = hs.get_notifier()
         self.attestations = hs.get_groups_attestation_signing()
 
+        handlers = hs.get_handlers()
+        self.profile_handler = handlers.profile_handler
+
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
 
@@ -123,6 +126,7 @@ class GroupsLocalHandler(object):
 
         defer.returnValue(res)
 
+    @defer.inlineCallbacks
     def create_group(self, group_id, user_id, content):
         """Create a group
         """
@@ -130,13 +134,16 @@ class GroupsLocalHandler(object):
         logger.info("Asking to create group with ID: %r", group_id)
 
         if self.is_mine_id(group_id):
-            return self.groups_server_handler.create_group(
+            res = yield self.groups_server_handler.create_group(
                 group_id, user_id, content
             )
+            defer.returnValue(res)
 
-        return self.transport_client.create_group(
+        content["user_profile"] = yield self.profile_handler.get_profile(user_id)
+        res = yield self.transport_client.create_group(
             get_domain_from_id(group_id), group_id, user_id, content,
-        )  # TODO
+        )
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -265,7 +272,9 @@ class GroupsLocalHandler(object):
             "groups_key", token, users=[user_id],
         )
 
-        defer.returnValue({"state": "invite"})
+        user_profile = yield self.profile_handler.get_profile(user_id)
+
+        defer.returnValue({"state": "invite", "user_profile": user_profile})
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 7abee98dea..57e22edb0d 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -19,7 +19,7 @@ from twisted.internet import defer
 
 import synapse.types
 from synapse.api.errors import SynapseError, AuthError, CodeMessageException
-from synapse.types import UserID
+from synapse.types import UserID, get_domain_from_id
 from ._base import BaseHandler
 
 
@@ -27,15 +27,53 @@ logger = logging.getLogger(__name__)
 
 
 class ProfileHandler(BaseHandler):
+    PROFILE_UPDATE_MS = 60 * 1000
+    PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
 
     def __init__(self, hs):
         super(ProfileHandler, self).__init__(hs)
 
+        self.clock = hs.get_clock()
+
         self.federation = hs.get_replication_layer()
         self.federation.register_query_handler(
             "profile", self.on_profile_query
         )
 
+        self.clock.looping_call(self._update_remote_profile_cache, self.PROFILE_UPDATE_MS)
+
+    @defer.inlineCallbacks
+    def get_profile(self, user_id):
+        target_user = UserID.from_string(user_id)
+        if self.hs.is_mine(target_user):
+            displayname = yield self.store.get_profile_displayname(
+                target_user.localpart
+            )
+            avatar_url = yield self.store.get_profile_avatar_url(
+                target_user.localpart
+            )
+
+            defer.returnValue({
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+            })
+        else:
+            try:
+                result = yield self.federation.make_query(
+                    destination=target_user.domain,
+                    query_type="profile",
+                    args={
+                        "user_id": user_id,
+                    },
+                    ignore_backoff=True,
+                )
+                defer.returnValue(result)
+            except CodeMessageException as e:
+                if e.code != 404:
+                    logger.exception("Failed to get displayname")
+
+                raise
+
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
         if self.hs.is_mine(target_user):
@@ -182,3 +220,44 @@ class ProfileHandler(BaseHandler):
                     "Failed to update join event for room %s - %s",
                     room_id, str(e.message)
                 )
+
+    def _update_remote_profile_cache(self):
+        """Called periodically to check profiles of remote users we havent'
+        checked in a while.
+        """
+        entries = yield self.store.get_remote_profile_cache_entries_that_expire(
+            last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS
+        )
+
+        for user_id, displayname, avatar_url in entries:
+            is_subcscribed = yield self.store.is_subscribed_remote_profile_for_user(
+                user_id,
+            )
+            if not is_subcscribed:
+                yield self.store.maybe_delete_remote_profile_cache(user_id)
+                continue
+
+            try:
+                profile = yield self.federation.make_query(
+                    destination=get_domain_from_id(user_id),
+                    query_type="profile",
+                    args={
+                        "user_id": user_id,
+                    },
+                    ignore_backoff=True,
+                )
+            except:
+                logger.exception("Failed to get avatar_url")
+
+                yield self.store.update_remote_profile_cache(
+                    user_id, displayname, avatar_url
+                )
+                continue
+
+            new_name = profile.get("displayname")
+            new_avatar = profile.get("avatar_url")
+
+            # We always hit update to update the last_check timestamp
+            yield self.store.update_remote_profile_cache(
+                user_id, new_name, new_avatar
+            )
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index 26a40905ae..dca6af8a77 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from twisted.internet import defer
+
 from ._base import SQLBaseStore
 
 
@@ -55,3 +57,99 @@ class ProfileStore(SQLBaseStore):
             updatevalues={"avatar_url": new_avatar_url},
             desc="set_profile_avatar_url",
         )
+
+    def get_from_remote_profile_cache(self, user_id):
+        return self._simple_select_one(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            retcols=("displayname", "avatar_url", "last_check"),
+            allow_none=True,
+            desc="get_from_remote_profile_cache",
+        )
+
+    def add_remote_profile_cache(self, user_id, displayname, avatar_url):
+        """Ensure we are caching the remote user's profiles.
+
+        This should only be called when `is_subscribed_remote_profile_for_user`
+        would return true for the user.
+        """
+        return self._simple_upsert(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            values={
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+                "last_check": self._clock.time_msec(),
+            },
+            desc="add_remote_profile_cache",
+        )
+
+    def update_remote_profile_cache(self, user_id, displayname, avatar_url):
+        return self._simple_update(
+            table="remote_profile_cache",
+            keyvalues={"user_id": user_id},
+            values={
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+                "last_check": self._clock.time_msec(),
+            },
+            desc="update_remote_profile_cache",
+        )
+
+    @defer.inlineCallbacks
+    def maybe_delete_remote_profile_cache(self, user_id):
+        """Check if we still care about the remote user's profile, and if we
+        don't then remove their profile from the cache
+        """
+        subscribed = yield self.is_subscribed_remote_profile_for_user(user_id)
+        if not subscribed:
+            yield self._simple_delete(
+                table="remote_profile_cache",
+                keyvalues={"user_id": user_id},
+                desc="delete_remote_profile_cache",
+            )
+
+    def get_remote_profile_cache_entries_that_expire(self, last_checked):
+        """Get all users who haven't been checked since `last_checked`
+        """
+        def _get_remote_profile_cache_entries_that_expire_txn(txn):
+            sql = """
+                SELECT user_id, displayname, avatar_url
+                FROM remote_profile_cache
+                WHERE last_check < ?
+            """
+
+            txn.execute(sql, (last_checked,))
+
+            return self.cursor_to_dict(txn)
+
+        return self.runInteraction(
+            "get_remote_profile_cache_entries_that_expire",
+            _get_remote_profile_cache_entries_that_expire_txn,
+        )
+
+    @defer.inlineCallbacks
+    def is_subscribed_remote_profile_for_user(self, user_id):
+        """Check whether we are interested in a remote user's profile.
+        """
+        res = yield self._simple_select_one_onecol(
+            table="group_users",
+            keyvalues={"user_id": user_id},
+            retcol="user_id",
+            allow_none=True,
+            desc="should_update_remote_profile_cache_for_user",
+        )
+
+        if res:
+            defer.returnValue(True)
+
+        res = yield self._simple_select_one_onecol(
+            table="group_invites",
+            keyvalues={"user_id": user_id},
+            retcol="user_id",
+            allow_none=True,
+            desc="should_update_remote_profile_cache_for_user",
+        )
+
+        if res:
+            defer.returnValue(True)
diff --git a/synapse/storage/schema/delta/43/profile_cache.sql b/synapse/storage/schema/delta/43/profile_cache.sql
new file mode 100644
index 0000000000..e5ddc84df0
--- /dev/null
+++ b/synapse/storage/schema/delta/43/profile_cache.sql
@@ -0,0 +1,28 @@
+/* Copyright 2017 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- A subset of remote users whose profiles we have cached.
+-- Whether a user is in this table or not is defined by the storage function
+-- `is_subscribed_remote_profile_for_user`
+CREATE TABLE remote_profile_cache (
+    user_id TEXT NOT NULL,
+    displayname TEXT,
+    avatar_url TEXT,
+    last_check BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache(user_id);
+CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check);
-- 
cgit 1.4.1


From 4a9b1cf25300eedf66aaefcb36e23f5fadf2b57a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Aug 2017 16:23:58 +0100
Subject: Add user profiles to summary from group server

---
 synapse/groups/groups_server.py |  7 ++++++-
 synapse/handlers/profile.py     | 23 +++++++++++++++++++++++
 synapse/storage/profile.py      |  2 +-
 3 files changed, 30 insertions(+), 2 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 6bccae4bfb..94cf9788bb 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -45,6 +45,7 @@ class GroupsServerHandler(object):
         self.server_name = hs.hostname
         self.attestations = hs.get_groups_attestation_signing()
         self.transport_client = hs.get_federation_transport_client()
+        self.profile_handler = hs.get_profile_handler()
 
         # Ensure attestations get renewed
         hs.get_groups_attestation_renewer()
@@ -128,6 +129,9 @@ class GroupsServerHandler(object):
                     group_id, user_id,
                 )
 
+            user_profile = yield self.profile_handler.get_profile_from_cache(user_id)
+            entry.update(user_profile)
+
         users.sort(key=lambda e: e.get("order", 0))
 
         membership_info = yield self.store.get_users_membership_info_in_group(
@@ -387,7 +391,8 @@ class GroupsServerHandler(object):
 
             entry = {"user_id": g_user_id}
 
-            # TODO: Get profile information
+            profile = yield self.profile_handler.get_profile_from_cache(g_user_id)
+            entry.update(profile)
 
             if not is_public:
                 entry["is_public"] = False
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index c3cee38a43..e56e0a52bf 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -71,6 +71,29 @@ class ProfileHandler(BaseHandler):
 
                 raise
 
+    @defer.inlineCallbacks
+    def get_profile_from_cache(self, user_id):
+        """Get the profile information from our local cache. If the user is
+        ours then the profile information will always be corect. Otherwise,
+        it may be out of date/missing.
+        """
+        target_user = UserID.from_string(user_id)
+        if self.hs.is_mine(target_user):
+            displayname = yield self.store.get_profile_displayname(
+                target_user.localpart
+            )
+            avatar_url = yield self.store.get_profile_avatar_url(
+                target_user.localpart
+            )
+
+            defer.returnValue({
+                "displayname": displayname,
+                "avatar_url": avatar_url,
+            })
+        else:
+            profile = yield self.store.get_from_remote_profile_cache(user_id)
+            defer.returnValue(profile or {})
+
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
         if self.hs.is_mine(target_user):
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index dca6af8a77..beea3102fc 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -62,7 +62,7 @@ class ProfileStore(SQLBaseStore):
         return self._simple_select_one(
             table="remote_profile_cache",
             keyvalues={"user_id": user_id},
-            retcols=("displayname", "avatar_url", "last_check"),
+            retcols=("displayname", "avatar_url",),
             allow_none=True,
             desc="get_from_remote_profile_cache",
         )
-- 
cgit 1.4.1


From ae8d4bb0f0e39fc10275bdcc69ba08d98497624f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Sep 2017 15:55:18 +0100
Subject: Keep room_id's in group summary

---
 synapse/groups/groups_server.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 94cf9788bb..25f48a11ab 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -109,7 +109,6 @@ class GroupsServerHandler(object):
                 room_id, len(joined_users),
                 with_alias=False, allow_private=True,
             )
-            entry.pop("room_id", None)
 
             room_entry["profile"] = entry
 
-- 
cgit 1.4.1


From bb746a9de109a6c8643cbf21bbd876a67b9f5c9d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 21 Sep 2017 15:57:22 +0100
Subject: Revert: Keep room_id's in group summary

---
 synapse/groups/groups_server.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 25f48a11ab..94cf9788bb 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -109,6 +109,7 @@ class GroupsServerHandler(object):
                 room_id, len(joined_users),
                 with_alias=False, allow_private=True,
             )
+            entry.pop("room_id", None)
 
             room_entry["profile"] = entry
 
-- 
cgit 1.4.1


From a8e2a3df32f3584d728021d5feafecf78b0f37d1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 26 Sep 2017 15:39:13 +0100
Subject: Add unique index to group_rooms table

---
 synapse/groups/groups_server.py                  | 2 --
 synapse/storage/schema/delta/43/group_server.sql | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 94cf9788bb..699d8a5265 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -466,8 +466,6 @@ class GroupsServerHandler(object):
             group_id, and_exists=True, and_is_admin=requester_user_id
         )
 
-        # TODO: Check if room has already been added
-
         is_public = _parse_visibility_from_contents(content)
 
         yield self.store.add_room_to_group(group_id, room_id, is_public=is_public)
diff --git a/synapse/storage/schema/delta/43/group_server.sql b/synapse/storage/schema/delta/43/group_server.sql
index e74554381f..b2333848a0 100644
--- a/synapse/storage/schema/delta/43/group_server.sql
+++ b/synapse/storage/schema/delta/43/group_server.sql
@@ -52,7 +52,7 @@ CREATE TABLE group_rooms (
     is_public BOOLEAN NOT NULL  -- whether the room can be seen by everyone
 );
 
-CREATE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
+CREATE UNIQUE INDEX groups_rooms_g_idx ON group_rooms(group_id, room_id);
 CREATE INDEX groups_rooms_r_idx ON group_rooms(room_id);
 
 
-- 
cgit 1.4.1


From 17b8e2bd02ad0abbd25103b637eb8490f3a53507 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 26 Sep 2017 15:52:41 +0100
Subject: Add remove room API

---
 synapse/federation/transport/client.py | 12 ++++++++++++
 synapse/federation/transport/server.py | 14 +++++++++++++-
 synapse/groups/groups_server.py        | 12 ++++++++++++
 synapse/handlers/groups_local.py       |  1 +
 synapse/rest/client/v2_alpha/groups.py | 11 +++++++++++
 synapse/storage/group_server.py        | 23 +++++++++++++++++++++++
 6 files changed, 72 insertions(+), 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ce68cc4937..36f6eb75e9 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -525,6 +525,18 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
+        """Remove a room from a group
+        """
+        path = PREFIX + "/groups/%s/room/%s" % (group_id, room_id,)
+
+        return self.client.delete_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            ignore_backoff=True,
+        )
+
     @log_function
     def get_users_in_group(self, destination, group_id, requester_user_id):
         """Get users in a group
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index b5f07c50bf..c7565e0737 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -674,7 +674,7 @@ class FederationGroupsRoomsServlet(BaseFederationServlet):
 
 
 class FederationGroupsAddRoomsServlet(BaseFederationServlet):
-    """Add room to group
+    """Add/remove room from group
     """
     PATH = "/groups/(?P<group_id>[^/]*)/room/(?<room_id>)$"
 
@@ -690,6 +690,18 @@ class FederationGroupsAddRoomsServlet(BaseFederationServlet):
 
         defer.returnValue((200, new_content))
 
+    @defer.inlineCallbacks
+    def on_DELETE(self, origin, content, query, group_id, room_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.remove_room_from_group(
+            group_id, requester_user_id, room_id,
+        )
+
+        defer.returnValue((200, new_content))
+
 
 class FederationGroupsUsersServlet(BaseFederationServlet):
     """Get the users in a group on behalf of a user
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 699d8a5265..10bf61d178 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -472,6 +472,18 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
+    @defer.inlineCallbacks
+    def remove_room_from_group(self, group_id, requester_user_id, room_id):
+        """Remove room from group
+        """
+        yield self.check_group_is_ours(
+            group_id, and_exists=True, and_is_admin=requester_user_id
+        )
+
+        yield self.store.remove_room_from_group(group_id, room_id)
+
+        defer.returnValue({})
+
     @defer.inlineCallbacks
     def invite_to_group(self, group_id, user_id, requester_user_id, content):
         """Invite user to group
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 14fdf06b58..a2bacbfc38 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -69,6 +69,7 @@ class GroupsLocalHandler(object):
     get_rooms_in_group = _create_rerouter("get_rooms_in_group")
 
     add_room_to_group = _create_rerouter("add_room_to_group")
+    remove_room_from_group = _create_rerouter("remove_room_from_group")
 
     update_group_summary_room = _create_rerouter("update_group_summary_room")
     delete_group_summary_room = _create_rerouter("delete_group_summary_room")
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index b469058e9d..8f3ce15b02 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -423,6 +423,17 @@ class GroupAdminRoomsServlet(RestServlet):
 
         defer.returnValue((200, result))
 
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, group_id, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        result = yield self.groups_handler.remove_room_from_group(
+            group_id, user_id, room_id,
+        )
+
+        defer.returnValue((200, result))
+
 
 class GroupAdminUsersInviteServlet(RestServlet):
     """Invite a user to the group
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index d0b5ad231a..4fe9172adc 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -843,6 +843,29 @@ class GroupServerStore(SQLBaseStore):
             desc="add_room_to_group",
         )
 
+    def remove_room_from_group(self, group_id, room_id):
+        def _remove_room_from_group_txn(txn):
+            self._simple_delete_txn(
+                txn,
+                table="group_rooms",
+                keyvalues={
+                    "group_id": group_id,
+                    "room_id": room_id,
+                },
+            )
+
+            self._simple_delete_txn(
+                txn,
+                table="group_summary_rooms",
+                keyvalues={
+                    "group_id": group_id,
+                    "room_id": room_id,
+                },
+            )
+        return self.runInteraction(
+            "remove_room_from_group", _remove_room_from_group_txn,
+        )
+
     def get_publicised_groups_for_user(self, user_id):
         """Get all groups a user is publicising
         """
-- 
cgit 1.4.1


From eaaa837e002fa068f23b04b140d538e91ccc2eab Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 5 Oct 2017 11:43:22 +0100
Subject: Don't corrupt cache

---
 synapse/groups/groups_server.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 10bf61d178..991cc12cce 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -109,6 +109,7 @@ class GroupsServerHandler(object):
                 room_id, len(joined_users),
                 with_alias=False, allow_private=True,
             )
+            entry = dict(entry)  # so we don't change whats cached
             entry.pop("room_id", None)
 
             room_entry["profile"] = entry
-- 
cgit 1.4.1


From ec954f47fb7a1aaa176a7fbf7ca8e683cf428af8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Oct 2017 13:15:44 +0100
Subject: Validate room ids

---
 synapse/groups/groups_server.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'synapse/groups')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 991cc12cce..6a85908dd6 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
-from synapse.types import UserID, get_domain_from_id
+from synapse.types import UserID, get_domain_from_id, RoomID
 
 
 import logging
@@ -160,6 +160,8 @@ class GroupsServerHandler(object):
         """
         yield self.check_group_is_ours(group_id, and_exists=True, and_is_admin=user_id)
 
+        RoomID.from_string(room_id)  # Ensure valid room id
+
         order = content.get("order", None)
 
         is_public = _parse_visibility_from_contents(content)
@@ -463,6 +465,8 @@ class GroupsServerHandler(object):
     def add_room_to_group(self, group_id, requester_user_id, room_id, content):
         """Add room to group
         """
+        RoomID.from_string(room_id)  # Ensure valid room id
+
         yield self.check_group_is_ours(
             group_id, and_exists=True, and_is_admin=requester_user_id
         )
-- 
cgit 1.4.1