Merge branch 'develop' into anoa/widgets_room_upgrade github/anoa/widgets_room_upgrade anoa/widgets_room_upgrade

17 files changed, 357 insertions, 71 deletions

diff --git a/changelog.d/4555.bugfix b/changelog.d/4555.bugfix
new file mode 100644
index 0000000000..d596022c3f
--- /dev/null
+++ b/changelog.d/4555.bugfix
@@ -0,0 +1 @@
+Avoid redundant URL encoding of redirect URL for SSO login in the fallback login page. Fixes a regression introduced in [#4220](https://github.com/matrix-org/synapse/pull/4220). Contributed by Marcel Fabian Krüger ("[zaugin](https://github.com/zauguin)").
diff --git a/changelog.d/4987.misc b/changelog.d/4987.misc
new file mode 100644
index 0000000000..33490e146f
--- /dev/null
+++ b/changelog.d/4987.misc
@@ -0,0 +1 @@
+README updates: Corrected the default POSTGRES_USER. Added port forwarding hint in TLS section.
diff --git a/changelog.d/4999.bugfix b/changelog.d/4999.bugfix
new file mode 100644
index 0000000000..acbc191960
--- /dev/null
+++ b/changelog.d/4999.bugfix
@@ -0,0 +1 @@
+Prevent the ability to kick users from a room they aren't in.
diff --git a/changelog.d/5002.feature b/changelog.d/5002.feature
new file mode 100644
index 0000000000..d8f50e963f
--- /dev/null
+++ b/changelog.d/5002.feature
@@ -0,0 +1 @@
+Add a delete group admin API.
diff --git a/changelog.d/5003.bugfix b/changelog.d/5003.bugfix
new file mode 100644
index 0000000000..9955dc871f
--- /dev/null
+++ b/changelog.d/5003.bugfix
@@ -0,0 +1 @@
+Fix issue #4596 so synapse_port_db script works with --curses option on Python 3. Contributed by Anders Jensen-Waud <anders@jensenwaud.com>.
diff --git a/changelog.d/5007.misc b/changelog.d/5007.misc
new file mode 100644
index 0000000000..05b6ce2c26
--- /dev/null
+++ b/changelog.d/5007.misc
@@ -0,0 +1 @@
+Refactor synapse.storage._base._simple_select_list_paginate.
\ No newline at end of file
diff --git a/docker/README.md b/docker/README.md
index 44ade63f27..b48d74e09c 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -60,7 +60,8 @@ Synapse requires a valid TLS certificate. You can do one of the following:
  * Provide your own certificate and key (as
    `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.crt` and
    `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.key`, or elsewhere by providing an
-   entire config as `${SYNAPSE_CONFIG_PATH}`).
+   entire config as `${SYNAPSE_CONFIG_PATH}`). In this case, you should forward
+   traffic to port 8448 in the container, for example with `-p 443:8448`.
 
  * Use a reverse proxy to terminate incoming TLS, and forward the plain http
    traffic to port 8008 in the container. In this case you should set `-e
@@ -138,7 +139,7 @@ Database specific values (will use SQLite if not set):
   **NOTE**: You are highly encouraged to use postgresql! Please use the compose
   file to make it easier to deploy.
 * `POSTGRES_USER` - The user for the synapse postgres database. [default:
-  `matrix`]
+  `synapse`]
 
 Mail server specific values (will not send emails if not set):
 
diff --git a/docs/admin_api/delete_group.md b/docs/admin_api/delete_group.md
new file mode 100644
index 0000000000..d703d108b0
--- /dev/null
+++ b/docs/admin_api/delete_group.md
@@ -0,0 +1,14 @@
+# Delete a local group
+
+This API lets a server admin delete a local group. Doing so will kick all
+users out of the group so that their clients will correctly handle the group
+being deleted.
+
+
+The API is:
+
+```
+POST /_matrix/client/r0/admin/delete_group/<group_id>
+```
+
+including an `access_token` of a server admin.
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 2fa01d1a18..3de394b035 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -811,7 +811,7 @@ class CursesProgress(Progress):
         middle_space = 1
 
         items = self.tables.items()
-        items.sort(key=lambda i: (i[1]["perc"], i[0]))
+        items = sorted(items, key=lambda i: (i[1]["perc"], i[0]))
 
         for i, (table, data) in enumerate(items):
             if i + 2 >= rows:
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index a7eaead56b..817be40360 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -22,6 +22,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
 from synapse.types import GroupID, RoomID, UserID, get_domain_from_id
+from synapse.util.async_helpers import concurrently_execute
 
 logger = logging.getLogger(__name__)
 
@@ -896,6 +897,78 @@ class GroupsServerHandler(object):
             "group_id": group_id,
         })
 
+    @defer.inlineCallbacks
+    def delete_group(self, group_id, requester_user_id):
+        """Deletes a group, kicking out all current members.
+
+        Only group admins or server admins can call this request
+
+        Args:
+            group_id (str)
+            request_user_id (str)
+
+        Returns:
+            Deferred
+        """
+
+        yield self.check_group_is_ours(
+            group_id, requester_user_id,
+            and_exists=True,
+        )
+
+        # Only server admins or group admins can delete groups.
+
+        is_admin = yield self.store.is_user_admin_in_group(
+            group_id, requester_user_id
+        )
+
+        if not is_admin:
+            is_admin = yield self.auth.is_server_admin(
+                UserID.from_string(requester_user_id),
+            )
+
+        if not is_admin:
+            raise SynapseError(403, "User is not an admin")
+
+        # Before deleting the group lets kick everyone out of it
+        users = yield self.store.get_users_in_group(
+            group_id, include_private=True,
+        )
+
+        @defer.inlineCallbacks
+        def _kick_user_from_group(user_id):
+            if self.hs.is_mine_id(user_id):
+                groups_local = self.hs.get_groups_local_handler()
+                yield groups_local.user_removed_from_group(group_id, user_id, {})
+            else:
+                yield self.transport_client.remove_user_from_group_notification(
+                    get_domain_from_id(user_id), group_id, user_id, {}
+                )
+                yield self.store.maybe_delete_remote_profile_cache(user_id)
+
+        # We kick users out in the order of:
+        #   1. Non-admins
+        #   2. Other admins
+        #   3. The requester
+        #
+        # This is so that if the deletion fails for some reason other admins or
+        # the requester still has auth to retry.
+        non_admins = []
+        admins = []
+        for u in users:
+            if u["user_id"] == requester_user_id:
+                continue
+            if u["is_admin"]:
+                admins.append(u["user_id"])
+            else:
+                non_admins.append(u["user_id"])
+
+        yield concurrently_execute(_kick_user_from_group, non_admins, 10)
+        yield concurrently_execute(_kick_user_from_group, admins, 10)
+        yield _kick_user_from_group(requester_user_id)
+
+        yield self.store.delete_group(group_id)
+
 
 def _parse_join_policy_from_contents(content):
     """Given a content for a request, return the specified join policy or None
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 71ce5b54e5..e432740832 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -421,6 +421,9 @@ class RoomMemberHandler(object):
             room_id, latest_event_ids=latest_event_ids,
         )
 
+        # TODO: Refactor into dictionary of explicitly allowed transitions
+        # between old and new state, with specific error messages for some
+        # transitions and generic otherwise
         old_state_id = current_state_ids.get((EventTypes.Member, target.to_string()))
         if old_state_id:
             old_state = yield self.store.get_event(old_state_id, allow_none=True)
@@ -446,6 +449,9 @@ class RoomMemberHandler(object):
                 if same_sender and same_membership and same_content:
                     defer.returnValue(old_state)
 
+            if old_membership in ["ban", "leave"] and action == "kick":
+                raise AuthError(403, "The target user is not in the room")
+
             # we don't allow people to reject invites to the server notice
             # room, but they can leave it once they are joined.
             if (
@@ -459,6 +465,9 @@ class RoomMemberHandler(object):
                         "You cannot reject this invite",
                         errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM,
                     )
+        else:
+            if action == "kick":
+                raise AuthError(403, "The target user is not in the room")
 
         is_host_in_room = yield self._is_host_in_room(current_state_ids)
 
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 59526f707e..7d7a75fc30 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -784,6 +784,31 @@ class SearchUsersRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
+class DeleteGroupAdminRestServlet(ClientV1RestServlet):
+    """Allows deleting of local groups
+    """
+    PATTERNS = client_path_patterns("/admin/delete_group/(?P<group_id>[^/]*)")
+
+    def __init__(self, hs):
+        super(DeleteGroupAdminRestServlet, self).__init__(hs)
+        self.group_server = hs.get_groups_server_handler()
+        self.is_mine_id = hs.is_mine_id
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        if not self.is_mine_id(group_id):
+            raise SynapseError(400, "Can only delete local groups")
+
+        yield self.group_server.delete_group(group_id, requester.user.to_string())
+        defer.returnValue((200, {}))
+
+
 def register_servlets(hs, http_server):
     WhoisRestServlet(hs).register(http_server)
     PurgeMediaCacheRestServlet(hs).register(http_server)
@@ -799,3 +824,4 @@ def register_servlets(hs, http_server):
     ListMediaInRoom(hs).register(http_server)
     UserRegisterServlet(hs).register(http_server)
     VersionServlet(hs).register(http_server)
+    DeleteGroupAdminRestServlet(hs).register(http_server)
diff --git a/synapse/static/client/login/js/login.js b/synapse/static/client/login/js/login.js
index 3a958749a1..e02663f50e 100644
--- a/synapse/static/client/login/js/login.js
+++ b/synapse/static/client/login/js/login.js
@@ -49,7 +49,7 @@ var show_login = function() {
     $("#loading").hide();
 
     var this_page = window.location.origin + window.location.pathname;
-    $("#sso_redirect_url").val(encodeURIComponent(this_page));
+    $("#sso_redirect_url").val(this_page);
 
     if (matrixLogin.serverAcceptsPassword) {
         $("#password_flow").show();
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e9aa2fc9dd..c432041b4e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -18,6 +18,8 @@ import calendar
 import logging
 import time
 
+from twisted.internet import defer
+
 from synapse.api.constants import PresenceState
 from synapse.storage.devices import DeviceStore
 from synapse.storage.user_erasure_store import UserErasureStore
@@ -453,6 +455,7 @@ class DataStore(
             desc="get_users",
         )
 
+    @defer.inlineCallbacks
     def get_users_paginate(self, order, start, limit):
         """Function to reterive a paginated list of users from
         users list. This will return a json object, which contains
@@ -465,16 +468,19 @@ class DataStore(
         Returns:
             defer.Deferred: resolves to json object {list[dict[str, Any]], count}
         """
-        is_guest = 0
-        i_start = (int)(start)
-        i_limit = (int)(limit)
-        return self.get_user_list_paginate(
+        users = yield self.runInteraction(
+            "get_users_paginate",
+            self._simple_select_list_paginate_txn,
             table="users",
-            keyvalues={"is_guest": is_guest},
-            pagevalues=[order, i_limit, i_start],
+            keyvalues={"is_guest": False},
+            orderby=order,
+            start=start,
+            limit=limit,
             retcols=["name", "password_hash", "is_guest", "admin"],
-            desc="get_users_paginate",
         )
+        count = yield self.runInteraction("get_users_paginate", self.get_user_count_txn)
+        retval = {"users": users, "total": count}
+        defer.returnValue(retval)
 
     def search_users(self, term):
         """Function to search users list for one or more users with
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 131820628a..983ce026e1 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -595,7 +595,7 @@ class SQLBaseStore(object):
 
         Args:
             table (str): The table to upsert into
-            keyvalues (dict): The unique key tables and their new values
+            keyvalues (dict): The unique key columns and their new values
             values (dict): The nonunique columns and their new values
             insertion_values (dict): additional key/values to use only when
                 inserting
@@ -627,7 +627,7 @@ class SQLBaseStore(object):
 
                 # presumably we raced with another transaction: let's retry.
                 logger.warn(
-                    "%s when upserting into %s; retrying: %s", e.__name__, table, e
+                    "IntegrityError when upserting into %s; retrying: %s", table, e
                 )
 
     def _simple_upsert_txn(
@@ -1398,21 +1398,31 @@ class SQLBaseStore(object):
             return 0
 
     def _simple_select_list_paginate(
-        self, table, keyvalues, pagevalues, retcols, desc="_simple_select_list_paginate"
+        self,
+        table,
+        keyvalues,
+        orderby,
+        start,
+        limit,
+        retcols,
+        order_direction="ASC",
+        desc="_simple_select_list_paginate",
     ):
-        """Executes a SELECT query on the named table with start and limit,
+        """
+        Executes a SELECT query on the named table with start and limit,
         of row numbers, which may return zero or number of rows from start to limit,
         returning the result as a list of dicts.
 
         Args:
             table (str): the table name
-            keyvalues (dict[str, Any] | None):
+            keyvalues (dict[str, T] | None):
                 column names and values to select the rows with, or None to not
                 apply a WHERE clause.
+            orderby (str): Column to order the results by.
+            start (int): Index to begin the query at.
+            limit (int): Number of results to return.
             retcols (iterable[str]): the names of the columns to return
-            order (str): order the select by this column
-            start (int): start number to begin the query from
-            limit (int): number of rows to reterive
+            order_direction (str): Whether the results should be ordered "ASC" or "DESC".
         Returns:
             defer.Deferred: resolves to list[dict[str, Any]]
         """
@@ -1421,15 +1431,27 @@ class SQLBaseStore(object):
             self._simple_select_list_paginate_txn,
             table,
             keyvalues,
-            pagevalues,
+            orderby,
+            start,
+            limit,
             retcols,
+            order_direction=order_direction,
         )
 
     @classmethod
     def _simple_select_list_paginate_txn(
-        cls, txn, table, keyvalues, pagevalues, retcols
+        cls,
+        txn,
+        table,
+        keyvalues,
+        orderby,
+        start,
+        limit,
+        retcols,
+        order_direction="ASC",
     ):
-        """Executes a SELECT query on the named table with start and limit,
+        """
+        Executes a SELECT query on the named table with start and limit,
         of row numbers, which may return zero or number of rows from start to limit,
         returning the result as a list of dicts.
 
@@ -1439,64 +1461,32 @@ class SQLBaseStore(object):
             keyvalues (dict[str, T] | None):
                 column names and values to select the rows with, or None to not
                 apply a WHERE clause.
-            pagevalues ([]):
-                order (str): order the select by this column
-                start (int): start number to begin the query from
-                limit (int): number of rows to reterive
+            orderby (str): Column to order the results by.
+            start (int): Index to begin the query at.
+            limit (int): Number of results to return.
             retcols (iterable[str]): the names of the columns to return
+            order_direction (str): Whether the results should be ordered "ASC" or "DESC".
         Returns:
             defer.Deferred: resolves to list[dict[str, Any]]
-
         """
+        if order_direction not in ["ASC", "DESC"]:
+            raise ValueError("order_direction must be one of 'ASC' or 'DESC'.")
+
         if keyvalues:
-            sql = "SELECT %s FROM %s WHERE %s ORDER BY %s" % (
-                ", ".join(retcols),
-                table,
-                " AND ".join("%s = ?" % (k,) for k in keyvalues),
-                " ? ASC LIMIT ? OFFSET ?",
-            )
-            txn.execute(sql, list(keyvalues.values()) + list(pagevalues))
+            where_clause = "WHERE " + " AND ".join("%s = ?" % (k,) for k in keyvalues)
         else:
-            sql = "SELECT %s FROM %s ORDER BY %s" % (
-                ", ".join(retcols),
-                table,
-                " ? ASC LIMIT ? OFFSET ?",
-            )
-            txn.execute(sql, pagevalues)
+            where_clause = ""
 
-        return cls.cursor_to_dict(txn)
-
-    @defer.inlineCallbacks
-    def get_user_list_paginate(
-        self, table, keyvalues, pagevalues, retcols, desc="get_user_list_paginate"
-    ):
-        """Get a list of users from start row to a limit number of rows. This will
-        return a json object with users and total number of users in users list.
-
-        Args:
-            table (str): the table name
-            keyvalues (dict[str, Any] | None):
-                column names and values to select the rows with, or None to not
-                apply a WHERE clause.
-            pagevalues ([]):
-                order (str): order the select by this column
-                start (int): start number to begin the query from
-                limit (int): number of rows to reterive
-            retcols (iterable[str]): the names of the columns to return
-        Returns:
-            defer.Deferred: resolves to json object {list[dict[str, Any]], count}
-        """
-        users = yield self.runInteraction(
-            desc,
-            self._simple_select_list_paginate_txn,
+        sql = "SELECT %s FROM %s %s ORDER BY %s %s LIMIT ? OFFSET ?" % (
+            ", ".join(retcols),
             table,
-            keyvalues,
-            pagevalues,
-            retcols,
+            where_clause,
+            orderby,
+            order_direction,
         )
-        count = yield self.runInteraction(desc, self.get_user_count_txn)
-        retval = {"users": users, "total": count}
-        defer.returnValue(retval)
+        txn.execute(sql, list(keyvalues.values()) + [limit, start])
+
+        return cls.cursor_to_dict(txn)
 
     def get_user_count_txn(self, txn):
         """Get a total number of registered users in the users list.
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 80102b02e0..dce6a43ac1 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -1150,3 +1150,40 @@ class GroupServerStore(SQLBaseStore):
 
     def get_group_stream_token(self):
         return self._group_updates_id_gen.get_current_token()
+
+    def delete_group(self, group_id):
+        """Deletes a group fully from the database.
+
+        Args:
+            group_id (str)
+
+        Returns:
+            Deferred
+        """
+
+        def _delete_group_txn(txn):
+            tables = [
+                "groups",
+                "group_users",
+                "group_invites",
+                "group_rooms",
+                "group_summary_rooms",
+                "group_summary_room_categories",
+                "group_room_categories",
+                "group_summary_users",
+                "group_summary_roles",
+                "group_roles",
+                "group_attestations_renewals",
+                "group_attestations_remote",
+            ]
+
+            for table in tables:
+                self._simple_delete_txn(
+                    txn,
+                    table=table,
+                    keyvalues={"group_id": group_id},
+                )
+
+        return self.runInteraction(
+            "delete_group", _delete_group_txn
+        )
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index ef38473bd6..c00ef21d75 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -21,6 +21,7 @@ from mock import Mock
 
 from synapse.api.constants import UserTypes
 from synapse.rest.client.v1 import admin, events, login, room
+from synapse.rest.client.v2_alpha import groups
 
 from tests import unittest
 
@@ -490,3 +491,126 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase):
         self.assertEqual(
             expect_code, int(channel.result["code"]), msg=channel.result["body"],
         )
+
+
+class DeleteGroupTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        groups.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+    def test_delete_group(self):
+        # Create a new group
+        request, channel = self.make_request(
+            "POST",
+            "/create_group".encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={
+                "localpart": "test",
+            }
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        group_id = channel.json_body["group_id"]
+
+        self._check_group(group_id, expect_code=200)
+
+        # Invite/join another user
+
+        url = "/groups/%s/admin/users/invite/%s" % (group_id, self.other_user)
+        request, channel = self.make_request(
+            "PUT",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={}
+        )
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        url = "/groups/%s/self/accept_invite" % (group_id,)
+        request, channel = self.make_request(
+            "PUT",
+            url.encode('ascii'),
+            access_token=self.other_user_token,
+            content={}
+        )
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        # Check other user knows they're in the group
+        self.assertIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
+        self.assertIn(group_id, self._get_groups_user_is_in(self.other_user_token))
+
+        # Now delete the group
+        url = "/admin/delete_group/" + group_id
+        request, channel = self.make_request(
+            "POST",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={
+                "localpart": "test",
+            }
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        # Check group returns 404
+        self._check_group(group_id, expect_code=404)
+
+        # Check users don't think they're in the group
+        self.assertNotIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
+        self.assertNotIn(group_id, self._get_groups_user_is_in(self.other_user_token))
+
+    def _check_group(self, group_id, expect_code):
+        """Assert that trying to fetch the given group results in the given
+        HTTP status code
+        """
+
+        url = "/groups/%s/profile" % (group_id,)
+        request, channel = self.make_request(
+            "GET",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+        )
+
+        self.render(request)
+        self.assertEqual(
+            expect_code, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+    def _get_groups_user_is_in(self, access_token):
+        """Returns the list of groups the user is in (given their access token)
+        """
+        request, channel = self.make_request(
+            "GET",
+            "/joined_groups".encode('ascii'),
+            access_token=access_token,
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        return channel.json_body["groups"]