From 8d6dc106d188308f687db1f696cad7ad1b9ae83d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 12:02:44 +0100
Subject: Don't use _cursor_to_dict in find_next_generated_user_id_localpart

---
 synapse/storage/registration.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index d809b2ba46..6b557ca0cf 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -460,14 +460,12 @@ class RegistrationStore(RegistrationWorkerStore,
         """
         def _find_next_generated_user_id(txn):
             txn.execute("SELECT name FROM users")
-            rows = self.cursor_to_dict(txn)
 
             regex = re.compile("^@(\d+):")
 
             found = set()
 
-            for r in rows:
-                user_id = r["name"]
+            for user_id, in txn:
                 match = regex.search(user_id)
                 if match:
                     found.add(int(match.group(1)))
-- 
cgit 1.5.1


From fecb45e0c38912f72655d53faa953e7713511e7f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 13:31:49 +0100
Subject: Remove last usage of ujson

---
 synapse/python_dependencies.py  | 1 -
 synapse/storage/group_server.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 91179ce532..40eedb63cb 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,7 +34,6 @@ REQUIREMENTS = {
     "bcrypt": ["bcrypt>=3.1.0"],
     "pillow": ["PIL"],
     "pydenticon": ["pydenticon"],
-    "ujson": ["ujson"],
     "blist": ["blist"],
     "pysaml2>=3.0.0": ["saml2>=3.0.0"],
     "pymacaroons-pynacl": ["pymacaroons"],
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8fde1aab8e..d03858234b 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError
 
 from ._base import SQLBaseStore
 
-import ujson as json
+import simplejson as json
 
 
 # The category ID for the "default" category. We don't store as null in the
-- 
cgit 1.5.1


From 3f49e131d9a31d5bf198f2aa6b102562b2e747d0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 10:30:03 +0100
Subject: Add counter metrics for calculating state delta

This will allow us to measure how often we calculate state deltas in
event persistence that we would have been able to calculate at the same
time we calculated the state for the event.
---
 synapse/storage/events.py | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 85ce6bea1a..f3a362db20 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -52,6 +52,15 @@ persist_event_counter = metrics.register_counter("persisted_events")
 event_counter = metrics.register_counter(
     "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
 )
+state_delta_counter = metrics.register_counter(
+    "state_delta",
+)
+state_delta_single_event_counter = metrics.register_counter(
+    "state_delta_single_event",
+)
+state_delta_reuse_delta_counter = metrics.register_counter(
+    "state_delta_reuse_delta",
+)
 
 
 def encode_json(json_object):
@@ -368,7 +377,8 @@ class EventsStore(EventsWorkerStore):
                                 room_id, ev_ctx_rm, latest_event_ids
                             )
 
-                            if new_latest_event_ids == set(latest_event_ids):
+                            latest_event_ids = set(latest_event_ids)
+                            if new_latest_event_ids == latest_event_ids:
                                 # No change in extremities, so no change in state
                                 continue
 
@@ -389,6 +399,25 @@ class EventsStore(EventsWorkerStore):
                                 if all_single_prev_not_state:
                                     continue
 
+                            state_delta_counter.inc()
+                            if len(new_latest_event_ids) == 1:
+                                state_delta_single_event_counter.inc()
+
+                                # This is a fairly handwavey check to see if we could
+                                # have guessed what the delta would have been when
+                                # processing one of these events.
+                                # What we're interested in is if the latest extremities
+                                # were the same when we created the event as they are
+                                # now. We guess this by looking at the prev events and
+                                # checking if they match up, as when this server creates
+                                # a new event it will use the extremities as the prev
+                                # events.
+                                for ev, _ in ev_ctx_rm:
+                                    prev_event_ids = set(e for e, _ in ev.prev_events)
+                                    if latest_event_ids == prev_event_ids:
+                                        state_delta_reuse_delta_counter.inc()
+                                    break
+
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
                             )
-- 
cgit 1.5.1


From e70287cff3adeeeff2a7a1e95fc5845feb365710 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:13:38 +0100
Subject: Comment

---
 synapse/storage/events.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f3a362db20..ee8b3c46d3 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -52,12 +52,19 @@ persist_event_counter = metrics.register_counter("persisted_events")
 event_counter = metrics.register_counter(
     "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
 )
+
+# The number of times we are recalculating the current state
 state_delta_counter = metrics.register_counter(
     "state_delta",
 )
+# The number of times we are recalculating state when there is only a
+# single forward extremity
 state_delta_single_event_counter = metrics.register_counter(
     "state_delta_single_event",
 )
+# The number of times we are reculating state when we could have resonably
+# calculated the delta when we calculated the state for an event we were
+# persisting.
 state_delta_reuse_delta_counter = metrics.register_counter(
     "state_delta_reuse_delta",
 )
-- 
cgit 1.5.1


From 152c2ac19e6af373a9e79a8f4da80d9e77a5396f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:13:46 +0100
Subject: Fix indent

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ee8b3c46d3..95b36d669b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -423,7 +423,7 @@ class EventsStore(EventsWorkerStore):
                                     prev_event_ids = set(e for e, _ in ev.prev_events)
                                     if latest_event_ids == prev_event_ids:
                                         state_delta_reuse_delta_counter.inc()
-                                    break
+                                        break
 
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
-- 
cgit 1.5.1


From 800cfd5774d9726a00c53f2d84ce0286ab64c17b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:30:39 +0100
Subject: Comment

---
 synapse/storage/events.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 95b36d669b..f3d65f4338 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -415,10 +415,11 @@ class EventsStore(EventsWorkerStore):
                                 # processing one of these events.
                                 # What we're interested in is if the latest extremities
                                 # were the same when we created the event as they are
-                                # now. We guess this by looking at the prev events and
-                                # checking if they match up, as when this server creates
-                                # a new event it will use the extremities as the prev
-                                # events.
+                                # now. When this server creates a new event (as opposed
+                                # to receiving it over federation) it will use the
+                                # forward extremities as the prev_events, so we can
+                                # guess this by looking at the prev_events and checking
+                                # if they match the current forward extremities.
                                 for ev, _ in ev_ctx_rm:
                                     prev_event_ids = set(e for e, _ in ev.prev_events)
                                     if latest_event_ids == prev_event_ids:
-- 
cgit 1.5.1


From 9187e0762f0b4f028d15fac4502e458f513d6642 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 10:02:32 +0100
Subject: count_daily_users failed if db was sqlite due to type failure -
 presumably this prevcented all sqlite homeservers reporting home

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index de00cae447..b97e5e5ff4 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -260,7 +260,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 ) u
             """
 
-            txn.execute(sql, (yesterday,))
+            txn.execute(sql, (str(yesterday),))
             count, = txn.fetchone()
             return count
 
-- 
cgit 1.5.1


From a32d2548d986f7075e8310184ce0b70c69513a02 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 10:39:13 +0100
Subject: query and call for r30 stats

---
 synapse/app/homeserver.py   |  2 ++
 synapse/storage/__init__.py | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index c00afbba28..8bce9f1ace 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -425,6 +425,8 @@ def run(hs):
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
+        stats["r30_users"] = yield hs.get_datastore().count_r30_users()
+
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b97e5e5ff4..10f99c3cd5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -267,6 +267,42 @@ class DataStore(RoomMemberStore, RoomStore,
         ret = yield self.runInteraction("count_users", _count_users)
         defer.returnValue(ret)
 
+    @defer.inlineCallbacks
+    def count_r30_users(self):
+        """
+        Counts the number of 30 day retained users, defined as:-
+         * Users who have created their accounts more than 30 days
+         * Where last seen at most 30 days ago
+         * Where account creation and last_seen are > 30 days
+        """
+        def _count_r30_users(txn):
+            thirty_days_in_secs = 86400 * 30
+            now = int(self._clock.time_msec())
+            thirty_days_ago_in_secs = now - thirty_days_in_secs
+
+            sql = """
+                SELECT COALESCE(count(*), 0) FROM (
+                    SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
+                    FROM users, user_ips
+                    WHERE users.name = user_ips.user_id
+                    AND appservice_id is NULL
+                    AND users.creation_ts < ?
+                    AND user_ips.last_seen/1000 > ?
+                    AND (user_ips.last_seen/1000) - users.creation_ts > ?
+                    GROUP BY users.name, users.creation_ts
+                ) u
+            """
+
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs,
+                              thirty_days_in_secs))
+
+            count, = txn.fetchone()
+            return count
+
+        ret = yield self.runInteraction("count_r30_users", _count_r30_users)
+        defer.returnValue(ret)
+
     def get_users(self):
         """Function to reterive a list of users in users table.
 
-- 
cgit 1.5.1


From 545001b9e4b1d6710145d3efe2117fbdf823fb38 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 11:19:45 +0100
Subject: Fix search_user_dir multiple sqlite versions do different things

---
 synapse/storage/user_directory.py    | 4 ++--
 tests/storage/test_user_directory.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index dfdcbb3181..d6e289ffbe 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -667,7 +667,7 @@ class UserDirectoryStore(SQLBaseStore):
             # The array of numbers are the weights for the various part of the
             # search: (domain, _, display name, localpart)
             sql = """
-                SELECT d.user_id, display_name, avatar_url
+                SELECT d.user_id AS user_id, display_name, avatar_url
                 FROM user_directory_search
                 INNER JOIN user_directory AS d USING (user_id)
                 %s
@@ -702,7 +702,7 @@ class UserDirectoryStore(SQLBaseStore):
             search_query = _parse_query_sqlite(search_term)
 
             sql = """
-                SELECT d.user_id, display_name, avatar_url
+                SELECT d.user_id AS user_id, display_name, avatar_url
                 FROM user_directory_search
                 INNER JOIN user_directory AS d USING (user_id)
                 %s
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 88add45217..0891308f25 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -62,7 +62,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase):
         self.assertFalse(r["limited"])
         self.assertEqual(1, len(r["results"]))
         self.assertDictEqual(r["results"][0], {
-            "d.user_id": BOB,
+            "user_id": BOB,
             "display_name": "bob",
             "avatar_url": None,
         })
-- 
cgit 1.5.1


From 0f890f477eb2ed03b8fd48710d1960210f44a334 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 11:49:57 +0100
Subject: No need to cast in count_daily_users

---
 synapse/storage/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 10f99c3cd5..ba43b2d8ec 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -250,7 +250,7 @@ class DataStore(RoomMemberStore, RoomStore,
         Counts the number of users who used this homeserver in the last 24 hours.
         """
         def _count_users(txn):
-            yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24),
+            yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24)
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
@@ -260,7 +260,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 ) u
             """
 
-            txn.execute(sql, (str(yesterday),))
+            txn.execute(sql, (yesterday,))
             count, = txn.fetchone()
             return count
 
-- 
cgit 1.5.1


From 788e69098c93f2433ef907015666c624bb39318f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 12:03:13 +0100
Subject: Add user_ips last seen index

---
 synapse/storage/client_ips.py                           |  7 +++++++
 .../schema/delta/48/add_user_ips_last_seen_index.sql    | 17 +++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index a03d1d6104..7b44dae0fc 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -48,6 +48,13 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
             columns=["user_id", "device_id", "last_seen"],
         )
 
+        self.register_background_index_update(
+            "user_ips_last_seen_index",
+            index_name="user_ips_last_seen",
+            table="user_ips",
+            columns=["user_id", "last_seen"],
+        )
+
         # (user_id, access_token, ip) -> (user_agent, device_id, last_seen)
         self._batch_row_update = {}
 
diff --git a/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql
new file mode 100644
index 0000000000..9248b0b24a
--- /dev/null
+++ b/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql
@@ -0,0 +1,17 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT into background_updates (update_name, progress_json)
+    VALUES ('user_ips_last_seen_index', '{}');
-- 
cgit 1.5.1


From 79452edeee94a09a826ee2b41a08811b823a3ad6 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:03:37 +0100
Subject: Add joinability for groups

Adds API to set the 'joinable' flag, and corresponding flag in the
table.
---
 synapse/federation/transport/client.py | 17 +++++++++++++++++
 synapse/federation/transport/server.py | 20 ++++++++++++++++++++
 synapse/groups/groups_server.py        | 19 +++++++++++++++++++
 synapse/handlers/groups_local.py       |  3 +++
 synapse/rest/client/v2_alpha/groups.py | 28 ++++++++++++++++++++++++++++
 synapse/storage/group_server.py        | 13 +++++++++++++
 synapse/storage/prepare_database.py    |  3 ++-
 7 files changed, 102 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 5488e82985..46a797b4ba 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -856,6 +857,22 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    @log_function
+    def set_group_joinable(self, destination, group_id, requester_user_id,
+                                  content):
+        """Sets whether a group is joinable without an invite or knock
+        """
+        path = PREFIX + "/groups/%s/joinable" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+
     @log_function
     def delete_group_summary_user(self, destination, group_id, requester_user_id,
                                   user_id, role_id):
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a66a6b0692..107deb4e1e 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -1124,6 +1125,24 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
         defer.returnValue((200, resp))
 
 
+class FederationGroupsJoinableServlet(BaseFederationServlet):
+    """Sets whether a group is joinable without an invite or knock
+    """
+    PATH = "/groups/(?P<group_id>[^/]*)/joinable$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.set_group_joinable(
+            group_id, requester_user_id, content
+        )
+
+        defer.returnValue((200, new_content))
+
+
 FEDERATION_SERVLET_CLASSES = (
     FederationSendServlet,
     FederationPullServlet,
@@ -1172,6 +1191,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsSummaryUsersServlet,
     FederationGroupsAddRoomsServlet,
     FederationGroupsAddRoomsConfigServlet,
+    FederationGroupsJoinableServlet,
 )
 
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 0b995aed70..25cbfb1691 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -205,6 +206,24 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
+    @defer.inlineCallbacks
+    def set_group_joinable(self, group_id, requester_user_id, content):
+        """Sets whether a group is joinable without an invite or knock
+        """
+        yield self.check_group_is_ours(
+            group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
+        )
+
+        is_joinable = content.get('joinable')
+        if is_joinable is None:
+            raise SynapseError(
+                400, "No value specified for 'joinable'"
+            )
+
+        yield self.store.set_group_joinable(group_id, is_joinable=is_joinable)
+
+        defer.returnValue({})
+
     @defer.inlineCallbacks
     def get_group_categories(self, group_id, requester_user_id):
         """Get all categories in a group (as seen by user)
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index e4d0cc8b02..c9671b9046 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -90,6 +91,8 @@ class GroupsLocalHandler(object):
     get_group_role = _create_rerouter("get_group_role")
     get_group_roles = _create_rerouter("get_group_roles")
 
+    set_group_joinable = _create_rerouter("set_group_joinable")
+
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
         """Get the group summary for a group.
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index f762dbfa9a..dc8247d172 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -401,6 +402,32 @@ class GroupInvitedUsersServlet(RestServlet):
         defer.returnValue((200, result))
 
 
+class GroupJoinableServlet(RestServlet):
+    """Set whether a group is joinable without an invite
+    """
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/joinable$")
+
+    def __init__(self, hs):
+        super(GroupJoinableServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+
+        result = yield self.groups_handler.set_group_joinable(
+            group_id,
+            requester_user_id,
+            content,
+        )
+
+        defer.returnValue((200, result))
+
+
 class GroupCreateServlet(RestServlet):
     """Create a group
     """
@@ -738,6 +765,7 @@ def register_servlets(hs, http_server):
     GroupInvitedUsersServlet(hs).register(http_server)
     GroupUsersServlet(hs).register(http_server)
     GroupRoomServlet(hs).register(http_server)
+    GroupJoinableServlet(hs).register(http_server)
     GroupCreateServlet(hs).register(http_server)
     GroupAdminRoomsServlet(hs).register(http_server)
     GroupAdminRoomsConfigServlet(hs).register(http_server)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8fde1aab8e..96553d4fb1 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,6 +30,18 @@ _DEFAULT_ROLE_ID = ""
 
 
 class GroupServerStore(SQLBaseStore):
+    def set_group_joinable(self, group_id, is_joinable):
+        return self._simple_update_one(
+            table="groups",
+            keyvalues={
+                "group_id": group_id,
+            },
+            updatevalues={
+                "is_joinable": is_joinable,
+            },
+            desc="set_group_joinable",
+        )
+
     def get_group(self, group_id):
         return self._simple_select_one(
             table="groups",
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index c845a0cec5..04411a665f 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 47
+SCHEMA_VERSION = 48
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
-- 
cgit 1.5.1


From 352e1ff9ed945fd7f2655bf47d591184fc980afb Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:07:57 +0100
Subject: Add schema delta file

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/groups_joinable.sql

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
new file mode 100644
index 0000000000..fb9c7a8d1c
--- /dev/null
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -0,0 +1,16 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT 0;
-- 
cgit 1.5.1


From a1642708331ef64e38f4d2708cee9eefbc3d391e Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:23:00 +0100
Subject: Make column definition that works on both dbs

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index fb9c7a8d1c..39c8fed46c 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT 0;
+ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT (CAST(0 AS BOOLEAN));
-- 
cgit 1.5.1


From 86932be2cb1837688d154ff78fb6418f78483133 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 14:36:53 +0100
Subject: Support multi client R30 for psql

---
 synapse/app/homeserver.py   |  4 +++-
 synapse/storage/__init__.py | 36 ++++++++++++++++++++++++++++--------
 2 files changed, 31 insertions(+), 9 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 286f4dcf7b..35e2b00f1b 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -425,7 +425,9 @@ def run(hs):
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
-        stats["r30_users_all"] = yield hs.get_datastore().count_r30_users()
+        r30_results = yield hs.get_datastore().count_r30_users()
+        for name, count in r30_results.items():
+            stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index ba43b2d8ec..b651973c79 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,6 +280,15 @@ class DataStore(RoomMemberStore, RoomStore,
             now = int(self._clock.time_msec())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
 
+            # Are these filters sufficiently robust?
+            filters = {
+                "ALL": "",
+                "IOS": "^(Vector|Riot|Riot\.im)\/.* iOS",
+                "ANDROID": "^(Dalvik|Riot|Riot\.im)\/.* Android",
+                "ELECTRON": "Electron",
+                "WEB": "(Gecko|Mozilla)",
+            }
+
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
                     SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
@@ -289,16 +298,27 @@ class DataStore(RoomMemberStore, RoomStore,
                     AND users.creation_ts < ?
                     AND user_ips.last_seen/1000 > ?
                     AND (user_ips.last_seen/1000) - users.creation_ts > ?
-                    GROUP BY users.name, users.creation_ts
-                ) u
             """
 
-            txn.execute(sql, (thirty_days_ago_in_secs,
-                              thirty_days_ago_in_secs,
-                              thirty_days_in_secs))
-
-            count, = txn.fetchone()
-            return count
+            if isinstance(self.database_engine, PostgresEngine):
+                sql = sql + "AND user_ips.user_agent ~ ? "
+            sql = sql + "GROUP BY users.name, users.creation_ts ) u"
+
+            results = {}
+            if isinstance(self.database_engine, PostgresEngine):
+                for filter_name, user_agent_filter in filters.items():
+                    txn.execute(sql, (thirty_days_ago_in_secs,
+                                      thirty_days_ago_in_secs,
+                                      thirty_days_in_secs,
+                                      user_agent_filter))
+                    results[filter_name], = txn.fetchone()
+
+            else:
+                txn.execute(sql, (thirty_days_ago_in_secs,
+                                  thirty_days_ago_in_secs,
+                                  thirty_days_in_secs))
+                results["ALL"], = txn.fetchone()
+            return results
 
         ret = yield self.runInteraction("count_r30_users", _count_r30_users)
         defer.returnValue(ret)
-- 
cgit 1.5.1


From 4262aba17b643bc82c5cce92298dac0a27b2727c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 14:40:03 +0100
Subject: bump schema version

---
 synapse/storage/prepare_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index c845a0cec5..68675e15d2 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 47
+SCHEMA_VERSION = 48
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
-- 
cgit 1.5.1


From a838444a70195588de55a514524c4af720099177 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:50:30 +0100
Subject: Grr. Copy the definition from is_admin

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 39c8fed46c..9e106e909c 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT (CAST(0 AS BOOLEAN));
+ALTER TABLE groups ADD COLUMN is_joinable BOOL DEFAULT 0 NOT NULL;
-- 
cgit 1.5.1


From 929b34963d320f571512453dac980ef235914956 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:53:55 +0100
Subject: OK, smallint it is then

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 9e106e909c..ace7d0a723 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOL DEFAULT 0 NOT NULL;
+ALTER TABLE groups ADD COLUMN is_joinable SMALLINT DEFAULT 0 NOT NULL;
-- 
cgit 1.5.1


From 241e4e86873d5880f564791e3768247fa55c3fa8 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 16:25:53 +0100
Subject: remove twisted deferral cruft

---
 synapse/storage/__init__.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b651973c79..b2b85e266d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -244,7 +244,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return [UserPresenceState(**row) for row in rows]
 
-    @defer.inlineCallbacks
     def count_daily_users(self):
         """
         Counts the number of users who used this homeserver in the last 24 hours.
@@ -264,10 +263,9 @@ class DataStore(RoomMemberStore, RoomStore,
             count, = txn.fetchone()
             return count
 
-        ret = yield self.runInteraction("count_users", _count_users)
-        defer.returnValue(ret)
+        return self.runInteraction("count_users", _count_users)
+
 
-    @defer.inlineCallbacks
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
@@ -320,8 +318,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 results["ALL"], = txn.fetchone()
             return results
 
-        ret = yield self.runInteraction("count_r30_users", _count_r30_users)
-        defer.returnValue(ret)
+        return self.runInteraction("count_r30_users", _count_r30_users)
 
     def get_users(self):
         """Function to reterive a list of users in users table.
-- 
cgit 1.5.1


From dc7c020b33dc9606089fa66fdec2dacb7f807f6d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 17:25:15 +0100
Subject:  fix pep8 errors

---
 synapse/storage/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b2b85e266d..70c6171404 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 from synapse.storage.devices import DeviceStore
 from .appservice import (
     ApplicationServiceStore, ApplicationServiceTransactionStore
@@ -265,7 +263,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return self.runInteraction("count_users", _count_users)
 
-
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
-- 
cgit 1.5.1


From 9ee44a372d4fcf6a461b610230a285610613e8ac Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 29 Mar 2018 16:45:34 +0100
Subject: Remove need for sqlite specific query

---
 synapse/storage/__init__.py | 87 +++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 30 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 70c6171404..0b4693041f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -269,50 +269,77 @@ class DataStore(RoomMemberStore, RoomStore,
          * Users who have created their accounts more than 30 days
          * Where last seen at most 30 days ago
          * Where account creation and last_seen are > 30 days
+
+         Returns counts globaly for a given user as well as breaking
+         by platform
         """
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time_msec())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
 
-            # Are these filters sufficiently robust?
-            filters = {
-                "ALL": "",
-                "IOS": "^(Vector|Riot|Riot\.im)\/.* iOS",
-                "ANDROID": "^(Dalvik|Riot|Riot\.im)\/.* Android",
-                "ELECTRON": "Electron",
-                "WEB": "(Gecko|Mozilla)",
-            }
+            sql = """
+                SELECT platform, COALESCE(count(*), 0) FROM (
+                     SELECT users.name, platform, users.creation_ts * 1000, MAX(uip.last_seen)
+                     FROM users
+                     INNER JOIN (
+                         SELECT
+                         user_id,
+                         last_seen,
+                         CASE
+                             WHEN user_agent LIKE '%Android%' THEN 'android'
+                             WHEN user_agent LIKE '%iOS%' THEN 'ios'
+                             WHEN user_agent LIKE '%Electron%' THEN 'electron'
+                             WHEN user_agent LIKE '%Mozilla%' THEN 'web'
+                             WHEN user_agent LIKE '%Gecko%' THEN 'web'
+                             ELSE 'unknown'
+                         END
+                         AS platform
+                         FROM user_ips
+                     ) uip
+                     ON users.name = uip.user_id
+                     AND users.appservice_id is NULL
+                     AND users.creation_ts < ?
+                     AND uip.last_seen/1000 > ?
+                     AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
+                     GROUP BY users.name, platform, users.creation_ts
+                ) u GROUP BY platform
+            """
+
+            results = {}
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs))
+            rows = txn.fetchall()
+            for row in rows:
+                if row[0] is 'unknown':
+                    pass
+                results[row[0]] = row[1]
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
-                    SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
-                    FROM users, user_ips
-                    WHERE users.name = user_ips.user_id
+                    SELECT users.name, users.creation_ts * 1000, MAX(uip.last_seen)
+                    FROM users
+                    INNER JOIN (
+                        SELECT
+                        user_id,
+                        last_seen
+                        FROM user_ips
+                    ) uip
+                    ON users.name = uip.user_id
                     AND appservice_id is NULL
                     AND users.creation_ts < ?
-                    AND user_ips.last_seen/1000 > ?
-                    AND (user_ips.last_seen/1000) - users.creation_ts > ?
+                    AND uip.last_seen/1000 > ?
+                    AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
+                    GROUP BY users.name, users.creation_ts
+                ) u
             """
 
-            if isinstance(self.database_engine, PostgresEngine):
-                sql = sql + "AND user_ips.user_agent ~ ? "
-            sql = sql + "GROUP BY users.name, users.creation_ts ) u"
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs))
+
+            count, = txn.fetchone()
+            results['all'] = count
 
-            results = {}
-            if isinstance(self.database_engine, PostgresEngine):
-                for filter_name, user_agent_filter in filters.items():
-                    txn.execute(sql, (thirty_days_ago_in_secs,
-                                      thirty_days_ago_in_secs,
-                                      thirty_days_in_secs,
-                                      user_agent_filter))
-                    results[filter_name], = txn.fetchone()
-
-            else:
-                txn.execute(sql, (thirty_days_ago_in_secs,
-                                  thirty_days_ago_in_secs,
-                                  thirty_days_in_secs))
-                results["ALL"], = txn.fetchone()
             return results
 
         return self.runInteraction("count_r30_users", _count_r30_users)
-- 
cgit 1.5.1


From b4e37c6f50b91dd0ea90c773185884659e3a738a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 29 Mar 2018 17:27:39 +0100
Subject: pep8

---
 synapse/storage/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 0b4693041f..f68e436df0 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,7 +280,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-                     SELECT users.name, platform, users.creation_ts * 1000, MAX(uip.last_seen)
+                     SELECT users.name, platform, users.creation_ts * 1000,
+                                                            MAX(uip.last_seen)
                      FROM users
                      INNER JOIN (
                          SELECT
@@ -317,7 +318,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
-                    SELECT users.name, users.creation_ts * 1000, MAX(uip.last_seen)
+                    SELECT users.name, users.creation_ts * 1000,
+                                                        MAX(uip.last_seen)
                     FROM users
                     INNER JOIN (
                         SELECT
-- 
cgit 1.5.1


From fcfe7f6ad3a2a9c285ac96008395fc47e096ff4b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 22:45:52 +0100
Subject: Use simplejson throughout

Let's use simplejson rather than json, for consistency.
---
 synapse/api/errors.py                                    | 3 ++-
 synapse/handlers/identity.py                             | 8 +++++---
 synapse/storage/schema/delta/14/upgrade_appservice_db.py | 3 ++-
 3 files changed, 9 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index aa15f73f36..bee59e80dd 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -15,9 +15,10 @@
 
 """Contains exceptions and error codes."""
 
-import json
 import logging
 
+import simplejson as json
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 9efcdff1d6..91a0898860 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -15,6 +15,11 @@
 # limitations under the License.
 
 """Utilities for interacting with Identity Servers"""
+
+import logging
+
+import simplejson as json
+
 from twisted.internet import defer
 
 from synapse.api.errors import (
@@ -24,9 +29,6 @@ from ._base import BaseHandler
 from synapse.util.async import run_on_reactor
 from synapse.api.errors import SynapseError, Codes
 
-import json
-import logging
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
index 8755bb2e49..4d725b92fe 100644
--- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py
+++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import logging
 
+import simplejson as json
+
 logger = logging.getLogger(__name__)
 
 
-- 
cgit 1.5.1


From 05630758f25d958bf60fde4df5f80a89e4a9a0ac Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 22:57:28 +0100
Subject: Use static JSONEncoders

using json.dumps with custom options requires us to create a new JSONEncoder on
each call. It's more efficient to create one upfront and reuse it.
---
 synapse/handlers/message.py         |  4 ++--
 synapse/replication/tcp/commands.py |  8 +++++---
 synapse/storage/events.py           | 23 ++++++++---------------
 synapse/util/frozenutils.py         | 19 +++++++++++++++++++
 4 files changed, 34 insertions(+), 20 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 5a8ddc253e..6de6e13b7b 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -27,7 +27,7 @@ from synapse.types import (
 from synapse.util.async import run_on_reactor, ReadWriteLock, Limiter
 from synapse.util.logcontext import preserve_fn, run_in_background
 from synapse.util.metrics import measure_func
-from synapse.util.frozenutils import unfreeze
+from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.stringutils import random_string
 from synapse.visibility import filter_events_for_client
 from synapse.replication.http.send_event import send_event_to_master
@@ -678,7 +678,7 @@ class EventCreationHandler(object):
 
         # Ensure that we can round trip before trying to persist in db
         try:
-            dump = simplejson.dumps(unfreeze(event.content))
+            dump = frozendict_json_encoder.encode(event.content)
             simplejson.loads(dump)
         except Exception:
             logger.exception("Failed to encode content: %r", event.content)
diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 0005ad5879..34bcf903a3 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -24,6 +24,8 @@ import simplejson
 
 logger = logging.getLogger(__name__)
 
+_json_encoder = simplejson.JSONEncoder(namedtuple_as_object=False)
+
 
 class Command(object):
     """The base command class.
@@ -107,7 +109,7 @@ class RdataCommand(Command):
         return " ".join((
             self.stream_name,
             str(self.token) if self.token is not None else "batch",
-            simplejson.dumps(self.row, namedtuple_as_object=False),
+            _json_encoder.dumps(self.row),
         ))
 
 
@@ -302,7 +304,7 @@ class InvalidateCacheCommand(Command):
 
     def to_line(self):
         return " ".join((
-            self.cache_func, simplejson.dumps(self.keys, namedtuple_as_object=False)
+            self.cache_func, _json_encoder.encode(self.keys),
         ))
 
 
@@ -334,7 +336,7 @@ class UserIpCommand(Command):
         )
 
     def to_line(self):
-        return self.user_id + " " + simplejson.dumps((
+        return self.user_id + " " + _json_encoder.encode((
             self.access_token, self.ip, self.user_agent, self.device_id,
             self.last_seen,
         ))
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f3d65f4338..ece5e6c41f 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -14,15 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.events_worker import EventsWorkerStore
+from collections import OrderedDict, deque, namedtuple
+from functools import wraps
+import logging
 
+import simplejson as json
 from twisted.internet import defer
 
-from synapse.events import USE_FROZEN_DICTS
 
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.util.async import ObservableDeferred
+from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import (
-    PreserveLoggingContext, make_deferred_yieldable
+    PreserveLoggingContext, make_deferred_yieldable,
 )
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
@@ -30,16 +34,8 @@ from synapse.api.constants import EventTypes
 from synapse.api.errors import SynapseError
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.types import get_domain_from_id
-
-from canonicaljson import encode_canonical_json
-from collections import deque, namedtuple, OrderedDict
-from functools import wraps
-
 import synapse.metrics
 
-import logging
-import simplejson as json
-
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
 from synapse.events.snapshot import EventContext   # noqa: F401
@@ -71,10 +67,7 @@ state_delta_reuse_delta_counter = metrics.register_counter(
 
 
 def encode_json(json_object):
-    if USE_FROZEN_DICTS:
-        return encode_canonical_json(json_object)
-    else:
-        return json.dumps(json_object, ensure_ascii=False)
+    return frozendict_json_encoder.encode(json_object)
 
 
 class _EventPeristenceQueue(object):
diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py
index 6322f0f55c..f497b51f4a 100644
--- a/synapse/util/frozenutils.py
+++ b/synapse/util/frozenutils.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from frozendict import frozendict
+import simplejson as json
 
 
 def freeze(o):
@@ -49,3 +50,21 @@ def unfreeze(o):
         pass
 
     return o
+
+
+def _handle_frozendict(obj):
+    """Helper for EventEncoder. Makes frozendicts serializable by returning
+    the underlying dict
+    """
+    if type(obj) is frozendict:
+        # fishing the protected dict out of the object is a bit nasty,
+        # but we don't really want the overhead of copying the dict.
+        return obj._dict
+    raise TypeError('Object of type %s is not JSON serializable' %
+                    obj.__class__.__name__)
+
+
+# A JSONEncoder which is capable of encoding frozendics without barfing
+frozendict_json_encoder = json.JSONEncoder(
+    default=_handle_frozendict,
+)
-- 
cgit 1.5.1


From 2fe3f848b92ee9493a724935167fad84678a7eb2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 23:05:33 +0100
Subject: Remove uses of events.content

---
 synapse/storage/room.py       | 7 ++++---
 synapse/storage/roommember.py | 6 ++++--
 synapse/storage/search.py     | 6 ++++--
 3 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 908551d6d9..740c036975 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -594,7 +594,8 @@ class RoomStore(RoomWorkerStore, SearchStore):
 
         while next_token:
             sql = """
-                SELECT stream_ordering, content FROM events
+                SELECT stream_ordering, json FROM events
+                JOIN event_json USING (event_id)
                 WHERE room_id = ?
                     AND stream_ordering < ?
                     AND contains_url = ? AND outlier = ?
@@ -606,8 +607,8 @@ class RoomStore(RoomWorkerStore, SearchStore):
             next_token = None
             for stream_ordering, content_json in txn:
                 next_token = stream_ordering
-                content = json.loads(content_json)
-
+                event_json = json.loads(content_json)
+                content = event_json["content"]
                 content_url = content.get("url")
                 thumbnail_url = content.get("info", {}).get("thumbnail_url")
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index d662d1cfc0..6a861943a2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -645,8 +645,9 @@ class RoomMemberStore(RoomMemberWorkerStore):
 
         def add_membership_profile_txn(txn):
             sql = ("""
-                SELECT stream_ordering, event_id, events.room_id, content
+                SELECT stream_ordering, event_id, events.room_id, event_json.json
                 FROM events
+                INNER JOIN event_json USING (event_id)
                 INNER JOIN room_memberships USING (event_id)
                 WHERE ? <= stream_ordering AND stream_ordering < ?
                 AND type = 'm.room.member'
@@ -667,7 +668,8 @@ class RoomMemberStore(RoomMemberWorkerStore):
                 event_id = row["event_id"]
                 room_id = row["room_id"]
                 try:
-                    content = json.loads(row["content"])
+                    event_json = json.loads(row["json"])
+                    content = event_json['content']
                 except Exception:
                     continue
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 984643b057..426cbe6e1a 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -75,8 +75,9 @@ class SearchStore(BackgroundUpdateStore):
 
         def reindex_search_txn(txn):
             sql = (
-                "SELECT stream_ordering, event_id, room_id, type, content, "
+                "SELECT stream_ordering, event_id, room_id, type, json, "
                 " origin_server_ts FROM events"
+                " JOIN event_json USING (event_id)"
                 " WHERE ? <= stream_ordering AND stream_ordering < ?"
                 " AND (%s)"
                 " ORDER BY stream_ordering DESC"
@@ -104,7 +105,8 @@ class SearchStore(BackgroundUpdateStore):
                     stream_ordering = row["stream_ordering"]
                     origin_server_ts = row["origin_server_ts"]
                     try:
-                        content = json.loads(row["content"])
+                        event_json = json.loads(row["json"])
+                        content = event_json["content"]
                     except Exception:
                         continue
 
-- 
cgit 1.5.1


From eb8d8d6f57c7f6017548aa95409bb8cc346a5ae0 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Tue, 3 Apr 2018 15:40:43 +0100
Subject: Use join_policy API instead of joinable

The API is now under
 /groups/$group_id/setting/m.join_policy

and expects a JSON blob of the shape

```json
{
  "m.join_policy": {
    "type": "invite"
  }
}
```

where "invite" could alternatively be "open".
---
 synapse/federation/transport/client.py             |  4 +--
 synapse/federation/transport/server.py             |  8 ++---
 synapse/groups/groups_server.py                    | 41 ++++++++++++++++++----
 synapse/handlers/groups_local.py                   |  2 +-
 synapse/rest/client/v2_alpha/groups.py             | 12 +++----
 synapse/storage/group_server.py                    |  6 ++--
 .../storage/schema/delta/48/groups_joinable.sql    |  8 ++++-
 7 files changed, 58 insertions(+), 23 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 5a6b63350b..0f7f656824 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -860,9 +860,9 @@ class TransportLayerClient(object):
     @log_function
     def set_group_joinable(self, destination, group_id, requester_user_id,
                            content):
-        """Sets whether a group is joinable without an invite or knock
+        """Sets the join policy for a group
         """
-        path = PREFIX + "/groups/%s/joinable" % (group_id,)
+        path = PREFIX + "/groups/%s/setting/m.join_policy" % (group_id,)
 
         return self.client.post_json(
             destination=destination,
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 107deb4e1e..a52d3948f4 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1125,10 +1125,10 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
         defer.returnValue((200, resp))
 
 
-class FederationGroupsJoinableServlet(BaseFederationServlet):
+class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
     """Sets whether a group is joinable without an invite or knock
     """
-    PATH = "/groups/(?P<group_id>[^/]*)/joinable$"
+    PATH = "/groups/(?P<group_id>[^/]*)/setting/m.join_policy$"
 
     @defer.inlineCallbacks
     def on_POST(self, origin, content, query, group_id):
@@ -1136,7 +1136,7 @@ class FederationGroupsJoinableServlet(BaseFederationServlet):
         if get_domain_from_id(requester_user_id) != origin:
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
-        new_content = yield self.handler.set_group_joinable(
+        new_content = yield self.handler.set_group_join_policy(
             group_id, requester_user_id, content
         )
 
@@ -1191,7 +1191,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsSummaryUsersServlet,
     FederationGroupsAddRoomsServlet,
     FederationGroupsAddRoomsConfigServlet,
-    FederationGroupsJoinableServlet,
+    FederationGroupsSettingJoinPolicyServlet,
 )
 
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 25cbfb1691..70781e1854 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -207,20 +207,24 @@ class GroupsServerHandler(object):
         defer.returnValue({})
 
     @defer.inlineCallbacks
-    def set_group_joinable(self, group_id, requester_user_id, content):
-        """Sets whether a group is joinable without an invite or knock
+    def set_group_join_policy(self, group_id, requester_user_id, content):
+        """Sets the group join policy.
+
+        Currently supported policies are:
+         - "invite": an invite must be received and accepted in order to join.
+         - "open": anyone can join.
         """
         yield self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
 
-        is_joinable = content.get('joinable')
-        if is_joinable is None:
+        join_policy = _parse_join_policy_from_contents(content)
+        if join_policy is None:
             raise SynapseError(
-                400, "No value specified for 'joinable'"
+                400, "No value specified for 'm.join_policy'"
             )
 
-        yield self.store.set_group_joinable(group_id, is_joinable=is_joinable)
+        yield self.store.set_group_join_policy(group_id, join_policy=join_policy)
 
         defer.returnValue({})
 
@@ -854,6 +858,31 @@ class GroupsServerHandler(object):
         })
 
 
+def _parse_join_policy_from_contents(content):
+    """Given a content for a request, return the specified join policy or None
+    """
+
+    join_policy_dict = content.get("m.join_policy")
+    if join_policy_dict:
+        return _parse_join_policy_dict(join_policy_dict)
+    else:
+        return None
+
+
+def _parse_join_policy_dict(join_policy_dict):
+    """Given a dict for the "m.join_policy" config return the join policy specified
+    """
+    join_policy_type = join_policy_dict.get("type")
+    if not join_policy_type:
+        return True
+
+    if join_policy_type not in ("invite", "open"):
+        raise SynapseError(
+            400, "Synapse only supports 'invite'/'open' join rule"
+        )
+    return join_policy_type
+
+
 def _parse_visibility_from_contents(content):
     """Given a content for a request parse out whether the entity should be
     public or not
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index c9671b9046..5f7b0ff305 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -91,7 +91,7 @@ class GroupsLocalHandler(object):
     get_group_role = _create_rerouter("get_group_role")
     get_group_roles = _create_rerouter("get_group_roles")
 
-    set_group_joinable = _create_rerouter("set_group_joinable")
+    set_group_join_policy = _create_rerouter("set_group_join_policy")
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index aa94130e57..8faaa1d6a0 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -402,13 +402,13 @@ class GroupInvitedUsersServlet(RestServlet):
         defer.returnValue((200, result))
 
 
-class GroupJoinableServlet(RestServlet):
-    """Set whether a group is joinable without an invite
+class GroupSettingJoinPolicyServlet(RestServlet):
+    """Set group join policy
     """
-    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/joinable$")
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/setting/m.join_policy$")
 
     def __init__(self, hs):
-        super(GroupJoinableServlet, self).__init__()
+        super(GroupSettingJoinPolicyServlet, self).__init__()
         self.auth = hs.get_auth()
         self.groups_handler = hs.get_groups_local_handler()
 
@@ -419,7 +419,7 @@ class GroupJoinableServlet(RestServlet):
 
         content = parse_json_object_from_request(request)
 
-        result = yield self.groups_handler.set_group_joinable(
+        result = yield self.groups_handler.set_group_join_policy(
             group_id,
             requester_user_id,
             content,
@@ -765,7 +765,7 @@ def register_servlets(hs, http_server):
     GroupInvitedUsersServlet(hs).register(http_server)
     GroupUsersServlet(hs).register(http_server)
     GroupRoomServlet(hs).register(http_server)
-    GroupJoinableServlet(hs).register(http_server)
+    GroupSettingJoinPolicyServlet(hs).register(http_server)
     GroupCreateServlet(hs).register(http_server)
     GroupAdminRoomsServlet(hs).register(http_server)
     GroupAdminRoomsConfigServlet(hs).register(http_server)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 96553d4fb1..db66ea1eb0 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -30,16 +30,16 @@ _DEFAULT_ROLE_ID = ""
 
 
 class GroupServerStore(SQLBaseStore):
-    def set_group_joinable(self, group_id, is_joinable):
+    def set_group_join_policy(self, group_id, join_policy):
         return self._simple_update_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
             },
             updatevalues={
-                "is_joinable": is_joinable,
+                "join_policy": join_policy,
             },
-            desc="set_group_joinable",
+            desc="set_group_join_policy",
         )
 
     def get_group(self, group_id):
diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index ace7d0a723..ab3b00286d 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,10 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable SMALLINT DEFAULT 0 NOT NULL;
+/* 
+ * This isn't a real ENUM because sqlite doesn't support it
+ * and we use a default of NULL for inserted rows and interpret
+ * NULL at the python store level as necessary so that existing
+ * rows are given the correct default policy.
+ */
+ALTER TABLE groups ADD COLUMN join_policy TEXT DEFAULT NULL;
-- 
cgit 1.5.1


From 0e5f479fc05ef9257c1bfce033c8fb91e6244ffe Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 5 Apr 2018 12:16:46 +0100
Subject: Review comments

Use iteritems over item to loop over dict
formatting
---
 synapse/app/homeserver.py   | 2 +-
 synapse/storage/__init__.py | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 35e2b00f1b..777e9c529a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -426,7 +426,7 @@ def run(hs):
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
         r30_results = yield hs.get_datastore().count_r30_users()
-        for name, count in r30_results.items():
+        for name, count in r30_results.iteritems():
             stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index f68e436df0..4800584b59 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,8 +280,9 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-                     SELECT users.name, platform, users.creation_ts * 1000,
-                                                            MAX(uip.last_seen)
+                     SELECT
+                        users.name, platform, users.creation_ts * 1000,
+                        MAX(uip.last_seen)
                      FROM users
                      INNER JOIN (
                          SELECT
@@ -310,8 +311,8 @@ class DataStore(RoomMemberStore, RoomStore,
             results = {}
             txn.execute(sql, (thirty_days_ago_in_secs,
                               thirty_days_ago_in_secs))
-            rows = txn.fetchall()
-            for row in rows:
+
+            for row in txn:
                 if row[0] is 'unknown':
                     pass
                 results[row[0]] = row[1]
-- 
cgit 1.5.1


From b214a04ffc1200535f1d9c6ec45717cd266f36e5 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 13:29:16 +0100
Subject: Document set_group_join_policy

---
 synapse/storage/group_server.py | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'synapse/storage')

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index db66ea1eb0..ab4f710f7d 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -31,6 +31,12 @@ _DEFAULT_ROLE_ID = ""
 
 class GroupServerStore(SQLBaseStore):
     def set_group_join_policy(self, group_id, join_policy):
+        """Set the join policy of a group.
+
+        join_policy can be one of:
+         * "invite"
+         * "open"
+        """
         return self._simple_update_one(
             table="groups",
             keyvalues={
-- 
cgit 1.5.1


From 700e5e719875dd7008791f52828bb3cd92d6ce21 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 14:01:17 +0100
Subject: Use DEFAULT join_policy of "invite" in db

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index ab3b00286d..53add94367 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -19,4 +19,4 @@
  * NULL at the python store level as necessary so that existing
  * rows are given the correct default policy.
  */
-ALTER TABLE groups ADD COLUMN join_policy TEXT DEFAULT NULL;
+ALTER TABLE groups ADD COLUMN join_policy TEXT NON NULL DEFAULT 'invite';
-- 
cgit 1.5.1


From 917380e89d2d323be1a6ea03e53a31ed335c80df Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 14:32:12 +0100
Subject: NON NULL -> NOT NULL

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 53add94367..ce26eaf0c9 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -19,4 +19,4 @@
  * NULL at the python store level as necessary so that existing
  * rows are given the correct default policy.
  */
-ALTER TABLE groups ADD COLUMN join_policy TEXT NON NULL DEFAULT 'invite';
+ALTER TABLE groups ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'invite';
-- 
cgit 1.5.1


From b370fe61c0aeccac7745ad404dad925ec217ba6d Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 17:18:02 +0100
Subject: Implement group join API

---
 synapse/federation/transport/client.py | 13 ++++++++++
 synapse/federation/transport/server.py | 18 ++++++++++++++
 synapse/groups/groups_server.py        | 45 ++++++++++++++++++++++++++++++++++
 synapse/handlers/groups_local.py       | 40 +++++++++++++++++++++++++++++-
 synapse/storage/group_server.py        | 12 ++++++---
 5 files changed, 124 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 3beab47832..370f7ba78b 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -614,6 +614,19 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    @log_function
+    def join_group(self, destination, group_id, user_id, content):
+        """Attempts to join a group
+        """
+        path = PREFIX + "/groups/%s/users/%s/join" % (group_id, user_id)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
+
     @log_function
     def invite_to_group(self, destination, group_id, user_id, requester_user_id, content):
         """Invite a user to a group
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index b98e30459c..4c94d5a36c 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -803,6 +803,23 @@ class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsJoinServlet(BaseFederationServlet):
+    """Attempt to join a group
+    """
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id, user_id):
+        if get_domain_from_id(user_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        new_content = yield self.handler.join_group(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsRemoveUserServlet(BaseFederationServlet):
     """Leave or kick a user from the group
     """
@@ -1182,6 +1199,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsInvitedUsersServlet,
     FederationGroupsInviteServlet,
     FederationGroupsAcceptInviteServlet,
+    FederationGroupsJoinServlet,
     FederationGroupsRemoveUserServlet,
     FederationGroupsSummaryRoomsServlet,
     FederationGroupsCategoriesServlet,
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 70781e1854..77b6273e39 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -723,6 +723,51 @@ class GroupsServerHandler(object):
             "attestation": local_attestation,
         })
 
+    @defer.inlineCallbacks
+    def join_group(self, group_id, requester_user_id, content):
+        """User tries to join the group.
+
+        This will error if the group requires an invite/knock to join
+        """
+
+        yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+
+        group_info = yield self.store.get_group(
+            group_id,
+        )
+        if not group_info['is_joinable']:
+            raise SynapseError(403, "Group is not publicly joinable")
+
+        if not self.hs.is_mine_id(requester_user_id):
+            local_attestation = self.attestations.create_attestation(
+                group_id, requester_user_id,
+            )
+            remote_attestation = content["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                user_id=requester_user_id,
+                group_id=group_id,
+            )
+        else:
+            local_attestation = None
+            remote_attestation = None
+
+        is_public = _parse_visibility_from_contents(content)
+
+        yield self.store.add_user_to_group(
+            group_id, requester_user_id,
+            is_admin=False,
+            is_public=is_public,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({
+            "state": "join",
+            "attestation": local_attestation,
+        })
+
     @defer.inlineCallbacks
     def knock(self, group_id, requester_user_id, content):
         """A user requests becoming a member of the group
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 5f7b0ff305..977993e7d4 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -229,7 +229,45 @@ class GroupsLocalHandler(object):
     def join_group(self, group_id, user_id, content):
         """Request to join a group
         """
-        raise NotImplementedError()  # TODO
+        if self.is_mine_id(group_id):
+            yield self.groups_server_handler.join_group(
+                group_id, user_id, content
+            )
+            local_attestation = None
+            remote_attestation = None
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content["attestation"] = local_attestation
+
+            res = yield self.transport_client.join_group(
+                get_domain_from_id(group_id), group_id, user_id, content,
+            )
+
+            remote_attestation = res["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                group_id=group_id,
+                user_id=user_id,
+                server_name=get_domain_from_id(group_id),
+            )
+
+        # TODO: Check that the group is public and we're being added publically
+        is_publicised = content.get("publicise", False)
+
+        token = yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="join",
+            is_admin=False,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+            is_publicised=is_publicised,
+        )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
+
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index db316a27ec..16c11a056f 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -48,19 +48,25 @@ class GroupServerStore(SQLBaseStore):
             desc="set_group_join_policy",
         )
 
+    @defer.inlineCallbacks
     def get_group(self, group_id):
-        return self._simple_select_one(
+        ret = yield self._simple_select_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
             },
             retcols=(
-                "name", "short_description", "long_description", "avatar_url", "is_public"
+                "name", "short_description", "long_description", "avatar_url", "is_public", "is_joinable",
             ),
             allow_none=True,
-            desc="is_user_in_group",
+            desc="get_group",
         )
 
+        if ret and 'is_joinable' in ret:
+            ret['is_joinable'] = bool(ret['is_joinable'])
+
+        defer.returnValue(ret)
+
     def get_users_in_group(self, group_id, include_private=False):
         # TODO: Pagination
 
-- 
cgit 1.5.1


From edb45aae38b7bdeae1cbf18b435be5f8610c2d48 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 17:18:50 +0100
Subject: pep8

---
 synapse/storage/group_server.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 16c11a056f..5fbe0ada4e 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -56,7 +56,8 @@ class GroupServerStore(SQLBaseStore):
                 "group_id": group_id,
             },
             retcols=(
-                "name", "short_description", "long_description", "avatar_url", "is_public", "is_joinable",
+                "name", "short_description", "long_description",
+                "avatar_url", "is_public", "is_joinable",
             ),
             allow_none=True,
             desc="get_group",
-- 
cgit 1.5.1


From ae85c7804e733aa1adaed06a9de51445a084858e Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 16:31:57 +0100
Subject: is_joinable -> join_rule

---
 synapse/groups/groups_server.py | 2 +-
 synapse/storage/group_server.py | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 2c02da4725..507ec232c5 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -746,7 +746,7 @@ class GroupsServerHandler(object):
         group_info = yield self.store.get_group(
             group_id,
         )
-        if not group_info['is_joinable']:
+        if group_info['join_policy'] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
         local_attestation = yield self.add_user(group_id, requester_user_id, content)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 5fbe0ada4e..d81609dd16 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -57,15 +57,12 @@ class GroupServerStore(SQLBaseStore):
             },
             retcols=(
                 "name", "short_description", "long_description",
-                "avatar_url", "is_public", "is_joinable",
+                "avatar_url", "is_public", "join_rule",
             ),
             allow_none=True,
             desc="get_group",
         )
 
-        if ret and 'is_joinable' in ret:
-            ret['is_joinable'] = bool(ret['is_joinable'])
-
         defer.returnValue(ret)
 
     def get_users_in_group(self, group_id, include_private=False):
-- 
cgit 1.5.1


From 87c864b6984f6876f224e0d4ac2e2b872d2723ce Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 16:49:22 +0100
Subject: join_rule -> join_policy

---
 synapse/storage/group_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index d81609dd16..8bd8ca8124 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -57,7 +57,7 @@ class GroupServerStore(SQLBaseStore):
             },
             retcols=(
                 "name", "short_description", "long_description",
-                "avatar_url", "is_public", "join_rule",
+                "avatar_url", "is_public", "join_policy",
             ),
             allow_none=True,
             desc="get_group",
-- 
cgit 1.5.1


From cd087a265db8037ce2fe158ddc0e9b35a612d4bb Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 17:14:27 +0100
Subject: Don't use redundant inlineCallbacks

---
 synapse/storage/group_server.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8bd8ca8124..da05ccb027 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -48,9 +48,8 @@ class GroupServerStore(SQLBaseStore):
             desc="set_group_join_policy",
         )
 
-    @defer.inlineCallbacks
     def get_group(self, group_id):
-        ret = yield self._simple_select_one(
+        return self._simple_select_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
@@ -63,8 +62,6 @@ class GroupServerStore(SQLBaseStore):
             desc="get_group",
         )
 
-        defer.returnValue(ret)
-
     def get_users_in_group(self, group_id, include_private=False):
         # TODO: Pagination
 
-- 
cgit 1.5.1


From 64bc2162ef1c51cf5d5baa3cd3b303216b55df8c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 17:50:36 +0100
Subject: Fix psycopg2 interpolation

---
 synapse/storage/__init__.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 4800584b59..01c580b66e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -289,11 +289,11 @@ class DataStore(RoomMemberStore, RoomStore,
                          user_id,
                          last_seen,
                          CASE
-                             WHEN user_agent LIKE '%Android%' THEN 'android'
-                             WHEN user_agent LIKE '%iOS%' THEN 'ios'
-                             WHEN user_agent LIKE '%Electron%' THEN 'electron'
-                             WHEN user_agent LIKE '%Mozilla%' THEN 'web'
-                             WHEN user_agent LIKE '%Gecko%' THEN 'web'
+                             WHEN user_agent LIKE '%%Android%%' THEN 'android'
+                             WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
+                             WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
+                             WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
+                             WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
                              ELSE 'unknown'
                          END
                          AS platform
-- 
cgit 1.5.1


From 87770300d590eb105694e1cbc24aa798f788ca52 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:38:59 +0100
Subject: Fix msec to sec

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 01c580b66e..e736a857ea 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -276,7 +276,7 @@ class DataStore(RoomMemberStore, RoomStore,
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time_msec())
-            thirty_days_ago_in_secs = now - thirty_days_in_secs
+            thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-- 
cgit 1.5.1


From 61b439c90437527afdfa6b46e5ab88de6b074792 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:43:48 +0100
Subject: Fix msec to sec, again

---
 synapse/storage/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e736a857ea..417b9aa2a8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -275,9 +275,9 @@ class DataStore(RoomMemberStore, RoomStore,
         """
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
-            now = int(self._clock.time_msec())
-            thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs
-
+            now = int(self._clock.time())
+            thirty_days_ago_in_secs = now - thirty_days_in_secs
+            print str(thirty_days_ago_in_secs)
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
                      SELECT
-- 
cgit 1.5.1


From 41e06118954234878311cea3e7a55a88e6eec784 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:44:20 +0100
Subject: remove errant print

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/storage')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 417b9aa2a8..eacd49d6a5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -277,7 +277,7 @@ class DataStore(RoomMemberStore, RoomStore,
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
-            print str(thirty_days_ago_in_secs)
+
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
                      SELECT
-- 
cgit 1.5.1