diff --git a/changelog.d/3662.feature b/changelog.d/3662.feature
new file mode 100644
index 0000000000..daacef086d
--- /dev/null
+++ b/changelog.d/3662.feature
@@ -0,0 +1 @@
+Ability to whitelist specific threepids against monthly active user limiting
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3a67db8b30..a4a65e7286 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -518,6 +518,8 @@ def run(hs):
# If you increase the loop period, the accuracy of user_daily_visits
# table will decrease
clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
+
+ # monthly active user limiting functionality
clock.looping_call(
hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
)
@@ -530,9 +532,13 @@ def run(hs):
current_mau_gauge.set(float(count))
max_mau_value_gauge.set(float(hs.config.max_mau_value))
+ hs.get_datastore().initialise_reserved_users(
+ hs.config.mau_limits_reserved_threepids
+ )
generate_monthly_active_users()
if hs.config.limit_usage_by_mau:
clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+ # End of monthly active user settings
if hs.config.report_stats:
logger.info("Scheduling stats reporting for 3 hour intervals")
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 2e1e2f5961..3b078d72ca 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -74,6 +74,9 @@ class ServerConfig(Config):
self.max_mau_value = config.get(
"max_mau_value", 0,
)
+ self.mau_limits_reserved_threepids = config.get(
+ "mau_limit_reserved_threepids", []
+ )
# Options to disable HS
self.hs_disabled = config.get("hs_disabled", False)
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 8b3beaf26a..d47dcef3a0 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import logging
from twisted.internet import defer
@@ -19,6 +20,8 @@ from synapse.util.caches.descriptors import cached
from ._base import SQLBaseStore
+logger = logging.getLogger(__name__)
+
# Number of msec of granularity to store the monthly_active_user timestamp
# This means it is not necessary to update the table on every request
LAST_SEEN_GRANULARITY = 60 * 60 * 1000
@@ -29,7 +32,29 @@ class MonthlyActiveUsersStore(SQLBaseStore):
super(MonthlyActiveUsersStore, self).__init__(None, hs)
self._clock = hs.get_clock()
self.hs = hs
+ self.reserved_users = ()
+
+ @defer.inlineCallbacks
+ def initialise_reserved_users(self, threepids):
+ # TODO Why can't I do this in init?
+ store = self.hs.get_datastore()
+ reserved_user_list = []
+
+ # Do not add more reserved users than the total allowable number
+ for tp in threepids[:self.hs.config.max_mau_value]:
+ user_id = yield store.get_user_id_by_threepid(
+ tp["medium"], tp["address"]
+ )
+ if user_id:
+ self.upsert_monthly_active_user(user_id)
+ reserved_user_list.append(user_id)
+ else:
+ logger.warning(
+ "mau limit reserved threepid %s not found in db" % tp
+ )
+ self.reserved_users = tuple(reserved_user_list)
+ @defer.inlineCallbacks
def reap_monthly_active_users(self):
"""
Cleans out monthly active user table to ensure that no stale
@@ -44,8 +69,20 @@ class MonthlyActiveUsersStore(SQLBaseStore):
int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
)
# Purge stale users
- sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
- txn.execute(sql, (thirty_days_ago,))
+
+ # questionmarks is a hack to overcome sqlite not supporting
+ # tuples in 'WHERE IN %s'
+ questionmarks = '?' * len(self.reserved_users)
+ query_args = [thirty_days_ago]
+ query_args.extend(self.reserved_users)
+
+ sql = """
+ DELETE FROM monthly_active_users
+ WHERE timestamp < ?
+ AND user_id NOT IN ({})
+ """.format(','.join(questionmarks))
+
+ txn.execute(sql, query_args)
# If MAU user count still exceeds the MAU threshold, then delete on
# a least recently active basis.
@@ -55,6 +92,8 @@ class MonthlyActiveUsersStore(SQLBaseStore):
# While Postgres does not require 'LIMIT', but also does not support
# negative LIMIT values. So there is no way to write it that both can
# support
+ query_args = [self.hs.config.max_mau_value]
+ query_args.extend(self.reserved_users)
sql = """
DELETE FROM monthly_active_users
WHERE user_id NOT IN (
@@ -62,8 +101,9 @@ class MonthlyActiveUsersStore(SQLBaseStore):
ORDER BY timestamp DESC
LIMIT ?
)
- """
- txn.execute(sql, (self.hs.config.max_mau_value,))
+ AND user_id NOT IN ({})
+ """.format(','.join(questionmarks))
+ txn.execute(sql, query_args)
yield self.runInteraction("reap_monthly_active_users", _reap_users)
# It seems poor to invalidate the whole cache, Postgres supports
@@ -122,7 +162,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Arguments:
user_id (str): user to add/update
Return:
- int : timestamp since last seen, None if never seen
+ Deferred[int] : timestamp since last seen, None if never seen
"""
@@ -144,7 +184,6 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Args:
user_id(str): the user_id to query
"""
-
if self.hs.config.limit_usage_by_mau:
last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
now = self.hs.get_clock().time_msec()
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 0bfd24a7fb..cbd480cd42 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -19,6 +19,8 @@ import tests.unittest
import tests.utils
from tests.utils import setup_test_homeserver
+FORTY_DAYS = 40 * 24 * 60 * 60
+
class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
def __init__(self, *args, **kwargs):
@@ -30,6 +32,56 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
self.store = self.hs.get_datastore()
@defer.inlineCallbacks
+ def test_initialise_reserved_users(self):
+
+ user1 = "@user1:server"
+ user1_email = "user1@matrix.org"
+ user2 = "@user2:server"
+ user2_email = "user2@matrix.org"
+ threepids = [
+ {'medium': 'email', 'address': user1_email},
+ {'medium': 'email', 'address': user2_email}
+ ]
+ user_num = len(threepids)
+
+ yield self.store.register(
+ user_id=user1,
+ token="123",
+ password_hash=None)
+
+ yield self.store.register(
+ user_id=user2,
+ token="456",
+ password_hash=None)
+
+ now = int(self.hs.get_clock().time_msec())
+ yield self.store.user_add_threepid(user1, "email", user1_email, now, now)
+ yield self.store.user_add_threepid(user2, "email", user2_email, now, now)
+ yield self.store.initialise_reserved_users(threepids)
+
+ active_count = yield self.store.get_monthly_active_count()
+
+ # Test total counts
+ self.assertEquals(active_count, user_num)
+
+ # Test user is marked as active
+
+ timestamp = yield self.store._user_last_seen_monthly_active(user1)
+ self.assertTrue(timestamp)
+ timestamp = yield self.store._user_last_seen_monthly_active(user2)
+ self.assertTrue(timestamp)
+
+ # Test that users are never removed from the db.
+ self.hs.config.max_mau_value = 0
+
+ self.hs.get_clock().advance_time(FORTY_DAYS)
+
+ yield self.store.reap_monthly_active_users()
+
+ active_count = yield self.store.get_monthly_active_count()
+ self.assertEquals(active_count, user_num)
+
+ @defer.inlineCallbacks
def test_can_insert_and_count_mau(self):
count = yield self.store.get_monthly_active_count()
self.assertEqual(0, count)
@@ -63,4 +115,9 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
self.assertTrue(count, initial_users)
yield self.store.reap_monthly_active_users()
count = yield self.store.get_monthly_active_count()
- self.assertTrue(count, initial_users - self.hs.config.max_mau_value)
+ self.assertEquals(count, initial_users - self.hs.config.max_mau_value)
+
+ self.hs.get_clock().advance_time(FORTY_DAYS)
+ yield self.store.reap_monthly_active_users()
+ count = yield self.store.get_monthly_active_count()
+ self.assertEquals(count, 0)
diff --git a/tests/utils.py b/tests/utils.py
index a0aa38d264..5e4d77920c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -76,6 +76,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
config.limit_usage_by_mau = False
config.hs_disabled = False
config.hs_disabled_message = ""
+ config.max_mau_value = 50
+ config.mau_limits_reserved_threepids = []
# disable user directory updates, because they get done in the
# background, which upsets the test runner.
|
