diff --git a/changelog.d/3633.feature b/changelog.d/3633.feature
new file mode 100644
index 0000000000..8007a04840
--- /dev/null
+++ b/changelog.d/3633.feature
@@ -0,0 +1 @@
+Add ability to limit number of monthly active users on the server
diff --git a/changelog.d/3654.feature b/changelog.d/3654.feature
new file mode 100644
index 0000000000..35c95580bc
--- /dev/null
+++ b/changelog.d/3654.feature
@@ -0,0 +1 @@
+Basic support for room versioning
diff --git a/changelog.d/3658.bugfix b/changelog.d/3658.bugfix
new file mode 100644
index 0000000000..556011a150
--- /dev/null
+++ b/changelog.d/3658.bugfix
@@ -0,0 +1 @@
+Fix occasional glitches in the synapse_event_persisted_position metric
diff --git a/changelog.d/3662.feature b/changelog.d/3662.feature
new file mode 100644
index 0000000000..daacef086d
--- /dev/null
+++ b/changelog.d/3662.feature
@@ -0,0 +1 @@
+Ability to whitelist specific threepids against monthly active user limiting
diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 94a1de58f4..c58612594a 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -54,7 +54,7 @@
"gnetId": null,
"graphTooltip": 0,
"id": null,
- "iteration": 1533026624326,
+ "iteration": 1533598785368,
"links": [
{
"asDropdown": true,
@@ -4629,7 +4629,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 11
+ "y": 29
},
"id": 67,
"legend": {
@@ -4655,11 +4655,11 @@
"steppedLine": false,
"targets": [
{
- "expr": " synapse_event_persisted_position{instance=\"$instance\"} - ignoring(index, job, name) group_right(instance) synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": " synapse_event_persisted_position{instance=\"$instance\",job=\"synapse\"} - ignoring(index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}}",
+ "legendFormat": "{{job}}-{{index}} ",
"refId": "A"
}
],
@@ -4697,7 +4697,11 @@
"min": null,
"show": true
}
- ]
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
},
{
"aliasColors": {},
@@ -4710,7 +4714,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 11
+ "y": 29
},
"id": 71,
"legend": {
@@ -4778,7 +4782,11 @@
"min": null,
"show": true
}
- ]
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"title": "Event processing loop positions",
@@ -4957,5 +4965,5 @@
"timezone": "",
"title": "Synapse",
"uid": "000000012",
- "version": 125
+ "version": 127
}
\ No newline at end of file
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 5bbbe8e2e7..91b23ff1d7 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -213,7 +213,7 @@ class Auth(object):
default=[b""]
)[0]
if user and access_token and ip_addr:
- self.store.insert_client_ip(
+ yield self.store.insert_client_ip(
user_id=user.to_string(),
access_token=access_token,
ip=ip_addr,
@@ -773,3 +773,15 @@ class Auth(object):
raise AuthError(
403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
)
+
+ @defer.inlineCallbacks
+ def check_auth_blocking(self):
+ """Checks if the user should be rejected for some external reason,
+ such as monthly active user limiting or global disable flag
+ """
+ if self.hs.config.limit_usage_by_mau is True:
+ current_mau = yield self.store.get_monthly_active_count()
+ if current_mau >= self.hs.config.max_mau_value:
+ raise AuthError(
+ 403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+ )
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 4df930c8d1..b0da506f6d 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -94,3 +95,11 @@ class RoomCreationPreset(object):
class ThirdPartyEntityKind(object):
USER = "user"
LOCATION = "location"
+
+
+# the version we will give rooms which are created on this server
+DEFAULT_ROOM_VERSION = "1"
+
+# vdh-test-version is a placeholder to get room versioning support working and tested
+# until we have a working v2.
+KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b41d595059..70400347bc 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -56,6 +57,8 @@ class Codes(object):
CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
+ UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
+ INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
class CodeMessageException(RuntimeError):
@@ -285,6 +288,27 @@ class LimitExceededError(SynapseError):
)
+class IncompatibleRoomVersionError(SynapseError):
+ """A server is trying to join a room whose version it does not support."""
+
+ def __init__(self, room_version):
+ super(IncompatibleRoomVersionError, self).__init__(
+ code=400,
+ msg="Your homeserver does not support the features required to "
+ "join this room",
+ errcode=Codes.INCOMPATIBLE_ROOM_VERSION,
+ )
+
+ self._room_version = room_version
+
+ def error_dict(self):
+ return cs_error(
+ self.msg,
+ self.errcode,
+ room_version=self._room_version,
+ )
+
+
def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
""" Utility method for constructing an error response for client-server
interactions.
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index fba51c26e8..a4a65e7286 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -519,17 +519,26 @@ def run(hs):
# table will decrease
clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
+ # monthly active user limiting functionality
+ clock.looping_call(
+ hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
+ )
+
@defer.inlineCallbacks
def generate_monthly_active_users():
count = 0
if hs.config.limit_usage_by_mau:
- count = yield hs.get_datastore().count_monthly_users()
+ count = yield hs.get_datastore().get_monthly_active_count()
current_mau_gauge.set(float(count))
max_mau_value_gauge.set(float(hs.config.max_mau_value))
+ hs.get_datastore().initialise_reserved_users(
+ hs.config.mau_limits_reserved_threepids
+ )
generate_monthly_active_users()
if hs.config.limit_usage_by_mau:
clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+ # End of monthly active user settings
if hs.config.report_stats:
logger.info("Scheduling stats reporting for 3 hour intervals")
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 6a471a0a5e..114d7a9815 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -69,12 +69,15 @@ class ServerConfig(Config):
# Options to control access by tracking MAU
self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
+ self.max_mau_value = 0
if self.limit_usage_by_mau:
self.max_mau_value = config.get(
"max_mau_value", 0,
)
- else:
- self.max_mau_value = 0
+ self.mau_limits_reserved_threepids = config.get(
+ "mau_limit_reserved_threepids", []
+ )
+
# FIXME: federation_domain_whitelist needs sytests
self.federation_domain_whitelist = None
federation_domain_whitelist = config.get(
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index b32f64e729..6baeccca38 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -20,7 +20,7 @@ from signedjson.key import decode_verify_key_bytes
from signedjson.sign import SignatureVerifyException, verify_signed_json
from unpaddedbase64 import decode_base64
-from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, JoinRules, Membership
from synapse.api.errors import AuthError, EventSizeError, SynapseError
from synapse.types import UserID, get_domain_from_id
@@ -83,6 +83,14 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
403,
"Creation event's room_id domain does not match sender's"
)
+
+ room_version = event.content.get("room_version", "1")
+ if room_version not in KNOWN_ROOM_VERSIONS:
+ raise AuthError(
+ 403,
+ "room appears to have unsupported version %s" % (
+ room_version,
+ ))
# FIXME
logger.debug("Allowing! %s", event)
return
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 7550e11b6e..c9f3c2d352 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,7 +25,7 @@ from prometheus_client import Counter
from twisted.internet import defer
-from synapse.api.constants import Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
from synapse.api.errors import (
CodeMessageException,
FederationDeniedError,
@@ -518,10 +518,10 @@ class FederationClient(FederationBase):
description, destination, exc_info=1,
)
- raise RuntimeError("Failed to %s via any server", description)
+ raise RuntimeError("Failed to %s via any server" % (description, ))
def make_membership_event(self, destinations, room_id, user_id, membership,
- content={},):
+ content, params):
"""
Creates an m.room.member event, with context, without participating in the room.
@@ -537,8 +537,10 @@ class FederationClient(FederationBase):
user_id (str): The user whose membership is being evented.
membership (str): The "membership" property of the event. Must be
one of "join" or "leave".
- content (object): Any additional data to put into the content field
+ content (dict): Any additional data to put into the content field
of the event.
+ params (dict[str, str|Iterable[str]]): Query parameters to include in the
+ request.
Return:
Deferred: resolves to a tuple of (origin (str), event (object))
where origin is the remote homeserver which generated the event.
@@ -558,10 +560,12 @@ class FederationClient(FederationBase):
@defer.inlineCallbacks
def send_request(destination):
ret = yield self.transport_layer.make_membership_event(
- destination, room_id, user_id, membership
+ destination, room_id, user_id, membership, params,
)
- pdu_dict = ret["event"]
+ pdu_dict = ret.get("event", None)
+ if not isinstance(pdu_dict, dict):
+ raise InvalidResponseError("Bad 'event' field in response")
logger.debug("Got response to make_%s: %s", membership, pdu_dict)
@@ -605,6 +609,26 @@ class FederationClient(FederationBase):
Fails with a ``RuntimeError`` if no servers were reachable.
"""
+ def check_authchain_validity(signed_auth_chain):
+ for e in signed_auth_chain:
+ if e.type == EventTypes.Create:
+ create_event = e
+ break
+ else:
+ raise InvalidResponseError(
+ "no %s in auth chain" % (EventTypes.Create,),
+ )
+
+ # the room version should be sane.
+ room_version = create_event.content.get("room_version", "1")
+ if room_version not in KNOWN_ROOM_VERSIONS:
+ # This shouldn't be possible, because the remote server should have
+ # rejected the join attempt during make_join.
+ raise InvalidResponseError(
+ "room appears to have unsupported version %s" % (
+ room_version,
+ ))
+
@defer.inlineCallbacks
def send_request(destination):
time_now = self._clock.time_msec()
@@ -661,7 +685,7 @@ class FederationClient(FederationBase):
for s in signed_state:
s.internal_metadata = copy.deepcopy(s.internal_metadata)
- auth_chain.sort(key=lambda e: e.depth)
+ check_authchain_validity(signed_auth)
defer.returnValue({
"state": signed_state,
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index bf89d568af..2b62f687b6 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -27,7 +27,13 @@ from twisted.internet.abstract import isIPAddress
from twisted.python import failure
from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError, FederationError, NotFoundError, SynapseError
+from synapse.api.errors import (
+ AuthError,
+ FederationError,
+ IncompatibleRoomVersionError,
+ NotFoundError,
+ SynapseError,
+)
from synapse.crypto.event_signing import compute_event_signature
from synapse.federation.federation_base import FederationBase, event_from_pdu_json
from synapse.federation.persistence import TransactionActions
@@ -323,12 +329,21 @@ class FederationServer(FederationBase):
defer.returnValue((200, resp))
@defer.inlineCallbacks
- def on_make_join_request(self, origin, room_id, user_id):
+ def on_make_join_request(self, origin, room_id, user_id, supported_versions):
origin_host, _ = parse_server_name(origin)
yield self.check_server_matches_acl(origin_host, room_id)
+
+ room_version = yield self.store.get_room_version(room_id)
+ if room_version not in supported_versions:
+ logger.warn("Room version %s not in %s", room_version, supported_versions)
+ raise IncompatibleRoomVersionError(room_version=room_version)
+
pdu = yield self.handler.on_make_join_request(room_id, user_id)
time_now = self._clock.time_msec()
- defer.returnValue({"event": pdu.get_pdu_json(time_now)})
+ defer.returnValue({
+ "event": pdu.get_pdu_json(time_now),
+ "room_version": room_version,
+ })
@defer.inlineCallbacks
def on_invite_request(self, origin, content):
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 4529d454af..b4fbe2c9d5 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -195,7 +195,7 @@ class TransportLayerClient(object):
@defer.inlineCallbacks
@log_function
- def make_membership_event(self, destination, room_id, user_id, membership):
+ def make_membership_event(self, destination, room_id, user_id, membership, params):
"""Asks a remote server to build and sign us a membership event
Note that this does not append any events to any graphs.
@@ -205,6 +205,8 @@ class TransportLayerClient(object):
room_id (str): room to join/leave
user_id (str): user to be joined/left
membership (str): one of join/leave
+ params (dict[str, str|Iterable[str]]): Query parameters to include in the
+ request.
Returns:
Deferred: Succeeds when we get a 2xx HTTP response. The result
@@ -241,6 +243,7 @@ class TransportLayerClient(object):
content = yield self.client.get_json(
destination=destination,
path=path,
+ args=params,
retry_on_dns_fail=retry_on_dns_fail,
timeout=20000,
ignore_backoff=ignore_backoff,
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index eae5f2b427..77969a4f38 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -190,6 +190,41 @@ def _parse_auth_header(header_bytes):
class BaseFederationServlet(object):
+ """Abstract base class for federation servlet classes.
+
+ The servlet object should have a PATH attribute which takes the form of a regexp to
+ match against the request path (excluding the /federation/v1 prefix).
+
+ The servlet should also implement one or more of on_GET, on_POST, on_PUT, to match
+ the appropriate HTTP method. These methods have the signature:
+
+ on_<METHOD>(self, origin, content, query, **kwargs)
+
+ With arguments:
+
+ origin (unicode|None): The authenticated server_name of the calling server,
+ unless REQUIRE_AUTH is set to False and authentication failed.
+
+ content (unicode|None): decoded json body of the request. None if the
+ request was a GET.
+
+ query (dict[bytes, list[bytes]]): Query params from the request. url-decoded
+ (ie, '+' and '%xx' are decoded) but note that it is *not* utf8-decoded
+ yet.
+
+ **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+ components as specified in the path match regexp.
+
+ Returns:
+ Deferred[(int, object)|None]: either (response code, response object) to
+ return a JSON response, or None if the request has already been handled.
+
+ Raises:
+ SynapseError: to return an error code
+
+ Exception: other exceptions will be caught, logged, and a 500 will be
+ returned.
+ """
REQUIRE_AUTH = True
def __init__(self, handler, authenticator, ratelimiter, server_name):
@@ -204,6 +239,18 @@ class BaseFederationServlet(object):
@defer.inlineCallbacks
@functools.wraps(func)
def new_func(request, *args, **kwargs):
+ """ A callback which can be passed to HttpServer.RegisterPaths
+
+ Args:
+ request (twisted.web.http.Request):
+ *args: unused?
+ **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+ components as specified in the path match regexp.
+
+ Returns:
+ Deferred[(int, object)|None]: (response code, response object) as returned
+ by the callback method. None if the request has already been handled.
+ """
content = None
if request.method in ["PUT", "POST"]:
# TODO: Handle other method types? other content types?
@@ -384,9 +431,31 @@ class FederationMakeJoinServlet(BaseFederationServlet):
PATH = "/make_join/(?P<context>[^/]*)/(?P<user_id>[^/]*)"
@defer.inlineCallbacks
- def on_GET(self, origin, content, query, context, user_id):
+ def on_GET(self, origin, _content, query, context, user_id):
+ """
+ Args:
+ origin (unicode): The authenticated server_name of the calling server
+
+ _content (None): (GETs don't have bodies)
+
+ query (dict[bytes, list[bytes]]): Query params from the request.
+
+ **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+ components as specified in the path match regexp.
+
+ Returns:
+ Deferred[(int, object)|None]: either (response code, response object) to
+ return a JSON response, or None if the request has already been handled.
+ """
+ versions = query.get(b'ver')
+ if versions is not None:
+ supported_versions = [v.decode("utf-8") for v in versions]
+ else:
+ supported_versions = ["1"]
+
content = yield self.handler.on_make_join_request(
origin, context, user_id,
+ supported_versions=supported_versions,
)
defer.returnValue((200, content))
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 184eef09d0..7ea8ce9f94 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -520,7 +520,7 @@ class AuthHandler(BaseHandler):
"""
logger.info("Logging in user %s on device %s", user_id, device_id)
access_token = yield self.issue_access_token(user_id, device_id)
- yield self._check_mau_limits()
+ yield self.auth.check_auth_blocking()
# the device *should* have been registered before we got here; however,
# it's possible we raced against a DELETE operation. The thing we
@@ -734,7 +734,7 @@ class AuthHandler(BaseHandler):
@defer.inlineCallbacks
def validate_short_term_login_token_and_get_user_id(self, login_token):
- yield self._check_mau_limits()
+ yield self.auth.check_auth_blocking()
auth_api = self.hs.get_auth()
user_id = None
try:
@@ -907,19 +907,6 @@ class AuthHandler(BaseHandler):
else:
return defer.succeed(False)
- @defer.inlineCallbacks
- def _check_mau_limits(self):
- """
- Ensure that if mau blocking is enabled that invalid users cannot
- log in.
- """
- if self.hs.config.limit_usage_by_mau is True:
- current_mau = yield self.store.count_monthly_users()
- if current_mau >= self.hs.config.max_mau_value:
- raise AuthError(
- 403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
- )
-
@attr.s
class MacaroonGenerator(object):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 533b82c783..0dffd44e22 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -30,7 +30,12 @@ from unpaddedbase64 import decode_base64
from twisted.internet import defer
-from synapse.api.constants import EventTypes, Membership, RejectedReason
+from synapse.api.constants import (
+ KNOWN_ROOM_VERSIONS,
+ EventTypes,
+ Membership,
+ RejectedReason,
+)
from synapse.api.errors import (
AuthError,
CodeMessageException,
@@ -922,6 +927,9 @@ class FederationHandler(BaseHandler):
joinee,
"join",
content,
+ params={
+ "ver": KNOWN_ROOM_VERSIONS,
+ },
)
# This shouldn't happen, because the RoomMemberHandler has a
@@ -1187,13 +1195,14 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
- content={},):
+ content={}, params=None):
origin, pdu = yield self.federation_client.make_membership_event(
target_hosts,
room_id,
user_id,
membership,
content,
+ params=params,
)
logger.debug("Got response to make_%s: %s", membership, pdu)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 289704b241..0e16bbe0ee 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -540,9 +540,7 @@ class RegistrationHandler(BaseHandler):
Do not accept registrations if monthly active user limits exceeded
and limiting is enabled
"""
- if self.hs.config.limit_usage_by_mau is True:
- current_mau = yield self.store.count_monthly_users()
- if current_mau >= self.hs.config.max_mau_value:
- raise RegistrationError(
- 403, "MAU Limit Exceeded", Codes.MAU_LIMIT_EXCEEDED
- )
+ try:
+ yield self.auth.check_auth_blocking()
+ except AuthError as e:
+ raise RegistrationError(e.code, str(e), e.errcode)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 7b7804d9b2..6a17c42238 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -21,9 +21,17 @@ import math
import string
from collections import OrderedDict
+from six import string_types
+
from twisted.internet import defer
-from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset
+from synapse.api.constants import (
+ DEFAULT_ROOM_VERSION,
+ KNOWN_ROOM_VERSIONS,
+ EventTypes,
+ JoinRules,
+ RoomCreationPreset,
+)
from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID
from synapse.util import stringutils
@@ -99,6 +107,21 @@ class RoomCreationHandler(BaseHandler):
if ratelimit:
yield self.ratelimit(requester)
+ room_version = config.get("room_version", DEFAULT_ROOM_VERSION)
+ if not isinstance(room_version, string_types):
+ raise SynapseError(
+ 400,
+ "room_version must be a string",
+ Codes.BAD_JSON,
+ )
+
+ if room_version not in KNOWN_ROOM_VERSIONS:
+ raise SynapseError(
+ 400,
+ "Your homeserver does not support this room version",
+ Codes.UNSUPPORTED_ROOM_VERSION,
+ )
+
if "room_alias_name" in config:
for wchar in string.whitespace:
if wchar in config["room_alias_name"]:
@@ -184,6 +207,9 @@ class RoomCreationHandler(BaseHandler):
creation_content = config.get("creation_content", {})
+ # override any attempt to set room versions via the creation_content
+ creation_content["room_version"] = room_version
+
room_member_handler = self.hs.get_room_member_handler()
yield self._send_events_for_new_room(
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index bf1aa29502..b3f5415aa6 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -439,7 +439,7 @@ class MatrixFederationHttpClient(object):
defer.returnValue(json.loads(body))
@defer.inlineCallbacks
- def get_json(self, destination, path, args={}, retry_on_dns_fail=True,
+ def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
timeout=None, ignore_backoff=False):
""" GETs some json from the given host homeserver and path
@@ -447,7 +447,7 @@ class MatrixFederationHttpClient(object):
destination (str): The remote server to send the HTTP request
to.
path (str): The HTTP path.
- args (dict): A dictionary used to create query strings, defaults to
+ args (dict|None): A dictionary used to create query strings, defaults to
None.
timeout (int): How long to try (in ms) the destination for before
giving up. None indicates no timeout and that the request will
@@ -702,6 +702,9 @@ def check_content_type_is_json(headers):
def encode_query_args(args):
+ if args is None:
+ return b""
+
encoded_args = {}
for k, vs in args.items():
if isinstance(vs, string_types):
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index bdb5eee4af..4830c68f35 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -44,8 +44,8 @@ class SlavedEventStore(EventFederationWorkerStore,
RoomMemberWorkerStore,
EventPushActionsWorkerStore,
StreamWorkerStore,
- EventsWorkerStore,
StateGroupWorkerStore,
+ EventsWorkerStore,
SignatureWorkerStore,
UserErasureWorkerStore,
BaseSlavedStore):
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 134e4a80f1..23b4a8d76d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -39,6 +39,7 @@ from .filtering import FilteringStore
from .group_server import GroupServerStore
from .keys import KeyStore
from .media_repository import MediaRepositoryStore
+from .monthly_active_users import MonthlyActiveUsersStore
from .openid import OpenIdStore
from .presence import PresenceStore, UserPresenceState
from .profile import ProfileStore
@@ -87,6 +88,7 @@ class DataStore(RoomMemberStore, RoomStore,
UserDirectoryStore,
GroupServerStore,
UserErasureStore,
+ MonthlyActiveUsersStore,
):
def __init__(self, db_conn, hs):
@@ -94,7 +96,6 @@ class DataStore(RoomMemberStore, RoomStore,
self._clock = hs.get_clock()
self.database_engine = hs.database_engine
- self.db_conn = db_conn
self._stream_id_gen = StreamIdGenerator(
db_conn, "events", "stream_ordering",
extra_tables=[("local_invites", "stream_id")]
@@ -267,31 +268,6 @@ class DataStore(RoomMemberStore, RoomStore,
return self.runInteraction("count_users", _count_users)
- def count_monthly_users(self):
- """Counts the number of users who used this homeserver in the last 30 days
-
- This method should be refactored with count_daily_users - the only
- reason not to is waiting on definition of mau
-
- Returns:
- Defered[int]
- """
- def _count_monthly_users(txn):
- thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
- sql = """
- SELECT COALESCE(count(*), 0) FROM (
- SELECT user_id FROM user_ips
- WHERE last_seen > ?
- GROUP BY user_id
- ) u
- """
-
- txn.execute(sql, (thirty_days_ago,))
- count, = txn.fetchone()
- return count
-
- return self.runInteraction("count_monthly_users", _count_monthly_users)
-
def count_r30_users(self):
"""
Counts the number of 30 day retained users, defined as:-
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index b8cefd43d6..2489527f2c 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -35,6 +35,7 @@ LAST_SEEN_GRANULARITY = 120 * 1000
class ClientIpStore(background_updates.BackgroundUpdateStore):
def __init__(self, db_conn, hs):
+
self.client_ip_last_seen = Cache(
name="client_ip_last_seen",
keylen=4,
@@ -74,6 +75,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
"before", "shutdown", self._update_client_ips_batch
)
+ @defer.inlineCallbacks
def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
now=None):
if not now:
@@ -84,7 +86,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
last_seen = self.client_ip_last_seen.get(key)
except KeyError:
last_seen = None
-
+ yield self.populate_monthly_active_users(user_id)
# Rate-limited inserts
if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
return
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index e8e5a0fe44..ce32e8fefd 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -485,9 +485,14 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
new_forward_extremeties=new_forward_extremeties,
)
persist_event_counter.inc(len(chunk))
- synapse.metrics.event_persisted_position.set(
- chunk[-1][0].internal_metadata.stream_ordering,
- )
+
+ if not backfilled:
+ # backfilled events have negative stream orderings, so we don't
+ # want to set the event_persisted_position to that.
+ synapse.metrics.event_persisted_position.set(
+ chunk[-1][0].internal_metadata.stream_ordering,
+ )
+
for event, context in chunk:
if context.app_service:
origin_type = "local"
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
new file mode 100644
index 0000000000..d47dcef3a0
--- /dev/null
+++ b/synapse/storage/monthly_active_users.py
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from twisted.internet import defer
+
+from synapse.util.caches.descriptors import cached
+
+from ._base import SQLBaseStore
+
+logger = logging.getLogger(__name__)
+
+# Number of msec of granularity to store the monthly_active_user timestamp
+# This means it is not necessary to update the table on every request
+LAST_SEEN_GRANULARITY = 60 * 60 * 1000
+
+
+class MonthlyActiveUsersStore(SQLBaseStore):
+ def __init__(self, dbconn, hs):
+ super(MonthlyActiveUsersStore, self).__init__(None, hs)
+ self._clock = hs.get_clock()
+ self.hs = hs
+ self.reserved_users = ()
+
+ @defer.inlineCallbacks
+ def initialise_reserved_users(self, threepids):
+ # TODO Why can't I do this in init?
+ store = self.hs.get_datastore()
+ reserved_user_list = []
+
+ # Do not add more reserved users than the total allowable number
+ for tp in threepids[:self.hs.config.max_mau_value]:
+ user_id = yield store.get_user_id_by_threepid(
+ tp["medium"], tp["address"]
+ )
+ if user_id:
+ self.upsert_monthly_active_user(user_id)
+ reserved_user_list.append(user_id)
+ else:
+ logger.warning(
+ "mau limit reserved threepid %s not found in db" % tp
+ )
+ self.reserved_users = tuple(reserved_user_list)
+
+ @defer.inlineCallbacks
+ def reap_monthly_active_users(self):
+ """
+ Cleans out monthly active user table to ensure that no stale
+ entries exist.
+
+ Returns:
+ Deferred[]
+ """
+ def _reap_users(txn):
+
+ thirty_days_ago = (
+ int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
+ )
+ # Purge stale users
+
+ # questionmarks is a hack to overcome sqlite not supporting
+ # tuples in 'WHERE IN %s'
+ questionmarks = '?' * len(self.reserved_users)
+ query_args = [thirty_days_ago]
+ query_args.extend(self.reserved_users)
+
+ sql = """
+ DELETE FROM monthly_active_users
+ WHERE timestamp < ?
+ AND user_id NOT IN ({})
+ """.format(','.join(questionmarks))
+
+ txn.execute(sql, query_args)
+
+ # If MAU user count still exceeds the MAU threshold, then delete on
+ # a least recently active basis.
+ # Note it is not possible to write this query using OFFSET due to
+ # incompatibilities in how sqlite and postgres support the feature.
+ # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
+ # While Postgres does not require 'LIMIT', but also does not support
+ # negative LIMIT values. So there is no way to write it that both can
+ # support
+ query_args = [self.hs.config.max_mau_value]
+ query_args.extend(self.reserved_users)
+ sql = """
+ DELETE FROM monthly_active_users
+ WHERE user_id NOT IN (
+ SELECT user_id FROM monthly_active_users
+ ORDER BY timestamp DESC
+ LIMIT ?
+ )
+ AND user_id NOT IN ({})
+ """.format(','.join(questionmarks))
+ txn.execute(sql, query_args)
+
+ yield self.runInteraction("reap_monthly_active_users", _reap_users)
+ # It seems poor to invalidate the whole cache, Postgres supports
+ # 'Returning' which would allow me to invalidate only the
+ # specific users, but sqlite has no way to do this and instead
+ # I would need to SELECT and the DELETE which without locking
+ # is racy.
+ # Have resolved to invalidate the whole cache for now and do
+ # something about it if and when the perf becomes significant
+ self._user_last_seen_monthly_active.invalidate_all()
+ self.get_monthly_active_count.invalidate_all()
+
+ @cached(num_args=0)
+ def get_monthly_active_count(self):
+ """Generates current count of monthly active users
+
+ Returns:
+ Defered[int]: Number of current monthly active users
+ """
+
+ def _count_users(txn):
+ sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
+
+ txn.execute(sql)
+ count, = txn.fetchone()
+ return count
+ return self.runInteraction("count_users", _count_users)
+
+ def upsert_monthly_active_user(self, user_id):
+ """
+ Updates or inserts monthly active user member
+ Arguments:
+ user_id (str): user to add/update
+ Deferred[bool]: True if a new entry was created, False if an
+ existing one was updated.
+ """
+ is_insert = self._simple_upsert(
+ desc="upsert_monthly_active_user",
+ table="monthly_active_users",
+ keyvalues={
+ "user_id": user_id,
+ },
+ values={
+ "timestamp": int(self._clock.time_msec()),
+ },
+ lock=False,
+ )
+ if is_insert:
+ self._user_last_seen_monthly_active.invalidate((user_id,))
+ self.get_monthly_active_count.invalidate(())
+
+ @cached(num_args=1)
+ def _user_last_seen_monthly_active(self, user_id):
+ """
+ Checks if a given user is part of the monthly active user group
+ Arguments:
+ user_id (str): user to add/update
+ Return:
+ Deferred[int] : timestamp since last seen, None if never seen
+
+ """
+
+ return(self._simple_select_one_onecol(
+ table="monthly_active_users",
+ keyvalues={
+ "user_id": user_id,
+ },
+ retcol="timestamp",
+ allow_none=True,
+ desc="_user_last_seen_monthly_active",
+ ))
+
+ @defer.inlineCallbacks
+ def populate_monthly_active_users(self, user_id):
+ """Checks on the state of monthly active user limits and optionally
+ add the user to the monthly active tables
+
+ Args:
+ user_id(str): the user_id to query
+ """
+ if self.hs.config.limit_usage_by_mau:
+ last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
+ now = self.hs.get_clock().time_msec()
+
+ # We want to reduce to the total number of db writes, and are happy
+ # to trade accuracy of timestamp in order to lighten load. This means
+ # We always insert new users (where MAU threshold has not been reached),
+ # but only update if we have not previously seen the user for
+ # LAST_SEEN_GRANULARITY ms
+ if last_seen_timestamp is None:
+ count = yield self.get_monthly_active_count()
+ if count < self.hs.config.max_mau_value:
+ yield self.upsert_monthly_active_user(user_id)
+ elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
+ yield self.upsert_monthly_active_user(user_id)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index b290f834b3..b364719312 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 50
+SCHEMA_VERSION = 51
dir_path = os.path.abspath(os.path.dirname(__file__))
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
new file mode 100644
index 0000000000..c9d537d5a3
--- /dev/null
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -0,0 +1,27 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- a table of monthly active users, for use where blocking based on mau limits
+CREATE TABLE monthly_active_users (
+ user_id TEXT NOT NULL,
+ -- Last time we saw the user. Not guaranteed to be accurate due to rate limiting
+ -- on updates, Granularity of updates governed by
+ -- synapse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
+ -- Measured in ms since epoch.
+ timestamp BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
+CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index b27b3ae144..17b14d464b 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -21,15 +21,17 @@ from six.moves import range
from twisted.internet import defer
+from synapse.api.constants import EventTypes
+from synapse.api.errors import NotFoundError
+from synapse.storage._base import SQLBaseStore
from synapse.storage.background_updates import BackgroundUpdateStore
from synapse.storage.engines import PostgresEngine
+from synapse.storage.events_worker import EventsWorkerStore
from synapse.util.caches import get_cache_factor_for, intern_string
from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.caches.dictionary_cache import DictionaryCache
from synapse.util.stringutils import to_ascii
-from ._base import SQLBaseStore
-
logger = logging.getLogger(__name__)
@@ -46,7 +48,8 @@ class _GetStateGroupDelta(namedtuple("_GetStateGroupDelta", ("prev_group", "delt
return len(self.delta_ids) if self.delta_ids else 0
-class StateGroupWorkerStore(SQLBaseStore):
+# this inherits from EventsWorkerStore because it calls self.get_events
+class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
"""The parts of StateGroupStore that can be called from workers.
"""
@@ -61,6 +64,30 @@ class StateGroupWorkerStore(SQLBaseStore):
"*stateGroupCache*", 500000 * get_cache_factor_for("stateGroupCache")
)
+ @defer.inlineCallbacks
+ def get_room_version(self, room_id):
+ """Get the room_version of a given room
+
+ Args:
+ room_id (str)
+
+ Returns:
+ Deferred[str]
+
+ Raises:
+ NotFoundError if the room is unknown
+ """
+ # for now we do this by looking at the create event. We may want to cache this
+ # more intelligently in future.
+ state_ids = yield self.get_current_state_ids(room_id)
+ create_id = state_ids.get((EventTypes.Create, ""))
+
+ if not create_id:
+ raise NotFoundError("Unknown room")
+
+ create_event = yield self.get_event(create_id)
+ defer.returnValue(create_event.content.get("room_version", "1"))
+
@cached(max_entries=100000, iterable=True)
def get_current_state_ids(self, room_id):
"""Get the current state event ids for a room based on the
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index a82d737e71..5dc3398300 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -444,3 +444,28 @@ class AuthTestCase(unittest.TestCase):
self.assertEqual("Guest access token used for regular user", cm.exception.msg)
self.store.get_user_by_id.assert_called_with(USER_ID)
+
+ @defer.inlineCallbacks
+ def test_blocking_mau(self):
+ self.hs.config.limit_usage_by_mau = False
+ self.hs.config.max_mau_value = 50
+ lots_of_users = 100
+ small_number_of_users = 1
+
+ # Ensure no error thrown
+ yield self.auth.check_auth_blocking()
+
+ self.hs.config.limit_usage_by_mau = True
+
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(lots_of_users)
+ )
+
+ with self.assertRaises(AuthError):
+ yield self.auth.check_auth_blocking()
+
+ # Ensure does not throw an error
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(small_number_of_users)
+ )
+ yield self.auth.check_auth_blocking()
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 55eab9e9cf..8a9bf2d5fd 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -132,14 +132,14 @@ class AuthTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_mau_limits_exceeded(self):
self.hs.config.limit_usage_by_mau = True
- self.hs.get_datastore().count_monthly_users = Mock(
+ self.hs.get_datastore().get_monthly_active_count = Mock(
return_value=defer.succeed(self.large_number_of_users)
)
with self.assertRaises(AuthError):
yield self.auth_handler.get_access_token_for_user_id('user_a')
- self.hs.get_datastore().count_monthly_users = Mock(
+ self.hs.get_datastore().get_monthly_active_count = Mock(
return_value=defer.succeed(self.large_number_of_users)
)
with self.assertRaises(AuthError):
@@ -151,13 +151,13 @@ class AuthTestCase(unittest.TestCase):
def test_mau_limits_not_exceeded(self):
self.hs.config.limit_usage_by_mau = True
- self.hs.get_datastore().count_monthly_users = Mock(
+ self.hs.get_datastore().get_monthly_active_count = Mock(
return_value=defer.succeed(self.small_number_of_users)
)
# Ensure does not raise exception
yield self.auth_handler.get_access_token_for_user_id('user_a')
- self.hs.get_datastore().count_monthly_users = Mock(
+ self.hs.get_datastore().get_monthly_active_count = Mock(
return_value=defer.succeed(self.small_number_of_users)
)
yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 0937d71cf6..4ea59a58de 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -50,6 +50,10 @@ class RegistrationTestCase(unittest.TestCase):
self.hs.get_macaroon_generator = Mock(return_value=self.macaroon_generator)
self.hs.handlers = RegistrationHandlers(self.hs)
self.handler = self.hs.get_handlers().registration_handler
+ self.store = self.hs.get_datastore()
+ self.hs.config.max_mau_value = 50
+ self.lots_of_users = 100
+ self.small_number_of_users = 1
@defer.inlineCallbacks
def test_user_is_created_and_logged_in_if_doesnt_exist(self):
@@ -80,51 +84,43 @@ class RegistrationTestCase(unittest.TestCase):
self.assertEquals(result_token, 'secret')
@defer.inlineCallbacks
- def test_cannot_register_when_mau_limits_exceeded(self):
- local_part = "someone"
- display_name = "someone"
- requester = create_requester("@as:test")
- store = self.hs.get_datastore()
+ def test_mau_limits_when_disabled(self):
self.hs.config.limit_usage_by_mau = False
- self.hs.config.max_mau_value = 50
- lots_of_users = 100
- small_number_users = 1
-
- store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
-
# Ensure does not throw exception
- yield self.handler.get_or_create_user(requester, 'a', display_name)
+ yield self.handler.get_or_create_user("requester", 'a', "display_name")
+ @defer.inlineCallbacks
+ def test_get_or_create_user_mau_not_blocked(self):
self.hs.config.limit_usage_by_mau = True
-
- with self.assertRaises(RegistrationError):
- yield self.handler.get_or_create_user(requester, 'b', display_name)
-
- store.count_monthly_users = Mock(return_value=defer.succeed(small_number_users))
-
- self._macaroon_mock_generator("another_secret")
-
+ self.store.count_monthly_users = Mock(
+ return_value=defer.succeed(self.small_number_of_users)
+ )
# Ensure does not throw exception
- yield self.handler.get_or_create_user("@neil:matrix.org", 'c', "Neil")
-
- self._macaroon_mock_generator("another another secret")
- store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+ yield self.handler.get_or_create_user("@user:server", 'c', "User")
+ @defer.inlineCallbacks
+ def test_get_or_create_user_mau_blocked(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(self.lots_of_users)
+ )
with self.assertRaises(RegistrationError):
- yield self.handler.register(localpart=local_part)
+ yield self.handler.get_or_create_user("requester", 'b', "display_name")
- self._macaroon_mock_generator("another another secret")
- store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+ @defer.inlineCallbacks
+ def test_register_mau_blocked(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(self.lots_of_users)
+ )
+ with self.assertRaises(RegistrationError):
+ yield self.handler.register(localpart="local_part")
+ @defer.inlineCallbacks
+ def test_register_saml2_mau_blocked(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(self.lots_of_users)
+ )
with self.assertRaises(RegistrationError):
- yield self.handler.register_saml2(local_part)
-
- def _macaroon_mock_generator(self, secret):
- """
- Reset macaroon generator in the case where the test creates multiple users
- """
- macaroon_generator = Mock(
- generate_access_token=Mock(return_value=secret))
- self.hs.get_macaroon_generator = Mock(return_value=macaroon_generator)
- self.hs.handlers = RegistrationHandlers(self.hs)
- self.handler = self.hs.get_handlers().registration_handler
+ yield self.handler.register_saml2(localpart="local_part")
diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
deleted file mode 100644
index f19cb1265c..0000000000
--- a/tests/storage/test__init__.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from twisted.internet import defer
-
-import tests.utils
-
-
-class InitTestCase(tests.unittest.TestCase):
- def __init__(self, *args, **kwargs):
- super(InitTestCase, self).__init__(*args, **kwargs)
- self.store = None # type: synapse.storage.DataStore
-
- @defer.inlineCallbacks
- def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
-
- hs.config.max_mau_value = 50
- hs.config.limit_usage_by_mau = True
- self.store = hs.get_datastore()
- self.clock = hs.get_clock()
-
- @defer.inlineCallbacks
- def test_count_monthly_users(self):
- count = yield self.store.count_monthly_users()
- self.assertEqual(0, count)
-
- yield self._insert_user_ips("@user:server1")
- yield self._insert_user_ips("@user:server2")
-
- count = yield self.store.count_monthly_users()
- self.assertEqual(2, count)
-
- @defer.inlineCallbacks
- def _insert_user_ips(self, user):
- """
- Helper function to populate user_ips without using batch insertion infra
- args:
- user (str): specify username i.e. @user:server.com
- """
- yield self.store._simple_upsert(
- table="user_ips",
- keyvalues={
- "user_id": user,
- "access_token": "access_token",
- "ip": "ip",
- "user_agent": "user_agent",
- "device_id": "device_id",
- },
- values={
- "last_seen": self.clock.time_msec(),
- }
- )
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index bd6fda6cb1..7a58c6eb24 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from mock import Mock
from twisted.internet import defer
@@ -27,9 +28,9 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
- hs = yield tests.utils.setup_test_homeserver()
- self.store = hs.get_datastore()
- self.clock = hs.get_clock()
+ self.hs = yield tests.utils.setup_test_homeserver()
+ self.store = self.hs.get_datastore()
+ self.clock = self.hs.get_clock()
@defer.inlineCallbacks
def test_insert_new_client_ip(self):
@@ -54,3 +55,62 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
},
r
)
+
+ @defer.inlineCallbacks
+ def test_disabled_monthly_active_user(self):
+ self.hs.config.limit_usage_by_mau = False
+ self.hs.config.max_mau_value = 50
+ user_id = "@user:server"
+ yield self.store.insert_client_ip(
+ user_id, "access_token", "ip", "user_agent", "device_id",
+ )
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertFalse(active)
+
+ @defer.inlineCallbacks
+ def test_adding_monthly_active_user_when_full(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.hs.config.max_mau_value = 50
+ lots_of_users = 100
+ user_id = "@user:server"
+
+ self.store.get_monthly_active_count = Mock(
+ return_value=defer.succeed(lots_of_users)
+ )
+ yield self.store.insert_client_ip(
+ user_id, "access_token", "ip", "user_agent", "device_id",
+ )
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertFalse(active)
+
+ @defer.inlineCallbacks
+ def test_adding_monthly_active_user_when_space(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.hs.config.max_mau_value = 50
+ user_id = "@user:server"
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertFalse(active)
+
+ yield self.store.insert_client_ip(
+ user_id, "access_token", "ip", "user_agent", "device_id",
+ )
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertTrue(active)
+
+ @defer.inlineCallbacks
+ def test_updating_monthly_active_user_when_space(self):
+ self.hs.config.limit_usage_by_mau = True
+ self.hs.config.max_mau_value = 50
+ user_id = "@user:server"
+
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertFalse(active)
+
+ yield self.store.insert_client_ip(
+ user_id, "access_token", "ip", "user_agent", "device_id",
+ )
+ yield self.store.insert_client_ip(
+ user_id, "access_token", "ip", "user_agent", "device_id",
+ )
+ active = yield self.store._user_last_seen_monthly_active(user_id)
+ self.assertTrue(active)
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
new file mode 100644
index 0000000000..cbd480cd42
--- /dev/null
+++ b/tests/storage/test_monthly_active_users.py
@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import tests.unittest
+import tests.utils
+from tests.utils import setup_test_homeserver
+
+FORTY_DAYS = 40 * 24 * 60 * 60
+
+
+class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
+ def __init__(self, *args, **kwargs):
+ super(MonthlyActiveUsersTestCase, self).__init__(*args, **kwargs)
+
+ @defer.inlineCallbacks
+ def setUp(self):
+ self.hs = yield setup_test_homeserver()
+ self.store = self.hs.get_datastore()
+
+ @defer.inlineCallbacks
+ def test_initialise_reserved_users(self):
+
+ user1 = "@user1:server"
+ user1_email = "user1@matrix.org"
+ user2 = "@user2:server"
+ user2_email = "user2@matrix.org"
+ threepids = [
+ {'medium': 'email', 'address': user1_email},
+ {'medium': 'email', 'address': user2_email}
+ ]
+ user_num = len(threepids)
+
+ yield self.store.register(
+ user_id=user1,
+ token="123",
+ password_hash=None)
+
+ yield self.store.register(
+ user_id=user2,
+ token="456",
+ password_hash=None)
+
+ now = int(self.hs.get_clock().time_msec())
+ yield self.store.user_add_threepid(user1, "email", user1_email, now, now)
+ yield self.store.user_add_threepid(user2, "email", user2_email, now, now)
+ yield self.store.initialise_reserved_users(threepids)
+
+ active_count = yield self.store.get_monthly_active_count()
+
+ # Test total counts
+ self.assertEquals(active_count, user_num)
+
+ # Test user is marked as active
+
+ timestamp = yield self.store._user_last_seen_monthly_active(user1)
+ self.assertTrue(timestamp)
+ timestamp = yield self.store._user_last_seen_monthly_active(user2)
+ self.assertTrue(timestamp)
+
+ # Test that users are never removed from the db.
+ self.hs.config.max_mau_value = 0
+
+ self.hs.get_clock().advance_time(FORTY_DAYS)
+
+ yield self.store.reap_monthly_active_users()
+
+ active_count = yield self.store.get_monthly_active_count()
+ self.assertEquals(active_count, user_num)
+
+ @defer.inlineCallbacks
+ def test_can_insert_and_count_mau(self):
+ count = yield self.store.get_monthly_active_count()
+ self.assertEqual(0, count)
+
+ yield self.store.upsert_monthly_active_user("@user:server")
+ count = yield self.store.get_monthly_active_count()
+
+ self.assertEqual(1, count)
+
+ @defer.inlineCallbacks
+ def test__user_last_seen_monthly_active(self):
+ user_id1 = "@user1:server"
+ user_id2 = "@user2:server"
+ user_id3 = "@user3:server"
+ result = yield self.store._user_last_seen_monthly_active(user_id1)
+ self.assertFalse(result == 0)
+ yield self.store.upsert_monthly_active_user(user_id1)
+ yield self.store.upsert_monthly_active_user(user_id2)
+ result = yield self.store._user_last_seen_monthly_active(user_id1)
+ self.assertTrue(result > 0)
+ result = yield self.store._user_last_seen_monthly_active(user_id3)
+ self.assertFalse(result == 0)
+
+ @defer.inlineCallbacks
+ def test_reap_monthly_active_users(self):
+ self.hs.config.max_mau_value = 5
+ initial_users = 10
+ for i in range(initial_users):
+ yield self.store.upsert_monthly_active_user("@user%d:server" % i)
+ count = yield self.store.get_monthly_active_count()
+ self.assertTrue(count, initial_users)
+ yield self.store.reap_monthly_active_users()
+ count = yield self.store.get_monthly_active_count()
+ self.assertEquals(count, initial_users - self.hs.config.max_mau_value)
+
+ self.hs.get_clock().advance_time(FORTY_DAYS)
+ yield self.store.reap_monthly_active_users()
+ count = yield self.store.get_monthly_active_count()
+ self.assertEquals(count, 0)
diff --git a/tests/utils.py b/tests/utils.py
index 9bff3ff3b9..151db4b890 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -73,6 +73,13 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
config.block_events_without_consent_error = None
config.media_storage_providers = []
config.auto_join_rooms = []
+ config.limit_usage_by_mau = False
+ config.max_mau_value = 50
+ config.mau_limits_reserved_threepids = []
+
+ # we need a sane default_room_version, otherwise attempts to create rooms will
+ # fail.
+ config.default_room_version = "1"
# disable user directory updates, because they get done in the
# background, which upsets the test runner.
|
