diff --git a/.gitignore b/.gitignore
index 9f42a7568f..1718185384 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@ media_store/
build/
venv/
venv*/
+*venv/
localhost-800*/
static/client/register/register_config.js
diff --git a/.travis.yml b/.travis.yml
index 11c76db2e5..ebc972ed24 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,9 +8,6 @@ before_script:
- git remote set-branches --add origin develop
- git fetch origin develop
-services:
- - postgresql
-
matrix:
fast_finish: true
include:
@@ -25,6 +22,8 @@ matrix:
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+ services:
+ - postgresql
- python: 3.6
env: TOX_ENV=py36
diff --git a/CHANGES.md b/CHANGES.md
index 59feb6b1a7..ee864c3c63 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,18 @@
+Synapse 0.33.4 (2018-09-07)
+===========================
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
+Synapse 0.33.4rc2 (2018-09-06)
+==============================
+
+Pull in security fixes from v0.33.3.1
+
+
Synapse 0.33.3.1 (2018-09-06)
=============================
@@ -7,11 +22,56 @@ SECURITY FIXES
- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+
Internal Changes
----------------
- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+Synapse 0.33.4rc1 (2018-09-04)
+==============================
+
+Features
+--------
+
+- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
+- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
+- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
+- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
+- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
+
+
+Bugfixes
+--------
+
+- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
+- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
+- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
+- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
+- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
+- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
+- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
+- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
+- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
+
+Internal Changes
+----------------
+
+- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
+- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
+- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
+- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
+- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
+- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
+- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
+- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
+- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
+- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
+- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
+- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
+
+
Synapse 0.33.3 (2018-08-22)
===========================
diff --git a/README.rst b/README.rst
index d6f34ba9d1..2471619706 100644
--- a/README.rst
+++ b/README.rst
@@ -742,6 +742,18 @@ so an example nginx configuration might look like::
}
}
+and an example apache configuration may look like::
+
+ <VirtualHost *:443>
+ SSLEngine on
+ ServerName matrix.example.com;
+
+ <Location /_matrix>
+ ProxyPass http://127.0.0.1:8008/_matrix nocanon
+ ProxyPassReverse http://127.0.0.1:8008/_matrix
+ </Location>
+ </VirtualHost>
+
You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
recorded correctly.
diff --git a/changelog.d/3378.misc b/changelog.d/3378.misc
deleted file mode 100644
index 8f88f88e69..0000000000
--- a/changelog.d/3378.misc
+++ /dev/null
@@ -1 +0,0 @@
-Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.
diff --git a/changelog.d/3659.feature b/changelog.d/3659.feature
deleted file mode 100644
index a5b4821c09..0000000000
--- a/changelog.d/3659.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support profile API endpoints on workers
diff --git a/changelog.d/3673.misc b/changelog.d/3673.misc
deleted file mode 100644
index d672111fb9..0000000000
--- a/changelog.d/3673.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor state module to support multiple room versions
diff --git a/changelog.d/3680.feature b/changelog.d/3680.feature
deleted file mode 100644
index 4edaaf76a8..0000000000
--- a/changelog.d/3680.feature
+++ /dev/null
@@ -1 +0,0 @@
-Server notices for resource limit blocking
diff --git a/changelog.d/3722.bugfix b/changelog.d/3722.bugfix
deleted file mode 100644
index 16cbaf76cb..0000000000
--- a/changelog.d/3722.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues
diff --git a/changelog.d/3724.feature b/changelog.d/3724.feature
deleted file mode 100644
index 1b374ccf47..0000000000
--- a/changelog.d/3724.feature
+++ /dev/null
@@ -1 +0,0 @@
-Allow guests to use /rooms/:roomId/event/:eventId
diff --git a/changelog.d/3725.misc b/changelog.d/3725.misc
deleted file mode 100644
index 91ab9d7137..0000000000
--- a/changelog.d/3725.misc
+++ /dev/null
@@ -1 +0,0 @@
-The synapse.storage module has been ported to Python 3.
diff --git a/changelog.d/3726.misc b/changelog.d/3726.misc
deleted file mode 100644
index c4f66ec998..0000000000
--- a/changelog.d/3726.misc
+++ /dev/null
@@ -1 +0,0 @@
-Split the state_group_cache into member and non-member state events (and so speed up LL /sync)
diff --git a/changelog.d/3727.misc b/changelog.d/3727.misc
deleted file mode 100644
index 0b83220d90..0000000000
--- a/changelog.d/3727.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log failure to authenticate remote servers as warnings (without stack traces)
diff --git a/changelog.d/3730.misc b/changelog.d/3730.misc
deleted file mode 100644
index b1ea84f732..0000000000
--- a/changelog.d/3730.misc
+++ /dev/null
@@ -1 +0,0 @@
-The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.
diff --git a/changelog.d/3734.misc b/changelog.d/3734.misc
deleted file mode 100644
index 4f6e4b3848..0000000000
--- a/changelog.d/3734.misc
+++ /dev/null
@@ -1 +0,0 @@
-Reference the need for an HTTP replication port when using the federation_reader worker
diff --git a/changelog.d/3735.misc b/changelog.d/3735.misc
deleted file mode 100644
index f17004be71..0000000000
--- a/changelog.d/3735.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix minor spelling error in federation client documentation.
diff --git a/changelog.d/3737.misc b/changelog.d/3737.misc
deleted file mode 100644
index 0361da4ebc..0000000000
--- a/changelog.d/3737.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove redundant state resolution function
diff --git a/changelog.d/3740.misc b/changelog.d/3740.misc
deleted file mode 100644
index 4dcb7fb5de..0000000000
--- a/changelog.d/3740.misc
+++ /dev/null
@@ -1 +0,0 @@
-The test suite now passes on PostgreSQL.
diff --git a/changelog.d/3746.misc b/changelog.d/3746.misc
deleted file mode 100644
index fc00ee773a..0000000000
--- a/changelog.d/3746.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix MAU cache invalidation due to missing yield
diff --git a/changelog.d/3747.bugfix b/changelog.d/3747.bugfix
deleted file mode 100644
index c41e2a1213..0000000000
--- a/changelog.d/3747.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where we resent "limit exceeded" server notices repeatedly
diff --git a/changelog.d/3749.feature b/changelog.d/3749.feature
deleted file mode 100644
index 9f8837b106..0000000000
--- a/changelog.d/3749.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add mau_trial_days config param, so that users only get counted as MAU after N days.
diff --git a/changelog.d/3751.feature b/changelog.d/3751.feature
deleted file mode 100644
index dc9742b15b..0000000000
--- a/changelog.d/3751.feature
+++ /dev/null
@@ -1 +0,0 @@
-Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).
diff --git a/changelog.d/3753.bugfix b/changelog.d/3753.bugfix
deleted file mode 100644
index b4301267df..0000000000
--- a/changelog.d/3753.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices
diff --git a/changelog.d/3754.bugfix b/changelog.d/3754.bugfix
deleted file mode 100644
index 6e3ec80194..0000000000
--- a/changelog.d/3754.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic
diff --git a/changelog.d/3755.bugfix b/changelog.d/3755.bugfix
deleted file mode 100644
index 6a1f83f0ce..0000000000
--- a/changelog.d/3755.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
diff --git a/changelog.d/3756.bugfix b/changelog.d/3756.bugfix
deleted file mode 100644
index 6a1f83f0ce..0000000000
--- a/changelog.d/3756.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
diff --git a/changelog.d/3758.bugfix b/changelog.d/3758.bugfix
deleted file mode 100644
index 862739bfe8..0000000000
--- a/changelog.d/3758.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.
diff --git a/changelog.d/3760.bugfix b/changelog.d/3760.bugfix
deleted file mode 100644
index ce61fb8a2b..0000000000
--- a/changelog.d/3760.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don't return non-LL-member state in incremental sync state blocks
diff --git a/changelog.d/3764.misc b/changelog.d/3764.misc
deleted file mode 100644
index f3614f1982..0000000000
--- a/changelog.d/3764.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make sure that we close db connections opened during init
\ No newline at end of file
diff --git a/changelog.d/3768.bugfix b/changelog.d/3768.bugfix
deleted file mode 100644
index a039a7fa68..0000000000
--- a/changelog.d/3768.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in sending presence over federation
diff --git a/changelog.d/3777.bugfix b/changelog.d/3777.bugfix
deleted file mode 100644
index 46efc543a7..0000000000
--- a/changelog.d/3777.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where preserved threepid user comes to sign up and server is mau blocked
diff --git a/changelog.d/3788.bugfix b/changelog.d/3788.bugfix
new file mode 100644
index 0000000000..72316fb881
--- /dev/null
+++ b/changelog.d/3788.bugfix
@@ -0,0 +1 @@
+Remove connection ID for replication prometheus metrics, as it creates a large number of new series.
diff --git a/changelog.d/3795.misc b/changelog.d/3795.misc
new file mode 100644
index 0000000000..9f64ee5e2b
--- /dev/null
+++ b/changelog.d/3795.misc
@@ -0,0 +1 @@
+Make /sync slightly faster by avoiding needless copies
diff --git a/changelog.d/3800.bugfix b/changelog.d/3800.bugfix
new file mode 100644
index 0000000000..6b2e18b4a6
--- /dev/null
+++ b/changelog.d/3800.bugfix
@@ -0,0 +1 @@
+guest users should not be part of mau total
diff --git a/changelog.d/3803.misc b/changelog.d/3803.misc
new file mode 100644
index 0000000000..2b60653c29
--- /dev/null
+++ b/changelog.d/3803.misc
@@ -0,0 +1 @@
+handlers/ is now ported to Python 3.
diff --git a/changelog.d/3804.bugfix b/changelog.d/3804.bugfix
new file mode 100644
index 0000000000..a0cef20e3f
--- /dev/null
+++ b/changelog.d/3804.bugfix
@@ -0,0 +1 @@
+Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.
diff --git a/changelog.d/3805.misc b/changelog.d/3805.misc
new file mode 100644
index 0000000000..257feeb071
--- /dev/null
+++ b/changelog.d/3805.misc
@@ -0,0 +1 @@
+Limit the number of PDUs/EDUs per federation transaction
diff --git a/changelog.d/3806.misc b/changelog.d/3806.misc
new file mode 100644
index 0000000000..3c722eef2d
--- /dev/null
+++ b/changelog.d/3806.misc
@@ -0,0 +1 @@
+Only start postgres instance for postgres tests on Travis CI
diff --git a/changelog.d/3808.misc b/changelog.d/3808.misc
new file mode 100644
index 0000000000..e5e1cd9e0e
--- /dev/null
+++ b/changelog.d/3808.misc
@@ -0,0 +1 @@
+tests/ is now ported to Python 3.
diff --git a/changelog.d/3810.bugfix b/changelog.d/3810.bugfix
new file mode 100644
index 0000000000..2b938a81ae
--- /dev/null
+++ b/changelog.d/3810.bugfix
@@ -0,0 +1 @@
+Fix existing room tags not coming down sync when joining a room
diff --git a/changelog.d/3822.misc b/changelog.d/3822.misc
new file mode 100644
index 0000000000..5250f31896
--- /dev/null
+++ b/changelog.d/3822.misc
@@ -0,0 +1 @@
+crypto/ is now ported to Python 3.
diff --git a/changelog.d/3823.misc b/changelog.d/3823.misc
new file mode 100644
index 0000000000..0da491ddaa
--- /dev/null
+++ b/changelog.d/3823.misc
@@ -0,0 +1 @@
+rest/ is now ported to Python 3.
diff --git a/changelog.d/3824.bugfix b/changelog.d/3824.bugfix
new file mode 100644
index 0000000000..99f199dcc6
--- /dev/null
+++ b/changelog.d/3824.bugfix
@@ -0,0 +1 @@
+Fix jwt import check
\ No newline at end of file
diff --git a/changelog.d/3827.misc b/changelog.d/3827.misc
new file mode 100644
index 0000000000..bc294706cf
--- /dev/null
+++ b/changelog.d/3827.misc
@@ -0,0 +1 @@
+speed up lazy loading by 2-3x
diff --git a/changelog.d/3834.misc b/changelog.d/3834.misc
new file mode 100644
index 0000000000..8902f8fba7
--- /dev/null
+++ b/changelog.d/3834.misc
@@ -0,0 +1 @@
+Improved Dockerfile to remove build requirements after building reducing the image size.
diff --git a/changelog.d/3840.misc b/changelog.d/3840.misc
new file mode 100644
index 0000000000..b9585ae9be
--- /dev/null
+++ b/changelog.d/3840.misc
@@ -0,0 +1 @@
+Disable lazy loading for incremental syncs for now
diff --git a/changelog.d/3845.bugfix b/changelog.d/3845.bugfix
new file mode 100644
index 0000000000..5b7e8f1934
--- /dev/null
+++ b/changelog.d/3845.bugfix
@@ -0,0 +1 @@
+Fix outbound requests occasionally wedging, which can result in federation breaking between servers.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 777976217d..20d3fe3bd8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,6 +1,8 @@
FROM docker.io/python:2-alpine3.8
-RUN apk add --no-cache --virtual .nacl_deps \
+COPY . /synapse
+
+RUN apk add --no-cache --virtual .build_deps \
build-base \
libffi-dev \
libjpeg-turbo-dev \
@@ -8,13 +10,16 @@ RUN apk add --no-cache --virtual .nacl_deps \
libxslt-dev \
linux-headers \
postgresql-dev \
- su-exec \
- zlib-dev
-
-COPY . /synapse
-
-# A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
+ zlib-dev \
+ && cd /synapse \
+ && apk add --no-cache --virtual .runtime_deps \
+ libffi \
+ libjpeg-turbo \
+ libressl \
+ libxslt \
+ libpq \
+ zlib \
+ su-exec \
&& pip install --upgrade \
lxml \
pip \
@@ -26,8 +31,9 @@ RUN cd /synapse \
&& rm -rf \
setup.cfg \
setup.py \
- synapse
-
+ synapse \
+ && apk del .build_deps
+
VOLUME ["/data"]
EXPOSE 8008/tcp 8448/tcp
diff --git a/synapse/__init__.py b/synapse/__init__.py
index e395a0a9ee..65a2b894cc 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
""" This is a reference implementation of a Matrix home server.
"""
-__version__ = "0.33.3.1"
+__version__ = "0.33.4"
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 2fd9c48abf..b8d5690f2b 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -21,7 +21,7 @@ from .consent_config import ConsentConfig
from .database import DatabaseConfig
from .emailconfig import EmailConfig
from .groups import GroupsConfig
-from .jwt import JWTConfig
+from .jwt_config import JWTConfig
from .key import KeyConfig
from .logger import LoggingConfig
from .metrics import MetricsConfig
diff --git a/synapse/config/jwt.py b/synapse/config/jwt_config.py
index 51e7f7e003..51e7f7e003 100644
--- a/synapse/config/jwt.py
+++ b/synapse/config/jwt_config.py
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 1a391adec1..02b76dfcfb 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -123,6 +123,6 @@ class ClientTLSOptionsFactory(object):
def get_options(self, host):
return ClientTLSOptions(
- host.decode('utf-8'),
+ host,
CertificateOptions(verify=False).getContext()
)
diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index e94400b8e2..57d4665e84 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -50,7 +50,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
defer.returnValue((server_response, server_certificate))
except SynapseKeyClientError as e:
logger.warn("Error getting key for %r: %s", server_name, e)
- if e.status.startswith("4"):
+ if e.status.startswith(b"4"):
# Don't retry for 4xx responses.
raise IOError("Cannot get key for %r" % server_name)
except (ConnectError, DomainError) as e:
@@ -82,6 +82,12 @@ class SynapseKeyClientProtocol(HTTPClient):
self._peer = self.transport.getPeer()
logger.debug("Connected to %s", self._peer)
+ if not isinstance(self.path, bytes):
+ self.path = self.path.encode('ascii')
+
+ if not isinstance(self.host, bytes):
+ self.host = self.host.encode('ascii')
+
self.sendCommand(b"GET", self.path)
if self.host:
self.sendHeader(b"Host", self.host)
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index a6b31e8b5b..d89f94c219 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -16,9 +16,10 @@
import hashlib
import logging
-import urllib
from collections import namedtuple
+from six.moves import urllib
+
from signedjson.key import (
decode_verify_key_bytes,
encode_verify_key_base64,
@@ -444,7 +445,7 @@ class Keyring(object):
# an incoming request.
query_response = yield self.client.post_json(
destination=perspective_name,
- path=b"/_matrix/key/v2/query",
+ path="/_matrix/key/v2/query",
data={
u"server_keys": {
server_name: {
@@ -525,8 +526,8 @@ class Keyring(object):
(response, tls_certificate) = yield fetch_server_key(
server_name, self.hs.tls_client_options_factory,
- path=(b"/_matrix/key/v2/server/%s" % (
- urllib.quote(requested_key_id),
+ path=("/_matrix/key/v2/server/%s" % (
+ urllib.parse.quote(requested_key_id),
)).encode("ascii"),
)
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 94d7423d01..8cbf8c4f7f 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -463,7 +463,19 @@ class TransactionQueue(object):
# pending_transactions flag.
pending_pdus = self.pending_pdus_by_dest.pop(destination, [])
+
+ # We can only include at most 50 PDUs per transactions
+ pending_pdus, leftover_pdus = pending_pdus[:50], pending_pdus[50:]
+ if leftover_pdus:
+ self.pending_pdus_by_dest[destination] = leftover_pdus
+
pending_edus = self.pending_edus_by_dest.pop(destination, [])
+
+ # We can only include at most 100 EDUs per transactions
+ pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:]
+ if leftover_edus:
+ self.pending_edus_by_dest[destination] = leftover_edus
+
pending_presence = self.pending_presence_by_dest.pop(destination, {})
pending_edus.extend(
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 4a81bd2ba9..2a5eab124f 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -895,22 +895,24 @@ class AuthHandler(BaseHandler):
Args:
password (unicode): Password to hash.
- stored_hash (unicode): Expected hash value.
+ stored_hash (bytes): Expected hash value.
Returns:
Deferred(bool): Whether self.hash(password) == stored_hash.
"""
-
def _do_validate_hash():
# Normalise the Unicode in the password
pw = unicodedata.normalize("NFKC", password)
return bcrypt.checkpw(
pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
- stored_hash.encode('utf8')
+ stored_hash
)
if stored_hash:
+ if not isinstance(stored_hash, bytes):
+ stored_hash = stored_hash.encode('ascii')
+
return make_deferred_yieldable(
threads.deferToThreadPool(
self.hs.get_reactor(),
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 5816bf8b4f..578e9250fb 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -330,7 +330,8 @@ class E2eKeysHandler(object):
(algorithm, key_id, ex_json, key)
)
else:
- new_keys.append((algorithm, key_id, encode_canonical_json(key)))
+ new_keys.append((
+ algorithm, key_id, encode_canonical_json(key).decode('ascii')))
yield self.store.add_e2e_one_time_keys(
user_id, device_id, time_now, new_keys
@@ -358,7 +359,7 @@ def _exception_to_failure(e):
# Note that some Exceptions (notably twisted's ResponseFailed etc) don't
# give a string for e.message, which json then fails to serialize.
return {
- "status": 503, "message": str(e.message),
+ "status": 503, "message": str(e),
}
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 3fa7a98445..0c68e8a472 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -594,7 +594,7 @@ class FederationHandler(BaseHandler):
required_auth = set(
a_id
- for event in events + state_events.values() + auth_events.values()
+ for event in events + list(state_events.values()) + list(auth_events.values())
for a_id, _ in event.auth_events
)
auth_events.update({
@@ -802,7 +802,7 @@ class FederationHandler(BaseHandler):
)
continue
except NotRetryingDestination as e:
- logger.info(e.message)
+ logger.info(str(e))
continue
except FederationDeniedError as e:
logger.info(e)
@@ -1358,7 +1358,7 @@ class FederationHandler(BaseHandler):
)
if state_groups:
- _, state = state_groups.items().pop()
+ _, state = list(state_groups.items()).pop()
results = state
if event.is_state():
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 37e41afd61..38e1737ec9 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -162,7 +162,7 @@ class RoomListHandler(BaseHandler):
# Filter out rooms that we don't want to return
rooms_to_scan = [
r for r in sorted_rooms
- if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0
+ if r not in newly_unpublished and rooms_to_num_joined[r] > 0
]
total_room_count = len(rooms_to_scan)
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index c464adbd0b..0c1d52fd11 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -54,7 +54,7 @@ class SearchHandler(BaseHandler):
batch_token = None
if batch:
try:
- b = decode_base64(batch)
+ b = decode_base64(batch).decode('ascii')
batch_group, batch_group_key, batch_token = b.split("\n")
assert batch_group is not None
@@ -258,18 +258,18 @@ class SearchHandler(BaseHandler):
# it returns more from the same group (if applicable) rather
# than reverting to searching all results again.
if batch_group and batch_group_key:
- global_next_batch = encode_base64("%s\n%s\n%s" % (
+ global_next_batch = encode_base64(("%s\n%s\n%s" % (
batch_group, batch_group_key, pagination_token
- ))
+ )).encode('ascii'))
else:
- global_next_batch = encode_base64("%s\n%s\n%s" % (
+ global_next_batch = encode_base64(("%s\n%s\n%s" % (
"all", "", pagination_token
- ))
+ )).encode('ascii'))
for room_id, group in room_groups.items():
- group["next_batch"] = encode_base64("%s\n%s\n%s" % (
+ group["next_batch"] = encode_base64(("%s\n%s\n%s" % (
"room_id", room_id, pagination_token
- ))
+ )).encode('ascii'))
allowed_events.extend(room_events)
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ef20c2296c..23983a51ab 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -24,6 +24,7 @@ from twisted.internet import defer
from synapse.api.constants import EventTypes, Membership
from synapse.push.clientformat import format_push_rules_for_user
+from synapse.storage.roommember import MemberSummary
from synapse.types import RoomStreamToken
from synapse.util.async_helpers import concurrently_execute
from synapse.util.caches.expiringcache import ExpiringCache
@@ -525,6 +526,8 @@ class SyncHandler(object):
A deferred dict describing the room summary
"""
+ # FIXME: we could/should get this from room_stats when matthew/stats lands
+
# FIXME: this promulgates https://github.com/matrix-org/synapse/issues/3305
last_events, _ = yield self.store.get_recent_event_ids_for_room(
room_id, end_token=now_token.room_key, limit=1,
@@ -537,44 +540,54 @@ class SyncHandler(object):
last_event = last_events[-1]
state_ids = yield self.store.get_state_ids_for_event(
last_event.event_id, [
- (EventTypes.Member, None),
(EventTypes.Name, ''),
(EventTypes.CanonicalAlias, ''),
]
)
- member_ids = {
- state_key: event_id
- for (t, state_key), event_id in state_ids.iteritems()
- if t == EventTypes.Member
- }
+ # this is heavily cached, thus: fast.
+ details = yield self.store.get_room_summary(room_id)
+
name_id = state_ids.get((EventTypes.Name, ''))
canonical_alias_id = state_ids.get((EventTypes.CanonicalAlias, ''))
summary = {}
-
- # FIXME: it feels very heavy to load up every single membership event
- # just to calculate the counts.
- member_events = yield self.store.get_events(member_ids.values())
-
- joined_user_ids = []
- invited_user_ids = []
-
- for ev in member_events.values():
- if ev.content.get("membership") == Membership.JOIN:
- joined_user_ids.append(ev.state_key)
- elif ev.content.get("membership") == Membership.INVITE:
- invited_user_ids.append(ev.state_key)
+ empty_ms = MemberSummary([], 0)
# TODO: only send these when they change.
- summary["m.joined_member_count"] = len(joined_user_ids)
- summary["m.invited_member_count"] = len(invited_user_ids)
+ summary["m.joined_member_count"] = (
+ details.get(Membership.JOIN, empty_ms).count
+ )
+ summary["m.invited_member_count"] = (
+ details.get(Membership.INVITE, empty_ms).count
+ )
if name_id or canonical_alias_id:
defer.returnValue(summary)
- # FIXME: order by stream ordering, not alphabetic
+ joined_user_ids = [
+ r[0] for r in details.get(Membership.JOIN, empty_ms).members
+ ]
+ invited_user_ids = [
+ r[0] for r in details.get(Membership.INVITE, empty_ms).members
+ ]
+ gone_user_ids = (
+ [r[0] for r in details.get(Membership.LEAVE, empty_ms).members] +
+ [r[0] for r in details.get(Membership.BAN, empty_ms).members]
+ )
+
+ # FIXME: only build up a member_ids list for our heroes
+ member_ids = {}
+ for membership in (
+ Membership.JOIN,
+ Membership.INVITE,
+ Membership.LEAVE,
+ Membership.BAN
+ ):
+ for user_id, event_id in details.get(membership, empty_ms).members:
+ member_ids[user_id] = event_id
+ # FIXME: order by stream ordering rather than as returned by SQL
me = sync_config.user.to_string()
if (joined_user_ids or invited_user_ids):
summary['m.heroes'] = sorted(
@@ -586,7 +599,11 @@ class SyncHandler(object):
)[0:5]
else:
summary['m.heroes'] = sorted(
- [user_id for user_id in member_ids.keys() if user_id != me]
+ [
+ user_id
+ for user_id in gone_user_ids
+ if user_id != me
+ ]
)[0:5]
if not sync_config.filter_collection.lazy_load_members():
@@ -719,6 +736,26 @@ class SyncHandler(object):
lazy_load_members=lazy_load_members,
)
elif batch.limited:
+ state_at_timeline_start = yield self.store.get_state_ids_for_event(
+ batch.events[0].event_id, types=types,
+ filtered_types=filtered_types,
+ )
+
+ # for now, we disable LL for gappy syncs - see
+ # https://github.com/vector-im/riot-web/issues/7211#issuecomment-419976346
+ # N.B. this slows down incr syncs as we are now processing way
+ # more state in the server than if we were LLing.
+ #
+ # We still have to filter timeline_start to LL entries (above) in order
+ # for _calculate_state's LL logic to work, as we have to include LL
+ # members for timeline senders in case they weren't loaded in the initial
+ # sync. We do this by (counterintuitively) by filtering timeline_start
+ # members to just be ones which were timeline senders, which then ensures
+ # all of the rest get included in the state block (if we need to know
+ # about them).
+ types = None
+ filtered_types = None
+
state_at_previous_sync = yield self.get_state_at(
room_id, stream_position=since_token, types=types,
filtered_types=filtered_types,
@@ -729,24 +766,21 @@ class SyncHandler(object):
filtered_types=filtered_types,
)
- state_at_timeline_start = yield self.store.get_state_ids_for_event(
- batch.events[0].event_id, types=types,
- filtered_types=filtered_types,
- )
-
state_ids = _calculate_state(
timeline_contains=timeline_state,
timeline_start=state_at_timeline_start,
previous=state_at_previous_sync,
current=current_state_ids,
+ # we have to include LL members in case LL initial sync missed them
lazy_load_members=lazy_load_members,
)
else:
state_ids = {}
if lazy_load_members:
if types:
- # We're returning an incremental sync, with no "gap" since
- # the previous sync, so normally there would be no state to return
+ # We're returning an incremental sync, with no
+ # "gap" since the previous sync, so normally there would be
+ # no state to return.
# But we're lazy-loading, so the client might need some more
# member events to understand the events in this timeline.
# So we fish out all the member events corresponding to the
@@ -774,7 +808,7 @@ class SyncHandler(object):
logger.debug("filtering state from %r...", state_ids)
state_ids = {
t: event_id
- for t, event_id in state_ids.iteritems()
+ for t, event_id in iteritems(state_ids)
if cache.get(t[1]) != event_id
}
logger.debug("...to %r", state_ids)
@@ -1575,6 +1609,19 @@ class SyncHandler(object):
newly_joined_room=newly_joined,
)
+ # When we join the room (or the client requests full_state), we should
+ # send down any existing tags. Usually the user won't have tags in a
+ # newly joined room, unless either a) they've joined before or b) the
+ # tag was added by synapse e.g. for server notice rooms.
+ if full_state:
+ user_id = sync_result_builder.sync_config.user.to_string()
+ tags = yield self.store.get_tags_for_room(user_id, room_id)
+
+ # If there aren't any tags, don't send the empty tags list down
+ # sync
+ if not tags:
+ tags = None
+
account_data_events = []
if tags is not None:
account_data_events.append({
@@ -1603,10 +1650,24 @@ class SyncHandler(object):
)
summary = {}
+
+ # we include a summary in room responses when we're lazy loading
+ # members (as the client otherwise doesn't have enough info to form
+ # the name itself).
if (
sync_config.filter_collection.lazy_load_members() and
(
+ # we recalulate the summary:
+ # if there are membership changes in the timeline, or
+ # if membership has changed during a gappy sync, or
+ # if this is an initial sync.
any(ev.type == EventTypes.Member for ev in batch.events) or
+ (
+ # XXX: this may include false positives in the form of LL
+ # members which have snuck into state
+ batch.limited and
+ any(t == EventTypes.Member for (t, k) in state)
+ ) or
since_token is None
)
):
@@ -1636,6 +1697,16 @@ class SyncHandler(object):
unread_notifications["highlight_count"] = notifs["highlight_count"]
sync_result_builder.joined.append(room_sync)
+
+ if batch.limited:
+ user_id = sync_result_builder.sync_config.user.to_string()
+ logger.info(
+ "Incremental syncing room %s for user %s with %d state events" % (
+ room_id,
+ user_id,
+ len(state),
+ )
+ )
elif room_builder.rtype == "archived":
room_sync = ArchivedSyncResult(
room_id=room_id,
@@ -1729,17 +1800,17 @@ def _calculate_state(
event_id_to_key = {
e: key
for key, e in itertools.chain(
- timeline_contains.items(),
- previous.items(),
- timeline_start.items(),
- current.items(),
+ iteritems(timeline_contains),
+ iteritems(previous),
+ iteritems(timeline_start),
+ iteritems(current),
)
}
- c_ids = set(e for e in current.values())
- ts_ids = set(e for e in timeline_start.values())
- p_ids = set(e for e in previous.values())
- tc_ids = set(e for e in timeline_contains.values())
+ c_ids = set(e for e in itervalues(current))
+ ts_ids = set(e for e in itervalues(timeline_start))
+ p_ids = set(e for e in itervalues(previous))
+ tc_ids = set(e for e in itervalues(timeline_contains))
# If we are lazyloading room members, we explicitly add the membership events
# for the senders in the timeline into the state block returned by /sync,
@@ -1753,7 +1824,7 @@ def _calculate_state(
if lazy_load_members:
p_ids.difference_update(
- e for t, e in timeline_start.iteritems()
+ e for t, e in iteritems(timeline_start)
if t[0] == EventTypes.Member
)
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 6a1fc8ca55..f9a1fbf95d 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -280,7 +280,10 @@ class MatrixFederationHttpClient(object):
# :'(
# Update transactions table?
with logcontext.PreserveLoggingContext():
- body = yield treq.content(response)
+ body = yield self._timeout_deferred(
+ treq.content(response),
+ timeout,
+ )
raise HttpResponseException(
response.code, response.phrase, body
)
@@ -394,7 +397,10 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext():
- body = yield treq.json_content(response)
+ body = yield self._timeout_deferred(
+ treq.json_content(response),
+ timeout,
+ )
defer.returnValue(body)
@defer.inlineCallbacks
@@ -444,7 +450,10 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext():
- body = yield treq.json_content(response)
+ body = yield self._timeout_deferred(
+ treq.json_content(response),
+ timeout,
+ )
defer.returnValue(body)
@@ -496,7 +505,10 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext():
- body = yield treq.json_content(response)
+ body = yield self._timeout_deferred(
+ treq.json_content(response),
+ timeout,
+ )
defer.returnValue(body)
@@ -543,7 +555,10 @@ class MatrixFederationHttpClient(object):
check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext():
- body = yield treq.json_content(response)
+ body = yield self._timeout_deferred(
+ treq.json_content(response),
+ timeout,
+ )
defer.returnValue(body)
@@ -585,8 +600,10 @@ class MatrixFederationHttpClient(object):
try:
with logcontext.PreserveLoggingContext():
- length = yield _readBodyToFile(
- response, output_stream, max_size
+ length = yield self._timeout_deferred(
+ _readBodyToFile(
+ response, output_stream, max_size
+ ),
)
except Exception:
logger.exception("Failed to download body")
@@ -594,6 +611,27 @@ class MatrixFederationHttpClient(object):
defer.returnValue((length, headers))
+ def _timeout_deferred(self, deferred, timeout_ms=None):
+ """Times the deferred out after `timeout_ms` ms
+
+ Args:
+ deferred (Deferred)
+ timeout_ms (int|None): Timeout in milliseconds. If None defaults
+ to 60 seconds.
+
+ Returns:
+ Deferred
+ """
+
+ add_timeout_to_deferred(
+ deferred,
+ timeout_ms / 1000. if timeout_ms else 60,
+ self.hs.get_reactor(),
+ cancelled_to_request_timed_out_error,
+ )
+
+ return deferred
+
class _ReadBodyToFileProtocol(protocol.Protocol):
def __init__(self, stream, deferred, max_size):
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 6dd5179320..0d8de600cf 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -42,8 +42,8 @@ REQUIREMENTS = {
"Twisted>=17.1.0": ["twisted>=17.1.0"],
"treq>=15.1": ["treq>=15.1"],
- # We use crypto.get_elliptic_curve which is only supported in >=0.15
- "pyopenssl>=0.15": ["OpenSSL>=0.15"],
+ # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
+ "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],
"pyyaml": ["yaml"],
"pyasn1": ["pyasn1"],
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 74e892c104..5dc7b3fffc 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -590,9 +590,9 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
pending_commands = LaterGauge(
"synapse_replication_tcp_protocol_pending_commands",
"",
- ["name", "conn_id"],
+ ["name"],
lambda: {
- (p.name, p.conn_id): len(p.pending_commands) for p in connected_connections
+ (p.name,): len(p.pending_commands) for p in connected_connections
},
)
@@ -607,9 +607,9 @@ def transport_buffer_size(protocol):
transport_send_buffer = LaterGauge(
"synapse_replication_tcp_protocol_transport_send_buffer",
"",
- ["name", "conn_id"],
+ ["name"],
lambda: {
- (p.name, p.conn_id): transport_buffer_size(p) for p in connected_connections
+ (p.name,): transport_buffer_size(p) for p in connected_connections
},
)
@@ -632,9 +632,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
tcp_transport_kernel_send_buffer = LaterGauge(
"synapse_replication_tcp_protocol_transport_kernel_send_buffer",
"",
- ["name", "conn_id"],
+ ["name"],
lambda: {
- (p.name, p.conn_id): transport_kernel_read_buffer_size(p, False)
+ (p.name,): transport_kernel_read_buffer_size(p, False)
for p in connected_connections
},
)
@@ -643,9 +643,9 @@ tcp_transport_kernel_send_buffer = LaterGauge(
tcp_transport_kernel_read_buffer = LaterGauge(
"synapse_replication_tcp_protocol_transport_kernel_read_buffer",
"",
- ["name", "conn_id"],
+ ["name"],
lambda: {
- (p.name, p.conn_id): transport_kernel_read_buffer_size(p, True)
+ (p.name,): transport_kernel_read_buffer_size(p, True)
for p in connected_connections
},
)
@@ -654,9 +654,9 @@ tcp_transport_kernel_read_buffer = LaterGauge(
tcp_inbound_commands = LaterGauge(
"synapse_replication_tcp_protocol_inbound_commands",
"",
- ["command", "name", "conn_id"],
+ ["command", "name"],
lambda: {
- (k[0], p.name, p.conn_id): count
+ (k[0], p.name,): count
for p in connected_connections
for k, count in iteritems(p.inbound_commands_counter)
},
@@ -665,9 +665,9 @@ tcp_inbound_commands = LaterGauge(
tcp_outbound_commands = LaterGauge(
"synapse_replication_tcp_protocol_outbound_commands",
"",
- ["command", "name", "conn_id"],
+ ["command", "name"],
lambda: {
- (k[0], p.name, p.conn_id): count
+ (k[0], p.name,): count
for p in connected_connections
for k, count in iteritems(p.outbound_commands_counter)
},
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index ad536ab570..41534b8c2a 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -101,7 +101,7 @@ class UserRegisterServlet(ClientV1RestServlet):
nonce = self.hs.get_secrets().token_hex(64)
self.nonces[nonce] = int(self.reactor.seconds())
- return (200, {"nonce": nonce.encode('ascii')})
+ return (200, {"nonce": nonce})
@defer.inlineCallbacks
def on_POST(self, request):
@@ -164,7 +164,7 @@ class UserRegisterServlet(ClientV1RestServlet):
key=self.hs.config.registration_shared_secret.encode(),
digestmod=hashlib.sha1,
)
- want_mac.update(nonce)
+ want_mac.update(nonce.encode('utf8'))
want_mac.update(b"\x00")
want_mac.update(username)
want_mac.update(b"\x00")
@@ -173,7 +173,10 @@ class UserRegisterServlet(ClientV1RestServlet):
want_mac.update(b"admin" if admin else b"notadmin")
want_mac = want_mac.hexdigest()
- if not hmac.compare_digest(want_mac, got_mac.encode('ascii')):
+ if not hmac.compare_digest(
+ want_mac.encode('ascii'),
+ got_mac.encode('ascii')
+ ):
raise SynapseError(403, "HMAC incorrect")
# Reuse the parts of RegisterRestServlet to reduce code duplication
diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py
index 0f3a2e8b51..cd9b3bdbd1 100644
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/v1/events.py
@@ -45,20 +45,20 @@ class EventStreamRestServlet(ClientV1RestServlet):
is_guest = requester.is_guest
room_id = None
if is_guest:
- if "room_id" not in request.args:
+ if b"room_id" not in request.args:
raise SynapseError(400, "Guest users must specify room_id param")
- if "room_id" in request.args:
- room_id = request.args["room_id"][0]
+ if b"room_id" in request.args:
+ room_id = request.args[b"room_id"][0].decode('ascii')
pagin_config = PaginationConfig.from_request(request)
timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS
- if "timeout" in request.args:
+ if b"timeout" in request.args:
try:
- timeout = int(request.args["timeout"][0])
+ timeout = int(request.args[b"timeout"][0])
except ValueError:
raise SynapseError(400, "timeout must be in milliseconds.")
- as_client_event = "raw" not in request.args
+ as_client_event = b"raw" not in request.args
chunk = yield self.event_stream_handler.get_stream(
requester.user.to_string(),
diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py
index fd5f85b53e..3ead75cb77 100644
--- a/synapse/rest/client/v1/initial_sync.py
+++ b/synapse/rest/client/v1/initial_sync.py
@@ -32,7 +32,7 @@ class InitialSyncRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def on_GET(self, request):
requester = yield self.auth.get_user_by_req(request)
- as_client_event = "raw" not in request.args
+ as_client_event = b"raw" not in request.args
pagination_config = PaginationConfig.from_request(request)
include_archived = parse_boolean(request, "archived", default=False)
content = yield self.initial_sync_handler.snapshot_all_rooms(
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index cb85fa1436..0010699d31 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -14,10 +14,9 @@
# limitations under the License.
import logging
-import urllib
import xml.etree.ElementTree as ET
-from six.moves.urllib import parse as urlparse
+from six.moves import urllib
from canonicaljson import json
from saml2 import BINDING_HTTP_POST, config
@@ -134,7 +133,7 @@ class LoginRestServlet(ClientV1RestServlet):
LoginRestServlet.SAML2_TYPE):
relay_state = ""
if "relay_state" in login_submission:
- relay_state = "&RelayState=" + urllib.quote(
+ relay_state = "&RelayState=" + urllib.parse.quote(
login_submission["relay_state"])
result = {
"uri": "%s%s" % (self.idp_redirect_url, relay_state)
@@ -366,7 +365,7 @@ class SAML2RestServlet(ClientV1RestServlet):
(user_id, token) = yield handler.register_saml2(username)
# Forward to the RelayState callback along with ava
if 'RelayState' in request.args:
- request.redirect(urllib.unquote(
+ request.redirect(urllib.parse.unquote(
request.args['RelayState'][0]) +
'?status=authenticated&access_token=' +
token + '&user_id=' + user_id + '&ava=' +
@@ -377,7 +376,7 @@ class SAML2RestServlet(ClientV1RestServlet):
"user_id": user_id, "token": token,
"ava": saml2_auth.ava}))
elif 'RelayState' in request.args:
- request.redirect(urllib.unquote(
+ request.redirect(urllib.parse.unquote(
request.args['RelayState'][0]) +
'?status=not_authenticated')
finish_request(request)
@@ -390,21 +389,22 @@ class CasRedirectServlet(ClientV1RestServlet):
def __init__(self, hs):
super(CasRedirectServlet, self).__init__(hs)
- self.cas_server_url = hs.config.cas_server_url
- self.cas_service_url = hs.config.cas_service_url
+ self.cas_server_url = hs.config.cas_server_url.encode('ascii')
+ self.cas_service_url = hs.config.cas_service_url.encode('ascii')
def on_GET(self, request):
args = request.args
- if "redirectUrl" not in args:
+ if b"redirectUrl" not in args:
return (400, "Redirect URL not specified for CAS auth")
- client_redirect_url_param = urllib.urlencode({
- "redirectUrl": args["redirectUrl"][0]
- })
- hs_redirect_url = self.cas_service_url + "/_matrix/client/api/v1/login/cas/ticket"
- service_param = urllib.urlencode({
- "service": "%s?%s" % (hs_redirect_url, client_redirect_url_param)
- })
- request.redirect("%s/login?%s" % (self.cas_server_url, service_param))
+ client_redirect_url_param = urllib.parse.urlencode({
+ b"redirectUrl": args[b"redirectUrl"][0]
+ }).encode('ascii')
+ hs_redirect_url = (self.cas_service_url +
+ b"/_matrix/client/api/v1/login/cas/ticket")
+ service_param = urllib.parse.urlencode({
+ b"service": b"%s?%s" % (hs_redirect_url, client_redirect_url_param)
+ }).encode('ascii')
+ request.redirect(b"%s/login?%s" % (self.cas_server_url, service_param))
finish_request(request)
@@ -422,11 +422,11 @@ class CasTicketServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def on_GET(self, request):
- client_redirect_url = request.args["redirectUrl"][0]
+ client_redirect_url = request.args[b"redirectUrl"][0]
http_client = self.hs.get_simple_http_client()
uri = self.cas_server_url + "/proxyValidate"
args = {
- "ticket": request.args["ticket"],
+ "ticket": request.args[b"ticket"][0].decode('ascii'),
"service": self.cas_service_url
}
try:
@@ -471,11 +471,11 @@ class CasTicketServlet(ClientV1RestServlet):
finish_request(request)
def add_login_token_to_redirect_url(self, url, token):
- url_parts = list(urlparse.urlparse(url))
- query = dict(urlparse.parse_qsl(url_parts[4]))
+ url_parts = list(urllib.parse.urlparse(url))
+ query = dict(urllib.parse.parse_qsl(url_parts[4]))
query.update({"loginToken": token})
- url_parts[4] = urllib.urlencode(query)
- return urlparse.urlunparse(url_parts)
+ url_parts[4] = urllib.parse.urlencode(query).encode('ascii')
+ return urllib.parse.urlunparse(url_parts)
def parse_cas_response(self, cas_response_body):
user = None
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 6e95d9bec2..9382b1f124 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -46,7 +46,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
try:
priority_class = _priority_class_from_spec(spec)
except InvalidRuleException as e:
- raise SynapseError(400, e.message)
+ raise SynapseError(400, str(e))
requester = yield self.auth.get_user_by_req(request)
@@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
content,
)
except InvalidRuleException as e:
- raise SynapseError(400, e.message)
+ raise SynapseError(400, str(e))
before = parse_string(request, "before")
if before:
@@ -95,9 +95,9 @@ class PushRuleRestServlet(ClientV1RestServlet):
)
self.notify_user(user_id)
except InconsistentRuleException as e:
- raise SynapseError(400, e.message)
+ raise SynapseError(400, str(e))
except RuleNotFoundException as e:
- raise SynapseError(400, e.message)
+ raise SynapseError(400, str(e))
defer.returnValue((200, {}))
@@ -142,10 +142,10 @@ class PushRuleRestServlet(ClientV1RestServlet):
PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR
)
- if path[0] == '':
+ if path[0] == b'':
defer.returnValue((200, rules))
- elif path[0] == 'global':
- path = path[1:]
+ elif path[0] == b'global':
+ path = [x.decode('ascii') for x in path[1:]]
result = _filter_ruleset_with_path(rules['global'], path)
defer.returnValue((200, result))
else:
@@ -192,10 +192,10 @@ class PushRuleRestServlet(ClientV1RestServlet):
def _rule_spec_from_path(path):
if len(path) < 2:
raise UnrecognizedRequestError()
- if path[0] != 'pushrules':
+ if path[0] != b'pushrules':
raise UnrecognizedRequestError()
- scope = path[1]
+ scope = path[1].decode('ascii')
path = path[2:]
if scope != 'global':
raise UnrecognizedRequestError()
@@ -203,13 +203,13 @@ def _rule_spec_from_path(path):
if len(path) == 0:
raise UnrecognizedRequestError()
- template = path[0]
+ template = path[0].decode('ascii')
path = path[1:]
if len(path) == 0 or len(path[0]) == 0:
raise UnrecognizedRequestError()
- rule_id = path[0]
+ rule_id = path[0].decode('ascii')
spec = {
'scope': scope,
@@ -220,7 +220,7 @@ def _rule_spec_from_path(path):
path = path[1:]
if len(path) > 0 and len(path[0]) > 0:
- spec['attr'] = path[0]
+ spec['attr'] = path[0].decode('ascii')
return spec
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 182a68b1e2..b84f0260f2 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -59,7 +59,7 @@ class PushersRestServlet(ClientV1RestServlet):
]
for p in pushers:
- for k, v in p.items():
+ for k, v in list(p.items()):
if k not in allowed_keys:
del p[k]
@@ -126,7 +126,7 @@ class PushersSetRestServlet(ClientV1RestServlet):
profile_tag=content.get('profile_tag', ""),
)
except PusherConfigException as pce:
- raise SynapseError(400, "Config Error: " + pce.message,
+ raise SynapseError(400, "Config Error: " + str(pce),
errcode=Codes.MISSING_PARAM)
self.notifier.on_new_replication_data()
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 976d98387d..663934efd0 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -207,7 +207,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
"sender": requester.user.to_string(),
}
- if 'ts' in request.args and requester.app_service:
+ if b'ts' in request.args and requester.app_service:
event_dict['origin_server_ts'] = parse_integer(request, "ts", 0)
event = yield self.event_creation_hander.create_and_send_nonmember_event(
@@ -255,7 +255,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet):
if RoomID.is_valid(room_identifier):
room_id = room_identifier
try:
- remote_room_hosts = request.args["server_name"]
+ remote_room_hosts = [
+ x.decode('ascii') for x in request.args[b"server_name"]
+ ]
except Exception:
remote_room_hosts = None
elif RoomAlias.is_valid(room_identifier):
@@ -461,10 +463,10 @@ class RoomMessageListRestServlet(ClientV1RestServlet):
pagination_config = PaginationConfig.from_request(
request, default_limit=10,
)
- as_client_event = "raw" not in request.args
- filter_bytes = parse_string(request, "filter")
+ as_client_event = b"raw" not in request.args
+ filter_bytes = parse_string(request, b"filter", encoding=None)
if filter_bytes:
- filter_json = urlparse.unquote(filter_bytes).decode("UTF-8")
+ filter_json = urlparse.unquote(filter_bytes.decode("UTF-8"))
event_filter = Filter(json.loads(filter_json))
else:
event_filter = None
@@ -560,7 +562,7 @@ class RoomEventContextServlet(ClientV1RestServlet):
# picking the API shape for symmetry with /messages
filter_bytes = parse_string(request, "filter")
if filter_bytes:
- filter_json = urlparse.unquote(filter_bytes).decode("UTF-8")
+ filter_json = urlparse.unquote(filter_bytes)
event_filter = Filter(json.loads(filter_json))
else:
event_filter = None
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 263d8eb73e..0251146722 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -89,7 +89,7 @@ class SyncRestServlet(RestServlet):
@defer.inlineCallbacks
def on_GET(self, request):
- if "from" in request.args:
+ if b"from" in request.args:
# /events used to use 'from', but /sync uses 'since'.
# Lets be helpful and whine if we see a 'from'.
raise SynapseError(
diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py
index d9d379182e..b9b5d07677 100644
--- a/synapse/rest/client/v2_alpha/thirdparty.py
+++ b/synapse/rest/client/v2_alpha/thirdparty.py
@@ -79,7 +79,7 @@ class ThirdPartyUserServlet(RestServlet):
yield self.auth.get_user_by_req(request, allow_guest=True)
fields = request.args
- fields.pop("access_token", None)
+ fields.pop(b"access_token", None)
results = yield self.appservice_handler.query_3pe(
ThirdPartyEntityKind.USER, protocol, fields
@@ -102,7 +102,7 @@ class ThirdPartyLocationServlet(RestServlet):
yield self.auth.get_user_by_req(request, allow_guest=True)
fields = request.args
- fields.pop("access_token", None)
+ fields.pop(b"access_token", None)
results = yield self.appservice_handler.query_3pe(
ThirdPartyEntityKind.LOCATION, protocol, fields
diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py
index b9ee6e1c13..38eb2ee23f 100644
--- a/synapse/rest/key/v1/server_key_resource.py
+++ b/synapse/rest/key/v1/server_key_resource.py
@@ -88,5 +88,5 @@ class LocalKey(Resource):
)
def getChild(self, name, request):
- if name == '':
+ if name == b'':
return self
diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py
index 3491fd2118..cb5abcf826 100644
--- a/synapse/rest/key/v2/__init__.py
+++ b/synapse/rest/key/v2/__init__.py
@@ -22,5 +22,5 @@ from .remote_key_resource import RemoteKey
class KeyApiV2Resource(Resource):
def __init__(self, hs):
Resource.__init__(self)
- self.putChild("server", LocalKey(hs))
- self.putChild("query", RemoteKey(hs))
+ self.putChild(b"server", LocalKey(hs))
+ self.putChild(b"query", RemoteKey(hs))
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index 7d67e4b064..eb8782aa6e 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -103,7 +103,7 @@ class RemoteKey(Resource):
def async_render_GET(self, request):
if len(request.postpath) == 1:
server, = request.postpath
- query = {server: {}}
+ query = {server.decode('ascii'): {}}
elif len(request.postpath) == 2:
server, key_id = request.postpath
minimum_valid_until_ts = parse_integer(
@@ -112,11 +112,12 @@ class RemoteKey(Resource):
arguments = {}
if minimum_valid_until_ts is not None:
arguments["minimum_valid_until_ts"] = minimum_valid_until_ts
- query = {server: {key_id: arguments}}
+ query = {server.decode('ascii'): {key_id.decode('ascii'): arguments}}
else:
raise SynapseError(
404, "Not found %r" % request.postpath, Codes.NOT_FOUND
)
+
yield self.query_keys(request, query, query_remote_on_cache_miss=True)
def render_POST(self, request):
@@ -135,6 +136,7 @@ class RemoteKey(Resource):
@defer.inlineCallbacks
def query_keys(self, request, query, query_remote_on_cache_miss=False):
logger.info("Handling query for keys %r", query)
+
store_queries = []
for server_name, key_ids in query.items():
if (
diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py
index f255f2883f..5a426ff2f6 100644
--- a/synapse/rest/media/v0/content_repository.py
+++ b/synapse/rest/media/v0/content_repository.py
@@ -56,7 +56,7 @@ class ContentRepoResource(resource.Resource):
# servers.
# TODO: A little crude here, we could do this better.
- filename = request.path.split('/')[-1]
+ filename = request.path.decode('ascii').split('/')[-1]
# be paranoid
filename = re.sub("[^0-9A-z.-_]", "", filename)
@@ -78,7 +78,7 @@ class ContentRepoResource(resource.Resource):
# select private. don't bother setting Expires as all our matrix
# clients are smart enough to be happy with Cache-Control (right?)
request.setHeader(
- "Cache-Control", "public,max-age=86400,s-maxage=86400"
+ b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
)
d = FileSender().beginFileTransfer(f, request)
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 65f4bd2910..76e479afa3 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -15,9 +15,8 @@
import logging
import os
-import urllib
-from six.moves.urllib import parse as urlparse
+from six.moves import urllib
from twisted.internet import defer
from twisted.protocols.basic import FileSender
@@ -35,10 +34,15 @@ def parse_media_id(request):
# This allows users to append e.g. /test.png to the URL. Useful for
# clients that parse the URL to see content type.
server_name, media_id = request.postpath[:2]
+
+ if isinstance(server_name, bytes):
+ server_name = server_name.decode('utf-8')
+ media_id = media_id.decode('utf8')
+
file_name = None
if len(request.postpath) > 2:
try:
- file_name = urlparse.unquote(request.postpath[-1]).decode("utf-8")
+ file_name = urllib.parse.unquote(request.postpath[-1].decode("utf-8"))
except UnicodeDecodeError:
pass
return server_name, media_id, file_name
@@ -93,22 +97,18 @@ def add_file_headers(request, media_type, file_size, upload_name):
file_size (int): Size in bytes of the media, if known.
upload_name (str): The name of the requested file, if any.
"""
+ def _quote(x):
+ return urllib.parse.quote(x.encode("utf-8"))
+
request.setHeader(b"Content-Type", media_type.encode("UTF-8"))
if upload_name:
if is_ascii(upload_name):
- request.setHeader(
- b"Content-Disposition",
- b"inline; filename=%s" % (
- urllib.quote(upload_name.encode("utf-8")),
- ),
- )
+ disposition = ("inline; filename=%s" % (_quote(upload_name),)).encode("ascii")
else:
- request.setHeader(
- b"Content-Disposition",
- b"inline; filename*=utf-8''%s" % (
- urllib.quote(upload_name.encode("utf-8")),
- ),
- )
+ disposition = (
+ "inline; filename*=utf-8''%s" % (_quote(upload_name),)).encode("ascii")
+
+ request.setHeader(b"Content-Disposition", disposition)
# cache for at least a day.
# XXX: we might want to turn this off for data we don't want to
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index fbfa85f74f..ca90964d1d 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -47,12 +47,12 @@ class DownloadResource(Resource):
def _async_render_GET(self, request):
set_cors_headers(request)
request.setHeader(
- "Content-Security-Policy",
- "default-src 'none';"
- " script-src 'none';"
- " plugin-types application/pdf;"
- " style-src 'unsafe-inline';"
- " object-src 'self';"
+ b"Content-Security-Policy",
+ b"default-src 'none';"
+ b" script-src 'none';"
+ b" plugin-types application/pdf;"
+ b" style-src 'unsafe-inline';"
+ b" object-src 'self';"
)
server_name, media_id, name = parse_media_id(request)
if server_name == self.server_name:
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 241c972070..a828ff4438 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -20,7 +20,7 @@ import logging
import os
import shutil
-from six import iteritems
+from six import PY3, iteritems
from six.moves.urllib import parse as urlparse
import twisted.internet.error
@@ -397,13 +397,13 @@ class MediaRepository(object):
yield finish()
- media_type = headers["Content-Type"][0]
+ media_type = headers[b"Content-Type"][0].decode('ascii')
time_now_ms = self.clock.time_msec()
- content_disposition = headers.get("Content-Disposition", None)
+ content_disposition = headers.get(b"Content-Disposition", None)
if content_disposition:
- _, params = cgi.parse_header(content_disposition[0],)
+ _, params = cgi.parse_header(content_disposition[0].decode('ascii'),)
upload_name = None
# First check if there is a valid UTF-8 filename
@@ -419,9 +419,13 @@ class MediaRepository(object):
upload_name = upload_name_ascii
if upload_name:
- upload_name = urlparse.unquote(upload_name)
+ if PY3:
+ upload_name = urlparse.unquote(upload_name)
+ else:
+ upload_name = urlparse.unquote(upload_name.encode('ascii'))
try:
- upload_name = upload_name.decode("utf-8")
+ if isinstance(upload_name, bytes):
+ upload_name = upload_name.decode("utf-8")
except UnicodeDecodeError:
upload_name = None
else:
@@ -755,14 +759,15 @@ class MediaRepositoryResource(Resource):
Resource.__init__(self)
media_repo = hs.get_media_repository()
- self.putChild("upload", UploadResource(hs, media_repo))
- self.putChild("download", DownloadResource(hs, media_repo))
- self.putChild("thumbnail", ThumbnailResource(
+
+ self.putChild(b"upload", UploadResource(hs, media_repo))
+ self.putChild(b"download", DownloadResource(hs, media_repo))
+ self.putChild(b"thumbnail", ThumbnailResource(
hs, media_repo, media_repo.media_storage,
))
- self.putChild("identicon", IdenticonResource())
+ self.putChild(b"identicon", IdenticonResource())
if hs.config.url_preview_enabled:
- self.putChild("preview_url", PreviewUrlResource(
+ self.putChild(b"preview_url", PreviewUrlResource(
hs, media_repo, media_repo.media_storage,
))
- self.putChild("config", MediaConfigResource(hs))
+ self.putChild(b"config", MediaConfigResource(hs))
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 778ef97337..cad2dec33a 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -261,7 +261,7 @@ class PreviewUrlResource(Resource):
logger.debug("Calculated OG for %s as %s" % (url, og))
- jsonog = json.dumps(og)
+ jsonog = json.dumps(og).encode('utf8')
# store OG in history-aware DB cache
yield self.store.store_url_cache(
@@ -301,20 +301,20 @@ class PreviewUrlResource(Resource):
logger.warn("Error downloading %s: %r", url, e)
raise SynapseError(
500, "Failed to download content: %s" % (
- traceback.format_exception_only(sys.exc_type, e),
+ traceback.format_exception_only(sys.exc_info()[0], e),
),
Codes.UNKNOWN,
)
yield finish()
try:
- if "Content-Type" in headers:
- media_type = headers["Content-Type"][0]
+ if b"Content-Type" in headers:
+ media_type = headers[b"Content-Type"][0].decode('ascii')
else:
media_type = "application/octet-stream"
time_now_ms = self.clock.time_msec()
- content_disposition = headers.get("Content-Disposition", None)
+ content_disposition = headers.get(b"Content-Disposition", None)
if content_disposition:
_, params = cgi.parse_header(content_disposition[0],)
download_name = None
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 8bf87f38f7..30ff87a4c4 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -930,6 +930,10 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
)
self._invalidate_cache_and_stream(
+ txn, self.get_room_summary, (room_id,)
+ )
+
+ self._invalidate_cache_and_stream(
txn, self.get_current_state_ids, (room_id,)
)
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index c7899d7fd2..b890c152db 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -199,10 +199,14 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Args:
user_id(str): the user_id to query
"""
+
if self.hs.config.limit_usage_by_mau:
+ # Trial users and guests should not be included as part of MAU group
+ is_guest = yield self.is_guest(user_id)
+ if is_guest:
+ return
is_trial = yield self.is_trial_user(user_id)
if is_trial:
- # we don't track trial users in the MAU table.
return
last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 9b4e6d6aa8..0707f9a86a 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -51,6 +51,12 @@ ProfileInfo = namedtuple(
"ProfileInfo", ("avatar_url", "display_name")
)
+# "members" points to a truncated list of (user_id, event_id) tuples for users of
+# a given membership type, suitable for use in calculating heroes for a room.
+# "count" points to the total numberr of users of a given membership type.
+MemberSummary = namedtuple(
+ "MemberSummary", ("members", "count")
+)
_MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
@@ -82,6 +88,65 @@ class RoomMemberWorkerStore(EventsWorkerStore):
return [to_ascii(r[0]) for r in txn]
return self.runInteraction("get_users_in_room", f)
+ @cached(max_entries=100000)
+ def get_room_summary(self, room_id):
+ """ Get the details of a room roughly suitable for use by the room
+ summary extension to /sync. Useful when lazy loading room members.
+ Args:
+ room_id (str): The room ID to query
+ Returns:
+ Deferred[dict[str, MemberSummary]:
+ dict of membership states, pointing to a MemberSummary named tuple.
+ """
+
+ def _get_room_summary_txn(txn):
+ # first get counts.
+ # We do this all in one transaction to keep the cache small.
+ # FIXME: get rid of this when we have room_stats
+ sql = """
+ SELECT count(*), m.membership FROM room_memberships as m
+ INNER JOIN current_state_events as c
+ ON m.event_id = c.event_id
+ AND m.room_id = c.room_id
+ AND m.user_id = c.state_key
+ WHERE c.type = 'm.room.member' AND c.room_id = ?
+ GROUP BY m.membership
+ """
+
+ txn.execute(sql, (room_id,))
+ res = {}
+ for count, membership in txn:
+ summary = res.setdefault(to_ascii(membership), MemberSummary([], count))
+
+ # we order by membership and then fairly arbitrarily by event_id so
+ # heroes are consistent
+ sql = """
+ SELECT m.user_id, m.membership, m.event_id
+ FROM room_memberships as m
+ INNER JOIN current_state_events as c
+ ON m.event_id = c.event_id
+ AND m.room_id = c.room_id
+ AND m.user_id = c.state_key
+ WHERE c.type = 'm.room.member' AND c.room_id = ?
+ ORDER BY
+ CASE m.membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
+ m.event_id ASC
+ LIMIT ?
+ """
+
+ # 6 is 5 (number of heroes) plus 1, in case one of them is the calling user.
+ txn.execute(sql, (room_id, Membership.JOIN, Membership.INVITE, 6))
+ for user_id, membership, event_id in txn:
+ summary = res[to_ascii(membership)]
+ # we will always have a summary for this membership type at this
+ # point given the summary currently contains the counts.
+ members = summary.members
+ members.append((to_ascii(user_id), to_ascii(event_id)))
+
+ return res
+
+ return self.runInteraction("get_room_summary", _get_room_summary_txn)
+
@cached()
def get_invited_rooms_for_user(self, user_id):
""" Get all the rooms the user is invited to
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index f65a27e5f1..379e9c4ab1 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -471,6 +471,7 @@ class AuthTestCase(unittest.TestCase):
def test_reserved_threepid(self):
self.hs.config.limit_usage_by_mau = True
self.hs.config.max_mau_value = 1
+ self.store.get_monthly_active_count = lambda: defer.succeed(2)
threepid = {'medium': 'email', 'address': 'reserved@server.com'}
unknown_threepid = {'medium': 'email', 'address': 'unreserved@server.com'}
self.hs.config.mau_limits_reserved_threepids = [threepid]
diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py
index 76b5090fff..a83f567ebd 100644
--- a/tests/app/test_frontend_proxy.py
+++ b/tests/app/test_frontend_proxy.py
@@ -47,7 +47,7 @@ class FrontendProxyTests(HomeserverTestCase):
self.assertEqual(len(self.reactor.tcpServers), 1)
site = self.reactor.tcpServers[0][1]
self.resource = (
- site.resource.children["_matrix"].children["client"].children["r0"]
+ site.resource.children[b"_matrix"].children[b"client"].children[b"r0"]
)
request, channel = self.make_request("PUT", "presence/a/status")
@@ -77,7 +77,7 @@ class FrontendProxyTests(HomeserverTestCase):
self.assertEqual(len(self.reactor.tcpServers), 1)
site = self.reactor.tcpServers[0][1]
self.resource = (
- site.resource.children["_matrix"].children["client"].children["r0"]
+ site.resource.children[b"_matrix"].children[b"client"].children[b"r0"]
)
request, channel = self.make_request("PUT", "presence/a/status")
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index c2d951b45f..36e136cded 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -43,9 +43,7 @@ def _expect_edu_transaction(edu_type, content, origin="test"):
def _make_edu_transaction_json(edu_type, content):
- return json.dumps(_expect_edu_transaction(edu_type, content)).encode(
- 'utf8'
- )
+ return json.dumps(_expect_edu_transaction(edu_type, content)).encode('utf8')
class TypingNotificationsTestCase(unittest.TestCase):
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 9fe0760496..359f7777ff 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -22,39 +22,24 @@ from six.moves.urllib import parse as urlparse
from twisted.internet import defer
-import synapse.rest.client.v1.room
from synapse.api.constants import Membership
-from synapse.http.server import JsonResource
-from synapse.types import UserID
-from synapse.util import Clock
+from synapse.rest.client.v1 import room
from tests import unittest
-from tests.server import (
- ThreadedMemoryReactorClock,
- make_request,
- render,
- setup_test_homeserver,
-)
-
-from .utils import RestHelper
PATH_PREFIX = b"/_matrix/client/api/v1"
-class RoomBase(unittest.TestCase):
+class RoomBase(unittest.HomeserverTestCase):
rmcreator_id = None
- def setUp(self):
+ servlets = [room.register_servlets, room.register_deprecated_servlets]
- self.clock = ThreadedMemoryReactorClock()
- self.hs_clock = Clock(self.clock)
+ def make_homeserver(self, reactor, clock):
- self.hs = setup_test_homeserver(
- self.addCleanup,
+ self.hs = self.setup_test_homeserver(
"red",
http_client=None,
- clock=self.hs_clock,
- reactor=self.clock,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]),
)
@@ -63,42 +48,21 @@ class RoomBase(unittest.TestCase):
self.hs.get_federation_handler = Mock(return_value=Mock())
- def get_user_by_access_token(token=None, allow_guest=False):
- return {
- "user": UserID.from_string(self.helper.auth_user_id),
- "token_id": 1,
- "is_guest": False,
- }
-
- def get_user_by_req(request, allow_guest=False, rights="access"):
- return synapse.types.create_requester(
- UserID.from_string(self.helper.auth_user_id), 1, False, None
- )
-
- self.hs.get_auth().get_user_by_req = get_user_by_req
- self.hs.get_auth().get_user_by_access_token = get_user_by_access_token
- self.hs.get_auth().get_access_token_from_request = Mock(return_value=b"1234")
-
def _insert_client_ip(*args, **kwargs):
return defer.succeed(None)
self.hs.get_datastore().insert_client_ip = _insert_client_ip
- self.resource = JsonResource(self.hs)
- synapse.rest.client.v1.room.register_servlets(self.hs, self.resource)
- synapse.rest.client.v1.room.register_deprecated_servlets(self.hs, self.resource)
- self.helper = RestHelper(self.hs, self.resource, self.user_id)
+ return self.hs
class RoomPermissionsTestCase(RoomBase):
""" Tests room permissions. """
- user_id = b"@sid1:red"
- rmcreator_id = b"@notme:red"
-
- def setUp(self):
+ user_id = "@sid1:red"
+ rmcreator_id = "@notme:red"
- super(RoomPermissionsTestCase, self).setUp()
+ def prepare(self, reactor, clock, hs):
self.helper.auth_user_id = self.rmcreator_id
# create some rooms under the name rmcreator_id
@@ -114,22 +78,20 @@ class RoomPermissionsTestCase(RoomBase):
self.created_rmid_msg_path = (
"rooms/%s/send/m.room.message/a1" % (self.created_rmid)
).encode('ascii')
- request, channel = make_request(
- b"PUT",
- self.created_rmid_msg_path,
- b'{"msgtype":"m.text","body":"test msg"}',
+ request, channel = self.make_request(
+ "PUT", self.created_rmid_msg_path, b'{"msgtype":"m.text","body":"test msg"}'
)
- render(request, self.resource, self.clock)
- self.assertEquals(channel.result["code"], b"200", channel.result)
+ self.render(request)
+ self.assertEquals(200, channel.code, channel.result)
# set topic for public room
- request, channel = make_request(
- b"PUT",
+ request, channel = self.make_request(
+ "PUT",
("rooms/%s/state/m.room.topic" % self.created_public_rmid).encode('ascii'),
b'{"topic":"Public Room Topic"}',
)
- render(request, self.resource, self.clock)
- self.assertEquals(channel.result["code"], b"200", channel.result)
+ self.render(request)
+ self.assertEquals(200, channel.code, channel.result)
# auth as user_id now
self.helper.auth_user_id = self.user_id
@@ -140,128 +102,128 @@ class RoomPermissionsTestCase(RoomBase):
seq = iter(range(100))
def send_msg_path():
- return b"/rooms/%s/send/m.room.message/mid%s" % (
+ return "/rooms/%s/send/m.room.message/mid%s" % (
self.created_rmid,
- str(next(seq)).encode('ascii'),
+ str(next(seq)),
)
# send message in uncreated room, expect 403
- request, channel = make_request(
- b"PUT",
- b"/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,),
+ request, channel = self.make_request(
+ "PUT",
+ "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,),
msg_content,
)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# send message in created room not joined (no state), expect 403
- request, channel = make_request(b"PUT", send_msg_path(), msg_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# send message in created room and invited, expect 403
self.helper.invite(
room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id
)
- request, channel = make_request(b"PUT", send_msg_path(), msg_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# send message in created room and joined, expect 200
self.helper.join(room=self.created_rmid, user=self.user_id)
- request, channel = make_request(b"PUT", send_msg_path(), msg_content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
# send message in created room and left, expect 403
self.helper.leave(room=self.created_rmid, user=self.user_id)
- request, channel = make_request(b"PUT", send_msg_path(), msg_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
def test_topic_perms(self):
topic_content = b'{"topic":"My Topic Name"}'
- topic_path = b"/rooms/%s/state/m.room.topic" % self.created_rmid
+ topic_path = "/rooms/%s/state/m.room.topic" % self.created_rmid
# set/get topic in uncreated room, expect 403
- request, channel = make_request(
- b"PUT", b"/rooms/%s/state/m.room.topic" % self.uncreated_rmid, topic_content
+ request, channel = self.make_request(
+ "PUT", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid, topic_content
)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
- request, channel = make_request(
- b"GET", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
+ request, channel = self.make_request(
+ "GET", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid
)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# set/get topic in created PRIVATE room not joined, expect 403
- request, channel = make_request(b"PUT", topic_path, topic_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
- request, channel = make_request(b"GET", topic_path)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", topic_path, topic_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
+ request, channel = self.make_request("GET", topic_path)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# set topic in created PRIVATE room and invited, expect 403
self.helper.invite(
room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id
)
- request, channel = make_request(b"PUT", topic_path, topic_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", topic_path, topic_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# get topic in created PRIVATE room and invited, expect 403
- request, channel = make_request(b"GET", topic_path)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", topic_path)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# set/get topic in created PRIVATE room and joined, expect 200
self.helper.join(room=self.created_rmid, user=self.user_id)
# Only room ops can set topic by default
self.helper.auth_user_id = self.rmcreator_id
- request, channel = make_request(b"PUT", topic_path, topic_content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", topic_path, topic_content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.helper.auth_user_id = self.user_id
- request, channel = make_request(b"GET", topic_path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
- self.assert_dict(json.loads(topic_content), channel.json_body)
+ request, channel = self.make_request("GET", topic_path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
+ self.assert_dict(json.loads(topic_content.decode('utf8')), channel.json_body)
# set/get topic in created PRIVATE room and left, expect 403
self.helper.leave(room=self.created_rmid, user=self.user_id)
- request, channel = make_request(b"PUT", topic_path, topic_content)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
- request, channel = make_request(b"GET", topic_path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", topic_path, topic_content)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
+ request, channel = self.make_request("GET", topic_path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
# get topic in PUBLIC room, not joined, expect 403
- request, channel = make_request(
- b"GET", b"/rooms/%s/state/m.room.topic" % self.created_public_rmid
+ request, channel = self.make_request(
+ "GET", "/rooms/%s/state/m.room.topic" % self.created_public_rmid
)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
# set topic in PUBLIC room, not joined, expect 403
- request, channel = make_request(
- b"PUT",
- b"/rooms/%s/state/m.room.topic" % self.created_public_rmid,
+ request, channel = self.make_request(
+ "PUT",
+ "/rooms/%s/state/m.room.topic" % self.created_public_rmid,
topic_content,
)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
def _test_get_membership(self, room=None, members=[], expect_code=None):
for member in members:
- path = b"/rooms/%s/state/m.room.member/%s" % (room, member)
- request, channel = make_request(b"GET", path)
- render(request, self.resource, self.clock)
- self.assertEquals(expect_code, int(channel.result["code"]))
+ path = "/rooms/%s/state/m.room.member/%s" % (room, member)
+ request, channel = self.make_request("GET", path)
+ self.render(request)
+ self.assertEquals(expect_code, channel.code)
def test_membership_basic_room_perms(self):
# === room does not exist ===
@@ -428,217 +390,211 @@ class RoomPermissionsTestCase(RoomBase):
class RoomsMemberListTestCase(RoomBase):
""" Tests /rooms/$room_id/members/list REST events."""
- user_id = b"@sid1:red"
+ user_id = "@sid1:red"
def test_get_member_list(self):
room_id = self.helper.create_room_as(self.user_id)
- request, channel = make_request(b"GET", b"/rooms/%s/members" % room_id)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
def test_get_member_list_no_room(self):
- request, channel = make_request(b"GET", b"/rooms/roomdoesnotexist/members")
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", "/rooms/roomdoesnotexist/members")
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
def test_get_member_list_no_permission(self):
- room_id = self.helper.create_room_as(b"@some_other_guy:red")
- request, channel = make_request(b"GET", b"/rooms/%s/members" % room_id)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ room_id = self.helper.create_room_as("@some_other_guy:red")
+ request, channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
def test_get_member_list_mixed_memberships(self):
- room_creator = b"@some_other_guy:red"
+ room_creator = "@some_other_guy:red"
room_id = self.helper.create_room_as(room_creator)
- room_path = b"/rooms/%s/members" % room_id
+ room_path = "/rooms/%s/members" % room_id
self.helper.invite(room=room_id, src=room_creator, targ=self.user_id)
# can't see list if you're just invited.
- request, channel = make_request(b"GET", room_path)
- render(request, self.resource, self.clock)
- self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", room_path)
+ self.render(request)
+ self.assertEquals(403, channel.code, msg=channel.result["body"])
self.helper.join(room=room_id, user=self.user_id)
# can see list now joined
- request, channel = make_request(b"GET", room_path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", room_path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.helper.leave(room=room_id, user=self.user_id)
# can see old list once left
- request, channel = make_request(b"GET", room_path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", room_path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
class RoomsCreateTestCase(RoomBase):
""" Tests /rooms and /rooms/$room_id REST events. """
- user_id = b"@sid1:red"
+ user_id = "@sid1:red"
def test_post_room_no_keys(self):
# POST with no config keys, expect new room id
- request, channel = make_request(b"POST", b"/createRoom", b"{}")
+ request, channel = self.make_request("POST", "/createRoom", "{}")
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), channel.result)
+ self.render(request)
+ self.assertEquals(200, channel.code, channel.result)
self.assertTrue("room_id" in channel.json_body)
def test_post_room_visibility_key(self):
# POST with visibility config key, expect new room id
- request, channel = make_request(
- b"POST", b"/createRoom", b'{"visibility":"private"}'
+ request, channel = self.make_request(
+ "POST", "/createRoom", b'{"visibility":"private"}'
)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertTrue("room_id" in channel.json_body)
def test_post_room_custom_key(self):
# POST with custom config keys, expect new room id
- request, channel = make_request(b"POST", b"/createRoom", b'{"custom":"stuff"}')
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ request, channel = self.make_request(
+ "POST", "/createRoom", b'{"custom":"stuff"}'
+ )
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertTrue("room_id" in channel.json_body)
def test_post_room_known_and_unknown_keys(self):
# POST with custom + known config keys, expect new room id
- request, channel = make_request(
- b"POST", b"/createRoom", b'{"visibility":"private","custom":"things"}'
+ request, channel = self.make_request(
+ "POST", "/createRoom", b'{"visibility":"private","custom":"things"}'
)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertTrue("room_id" in channel.json_body)
def test_post_room_invalid_content(self):
# POST with invalid content / paths, expect 400
- request, channel = make_request(b"POST", b"/createRoom", b'{"visibili')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]))
+ request, channel = self.make_request("POST", "/createRoom", b'{"visibili')
+ self.render(request)
+ self.assertEquals(400, channel.code)
- request, channel = make_request(b"POST", b"/createRoom", b'["hello"]')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]))
+ request, channel = self.make_request("POST", "/createRoom", b'["hello"]')
+ self.render(request)
+ self.assertEquals(400, channel.code)
class RoomTopicTestCase(RoomBase):
""" Tests /rooms/$room_id/topic REST events. """
- user_id = b"@sid1:red"
-
- def setUp(self):
-
- super(RoomTopicTestCase, self).setUp()
+ user_id = "@sid1:red"
+ def prepare(self, reactor, clock, hs):
# create the room
self.room_id = self.helper.create_room_as(self.user_id)
- self.path = b"/rooms/%s/state/m.room.topic" % (self.room_id,)
+ self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,)
def test_invalid_puts(self):
# missing keys or invalid json
- request, channel = make_request(b"PUT", self.path, '{}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, '{}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", self.path, '{"_name":"bob"}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, '{"_name":"bo"}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", self.path, '{"nao')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, '{"nao')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(
- b"PUT", self.path, '[{"_name":"bob"},{"_name":"jill"}]'
+ request, channel = self.make_request(
+ "PUT", self.path, '[{"_name":"bo"},{"_name":"jill"}]'
)
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", self.path, 'text only')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, 'text only')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", self.path, '')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, '')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
# valid key, wrong type
content = '{"topic":["Topic name"]}'
- request, channel = make_request(b"PUT", self.path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, content)
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
def test_rooms_topic(self):
# nothing should be there
- request, channel = make_request(b"GET", self.path)
- render(request, self.resource, self.clock)
- self.assertEquals(404, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", self.path)
+ self.render(request)
+ self.assertEquals(404, channel.code, msg=channel.result["body"])
# valid put
content = '{"topic":"Topic name"}'
- request, channel = make_request(b"PUT", self.path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
# valid get
- request, channel = make_request(b"GET", self.path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", self.path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.assert_dict(json.loads(content), channel.json_body)
def test_rooms_topic_with_extra_keys(self):
# valid put with extra keys
content = '{"topic":"Seasons","subtopic":"Summer"}'
- request, channel = make_request(b"PUT", self.path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", self.path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
# valid get
- request, channel = make_request(b"GET", self.path)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", self.path)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.assert_dict(json.loads(content), channel.json_body)
class RoomMemberStateTestCase(RoomBase):
""" Tests /rooms/$room_id/members/$user_id/state REST events. """
- user_id = b"@sid1:red"
-
- def setUp(self):
+ user_id = "@sid1:red"
- super(RoomMemberStateTestCase, self).setUp()
+ def prepare(self, reactor, clock, hs):
self.room_id = self.helper.create_room_as(self.user_id)
- def tearDown(self):
- pass
-
def test_invalid_puts(self):
path = "/rooms/%s/state/m.room.member/%s" % (self.room_id, self.user_id)
# missing keys or invalid json
- request, channel = make_request(b"PUT", path, '{}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, '{}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '{"_name":"bob"}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, '{"_name":"bo"}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '{"nao')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, '{"nao')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(
- b"PUT", path, b'[{"_name":"bob"},{"_name":"jill"}]'
+ request, channel = self.make_request(
+ "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]'
)
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, 'text only')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, 'text only')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, '')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
# valid keys, wrong types
content = '{"membership":["%s","%s","%s"]}' % (
@@ -646,9 +602,9 @@ class RoomMemberStateTestCase(RoomBase):
Membership.JOIN,
Membership.LEAVE,
)
- request, channel = make_request(b"PUT", path, content.encode('ascii'))
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, content.encode('ascii'))
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
def test_rooms_members_self(self):
path = "/rooms/%s/state/m.room.member/%s" % (
@@ -658,13 +614,13 @@ class RoomMemberStateTestCase(RoomBase):
# valid join message (NOOP since we made the room)
content = '{"membership":"%s"}' % Membership.JOIN
- request, channel = make_request(b"PUT", path, content.encode('ascii'))
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, content.encode('ascii'))
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"GET", path, None)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", path, None)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
expected_response = {"membership": Membership.JOIN}
self.assertEquals(expected_response, channel.json_body)
@@ -678,13 +634,13 @@ class RoomMemberStateTestCase(RoomBase):
# valid invite message
content = '{"membership":"%s"}' % Membership.INVITE
- request, channel = make_request(b"PUT", path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"GET", path, None)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", path, None)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.assertEquals(json.loads(content), channel.json_body)
def test_rooms_members_other_custom_keys(self):
@@ -699,13 +655,13 @@ class RoomMemberStateTestCase(RoomBase):
Membership.INVITE,
"Join us!",
)
- request, channel = make_request(b"PUT", path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"GET", path, None)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("GET", path, None)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
self.assertEquals(json.loads(content), channel.json_body)
@@ -714,60 +670,58 @@ class RoomMessagesTestCase(RoomBase):
user_id = "@sid1:red"
- def setUp(self):
- super(RoomMessagesTestCase, self).setUp()
-
+ def prepare(self, reactor, clock, hs):
self.room_id = self.helper.create_room_as(self.user_id)
def test_invalid_puts(self):
path = "/rooms/%s/send/m.room.message/mid1" % (urlparse.quote(self.room_id))
# missing keys or invalid json
- request, channel = make_request(b"PUT", path, '{}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, b'{}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '{"_name":"bob"}')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, b'{"_name":"bo"}')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '{"nao')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, b'{"nao')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(
- b"PUT", path, '[{"_name":"bob"},{"_name":"jill"}]'
+ request, channel = self.make_request(
+ "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]'
)
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, 'text only')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, b'text only')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
- request, channel = make_request(b"PUT", path, '')
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ request, channel = self.make_request("PUT", path, b'')
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
def test_rooms_messages_sent(self):
path = "/rooms/%s/send/m.room.message/mid1" % (urlparse.quote(self.room_id))
- content = '{"body":"test","msgtype":{"type":"a"}}'
- request, channel = make_request(b"PUT", path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+ content = b'{"body":"test","msgtype":{"type":"a"}}'
+ request, channel = self.make_request("PUT", path, content)
+ self.render(request)
+ self.assertEquals(400, channel.code, msg=channel.result["body"])
# custom message types
- content = '{"body":"test","msgtype":"test.custom.text"}'
- request, channel = make_request(b"PUT", path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ content = b'{"body":"test","msgtype":"test.custom.text"}'
+ request, channel = self.make_request("PUT", path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
# m.text message type
path = "/rooms/%s/send/m.room.message/mid2" % (urlparse.quote(self.room_id))
- content = '{"body":"test2","msgtype":"m.text"}'
- request, channel = make_request(b"PUT", path, content)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+ content = b'{"body":"test2","msgtype":"m.text"}'
+ request, channel = self.make_request("PUT", path, content)
+ self.render(request)
+ self.assertEquals(200, channel.code, msg=channel.result["body"])
class RoomInitialSyncTestCase(RoomBase):
@@ -775,16 +729,16 @@ class RoomInitialSyncTestCase(RoomBase):
user_id = "@sid1:red"
- def setUp(self):
- super(RoomInitialSyncTestCase, self).setUp()
-
+ def prepare(self, reactor, clock, hs):
# create the room
self.room_id = self.helper.create_room_as(self.user_id)
def test_initial_sync(self):
- request, channel = make_request(b"GET", "/rooms/%s/initialSync" % self.room_id)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ request, channel = self.make_request(
+ "GET", "/rooms/%s/initialSync" % self.room_id
+ )
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertEquals(self.room_id, channel.json_body["room_id"])
self.assertEquals("join", channel.json_body["membership"])
@@ -819,17 +773,16 @@ class RoomMessageListTestCase(RoomBase):
user_id = "@sid1:red"
- def setUp(self):
- super(RoomMessageListTestCase, self).setUp()
+ def prepare(self, reactor, clock, hs):
self.room_id = self.helper.create_room_as(self.user_id)
def test_topo_token_is_accepted(self):
token = "t1-0_0_0_0_0_0_0_0_0"
- request, channel = make_request(
- b"GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
+ request, channel = self.make_request(
+ "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertTrue("start" in channel.json_body)
self.assertEquals(token, channel.json_body['start'])
self.assertTrue("chunk" in channel.json_body)
@@ -837,11 +790,11 @@ class RoomMessageListTestCase(RoomBase):
def test_stream_token_is_accepted_for_fwd_pagianation(self):
token = "s0_0_0_0_0_0_0_0_0"
- request, channel = make_request(
- b"GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
+ request, channel = self.make_request(
+ "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
)
- render(request, self.resource, self.clock)
- self.assertEquals(200, int(channel.result["code"]))
+ self.render(request)
+ self.assertEquals(200, channel.code)
self.assertTrue("start" in channel.json_body)
self.assertEquals(token, channel.json_body['start'])
self.assertTrue("chunk" in channel.json_body)
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 560b1fba96..4c30c5f258 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -62,12 +62,6 @@ class FilterTestCase(unittest.HomeserverTestCase):
self.assertEqual(channel.code, 200)
self.assertTrue(
set(
- [
- "next_batch",
- "rooms",
- "account_data",
- "to_device",
- "device_lists",
- ]
+ ["next_batch", "rooms", "account_data", "to_device", "device_lists"]
).issubset(set(channel.json_body.keys()))
)
diff --git a/tests/server.py b/tests/server.py
index 615bba1b59..a2c3ca61f6 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -65,7 +65,7 @@ class FakeChannel(object):
def getPeer(self):
# We give an address so that getClientIP returns a non null entry,
# causing us to record the MAU
- return address.IPv4Address(b"TCP", "127.0.0.1", 3423)
+ return address.IPv4Address("TCP", "127.0.0.1", 3423)
def getHost(self):
return None
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index 5cc7fff39b..4701eedd45 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -80,12 +80,11 @@ class TestResourceLimitsServerNotices(unittest.TestCase):
self._rlsn._auth.check_auth_blocking = Mock()
mock_event = Mock(
- type=EventTypes.Message,
- content={"msgtype": ServerNoticeMsgType},
+ type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
+ )
+ self._rlsn._store.get_events = Mock(
+ return_value=defer.succeed({"123": mock_event})
)
- self._rlsn._store.get_events = Mock(return_value=defer.succeed(
- {"123": mock_event}
- ))
yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
# Would be better to check the content, but once == remove blocking event
@@ -99,12 +98,11 @@ class TestResourceLimitsServerNotices(unittest.TestCase):
)
mock_event = Mock(
- type=EventTypes.Message,
- content={"msgtype": ServerNoticeMsgType},
+ type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
+ )
+ self._rlsn._store.get_events = Mock(
+ return_value=defer.succeed({"123": mock_event})
)
- self._rlsn._store.get_events = Mock(return_value=defer.succeed(
- {"123": mock_event}
- ))
yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
self._send_notice.assert_not_called()
@@ -177,13 +175,9 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.TestCase):
@defer.inlineCallbacks
def test_server_notice_only_sent_once(self):
- self.store.get_monthly_active_count = Mock(
- return_value=1000,
- )
+ self.store.get_monthly_active_count = Mock(return_value=1000)
- self.store.user_last_seen_monthly_active = Mock(
- return_value=1000,
- )
+ self.store.user_last_seen_monthly_active = Mock(return_value=1000)
# Call the function multiple times to ensure we only send the notice once
yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
@@ -193,12 +187,12 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.TestCase):
# Now lets get the last load of messages in the service notice room and
# check that there is only one server notice
room_id = yield self.server_notices_manager.get_notice_room_for_user(
- self.user_id,
+ self.user_id
)
token = yield self.event_source.get_current_token()
events, _ = yield self.store.get_recent_events_for_room(
- room_id, limit=100, end_token=token.room_key,
+ room_id, limit=100, end_token=token.room_key
)
count = 0
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index c2e88bdbaf..c9b02a062b 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -101,6 +101,7 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
self.hs.config.limit_usage_by_mau = True
self.hs.config.max_mau_value = 50
user_id = "@user:server"
+ yield self.store.register(user_id=user_id, token="123", password_hash=None)
active = yield self.store.user_last_seen_monthly_active(user_id)
self.assertFalse(active)
@@ -108,8 +109,5 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id"
)
- yield self.store.insert_client_ip(
- user_id, "access_token", "ip", "user_agent", "device_id"
- )
active = yield self.store.user_last_seen_monthly_active(user_id)
self.assertTrue(active)
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 2036287288..ccfc21b7fc 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -12,6 +12,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from mock import Mock
+
+from twisted.internet import defer
from tests.unittest import HomeserverTestCase
@@ -23,7 +26,8 @@ class MonthlyActiveUsersTestCase(HomeserverTestCase):
hs = self.setup_test_homeserver()
self.store = hs.get_datastore()
-
+ hs.config.limit_usage_by_mau = True
+ hs.config.max_mau_value = 50
# Advance the clock a bit
reactor.advance(FORTY_DAYS)
@@ -73,7 +77,7 @@ class MonthlyActiveUsersTestCase(HomeserverTestCase):
active_count = self.store.get_monthly_active_count()
self.assertEquals(self.get_success(active_count), user_num)
- # Test that regalar users are removed from the db
+ # Test that regular users are removed from the db
ru_count = 2
self.store.upsert_monthly_active_user("@ru1:server")
self.store.upsert_monthly_active_user("@ru2:server")
@@ -139,3 +143,43 @@ class MonthlyActiveUsersTestCase(HomeserverTestCase):
count = self.store.get_monthly_active_count()
self.assertEquals(self.get_success(count), 0)
+
+ def test_populate_monthly_users_is_guest(self):
+ # Test that guest users are not added to mau list
+ user_id = "user_id"
+ self.store.register(
+ user_id=user_id, token="123", password_hash=None, make_guest=True
+ )
+ self.store.upsert_monthly_active_user = Mock()
+ self.store.populate_monthly_active_users(user_id)
+ self.pump()
+ self.store.upsert_monthly_active_user.assert_not_called()
+
+ def test_populate_monthly_users_should_update(self):
+ self.store.upsert_monthly_active_user = Mock()
+
+ self.store.is_trial_user = Mock(
+ return_value=defer.succeed(False)
+ )
+
+ self.store.user_last_seen_monthly_active = Mock(
+ return_value=defer.succeed(None)
+ )
+ self.store.populate_monthly_active_users('user_id')
+ self.pump()
+ self.store.upsert_monthly_active_user.assert_called_once()
+
+ def test_populate_monthly_users_should_not_update(self):
+ self.store.upsert_monthly_active_user = Mock()
+
+ self.store.is_trial_user = Mock(
+ return_value=defer.succeed(False)
+ )
+ self.store.user_last_seen_monthly_active = Mock(
+ return_value=defer.succeed(
+ self.hs.get_clock().time_msec()
+ )
+ )
+ self.store.populate_monthly_active_users('user_id')
+ self.pump()
+ self.store.upsert_monthly_active_user.assert_not_called()
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index d717b9f94e..b910965932 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -185,8 +185,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
# test _get_some_state_from_cache correctly filters out members with types=[]
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
- self.store._state_group_cache,
- group, [], filtered_types=[EventTypes.Member]
+ self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
)
self.assertEqual(is_all, True)
@@ -200,19 +199,20 @@ class StateStoreTestCase(tests.unittest.TestCase):
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [], filtered_types=[EventTypes.Member]
+ group,
+ [],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, True)
- self.assertDictEqual(
- {},
- state_dict,
- )
+ self.assertDictEqual({}, state_dict)
# test _get_some_state_from_cache correctly filters in members with wildcard types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_cache,
- group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+ group,
+ [(EventTypes.Member, None)],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, True)
@@ -226,7 +226,9 @@ class StateStoreTestCase(tests.unittest.TestCase):
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+ group,
+ [(EventTypes.Member, None)],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, True)
@@ -264,18 +266,15 @@ class StateStoreTestCase(tests.unittest.TestCase):
)
self.assertEqual(is_all, True)
- self.assertDictEqual(
- {
- (e5.type, e5.state_key): e5.event_id,
- },
- state_dict,
- )
+ self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
# test _get_some_state_from_cache correctly filters in members with specific types
# and no filtered_types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+ group,
+ [(EventTypes.Member, e5.state_key)],
+ filtered_types=None,
)
self.assertEqual(is_all, True)
@@ -305,9 +304,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
key=group,
value=state_dict_ids,
# list fetched keys so it knows it's partial
- fetched_keys=(
- (e1.type, e1.state_key),
- ),
+ fetched_keys=((e1.type, e1.state_key),),
)
(is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
@@ -315,20 +312,8 @@ class StateStoreTestCase(tests.unittest.TestCase):
)
self.assertEqual(is_all, False)
- self.assertEqual(
- known_absent,
- set(
- [
- (e1.type, e1.state_key),
- ]
- ),
- )
- self.assertDictEqual(
- state_dict_ids,
- {
- (e1.type, e1.state_key): e1.event_id,
- },
- )
+ self.assertEqual(known_absent, set([(e1.type, e1.state_key)]))
+ self.assertDictEqual(state_dict_ids, {(e1.type, e1.state_key): e1.event_id})
############################################
# test that things work with a partial cache
@@ -336,8 +321,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
# test _get_some_state_from_cache correctly filters out members with types=[]
room_id = self.room.to_string()
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
- self.store._state_group_cache,
- group, [], filtered_types=[EventTypes.Member]
+ self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
)
self.assertEqual(is_all, False)
@@ -346,7 +330,9 @@ class StateStoreTestCase(tests.unittest.TestCase):
room_id = self.room.to_string()
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [], filtered_types=[EventTypes.Member]
+ group,
+ [],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, True)
@@ -355,20 +341,19 @@ class StateStoreTestCase(tests.unittest.TestCase):
# test _get_some_state_from_cache correctly filters in members wildcard types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_cache,
- group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+ group,
+ [(EventTypes.Member, None)],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, False)
- self.assertDictEqual(
- {
- (e1.type, e1.state_key): e1.event_id,
- },
- state_dict,
- )
+ self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+ group,
+ [(EventTypes.Member, None)],
+ filtered_types=[EventTypes.Member],
)
self.assertEqual(is_all, True)
@@ -389,12 +374,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
)
self.assertEqual(is_all, False)
- self.assertDictEqual(
- {
- (e1.type, e1.state_key): e1.event_id,
- },
- state_dict,
- )
+ self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
@@ -404,18 +384,15 @@ class StateStoreTestCase(tests.unittest.TestCase):
)
self.assertEqual(is_all, True)
- self.assertDictEqual(
- {
- (e5.type, e5.state_key): e5.event_id,
- },
- state_dict,
- )
+ self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
# test _get_some_state_from_cache correctly filters in members with specific types
# and no filtered_types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_cache,
- group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+ group,
+ [(EventTypes.Member, e5.state_key)],
+ filtered_types=None,
)
self.assertEqual(is_all, False)
@@ -423,13 +400,10 @@ class StateStoreTestCase(tests.unittest.TestCase):
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_members_cache,
- group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+ group,
+ [(EventTypes.Member, e5.state_key)],
+ filtered_types=None,
)
self.assertEqual(is_all, True)
- self.assertDictEqual(
- {
- (e5.type, e5.state_key): e5.event_id,
- },
- state_dict,
- )
+ self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 0732615447..bdbacb8448 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -185,20 +185,20 @@ class TestMauLimit(unittest.TestCase):
self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
def create_user(self, localpart):
- request_data = json.dumps({
- "username": localpart,
- "password": "monkey",
- "auth": {"type": LoginType.DUMMY},
- })
+ request_data = json.dumps(
+ {
+ "username": localpart,
+ "password": "monkey",
+ "auth": {"type": LoginType.DUMMY},
+ }
+ )
- request, channel = make_request(b"POST", b"/register", request_data)
+ request, channel = make_request("POST", "/register", request_data)
render(request, self.resource, self.reactor)
- if channel.result["code"] != b"200":
+ if channel.code != 200:
raise HttpResponseException(
- int(channel.result["code"]),
- channel.result["reason"],
- channel.result["body"],
+ channel.code, channel.result["reason"], channel.result["body"]
).to_synapse_error()
access_token = channel.json_body["access_token"]
@@ -206,12 +206,12 @@ class TestMauLimit(unittest.TestCase):
return access_token
def do_sync_for_user(self, token):
- request, channel = make_request(b"GET", b"/sync", access_token=token)
+ request, channel = make_request(
+ "GET", "/sync", access_token=token.encode('ascii')
+ )
render(request, self.resource, self.reactor)
- if channel.result["code"] != b"200":
+ if channel.code != 200:
raise HttpResponseException(
- int(channel.result["code"]),
- channel.result["reason"],
- channel.result["body"],
+ channel.code, channel.result["reason"], channel.result["body"]
).to_synapse_error()
diff --git a/tests/test_state.py b/tests/test_state.py
index 452a123c3a..e20c33322a 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -180,7 +180,7 @@ class StateTestCase(unittest.TestCase):
graph = Graph(
nodes={
"START": DictObj(
- type=EventTypes.Create, state_key="", content={}, depth=1,
+ type=EventTypes.Create, state_key="", content={}, depth=1
),
"A": DictObj(type=EventTypes.Message, depth=2),
"B": DictObj(type=EventTypes.Message, depth=3),
diff --git a/tests/utils.py b/tests/utils.py
index b85017d279..215226debf 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -100,8 +100,13 @@ class TestHomeServer(HomeServer):
@defer.inlineCallbacks
def setup_test_homeserver(
- cleanup_func, name="test", datastore=None, config=None, reactor=None,
- homeserverToUse=TestHomeServer, **kargs
+ cleanup_func,
+ name="test",
+ datastore=None,
+ config=None,
+ reactor=None,
+ homeserverToUse=TestHomeServer,
+ **kargs
):
"""
Setup a homeserver suitable for running tests against. Keyword arguments
@@ -147,6 +152,7 @@ def setup_test_homeserver(
config.hs_disabled_message = ""
config.hs_disabled_limit_type = ""
config.max_mau_value = 50
+ config.mau_trial_days = 0
config.mau_limits_reserved_threepids = []
config.admin_contact = None
config.rc_messages_per_second = 10000
@@ -322,8 +328,7 @@ class MockHttpResource(HttpServer):
@patch('twisted.web.http.Request')
@defer.inlineCallbacks
def trigger(
- self, http_method, path, content, mock_request,
- federation_auth_origin=None,
+ self, http_method, path, content, mock_request, federation_auth_origin=None
):
""" Fire an HTTP event.
@@ -356,7 +361,7 @@ class MockHttpResource(HttpServer):
headers = {}
if federation_auth_origin is not None:
headers[b"Authorization"] = [
- b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin, )
+ b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin,)
]
mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
@@ -576,16 +581,16 @@ def create_room(hs, room_id, creator_id):
event_builder_factory = hs.get_event_builder_factory()
event_creation_handler = hs.get_event_creation_handler()
- builder = event_builder_factory.new({
- "type": EventTypes.Create,
- "state_key": "",
- "sender": creator_id,
- "room_id": room_id,
- "content": {},
- })
-
- event, context = yield event_creation_handler.create_new_client_event(
- builder
+ builder = event_builder_factory.new(
+ {
+ "type": EventTypes.Create,
+ "state_key": "",
+ "sender": creator_id,
+ "room_id": room_id,
+ "content": {},
+ }
)
+ event, context = yield event_creation_handler.create_new_client_event(builder)
+
yield store.persist_event(event, context)
|
