diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000000..ca333961f3
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,12 @@
+[run]
+branch = True
+parallel = True
+source = synapse
+
+[paths]
+source=
+ coverage
+
+[report]
+precision = 2
+ignore_errors = True
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000000..3edf9e717c
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,9 @@
+# EditorConfig https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
index 21acb3202a..756759c2d8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -1,3 +1,9 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
<!--
**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**:
@@ -11,38 +17,50 @@ the necessary data to fix your issue.
You can also preview your report before submitting it. You may remove sections
that aren't relevant to your particular case.
-Text between <!-- and --​> marks will be invisible in the report.
+Text between <!-- and --> marks will be invisible in the report.
-->
### Description
-Describe here the problem that you are experiencing, or the feature you are requesting.
+<!-- Describe here the problem that you are experiencing -->
### Steps to reproduce
-- For bugs, list the steps
+- list the steps
- that reproduce the bug
- using hyphens as bullet points
+<!--
Describe how what happens differs from what you expected.
-<!-- If you can identify any relevant log snippets from _homeserver.log_, please include
+If you can identify any relevant log snippets from _homeserver.log_, please include
those (please be careful to remove any personal or private data). Please surround them with
-``` (three backticks, on a line on their own), so that they are formatted legibly. -->
+``` (three backticks, on a line on their own), so that they are formatted legibly.
+-->
### Version information
<!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
-- **Homeserver**: Was this issue identified on matrix.org or another homeserver?
+<!-- Was this issue identified on matrix.org or another homeserver? -->
+- **Homeserver**:
If not matrix.org:
-- **Version**: What version of Synapse is running? <!--
+
+<!--
+What version of Synapse is running?
You can find the Synapse version by inspecting the server headers (replace matrix.org with
your own homeserver domain):
$ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
-->
-- **Install method**: package manager/git clone/pip
-- **Platform**: Tell us about the environment in which your homeserver is operating
- - distro, hardware, if it's running in a vm/container, etc.
+- **Version**:
+
+- **Install method**:
+<!-- examples: package manager/git clone/pip -->
+
+- **Platform**:
+<!--
+Tell us about the environment in which your homeserver is operating
+distro, hardware, if it's running in a vm/container, etc.
+-->
diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md
new file mode 100644
index 0000000000..150a46f505
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md
@@ -0,0 +1,9 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Description:**
+
+<!-- Describe here the feature you are requesting. -->
diff --git a/.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md b/.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md
new file mode 100644
index 0000000000..77581596c4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md
@@ -0,0 +1,9 @@
+---
+name: Support request
+about: I need support for Synapse
+
+---
+
+# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
+
+## Don't file an issue as a support request.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000000..aa883ba505
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,7 @@
+### Pull Request Checklist
+
+<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
+
+* [ ] Pull request is based on the develop branch
+* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog)
+* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off)
diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md
new file mode 100644
index 0000000000..7a4244f673
--- /dev/null
+++ b/.github/SUPPORT.md
@@ -0,0 +1,3 @@
+[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html
+
+It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix
diff --git a/.travis.yml b/.travis.yml
index 655fab9d8e..84d5efff9b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -36,24 +36,24 @@ matrix:
env: TOX_ENV="pep8,check_isort"
- python: 2.7
- env: TOX_ENV=py27 TRIAL_FLAGS="-j 2"
+ env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2"
- python: 2.7
env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2"
- python: 2.7
- env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+ env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4"
services:
- postgresql
- python: 3.5
- env: TOX_ENV=py35 TRIAL_FLAGS="-j 2"
+ env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2"
- python: 3.6
- env: TOX_ENV=py36 TRIAL_FLAGS="-j 2"
+ env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2"
- python: 3.6
- env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
+ env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4"
services:
- postgresql
diff --git a/MANIFEST.in b/MANIFEST.in
index 25cdf0a61b..ec18819bc9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -26,6 +26,7 @@ recursive-include synapse/static *.js
exclude Dockerfile
exclude .dockerignore
exclude test_postgresql.sh
+exclude .editorconfig
include pyproject.toml
recursive-include changelog.d *
@@ -34,6 +35,7 @@ prune .github
prune demo/etc
prune docker
prune .circleci
+prune .coveragerc
exclude jenkins*
recursive-exclude jenkins *.sh
diff --git a/README.rst b/README.rst
index 077939a45d..8c55dd7894 100644
--- a/README.rst
+++ b/README.rst
@@ -142,7 +142,7 @@ Installing prerequisites on openSUSE::
Installing prerequisites on OpenBSD::
doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
- libxslt
+ libxslt jpeg
To install the Synapse homeserver run::
@@ -729,9 +729,10 @@ port:
.. __: `key_management`_
-* Synapse does not currently support SNI on the federation protocol
- (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_), which
- means that using name-based virtual hosting is unreliable.
+* Until v0.33.3, Synapse did not support SNI on the federation port
+ (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_). This bug
+ is now fixed, but means that federating with older servers can be unreliable
+ when using name-based virtual hosting.
Furthermore, a number of the normal reasons for using a reverse-proxy do not
apply:
diff --git a/UPGRADE.rst b/UPGRADE.rst
index 55c77eedde..aa5473ec62 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -48,6 +48,15 @@ returned by the Client-Server API:
# configured on port 443.
curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
+Upgrading to v0.34.0
+====================
+
+This release removes the ``riot.im`` from the default list of trusted identity servers.
+
+If ``riot.im`` is in your homeserver's list of ``trusted_third_party_id_servers``,
+you should remove it. It was added in case a hypothetical future identity server was
+put there. If you don't remove it, users may be unable to deactivate their accounts.
+
Upgrading to v0.33.7
====================
diff --git a/changelog.d/3830.feature b/changelog.d/3830.feature
new file mode 100644
index 0000000000..af472cf763
--- /dev/null
+++ b/changelog.d/3830.feature
@@ -0,0 +1 @@
+Add option to track MAU stats (but not limit people)
diff --git a/changelog.d/4176.bugfix b/changelog.d/4176.bugfix
new file mode 100644
index 0000000000..3846f8a27b
--- /dev/null
+++ b/changelog.d/4176.bugfix
@@ -0,0 +1 @@
+The media repository now no longer fails to decode UTF-8 filenames when downloading remote media.
diff --git a/changelog.d/4180.misc b/changelog.d/4180.misc
new file mode 100644
index 0000000000..80194b3dc0
--- /dev/null
+++ b/changelog.d/4180.misc
@@ -0,0 +1 @@
+A coveragerc file, as well as the py36-coverage tox target, have been added.
diff --git a/changelog.d/4182.misc b/changelog.d/4182.misc
new file mode 100644
index 0000000000..62949a065a
--- /dev/null
+++ b/changelog.d/4182.misc
@@ -0,0 +1 @@
+Add a GitHub pull request template and add multiple issue templates
diff --git a/changelog.d/4183.bugfix b/changelog.d/4183.bugfix
new file mode 100644
index 0000000000..3e9ba3826f
--- /dev/null
+++ b/changelog.d/4183.bugfix
@@ -0,0 +1 @@
+URL previews now correctly decode non-UTF-8 text if the header contains a `<meta http-equiv="Content-Type"` header.
diff --git a/changelog.d/4188.misc b/changelog.d/4188.misc
new file mode 100644
index 0000000000..80c3995870
--- /dev/null
+++ b/changelog.d/4188.misc
@@ -0,0 +1 @@
+Update README to reflect the fact that #1491 is fixed
diff --git a/changelog.d/4189.misc b/changelog.d/4189.misc
new file mode 100644
index 0000000000..4a41357d94
--- /dev/null
+++ b/changelog.d/4189.misc
@@ -0,0 +1,2 @@
+Run the AS senders as background processes to fix warnings
+
diff --git a/changelog.d/4190.misc b/changelog.d/4190.misc
new file mode 100644
index 0000000000..6700a5150d
--- /dev/null
+++ b/changelog.d/4190.misc
@@ -0,0 +1 @@
+Add some diagnostics to the tests to detect logcontext problems
diff --git a/changelog.d/4192.bugfix b/changelog.d/4192.bugfix
new file mode 100644
index 0000000000..f346fe026a
--- /dev/null
+++ b/changelog.d/4192.bugfix
@@ -0,0 +1 @@
+Fix an issue where public consent URLs had two slashes.
diff --git a/changelog.d/4193.misc b/changelog.d/4193.misc
new file mode 100644
index 0000000000..23d86eab24
--- /dev/null
+++ b/changelog.d/4193.misc
@@ -0,0 +1 @@
+Add missing `jpeg` package prerequisite for OpenBSD in README.
diff --git a/changelog.d/4197.bugfix b/changelog.d/4197.bugfix
new file mode 100644
index 0000000000..c7c01da0f3
--- /dev/null
+++ b/changelog.d/4197.bugfix
@@ -0,0 +1 @@
+Fallback auth now accepts the session parameter on Python 3.
diff --git a/changelog.d/4200.misc b/changelog.d/4200.misc
new file mode 100644
index 0000000000..505c98edcf
--- /dev/null
+++ b/changelog.d/4200.misc
@@ -0,0 +1 @@
+Add a note saying you need to manually reclaim disk space after using the Purge History API
diff --git a/changelog.d/4204.misc b/changelog.d/4204.misc
new file mode 100644
index 0000000000..efd1f4abd6
--- /dev/null
+++ b/changelog.d/4204.misc
@@ -0,0 +1 @@
+Fix logcontext leaks in EmailPusher and in tests
\ No newline at end of file
diff --git a/changelog.d/4205.misc b/changelog.d/4205.misc
new file mode 100644
index 0000000000..bbdce2c7aa
--- /dev/null
+++ b/changelog.d/4205.misc
@@ -0,0 +1 @@
+More logcontext checking in unittests
diff --git a/changelog.d/4207.bugfix b/changelog.d/4207.bugfix
new file mode 100644
index 0000000000..c1ec9f6329
--- /dev/null
+++ b/changelog.d/4207.bugfix
@@ -0,0 +1 @@
+Remove riot.im from the list of trusted Identity Servers in the default configuration
diff --git a/changelog.d/4209.misc b/changelog.d/4209.misc
new file mode 100644
index 0000000000..efd1f4abd6
--- /dev/null
+++ b/changelog.d/4209.misc
@@ -0,0 +1 @@
+Fix logcontext leaks in EmailPusher and in tests
\ No newline at end of file
diff --git a/changelog.d/4211.bugfix b/changelog.d/4211.bugfix
new file mode 100644
index 0000000000..376f80c55a
--- /dev/null
+++ b/changelog.d/4211.bugfix
@@ -0,0 +1,2 @@
+fix start up failure when mau_limit_reserved_threepids set and db is postgres
+
diff --git a/changelog.d/4214.misc b/changelog.d/4214.misc
new file mode 100644
index 0000000000..b2f62060e3
--- /dev/null
+++ b/changelog.d/4214.misc
@@ -0,0 +1 @@
+Ignore __pycache__ directories in the database schema folder
diff --git a/changelog.d/4220.feature b/changelog.d/4220.feature
new file mode 100644
index 0000000000..e7a3e40483
--- /dev/null
+++ b/changelog.d/4220.feature
@@ -0,0 +1 @@
+Rename login type m.login.cas to m.login.sso
diff --git a/changelog.d/4223.bugfix b/changelog.d/4223.bugfix
new file mode 100644
index 0000000000..bab591a765
--- /dev/null
+++ b/changelog.d/4223.bugfix
@@ -0,0 +1 @@
+Fix auto join failures for servers that require user consent
diff --git a/changelog.d/4224.misc b/changelog.d/4224.misc
new file mode 100644
index 0000000000..1bfe2e5c31
--- /dev/null
+++ b/changelog.d/4224.misc
@@ -0,0 +1 @@
+Add note to UPGRADE.rst about removing riot.im from list of trusted identity servers
diff --git a/changelog.d/4225.misc b/changelog.d/4225.misc
new file mode 100644
index 0000000000..39062696ea
--- /dev/null
+++ b/changelog.d/4225.misc
@@ -0,0 +1 @@
+Added automated coverage reporting to CI.
diff --git a/changelog.d/4227.misc b/changelog.d/4227.misc
new file mode 100644
index 0000000000..7ebd51b6a4
--- /dev/null
+++ b/changelog.d/4227.misc
@@ -0,0 +1 @@
+Garbage-collect after each unit test to fix logcontext leaks
\ No newline at end of file
diff --git a/changelog.d/4230.feature b/changelog.d/4230.feature
new file mode 100644
index 0000000000..0ecb1d5ec6
--- /dev/null
+++ b/changelog.d/4230.feature
@@ -0,0 +1 @@
+Add an option to disable search for homeservers that may not be interested in it.
diff --git a/changelog.d/4234.misc b/changelog.d/4234.misc
new file mode 100644
index 0000000000..b5a01d38af
--- /dev/null
+++ b/changelog.d/4234.misc
@@ -0,0 +1 @@
+add more detail to logging regarding "More than one row matched" error
\ No newline at end of file
diff --git a/changelog.d/4241.bugfix b/changelog.d/4241.bugfix
new file mode 100644
index 0000000000..1158a5aa16
--- /dev/null
+++ b/changelog.d/4241.bugfix
@@ -0,0 +1 @@
+Fix exception caused by non-ascii event IDs
diff --git a/changelog.d/4244.misc b/changelog.d/4244.misc
new file mode 100644
index 0000000000..8236bbc28d
--- /dev/null
+++ b/changelog.d/4244.misc
@@ -0,0 +1 @@
+Drop sent_transactions table
diff --git a/changelog.d/4250.bugfix b/changelog.d/4250.bugfix
new file mode 100644
index 0000000000..1f60f5bd0a
--- /dev/null
+++ b/changelog.d/4250.bugfix
@@ -0,0 +1 @@
+Pushers can now be unsubscribed from on Python 3.
diff --git a/changelog.d/4253.bugfix b/changelog.d/4253.bugfix
new file mode 100644
index 0000000000..1796e95b86
--- /dev/null
+++ b/changelog.d/4253.bugfix
@@ -0,0 +1 @@
+Fix UnicodeDecodeError when postgres is configured to give non-English errors
diff --git a/changelog.d/4257.misc b/changelog.d/4257.misc
new file mode 100644
index 0000000000..43ac24cb7d
--- /dev/null
+++ b/changelog.d/4257.misc
@@ -0,0 +1 @@
+Add a basic .editorconfig
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index 1b0f655d26..d5074be6dd 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -150,10 +150,12 @@ enable_group_creation: true
# The list of identity servers trusted to verify third party
# identifiers by this server.
+#
+# Also defines the ID server which will be called when an account is
+# deactivated (one will be picked arbitrarily).
trusted_third_party_id_servers:
- matrix.org
- vector.im
- - riot.im
## Metrics ###
diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.rst
index 2da833c827..a5c3dc8149 100644
--- a/docs/admin_api/purge_history_api.rst
+++ b/docs/admin_api/purge_history_api.rst
@@ -61,3 +61,11 @@ the following:
}
The status will be one of ``active``, ``complete``, or ``failed``.
+
+Reclaim disk space (Postgres)
+-----------------------------
+
+To reclaim the disk space and return it to the operating system, you need to run
+`VACUUM FULL;` on the database.
+
+https://www.postgresql.org/docs/current/sql-vacuum.html
diff --git a/docs/log_contexts.rst b/docs/log_contexts.rst
index 82ac4f91e5..27cde11cf7 100644
--- a/docs/log_contexts.rst
+++ b/docs/log_contexts.rst
@@ -163,7 +163,7 @@ the logcontext was set, this will make things work out ok: provided
It's all too easy to forget to ``yield``: for instance if we forgot that
``do_some_stuff`` returned a deferred, we might plough on regardless. This
leads to a mess; it will probably work itself out eventually, but not before
-a load of stuff has been logged against the wrong content. (Normally, other
+a load of stuff has been logged against the wrong context. (Normally, other
things will break, more obviously, if you forget to ``yield``, so this tends
not to be a major problem in practice.)
@@ -440,3 +440,59 @@ To conclude: I think this scheme would have worked equally well, with less
danger of messing it up, and probably made some more esoteric code easier to
write. But again — changing the conventions of the entire Synapse codebase is
not a sensible option for the marginal improvement offered.
+
+
+A note on garbage-collection of Deferred chains
+-----------------------------------------------
+
+It turns out that our logcontext rules do not play nicely with Deferred
+chains which get orphaned and garbage-collected.
+
+Imagine we have some code that looks like this:
+
+.. code:: python
+
+ listener_queue = []
+
+ def on_something_interesting():
+ for d in listener_queue:
+ d.callback("foo")
+
+ @defer.inlineCallbacks
+ def await_something_interesting():
+ new_deferred = defer.Deferred()
+ listener_queue.append(new_deferred)
+
+ with PreserveLoggingContext():
+ yield new_deferred
+
+Obviously, the idea here is that we have a bunch of things which are waiting
+for an event. (It's just an example of the problem here, but a relatively
+common one.)
+
+Now let's imagine two further things happen. First of all, whatever was
+waiting for the interesting thing goes away. (Perhaps the request times out,
+or something *even more* interesting happens.)
+
+Secondly, let's suppose that we decide that the interesting thing is never
+going to happen, and we reset the listener queue:
+
+.. code:: python
+
+ def reset_listener_queue():
+ listener_queue.clear()
+
+So, both ends of the deferred chain have now dropped their references, and the
+deferred chain is now orphaned, and will be garbage-collected at some point.
+Note that ``await_something_interesting`` is a generator function, and when
+Python garbage-collects generator functions, it gives them a chance to clean
+up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case,
+that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will
+carefully restore the request context, but there is now nothing waiting for
+its return, so the request context is never cleared.
+
+To reiterate, this problem only arises when *both* ends of a deferred chain
+are dropped. Dropping the the reference to a deferred you're supposed to be
+calling is probably bad practice, so this doesn't actually happen too much.
+Unfortunately, when it does happen, it will lead to leaked logcontexts which
+are incredibly hard to track down.
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 415374a2ce..3e4dea2f19 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -535,7 +535,7 @@ def run(hs):
current_mau_count = 0
reserved_count = 0
store = hs.get_datastore()
- if hs.config.limit_usage_by_mau:
+ if hs.config.limit_usage_by_mau or hs.config.mau_stats_only:
current_mau_count = yield store.get_monthly_active_count()
reserved_count = yield store.get_registered_reserved_users_count()
current_mau_gauge.set(float(current_mau_count))
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 2430814796..685f15c061 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -53,8 +53,8 @@ import logging
from twisted.internet import defer
from synapse.appservice import ApplicationServiceState
+from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.util.logcontext import run_in_background
-from synapse.util.metrics import Measure
logger = logging.getLogger(__name__)
@@ -104,27 +104,34 @@ class _ServiceQueuer(object):
self.clock = clock
def enqueue(self, service, event):
- # if this service isn't being sent something
self.queued_events.setdefault(service.id, []).append(event)
- run_in_background(self._send_request, service)
- @defer.inlineCallbacks
- def _send_request(self, service):
+ # start a sender for this appservice if we don't already have one
+
if service.id in self.requests_in_flight:
return
+ run_as_background_process(
+ "as-sender-%s" % (service.id, ),
+ self._send_request, service,
+ )
+
+ @defer.inlineCallbacks
+ def _send_request(self, service):
+ # sanity-check: we shouldn't get here if this service already has a sender
+ # running.
+ assert(service.id not in self.requests_in_flight)
+
self.requests_in_flight.add(service.id)
try:
while True:
events = self.queued_events.pop(service.id, [])
if not events:
return
-
- with Measure(self.clock, "servicequeuer.send"):
- try:
- yield self.txn_ctrl.send(service, events)
- except Exception:
- logger.exception("AS request failed")
+ try:
+ yield self.txn_ctrl.send(service, events)
+ except Exception:
+ logger.exception("AS request failed")
finally:
self.requests_in_flight.discard(service.id)
@@ -223,7 +230,12 @@ class _Recoverer(object):
self.backoff_counter = 1
def recover(self):
- self.clock.call_later((2 ** self.backoff_counter), self.retry)
+ def _retry():
+ run_as_background_process(
+ "as-recoverer-%s" % (self.service.id,),
+ self.retry,
+ )
+ self.clock.call_later((2 ** self.backoff_counter), _retry)
def _backoff(self):
# cap the backoff to be around 8.5min => (2^9) = 512 secs
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index 7480ed5145..717bbfec61 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -93,10 +93,12 @@ class RegistrationConfig(Config):
# The list of identity servers trusted to verify third party
# identifiers by this server.
+ #
+ # Also defines the ID server which will be called when an account is
+ # deactivated (one will be picked arbitrarily).
trusted_third_party_id_servers:
- matrix.org
- vector.im
- - riot.im
# Users who register on this homeserver will automatically be joined
# to these rooms
diff --git a/synapse/config/server.py b/synapse/config/server.py
index c1c7c0105e..4a5b902f8e 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -62,6 +62,11 @@ class ServerConfig(Config):
# master, potentially causing inconsistency.
self.enable_media_repo = config.get("enable_media_repo", True)
+ # whether to enable search. If disabled, new entries will not be inserted
+ # into the search tables and they will not be indexed. Users will receive
+ # errors when attempting to search for messages.
+ self.enable_search = config.get("enable_search", True)
+
self.filter_timeline_limit = config.get("filter_timeline_limit", -1)
# Whether we should block invites sent to users on this server
@@ -77,6 +82,7 @@ class ServerConfig(Config):
self.max_mau_value = config.get(
"max_mau_value", 0,
)
+ self.mau_stats_only = config.get("mau_stats_only", False)
self.mau_limits_reserved_threepids = config.get(
"mau_limit_reserved_threepids", []
@@ -372,13 +378,23 @@ class ServerConfig(Config):
# max_mau_value: 50
# mau_trial_days: 2
#
+ # If enabled, the metrics for the number of monthly active users will
+ # be populated, however no one will be limited. If limit_usage_by_mau
+ # is true, this is implied to be true.
+ # mau_stats_only: False
+ #
# Sometimes the server admin will want to ensure certain accounts are
# never blocked by mau checking. These accounts are specified here.
#
# mau_limit_reserved_threepids:
# - medium: 'email'
# address: 'reserved_user@example.com'
-
+ #
+ # Room searching
+ #
+ # If disabled, new messages will not be indexed for searching and users
+ # will receive errors when searching for messages. Defaults to enabled.
+ # enable_search: true
""" % locals()
def read_arguments(self, args):
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index a958c45271..c6e89db4bc 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -473,7 +473,7 @@ class AuthHandler(BaseHandler):
"version": self.hs.config.user_consent_version,
"en": {
"name": self.hs.config.user_consent_policy_name,
- "url": "%s/_matrix/consent?v=%s" % (
+ "url": "%s_matrix/consent?v=%s" % (
self.hs.config.public_baseurl,
self.hs.config.user_consent_version,
),
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index d2beb275cf..015909bb26 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -217,7 +217,19 @@ class RegistrationHandler(BaseHandler):
user_id = None
token = None
attempts += 1
+ if not self.hs.config.user_consent_at_registration:
+ yield self._auto_join_rooms(user_id)
+ defer.returnValue((user_id, token))
+
+ @defer.inlineCallbacks
+ def _auto_join_rooms(self, user_id):
+ """Automatically joins users to auto join rooms - creating the room in the first place
+ if the user is the first to be created.
+
+ Args:
+ user_id(str): The user to join
+ """
# auto-join the user to any rooms we're supposed to dump them into
fake_requester = create_requester(user_id)
@@ -226,7 +238,6 @@ class RegistrationHandler(BaseHandler):
if self.hs.config.autocreate_auto_join_rooms:
count = yield self.store.count_all_users()
should_auto_create_rooms = count == 1
-
for r in self.hs.config.auto_join_rooms:
try:
if should_auto_create_rooms:
@@ -256,7 +267,15 @@ class RegistrationHandler(BaseHandler):
except Exception as e:
logger.error("Failed to join new user to %r: %r", r, e)
- defer.returnValue((user_id, token))
+ @defer.inlineCallbacks
+ def post_consent_actions(self, user_id):
+ """A series of registration actions that can only be carried out once consent
+ has been granted
+
+ Args:
+ user_id (str): The user to join
+ """
+ yield self._auto_join_rooms(user_id)
@defer.inlineCallbacks
def appservice_register(self, user_localpart, as_token):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 80e7b15de8..ec936bbb4e 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -50,6 +50,9 @@ class SearchHandler(BaseHandler):
dict to be returned to the client with results of search
"""
+ if not self.hs.config.enable_search:
+ raise SynapseError(400, "Search is disabled on this homeserver")
+
batch_group = None
batch_group_key = None
batch_token = None
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index ebcb93bfc7..1eb5be0957 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -36,6 +36,7 @@ from synapse.push.presentable_names import (
)
from synapse.types import UserID
from synapse.util.async_helpers import concurrently_execute
+from synapse.util.logcontext import make_deferred_yieldable
from synapse.visibility import filter_events_for_client
logger = logging.getLogger(__name__)
@@ -192,7 +193,7 @@ class Mailer(object):
logger.info("Sending email push notification to %s" % email_address)
- yield self.sendmail(
+ yield make_deferred_yieldable(self.sendmail(
self.hs.config.email_smtp_host,
raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
reactor=self.hs.get_reactor(),
@@ -201,7 +202,7 @@ class Mailer(object):
username=self.hs.config.email_smtp_user,
password=self.hs.config.email_smtp_pass,
requireTransportSecurity=self.hs.config.require_transport_security
- )
+ ))
@defer.inlineCallbacks
def get_room_vars(self, room_id, user_id, notifs, notif_events, room_state_ids):
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 0010699d31..f6b4a85e40 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -27,7 +27,7 @@ from twisted.web.client import PartialDownloadError
from synapse.api.errors import Codes, LoginError, SynapseError
from synapse.http.server import finish_request
-from synapse.http.servlet import parse_json_object_from_request
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.types import UserID
from synapse.util.msisdn import phone_number_to_msisdn
@@ -83,6 +83,7 @@ class LoginRestServlet(ClientV1RestServlet):
PATTERNS = client_path_patterns("/login$")
SAML2_TYPE = "m.login.saml2"
CAS_TYPE = "m.login.cas"
+ SSO_TYPE = "m.login.sso"
TOKEN_TYPE = "m.login.token"
JWT_TYPE = "m.login.jwt"
@@ -105,6 +106,10 @@ class LoginRestServlet(ClientV1RestServlet):
if self.saml2_enabled:
flows.append({"type": LoginRestServlet.SAML2_TYPE})
if self.cas_enabled:
+ flows.append({"type": LoginRestServlet.SSO_TYPE})
+
+ # we advertise CAS for backwards compat, though MSC1721 renamed it
+ # to SSO.
flows.append({"type": LoginRestServlet.CAS_TYPE})
# While its valid for us to advertise this login type generally,
@@ -384,11 +389,11 @@ class SAML2RestServlet(ClientV1RestServlet):
defer.returnValue((200, {"status": "not_authenticated"}))
-class CasRedirectServlet(ClientV1RestServlet):
- PATTERNS = client_path_patterns("/login/cas/redirect", releases=())
+class CasRedirectServlet(RestServlet):
+ PATTERNS = client_path_patterns("/login/(cas|sso)/redirect")
def __init__(self, hs):
- super(CasRedirectServlet, self).__init__(hs)
+ super(CasRedirectServlet, self).__init__()
self.cas_server_url = hs.config.cas_server_url.encode('ascii')
self.cas_service_url = hs.config.cas_service_url.encode('ascii')
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index b84f0260f2..4c07ae7f45 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -142,7 +142,7 @@ class PushersRemoveRestServlet(RestServlet):
To allow pusher to be delete by clicking a link (ie. GET request)
"""
PATTERNS = client_path_patterns("/pushers/remove$")
- SUCCESS_HTML = "<html><body>You have been unsubscribed</body><html>"
+ SUCCESS_HTML = b"<html><body>You have been unsubscribed</body><html>"
def __init__(self, hs):
super(PushersRemoveRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
index a8d8ed6590..fa73bdf3a1 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -21,7 +21,7 @@ from synapse.api.constants import LoginType
from synapse.api.errors import SynapseError
from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
from synapse.http.server import finish_request
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_string
from ._base import client_v2_patterns
@@ -131,16 +131,12 @@ class AuthRestServlet(RestServlet):
self.auth_handler = hs.get_auth_handler()
self.registration_handler = hs.get_handlers().registration_handler
- @defer.inlineCallbacks
def on_GET(self, request, stagetype):
- yield
- if stagetype == LoginType.RECAPTCHA:
- if ('session' not in request.args or
- len(request.args['session']) == 0):
- raise SynapseError(400, "No session supplied")
-
- session = request.args["session"][0]
+ session = parse_string(request, "session")
+ if not session:
+ raise SynapseError(400, "No session supplied")
+ if stagetype == LoginType.RECAPTCHA:
html = RECAPTCHA_TEMPLATE % {
'session': session,
'myurl': "%s/auth/%s/fallback/web" % (
@@ -155,13 +151,11 @@ class AuthRestServlet(RestServlet):
request.write(html_bytes)
finish_request(request)
- defer.returnValue(None)
+ return None
elif stagetype == LoginType.TERMS:
- session = request.args['session'][0]
-
html = TERMS_TEMPLATE % {
'session': session,
- 'terms_url': "%s/_matrix/consent?v=%s" % (
+ 'terms_url': "%s_matrix/consent?v=%s" % (
self.hs.config.public_baseurl,
self.hs.config.user_consent_version,
),
@@ -176,25 +170,25 @@ class AuthRestServlet(RestServlet):
request.write(html_bytes)
finish_request(request)
- defer.returnValue(None)
+ return None
else:
raise SynapseError(404, "Unknown auth stage type")
@defer.inlineCallbacks
def on_POST(self, request, stagetype):
- yield
+
+ session = parse_string(request, "session")
+ if not session:
+ raise SynapseError(400, "No session supplied")
+
if stagetype == LoginType.RECAPTCHA:
- if ('g-recaptcha-response' not in request.args or
- len(request.args['g-recaptcha-response'])) == 0:
- raise SynapseError(400, "No captcha response supplied")
- if ('session' not in request.args or
- len(request.args['session'])) == 0:
- raise SynapseError(400, "No session supplied")
+ response = parse_string(request, "g-recaptcha-response")
- session = request.args['session'][0]
+ if not response:
+ raise SynapseError(400, "No captcha response supplied")
authdict = {
- 'response': request.args['g-recaptcha-response'][0],
+ 'response': response,
'session': session,
}
@@ -242,7 +236,7 @@ class AuthRestServlet(RestServlet):
else:
html = TERMS_TEMPLATE % {
'session': session,
- 'terms_url': "%s/_matrix/consent?v=%s" % (
+ 'terms_url': "%s_matrix/consent?v=%s" % (
self.hs.config.public_baseurl,
self.hs.config.user_consent_version,
),
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 0515715f7c..aec0c6b075 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -457,6 +457,7 @@ class RegisterRestServlet(RestServlet):
yield self.store.user_set_consent_version(
registered_user_id, self.hs.config.user_consent_version,
)
+ yield self.registration_handler.post_consent_actions(registered_user_id)
defer.returnValue((200, return_dict))
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index ad525b22e1..80611cfe84 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -89,6 +89,7 @@ class ConsentResource(Resource):
self.hs = hs
self.store = hs.get_datastore()
+ self.registration_handler = hs.get_handlers().registration_handler
# this is required by the request_handler wrapper
self.clock = hs.get_clock()
@@ -199,6 +200,7 @@ class ConsentResource(Resource):
if e.code != 404:
raise
raise NotFoundError("Unknown user")
+ yield self.registration_handler.post_consent_actions(qualified_user_id)
try:
self._render_template(request, "success.html")
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 76e479afa3..efe42a429d 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -16,6 +16,7 @@
import logging
import os
+from six import PY3
from six.moves import urllib
from twisted.internet import defer
@@ -48,26 +49,21 @@ def parse_media_id(request):
return server_name, media_id, file_name
except Exception:
raise SynapseError(
- 404,
- "Invalid media id token %r" % (request.postpath,),
- Codes.UNKNOWN,
+ 404, "Invalid media id token %r" % (request.postpath,), Codes.UNKNOWN
)
def respond_404(request):
respond_with_json(
- request, 404,
- cs_error(
- "Not found %r" % (request.postpath,),
- code=Codes.NOT_FOUND,
- ),
- send_cors=True
+ request,
+ 404,
+ cs_error("Not found %r" % (request.postpath,), code=Codes.NOT_FOUND),
+ send_cors=True,
)
@defer.inlineCallbacks
-def respond_with_file(request, media_type, file_path,
- file_size=None, upload_name=None):
+def respond_with_file(request, media_type, file_path, file_size=None, upload_name=None):
logger.debug("Responding with %r", file_path)
if os.path.isfile(file_path):
@@ -97,31 +93,26 @@ def add_file_headers(request, media_type, file_size, upload_name):
file_size (int): Size in bytes of the media, if known.
upload_name (str): The name of the requested file, if any.
"""
+
def _quote(x):
return urllib.parse.quote(x.encode("utf-8"))
request.setHeader(b"Content-Type", media_type.encode("UTF-8"))
if upload_name:
if is_ascii(upload_name):
- disposition = ("inline; filename=%s" % (_quote(upload_name),)).encode("ascii")
+ disposition = "inline; filename=%s" % (_quote(upload_name),)
else:
- disposition = (
- "inline; filename*=utf-8''%s" % (_quote(upload_name),)).encode("ascii")
+ disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name),)
- request.setHeader(b"Content-Disposition", disposition)
+ request.setHeader(b"Content-Disposition", disposition.encode('ascii'))
# cache for at least a day.
# XXX: we might want to turn this off for data we don't want to
# recommend caching as it's sensitive or private - or at least
# select private. don't bother setting Expires as all our
# clients are smart enough to be happy with Cache-Control
- request.setHeader(
- b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
- )
-
- request.setHeader(
- b"Content-Length", b"%d" % (file_size,)
- )
+ request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400")
+ request.setHeader(b"Content-Length", b"%d" % (file_size,))
@defer.inlineCallbacks
@@ -153,6 +144,7 @@ class Responder(object):
Responder is a context manager which *must* be used, so that any resources
held can be cleaned up.
"""
+
def write_to_consumer(self, consumer):
"""Stream response into consumer
@@ -186,9 +178,18 @@ class FileInfo(object):
thumbnail_method (str)
thumbnail_type (str): Content type of thumbnail, e.g. image/png
"""
- def __init__(self, server_name, file_id, url_cache=False,
- thumbnail=False, thumbnail_width=None, thumbnail_height=None,
- thumbnail_method=None, thumbnail_type=None):
+
+ def __init__(
+ self,
+ server_name,
+ file_id,
+ url_cache=False,
+ thumbnail=False,
+ thumbnail_width=None,
+ thumbnail_height=None,
+ thumbnail_method=None,
+ thumbnail_type=None,
+ ):
self.server_name = server_name
self.file_id = file_id
self.url_cache = url_cache
@@ -197,3 +198,74 @@ class FileInfo(object):
self.thumbnail_height = thumbnail_height
self.thumbnail_method = thumbnail_method
self.thumbnail_type = thumbnail_type
+
+
+def get_filename_from_headers(headers):
+ """
+ Get the filename of the downloaded file by inspecting the
+ Content-Disposition HTTP header.
+
+ Args:
+ headers (twisted.web.http_headers.Headers): The HTTP
+ request headers.
+
+ Returns:
+ A Unicode string of the filename, or None.
+ """
+ content_disposition = headers.get(b"Content-Disposition", [b''])
+
+ # No header, bail out.
+ if not content_disposition[0]:
+ return
+
+ # dict of unicode: bytes, corresponding to the key value sections of the
+ # Content-Disposition header.
+ params = {}
+ parts = content_disposition[0].split(b";")
+ for i in parts:
+ # Split into key-value pairs, if able
+ # We don't care about things like `inline`, so throw it out
+ if b"=" not in i:
+ continue
+
+ key, value = i.strip().split(b"=")
+ params[key.decode('ascii')] = value
+
+ upload_name = None
+
+ # First check if there is a valid UTF-8 filename
+ upload_name_utf8 = params.get("filename*", None)
+ if upload_name_utf8:
+ if upload_name_utf8.lower().startswith(b"utf-8''"):
+ upload_name_utf8 = upload_name_utf8[7:]
+ # We have a filename*= section. This MUST be ASCII, and any UTF-8
+ # bytes are %-quoted.
+ if PY3:
+ try:
+ # Once it is decoded, we can then unquote the %-encoded
+ # parts strictly into a unicode string.
+ upload_name = urllib.parse.unquote(
+ upload_name_utf8.decode('ascii'), errors="strict"
+ )
+ except UnicodeDecodeError:
+ # Incorrect UTF-8.
+ pass
+ else:
+ # On Python 2, we first unquote the %-encoded parts and then
+ # decode it strictly using UTF-8.
+ try:
+ upload_name = urllib.parse.unquote(upload_name_utf8).decode('utf8')
+ except UnicodeDecodeError:
+ pass
+
+ # If there isn't check for an ascii name.
+ if not upload_name:
+ upload_name_ascii = params.get("filename", None)
+ if upload_name_ascii and is_ascii(upload_name_ascii):
+ # Make sure there's no %-quoted bytes. If there is, reject it as
+ # non-valid ASCII.
+ if b"%" not in upload_name_ascii:
+ upload_name = upload_name_ascii.decode('ascii')
+
+ # This may be None here, indicating we did not find a matching name.
+ return upload_name
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index d6c5f07af0..e117836e9a 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -14,14 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import cgi
import errno
import logging
import os
import shutil
-from six import PY3, iteritems
-from six.moves.urllib import parse as urlparse
+from six import iteritems
import twisted.internet.error
import twisted.web.http
@@ -34,14 +32,18 @@ from synapse.api.errors import (
NotFoundError,
SynapseError,
)
-from synapse.http.matrixfederationclient import MatrixFederationHttpClient
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.util import logcontext
from synapse.util.async_helpers import Linearizer
from synapse.util.retryutils import NotRetryingDestination
-from synapse.util.stringutils import is_ascii, random_string
+from synapse.util.stringutils import random_string
-from ._base import FileInfo, respond_404, respond_with_responder
+from ._base import (
+ FileInfo,
+ get_filename_from_headers,
+ respond_404,
+ respond_with_responder,
+)
from .config_resource import MediaConfigResource
from .download_resource import DownloadResource
from .filepath import MediaFilePaths
@@ -62,7 +64,7 @@ class MediaRepository(object):
def __init__(self, hs):
self.hs = hs
self.auth = hs.get_auth()
- self.client = MatrixFederationHttpClient(hs)
+ self.client = hs.get_http_client()
self.clock = hs.get_clock()
self.server_name = hs.hostname
self.store = hs.get_datastore()
@@ -397,39 +399,9 @@ class MediaRepository(object):
yield finish()
media_type = headers[b"Content-Type"][0].decode('ascii')
-
+ upload_name = get_filename_from_headers(headers)
time_now_ms = self.clock.time_msec()
- content_disposition = headers.get(b"Content-Disposition", None)
- if content_disposition:
- _, params = cgi.parse_header(content_disposition[0].decode('ascii'),)
- upload_name = None
-
- # First check if there is a valid UTF-8 filename
- upload_name_utf8 = params.get("filename*", None)
- if upload_name_utf8:
- if upload_name_utf8.lower().startswith("utf-8''"):
- upload_name = upload_name_utf8[7:]
-
- # If there isn't check for an ascii name.
- if not upload_name:
- upload_name_ascii = params.get("filename", None)
- if upload_name_ascii and is_ascii(upload_name_ascii):
- upload_name = upload_name_ascii
-
- if upload_name:
- if PY3:
- upload_name = urlparse.unquote(upload_name)
- else:
- upload_name = urlparse.unquote(upload_name.encode('ascii'))
- try:
- if isinstance(upload_name, bytes):
- upload_name = upload_name.decode("utf-8")
- except UnicodeDecodeError:
- upload_name = None
- else:
- upload_name = None
-
logger.info("Stored remote media in file %r", fname)
yield self.store.store_cached_remote_media(
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 91d1dafe64..d0ecf241b6 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import cgi
import datetime
import errno
import fnmatch
@@ -44,15 +43,19 @@ from synapse.http.server import (
)
from synapse.http.servlet import parse_integer, parse_string
from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.rest.media.v1._base import get_filename_from_headers
from synapse.util.async_helpers import ObservableDeferred
from synapse.util.caches.expiringcache import ExpiringCache
from synapse.util.logcontext import make_deferred_yieldable, run_in_background
-from synapse.util.stringutils import is_ascii, random_string
+from synapse.util.stringutils import random_string
from ._base import FileInfo
logger = logging.getLogger(__name__)
+_charset_match = re.compile(br"<\s*meta[^>]*charset\s*=\s*([a-z0-9-]+)", flags=re.I)
+_content_type_match = re.compile(r'.*; *charset="?(.*?)"?(;|$)', flags=re.I)
+
class PreviewUrlResource(Resource):
isLeaf = True
@@ -223,15 +226,25 @@ class PreviewUrlResource(Resource):
with open(media_info['filename'], 'rb') as file:
body = file.read()
- # clobber the encoding from the content-type, or default to utf-8
- # XXX: this overrides any <meta/> or XML charset headers in the body
- # which may pose problems, but so far seems to work okay.
- match = re.match(
- r'.*; *charset="?(.*?)"?(;|$)',
- media_info['media_type'],
- re.I
- )
- encoding = match.group(1) if match else "utf-8"
+ encoding = None
+
+ # Let's try and figure out if it has an encoding set in a meta tag.
+ # Limit it to the first 1kb, since it ought to be in the meta tags
+ # at the top.
+ match = _charset_match.search(body[:1000])
+
+ # If we find a match, it should take precedence over the
+ # Content-Type header, so set it here.
+ if match:
+ encoding = match.group(1).decode('ascii')
+
+ # If we don't find a match, we'll look at the HTTP Content-Type, and
+ # if that doesn't exist, we'll fall back to UTF-8.
+ if not encoding:
+ match = _content_type_match.match(
+ media_info['media_type']
+ )
+ encoding = match.group(1) if match else "utf-8"
og = decode_and_calc_og(body, media_info['uri'], encoding)
@@ -323,31 +336,7 @@ class PreviewUrlResource(Resource):
media_type = "application/octet-stream"
time_now_ms = self.clock.time_msec()
- content_disposition = headers.get(b"Content-Disposition", None)
- if content_disposition:
- _, params = cgi.parse_header(content_disposition[0],)
- download_name = None
-
- # First check if there is a valid UTF-8 filename
- download_name_utf8 = params.get("filename*", None)
- if download_name_utf8:
- if download_name_utf8.lower().startswith("utf-8''"):
- download_name = download_name_utf8[7:]
-
- # If there isn't check for an ascii name.
- if not download_name:
- download_name_ascii = params.get("filename", None)
- if download_name_ascii and is_ascii(download_name_ascii):
- download_name = download_name_ascii
-
- if download_name:
- download_name = urlparse.unquote(download_name)
- try:
- download_name = download_name.decode("utf-8")
- except UnicodeDecodeError:
- download_name = None
- else:
- download_name = None
+ download_name = get_filename_from_headers(headers)
yield self.store.store_local_media(
media_id=file_id,
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index 70a981f4a2..19e091ce3b 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -298,6 +298,8 @@ def _resolve_normal_events(events, auth_events):
def _ordered_events(events):
def key_func(e):
- return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
+ # we have to use utf-8 rather than ascii here because it turns out we allow
+ # people to send us events with non-ascii event IDs :/
+ return -int(e.depth), hashlib.sha1(e.event_id.encode('utf-8')).hexdigest()
return sorted(events, key=key_func)
diff --git a/synapse/static/client/login/index.html b/synapse/static/client/login/index.html
index 96c8723cab..bcb6bc6bb7 100644
--- a/synapse/static/client/login/index.html
+++ b/synapse/static/client/login/index.html
@@ -12,35 +12,30 @@
<h1>Log in with one of the following methods</h1>
<span id="feedback" style="color: #f00"></span>
- <br/>
- <br/>
<div id="loading">
<img src="spinner.gif" />
</div>
- <div id="cas_flow" class="login_flow" style="display:none"
- onclick="gotoCas(); return false;">
- CAS Authentication: <button id="cas_button" style="margin: 10px">Log in</button>
+ <div id="sso_flow" class="login_flow" style="display:none">
+ Single-sign on:
+ <form id="sso_form" action="/_matrix/client/r0/login/sso/redirect" method="get">
+ <input id="sso_redirect_url" type="hidden" name="redirectUrl" value=""/>
+ <input type="submit" value="Log in"/>
+ </form>
</div>
- <br/>
-
- <form id="password_form" class="login_flow" style="display:none"
- onsubmit="matrixLogin.password_login(); return false;">
- <div>
- Password Authentication:<br/>
-
- <div style="text-align: center">
- <input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
- <br/>
- <input id="password" size="32" type="password" placeholder="Password"/>
- <br/>
+ <div id="password_flow" class="login_flow" style="display:none">
+ Password Authentication:
+ <form onsubmit="matrixLogin.password_login(); return false;">
+ <input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
+ <br/>
+ <input id="password" size="32" type="password" placeholder="Password"/>
+ <br/>
- <button type="submit" style="margin: 10px">Log in</button>
- </div>
- </div>
- </form>
+ <input type="submit" value="Log in"/>
+ </form>
+ </div>
<div id="no_login_types" type="button" class="login_flow" style="display:none">
Log in currently unavailable.
diff --git a/synapse/static/client/login/js/login.js b/synapse/static/client/login/js/login.js
index bfb7386035..3a958749a1 100644
--- a/synapse/static/client/login/js/login.js
+++ b/synapse/static/client/login/js/login.js
@@ -1,7 +1,8 @@
window.matrixLogin = {
- endpoint: location.origin + "/_matrix/client/api/v1/login",
+ endpoint: location.origin + "/_matrix/client/r0/login",
serverAcceptsPassword: false,
- serverAcceptsCas: false
+ serverAcceptsCas: false,
+ serverAcceptsSso: false,
};
var submitPassword = function(user, pwd) {
@@ -40,12 +41,6 @@ var errorFunc = function(err) {
}
};
-var gotoCas = function() {
- var this_page = window.location.origin + window.location.pathname;
- var redirect_url = matrixLogin.endpoint + "/cas/redirect?redirectUrl=" + encodeURIComponent(this_page);
- window.location.replace(redirect_url);
-}
-
var setFeedbackString = function(text) {
$("#feedback").text(text);
};
@@ -53,12 +48,18 @@ var setFeedbackString = function(text) {
var show_login = function() {
$("#loading").hide();
+ var this_page = window.location.origin + window.location.pathname;
+ $("#sso_redirect_url").val(encodeURIComponent(this_page));
+
if (matrixLogin.serverAcceptsPassword) {
- $("#password_form").show();
+ $("#password_flow").show();
}
- if (matrixLogin.serverAcceptsCas) {
- $("#cas_flow").show();
+ if (matrixLogin.serverAcceptsSso) {
+ $("#sso_flow").show();
+ } else if (matrixLogin.serverAcceptsCas) {
+ $("#sso_form").attr("action", "/_matrix/client/r0/login/cas/redirect");
+ $("#sso_flow").show();
}
if (!matrixLogin.serverAcceptsPassword && !matrixLogin.serverAcceptsCas) {
@@ -67,8 +68,8 @@ var show_login = function() {
};
var show_spinner = function() {
- $("#password_form").hide();
- $("#cas_flow").hide();
+ $("#password_flow").hide();
+ $("#sso_flow").hide();
$("#no_login_types").hide();
$("#loading").show();
};
@@ -84,7 +85,10 @@ var fetch_info = function(cb) {
matrixLogin.serverAcceptsCas = true;
console.log("Server accepts CAS");
}
-
+ if ("m.login.sso" === flow.type) {
+ matrixLogin.serverAcceptsSso = true;
+ console.log("Server accepts SSO");
+ }
if ("m.login.password" === flow.type) {
matrixLogin.serverAcceptsPassword = true;
console.log("Server accepts password");
diff --git a/synapse/static/client/login/style.css b/synapse/static/client/login/style.css
index 73da0b5117..1cce5ed950 100644
--- a/synapse/static/client/login/style.css
+++ b/synapse/static/client/login/style.css
@@ -19,30 +19,23 @@ a:hover { color: #000; }
a:active { color: #000; }
input {
- width: 90%
-}
-
-textarea, input {
- font-family: inherit;
- font-size: inherit;
margin: 5px;
}
-.smallPrint {
- color: #888;
- font-size: 9pt ! important;
- font-style: italic ! important;
+textbox, input[type="text"], input[type="password"] {
+ width: 90%;
}
-.g-recaptcha div {
- margin: auto;
+form {
+ text-align: center;
+ margin: 10px 0 0 0;
}
.login_flow {
+ width: 300px;
text-align: left;
padding: 10px;
margin-bottom: 40px;
- display: inline-block;
-webkit-border-radius: 10px;
-moz-border-radius: 10px;
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 53c685c173..b23fb7e56c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -119,7 +119,6 @@ class DataStore(RoomMemberStore, RoomStore,
db_conn, "device_lists_stream", "stream_id",
)
- self._transaction_id_gen = IdGenerator(db_conn, "sent_transactions", "id")
self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id")
self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index d9d0255d0b..1d3069b143 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -29,6 +29,7 @@ from synapse.api.errors import StoreError
from synapse.storage.engines import PostgresEngine
from synapse.util.caches.descriptors import Cache
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
+from synapse.util.stringutils import exception_to_unicode
logger = logging.getLogger(__name__)
@@ -249,32 +250,32 @@ class SQLBaseStore(object):
except self.database_engine.module.OperationalError as e:
# This can happen if the database disappears mid
# transaction.
- logger.warn(
+ logger.warning(
"[TXN OPERROR] {%s} %s %d/%d",
- name, e, i, N
+ name, exception_to_unicode(e), i, N
)
if i < N:
i += 1
try:
conn.rollback()
except self.database_engine.module.Error as e1:
- logger.warn(
+ logger.warning(
"[TXN EROLL] {%s} %s",
- name, e1,
+ name, exception_to_unicode(e1),
)
continue
raise
except self.database_engine.module.DatabaseError as e:
if self.database_engine.is_deadlock(e):
- logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
+ logger.warning("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
if i < N:
i += 1
try:
conn.rollback()
except self.database_engine.module.Error as e1:
- logger.warn(
+ logger.warning(
"[TXN EROLL] {%s} %s",
- name, e1,
+ name, exception_to_unicode(e1),
)
continue
raise
@@ -849,9 +850,9 @@ class SQLBaseStore(object):
rowcount = cls._simple_update_txn(txn, table, keyvalues, updatevalues)
if rowcount == 0:
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if rowcount > 1:
- raise StoreError(500, "More than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
@staticmethod
def _simple_select_one_txn(txn, table, keyvalues, retcols,
@@ -868,9 +869,9 @@ class SQLBaseStore(object):
if not row:
if allow_none:
return None
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if txn.rowcount > 1:
- raise StoreError(500, "More than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
return dict(zip(retcols, row))
@@ -902,9 +903,9 @@ class SQLBaseStore(object):
txn.execute(sql, list(keyvalues.values()))
if txn.rowcount == 0:
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if txn.rowcount > 1:
- raise StoreError(500, "more than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
def _simple_delete(self, table, keyvalues, desc):
return self.runInteraction(
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index cf4104dc2e..479e01ddc1 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -34,8 +34,9 @@ class MonthlyActiveUsersStore(SQLBaseStore):
self.hs = hs
self.reserved_users = ()
# Do not add more reserved users than the total allowable number
- self._initialise_reserved_users(
- dbconn.cursor(),
+ self._new_transaction(
+ dbconn, "initialise_mau_threepids", [], [],
+ self._initialise_reserved_users,
hs.config.mau_limits_reserved_threepids[:self.hs.config.max_mau_value],
)
@@ -96,37 +97,38 @@ class MonthlyActiveUsersStore(SQLBaseStore):
txn.execute(sql, query_args)
- # If MAU user count still exceeds the MAU threshold, then delete on
- # a least recently active basis.
- # Note it is not possible to write this query using OFFSET due to
- # incompatibilities in how sqlite and postgres support the feature.
- # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
- # While Postgres does not require 'LIMIT', but also does not support
- # negative LIMIT values. So there is no way to write it that both can
- # support
- safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
- # Must be greater than zero for postgres
- safe_guard = safe_guard if safe_guard > 0 else 0
- query_args = [safe_guard]
-
- base_sql = """
- DELETE FROM monthly_active_users
- WHERE user_id NOT IN (
- SELECT user_id FROM monthly_active_users
- ORDER BY timestamp DESC
- LIMIT ?
+ if self.hs.config.limit_usage_by_mau:
+ # If MAU user count still exceeds the MAU threshold, then delete on
+ # a least recently active basis.
+ # Note it is not possible to write this query using OFFSET due to
+ # incompatibilities in how sqlite and postgres support the feature.
+ # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
+ # While Postgres does not require 'LIMIT', but also does not support
+ # negative LIMIT values. So there is no way to write it that both can
+ # support
+ safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
+ # Must be greater than zero for postgres
+ safe_guard = safe_guard if safe_guard > 0 else 0
+ query_args = [safe_guard]
+
+ base_sql = """
+ DELETE FROM monthly_active_users
+ WHERE user_id NOT IN (
+ SELECT user_id FROM monthly_active_users
+ ORDER BY timestamp DESC
+ LIMIT ?
+ )
+ """
+ # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
+ # when len(reserved_users) == 0. Works fine on sqlite.
+ if len(self.reserved_users) > 0:
+ query_args.extend(self.reserved_users)
+ sql = base_sql + """ AND user_id NOT IN ({})""".format(
+ ','.join(questionmarks)
)
- """
- # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
- # when len(reserved_users) == 0. Works fine on sqlite.
- if len(self.reserved_users) > 0:
- query_args.extend(self.reserved_users)
- sql = base_sql + """ AND user_id NOT IN ({})""".format(
- ','.join(questionmarks)
- )
- else:
- sql = base_sql
- txn.execute(sql, query_args)
+ else:
+ sql = base_sql
+ txn.execute(sql, query_args)
yield self.runInteraction("reap_monthly_active_users", _reap_users)
# It seems poor to invalidate the whole cache, Postgres supports
@@ -252,8 +254,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Args:
user_id(str): the user_id to query
"""
-
- if self.hs.config.limit_usage_by_mau:
+ if self.hs.config.limit_usage_by_mau or self.hs.config.mau_stats_only:
# Trial users and guests should not be included as part of MAU group
is_guest = yield self.is_guest(user_id)
if is_guest:
@@ -271,8 +272,14 @@ class MonthlyActiveUsersStore(SQLBaseStore):
# but only update if we have not previously seen the user for
# LAST_SEEN_GRANULARITY ms
if last_seen_timestamp is None:
- count = yield self.get_monthly_active_count()
- if count < self.hs.config.max_mau_value:
+ # In the case where mau_stats_only is True and limit_usage_by_mau is
+ # False, there is no point in checking get_monthly_active_count - it
+ # adds no value and will break the logic if max_mau_value is exceeded.
+ if not self.hs.config.limit_usage_by_mau:
yield self.upsert_monthly_active_user(user_id)
+ else:
+ count = yield self.get_monthly_active_count()
+ if count < self.hs.config.max_mau_value:
+ yield self.upsert_monthly_active_user(user_id)
elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
yield self.upsert_monthly_active_user(user_id)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index bd740e1e45..fa36daac52 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 52
+SCHEMA_VERSION = 53
dir_path = os.path.abspath(os.path.dirname(__file__))
@@ -257,7 +257,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
module.run_create(cur, database_engine)
if not is_empty:
module.run_upgrade(cur, database_engine, config=config)
- elif ext == ".pyc":
+ elif ext == ".pyc" or file_name == "__pycache__":
# Sometimes .pyc files turn up anyway even though we've
# disabled their generation; e.g. from distribution package
# installers. Silently skip it
diff --git a/synapse/storage/schema/delta/34/sent_txn_purge.py b/synapse/storage/schema/delta/34/sent_txn_purge.py
deleted file mode 100644
index 0ffab10b6f..0000000000
--- a/synapse/storage/schema/delta/34/sent_txn_purge.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import PostgresEngine
-
-logger = logging.getLogger(__name__)
-
-
-def run_create(cur, database_engine, *args, **kwargs):
- if isinstance(database_engine, PostgresEngine):
- cur.execute("TRUNCATE sent_transactions")
- else:
- cur.execute("DELETE FROM sent_transactions")
-
- cur.execute("CREATE INDEX sent_transactions_ts ON sent_transactions(ts)")
-
-
-def run_upgrade(cur, database_engine, *args, **kwargs):
- pass
diff --git a/synapse/storage/schema/delta/11/v11.sql b/synapse/storage/schema/delta/53/drop_sent_transactions.sql
index e7b4f90127..e372f5a44a 100644
--- a/synapse/storage/schema/delta/11/v11.sql
+++ b/synapse/storage/schema/delta/53/drop_sent_transactions.sql
@@ -1,4 +1,4 @@
-/* Copyright 2015, 2016 OpenMarket Ltd
+/* Copyright 2018 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,4 +13,4 @@
* limitations under the License.
*/
-CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
\ No newline at end of file
+DROP TABLE IF EXISTS sent_transactions;
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index a3f4a0a790..f6a058832e 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -25,25 +25,6 @@ CREATE TABLE IF NOT EXISTS received_transactions(
CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
--- Stores what transactions we've sent, what their response was (if we got one) and whether we have
--- since referenced the transaction in another outgoing transaction
-CREATE TABLE IF NOT EXISTS sent_transactions(
- id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
- transaction_id TEXT,
- destination TEXT,
- response_code INTEGER DEFAULT 0,
- response_json TEXT,
- ts BIGINT
-);
-
-CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
--- So that we can do an efficient look up of all transactions that have yet to be successfully
--- sent.
-CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
-
-
-- For sent transactions only.
CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
transaction_id INTEGER,
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index 14b67cce25..17e67bedac 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -25,25 +25,6 @@ CREATE TABLE IF NOT EXISTS received_transactions(
CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
--- Stores what transactions we've sent, what their response was (if we got one) and whether we have
--- since referenced the transaction in another outgoing transaction
-CREATE TABLE IF NOT EXISTS sent_transactions(
- id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
- transaction_id TEXT,
- destination TEXT,
- response_code INTEGER DEFAULT 0,
- response_json TEXT,
- ts BIGINT
-);
-
-CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
--- So that we can do an efficient look up of all transactions that have yet to be successfully
--- sent.
-CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
-
-
-- For sent transactions only.
CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
transaction_id INTEGER,
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index d5b5df93e6..c6420b2374 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -45,6 +45,10 @@ class SearchStore(BackgroundUpdateStore):
def __init__(self, db_conn, hs):
super(SearchStore, self).__init__(db_conn, hs)
+
+ if not hs.config.enable_search:
+ return
+
self.register_background_update_handler(
self.EVENT_SEARCH_UPDATE_NAME, self._background_reindex_search
)
@@ -316,6 +320,8 @@ class SearchStore(BackgroundUpdateStore):
entries (iterable[SearchEntry]):
entries to be added to the table
"""
+ if not self.hs.config.enable_search:
+ return
if isinstance(self.database_engine, PostgresEngine):
sql = (
"INSERT INTO event_search"
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 6f318c6a29..fdcb375f95 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -16,7 +16,8 @@
import random
import string
-from six import PY3
+import six
+from six import PY2, PY3
from six.moves import range
_string_with_symbols = (
@@ -71,3 +72,39 @@ def to_ascii(s):
return s.encode("ascii")
except UnicodeEncodeError:
return s
+
+
+def exception_to_unicode(e):
+ """Helper function to extract the text of an exception as a unicode string
+
+ Args:
+ e (Exception): exception to be stringified
+
+ Returns:
+ unicode
+ """
+ # urgh, this is a mess. The basic problem here is that psycopg2 constructs its
+ # exceptions with PyErr_SetString, with a (possibly non-ascii) argument. str() will
+ # then produce the raw byte sequence. Under Python 2, this will then cause another
+ # error if it gets mixed with a `unicode` object, as per
+ # https://github.com/matrix-org/synapse/issues/4252
+
+ # First of all, if we're under python3, everything is fine because it will sort this
+ # nonsense out for us.
+ if not PY2:
+ return str(e)
+
+ # otherwise let's have a stab at decoding the exception message. We'll circumvent
+ # Exception.__str__(), which would explode if someone raised Exception(u'non-ascii')
+ # and instead look at what is in the args member.
+
+ if len(e.args) == 0:
+ return u""
+ elif len(e.args) > 1:
+ return six.text_type(repr(e.args))
+
+ msg = e.args[0]
+ if isinstance(msg, bytes):
+ return msg.decode('utf-8', errors='replace')
+ else:
+ return msg
diff --git a/tests/__init__.py b/tests/__init__.py
index 9d9ca22829..d3181f9403 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,7 +16,9 @@
from twisted.trial import util
-from tests import utils
+import tests.patch_inline_callbacks
+
+# attempt to do the patch before we load any synapse code
+tests.patch_inline_callbacks.do_patch()
util.DEFAULT_TIMEOUT_DURATION = 10
-utils.setupdb()
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 8299dc72c8..d643bec887 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -63,6 +63,14 @@ class KeyringTestCase(unittest.TestCase):
keys = self.mock_perspective_server.get_verify_keys()
self.hs.config.perspectives = {self.mock_perspective_server.server_name: keys}
+ def assert_sentinel_context(self):
+ if LoggingContext.current_context() != LoggingContext.sentinel:
+ self.fail(
+ "Expected sentinel context but got %s" % (
+ LoggingContext.current_context(),
+ )
+ )
+
def check_context(self, _, expected):
self.assertEquals(
getattr(LoggingContext.current_context(), "request", None), expected
@@ -70,8 +78,6 @@ class KeyringTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_wait_for_previous_lookups(self):
- sentinel_context = LoggingContext.current_context()
-
kr = keyring.Keyring(self.hs)
lookup_1_deferred = defer.Deferred()
@@ -99,8 +105,10 @@ class KeyringTestCase(unittest.TestCase):
["server1"], {"server1": lookup_2_deferred}
)
self.assertFalse(wait_2_deferred.called)
+
# ... so we should have reset the LoggingContext.
- self.assertIs(LoggingContext.current_context(), sentinel_context)
+ self.assert_sentinel_context()
+
wait_2_deferred.addBoth(self.check_context, "two")
# let the first lookup complete (in the sentinel context)
@@ -198,8 +206,6 @@ class KeyringTestCase(unittest.TestCase):
json1 = {}
signedjson.sign.sign_json(json1, "server9", key1)
- sentinel_context = LoggingContext.current_context()
-
with LoggingContext("one") as context_one:
context_one.request = "one"
@@ -213,7 +219,7 @@ class KeyringTestCase(unittest.TestCase):
defer = kr.verify_json_for_server("server9", json1)
self.assertFalse(defer.called)
- self.assertIs(LoggingContext.current_context(), sentinel_context)
+ self.assert_sentinel_context()
yield defer
self.assertIs(LoggingContext.current_context(), context_one)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 3e9a190727..90a2a76475 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -150,7 +150,6 @@ class RegistrationTestCase(unittest.TestCase):
self.hs.config.auto_join_rooms = [room_alias_str]
res = yield self.handler.register(localpart='jeff')
rooms = yield self.store.get_rooms_for_user(res[0])
-
directory_handler = self.hs.get_handlers().directory_handler
room_alias = RoomAlias.from_string(room_alias_str)
room_id = yield directory_handler.get_association(room_alias)
@@ -184,3 +183,14 @@ class RegistrationTestCase(unittest.TestCase):
res = yield self.handler.register(localpart='jeff')
rooms = yield self.store.get_rooms_for_user(res[0])
self.assertEqual(len(rooms), 0)
+
+ @defer.inlineCallbacks
+ def test_auto_create_auto_join_where_no_consent(self):
+ self.hs.config.user_consent_at_registration = True
+ self.hs.config.block_events_without_consent_error = "Error"
+ room_alias_str = "#room:test"
+ self.hs.config.auto_join_rooms = [room_alias_str]
+ res = yield self.handler.register(localpart='jeff')
+ yield self.handler.post_consent_actions(res[0])
+ rooms = yield self.store.get_rooms_for_user(res[0])
+ self.assertEqual(len(rooms), 0)
diff --git a/tests/patch_inline_callbacks.py b/tests/patch_inline_callbacks.py
new file mode 100644
index 0000000000..0f613945c8
--- /dev/null
+++ b/tests/patch_inline_callbacks.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import functools
+import sys
+
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
+
+
+def do_patch():
+ """
+ Patch defer.inlineCallbacks so that it checks the state of the logcontext on exit
+ """
+
+ from synapse.util.logcontext import LoggingContext
+
+ orig_inline_callbacks = defer.inlineCallbacks
+
+ def new_inline_callbacks(f):
+
+ orig = orig_inline_callbacks(f)
+
+ @functools.wraps(f)
+ def wrapped(*args, **kwargs):
+ start_context = LoggingContext.current_context()
+
+ try:
+ res = orig(*args, **kwargs)
+ except Exception:
+ if LoggingContext.current_context() != start_context:
+ err = "%s changed context from %s to %s on exception" % (
+ f, start_context, LoggingContext.current_context()
+ )
+ print(err, file=sys.stderr)
+ raise Exception(err)
+ raise
+
+ if not isinstance(res, Deferred) or res.called:
+ if LoggingContext.current_context() != start_context:
+ err = "%s changed context from %s to %s" % (
+ f, start_context, LoggingContext.current_context()
+ )
+ # print the error to stderr because otherwise all we
+ # see in travis-ci is the 500 error
+ print(err, file=sys.stderr)
+ raise Exception(err)
+ return res
+
+ if LoggingContext.current_context() != LoggingContext.sentinel:
+ err = (
+ "%s returned incomplete deferred in non-sentinel context "
+ "%s (start was %s)"
+ ) % (
+ f, LoggingContext.current_context(), start_context,
+ )
+ print(err, file=sys.stderr)
+ raise Exception(err)
+
+ def check_ctx(r):
+ if LoggingContext.current_context() != start_context:
+ err = "%s completion of %s changed context from %s to %s" % (
+ "Failure" if isinstance(r, Failure) else "Success",
+ f, start_context, LoggingContext.current_context(),
+ )
+ print(err, file=sys.stderr)
+ raise Exception(err)
+ return r
+
+ res.addBoth(check_ctx)
+ return res
+
+ return wrapped
+
+ defer.inlineCallbacks = new_inline_callbacks
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index addc01ab7f..6dc45e8506 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -18,6 +18,7 @@ from mock import Mock
from twisted.internet.defer import Deferred
from synapse.rest.client.v1 import admin, login, room
+from synapse.util.logcontext import make_deferred_yieldable
from tests.unittest import HomeserverTestCase
@@ -47,7 +48,7 @@ class HTTPPusherTests(HomeserverTestCase):
def post_json_get_json(url, body):
d = Deferred()
self.push_attempts.append((d, url, body))
- return d
+ return make_deferred_yieldable(d)
m.post_json_get_json = post_json_get_json
diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py
new file mode 100644
index 0000000000..7fa120a10f
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_auth.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from twisted.internet.defer import succeed
+
+from synapse.api.constants import LoginType
+from synapse.rest.client.v1 import admin
+from synapse.rest.client.v2_alpha import auth, register
+
+from tests import unittest
+
+
+class FallbackAuthTests(unittest.HomeserverTestCase):
+
+ servlets = [
+ auth.register_servlets,
+ admin.register_servlets,
+ register.register_servlets,
+ ]
+ hijack_auth = False
+
+ def make_homeserver(self, reactor, clock):
+
+ config = self.default_config()
+
+ config.enable_registration_captcha = True
+ config.recaptcha_public_key = "brokencake"
+ config.registrations_require_3pid = []
+
+ hs = self.setup_test_homeserver(config=config)
+ return hs
+
+ def prepare(self, reactor, clock, hs):
+ auth_handler = hs.get_auth_handler()
+
+ self.recaptcha_attempts = []
+
+ def _recaptcha(authdict, clientip):
+ self.recaptcha_attempts.append((authdict, clientip))
+ return succeed(True)
+
+ auth_handler.checkers[LoginType.RECAPTCHA] = _recaptcha
+
+ @unittest.INFO
+ def test_fallback_captcha(self):
+
+ request, channel = self.make_request(
+ "POST",
+ "register",
+ {"username": "user", "type": "m.login.password", "password": "bar"},
+ )
+ self.render(request)
+
+ # Returns a 401 as per the spec
+ self.assertEqual(request.code, 401)
+ # Grab the session
+ session = channel.json_body["session"]
+ # Assert our configured public key is being given
+ self.assertEqual(
+ channel.json_body["params"]["m.login.recaptcha"]["public_key"], "brokencake"
+ )
+
+ request, channel = self.make_request(
+ "GET", "auth/m.login.recaptcha/fallback/web?session=" + session
+ )
+ self.render(request)
+ self.assertEqual(request.code, 200)
+
+ request, channel = self.make_request(
+ "POST",
+ "auth/m.login.recaptcha/fallback/web?session="
+ + session
+ + "&g-recaptcha-response=a",
+ )
+ self.render(request)
+ self.assertEqual(request.code, 200)
+
+ # The recaptcha handler is called with the response given
+ self.assertEqual(len(self.recaptcha_attempts), 1)
+ self.assertEqual(self.recaptcha_attempts[0][0]["response"], "a")
+
+ # Now we have fufilled the recaptcha fallback step, we can then send a
+ # request to the register API with the session in the authdict.
+ request, channel = self.make_request(
+ "POST", "register", {"auth": {"session": session}}
+ )
+ self.render(request)
+ self.assertEqual(channel.code, 200)
+
+ # We're given a registered user.
+ self.assertEqual(channel.json_body["user_id"], "@user:test")
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index a86901c2d8..ad5e9a612f 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -17,15 +17,21 @@
import os
import shutil
import tempfile
+from binascii import unhexlify
from mock import Mock
+from six.moves.urllib import parse
from twisted.internet import defer, reactor
+from twisted.internet.defer import Deferred
+from synapse.config.repository import MediaStorageProviderConfig
from synapse.rest.media.v1._base import FileInfo
from synapse.rest.media.v1.filepath import MediaFilePaths
from synapse.rest.media.v1.media_storage import MediaStorage
from synapse.rest.media.v1.storage_provider import FileStorageProviderBackend
+from synapse.util.logcontext import make_deferred_yieldable
+from synapse.util.module_loader import load_module
from tests import unittest
@@ -83,3 +89,143 @@ class MediaStorageTests(unittest.TestCase):
body = f.read()
self.assertEqual(test_body, body)
+
+
+class MediaRepoTests(unittest.HomeserverTestCase):
+
+ hijack_auth = True
+ user_id = "@test:user"
+
+ def make_homeserver(self, reactor, clock):
+
+ self.fetches = []
+
+ def get_file(destination, path, output_stream, args=None, max_size=None):
+ """
+ Returns tuple[int,dict,str,int] of file length, response headers,
+ absolute URI, and response code.
+ """
+
+ def write_to(r):
+ data, response = r
+ output_stream.write(data)
+ return response
+
+ d = Deferred()
+ d.addCallback(write_to)
+ self.fetches.append((d, destination, path, args))
+ return make_deferred_yieldable(d)
+
+ client = Mock()
+ client.get_file = get_file
+
+ self.storage_path = self.mktemp()
+ os.mkdir(self.storage_path)
+
+ config = self.default_config()
+ config.media_store_path = self.storage_path
+ config.thumbnail_requirements = {}
+ config.max_image_pixels = 2000000
+
+ provider_config = {
+ "module": "synapse.rest.media.v1.storage_provider.FileStorageProviderBackend",
+ "store_local": True,
+ "store_synchronous": False,
+ "store_remote": True,
+ "config": {"directory": self.storage_path},
+ }
+
+ loaded = list(load_module(provider_config)) + [
+ MediaStorageProviderConfig(False, False, False)
+ ]
+
+ config.media_storage_providers = [loaded]
+
+ hs = self.setup_test_homeserver(config=config, http_client=client)
+
+ return hs
+
+ def prepare(self, reactor, clock, hs):
+
+ self.media_repo = hs.get_media_repository_resource()
+ self.download_resource = self.media_repo.children[b'download']
+
+ # smol png
+ self.end_content = unhexlify(
+ b"89504e470d0a1a0a0000000d4948445200000001000000010806"
+ b"0000001f15c4890000000a49444154789c63000100000500010d"
+ b"0a2db40000000049454e44ae426082"
+ )
+
+ def _req(self, content_disposition):
+
+ request, channel = self.make_request(
+ "GET", "example.com/12345", shorthand=False
+ )
+ request.render(self.download_resource)
+ self.pump()
+
+ # We've made one fetch, to example.com, using the media URL, and asking
+ # the other server not to do a remote fetch
+ self.assertEqual(len(self.fetches), 1)
+ self.assertEqual(self.fetches[0][1], "example.com")
+ self.assertEqual(
+ self.fetches[0][2], "/_matrix/media/v1/download/example.com/12345"
+ )
+ self.assertEqual(self.fetches[0][3], {"allow_remote": "false"})
+
+ headers = {
+ b"Content-Length": [b"%d" % (len(self.end_content))],
+ b"Content-Type": [b'image/png'],
+ }
+ if content_disposition:
+ headers[b"Content-Disposition"] = [content_disposition]
+
+ self.fetches[0][0].callback(
+ (self.end_content, (len(self.end_content), headers))
+ )
+
+ self.pump()
+ self.assertEqual(channel.code, 200)
+
+ return channel
+
+ def test_disposition_filename_ascii(self):
+ """
+ If the filename is filename=<ascii> then Synapse will decode it as an
+ ASCII string, and use filename= in the response.
+ """
+ channel = self._req(b"inline; filename=out.png")
+
+ headers = channel.headers
+ self.assertEqual(headers.getRawHeaders(b"Content-Type"), [b"image/png"])
+ self.assertEqual(
+ headers.getRawHeaders(b"Content-Disposition"), [b"inline; filename=out.png"]
+ )
+
+ def test_disposition_filenamestar_utf8escaped(self):
+ """
+ If the filename is filename=*utf8''<utf8 escaped> then Synapse will
+ correctly decode it as the UTF-8 string, and use filename* in the
+ response.
+ """
+ filename = parse.quote(u"\u2603".encode('utf8')).encode('ascii')
+ channel = self._req(b"inline; filename*=utf-8''" + filename + b".png")
+
+ headers = channel.headers
+ self.assertEqual(headers.getRawHeaders(b"Content-Type"), [b"image/png"])
+ self.assertEqual(
+ headers.getRawHeaders(b"Content-Disposition"),
+ [b"inline; filename*=utf-8''" + filename + b".png"],
+ )
+
+ def test_disposition_none(self):
+ """
+ If there is no filename, one isn't passed on in the Content-Disposition
+ of the request.
+ """
+ channel = self._req(None)
+
+ headers = channel.headers
+ self.assertEqual(headers.getRawHeaders(b"Content-Type"), [b"image/png"])
+ self.assertEqual(headers.getRawHeaders(b"Content-Disposition"), None)
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 29579cf091..c62f71b44a 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -20,6 +20,7 @@ from mock import Mock
from twisted.internet.defer import Deferred
from synapse.config.repository import MediaStorageProviderConfig
+from synapse.util.logcontext import make_deferred_yieldable
from synapse.util.module_loader import load_module
from tests import unittest
@@ -77,7 +78,7 @@ class URLPreviewTests(unittest.HomeserverTestCase):
d = Deferred()
d.addCallback(write_to)
self.fetches.append((d, url))
- return d
+ return make_deferred_yieldable(d)
client = Mock()
client.get_file = get_file
@@ -162,3 +163,80 @@ class URLPreviewTests(unittest.HomeserverTestCase):
self.assertEqual(
channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
)
+
+ def test_non_ascii_preview_httpequiv(self):
+
+ request, channel = self.make_request(
+ "GET", "url_preview?url=matrix.org", shorthand=False
+ )
+ request.render(self.preview_url)
+ self.pump()
+
+ # We've made one fetch
+ self.assertEqual(len(self.fetches), 1)
+
+ end_content = (
+ b'<html><head>'
+ b'<meta http-equiv="Content-Type" content="text/html; charset=windows-1251"/>'
+ b'<meta property="og:title" content="\xe4\xea\xe0" />'
+ b'<meta property="og:description" content="hi" />'
+ b'</head></html>'
+ )
+
+ self.fetches[0][0].callback(
+ (
+ end_content,
+ (
+ len(end_content),
+ {
+ b"Content-Length": [b"%d" % (len(end_content))],
+ # This charset=utf-8 should be ignored, because the
+ # document has a meta tag overriding it.
+ b"Content-Type": [b'text/html; charset="utf8"'],
+ },
+ "https://example.com",
+ 200,
+ ),
+ )
+ )
+
+ self.pump()
+ self.assertEqual(channel.code, 200)
+ self.assertEqual(channel.json_body["og:title"], u"\u0434\u043a\u0430")
+
+ def test_non_ascii_preview_content_type(self):
+
+ request, channel = self.make_request(
+ "GET", "url_preview?url=matrix.org", shorthand=False
+ )
+ request.render(self.preview_url)
+ self.pump()
+
+ # We've made one fetch
+ self.assertEqual(len(self.fetches), 1)
+
+ end_content = (
+ b'<html><head>'
+ b'<meta property="og:title" content="\xe4\xea\xe0" />'
+ b'<meta property="og:description" content="hi" />'
+ b'</head></html>'
+ )
+
+ self.fetches[0][0].callback(
+ (
+ end_content,
+ (
+ len(end_content),
+ {
+ b"Content-Length": [b"%d" % (len(end_content))],
+ b"Content-Type": [b'text/html; charset="windows-1251"'],
+ },
+ "https://example.com",
+ 200,
+ ),
+ )
+ )
+
+ self.pump()
+ self.assertEqual(channel.code, 200)
+ self.assertEqual(channel.json_body["og:title"], u"\u0434\u043a\u0430")
diff --git a/tests/server.py b/tests/server.py
index 7919a1f124..ceec2f2d4e 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -14,6 +14,8 @@ from twisted.internet.error import DNSLookupError
from twisted.internet.interfaces import IReactorPluggableNameResolver
from twisted.python.failure import Failure
from twisted.test.proto_helpers import MemoryReactorClock
+from twisted.web.http import unquote
+from twisted.web.http_headers import Headers
from synapse.http.site import SynapseRequest
from synapse.util import Clock
@@ -50,6 +52,15 @@ class FakeChannel(object):
raise Exception("No result yet.")
return int(self.result["code"])
+ @property
+ def headers(self):
+ if not self.result:
+ raise Exception("No result yet.")
+ h = Headers()
+ for i in self.result["headers"]:
+ h.addRawHeader(*i)
+ return h
+
def writeHeaders(self, version, code, reason, headers):
self.result["version"] = version
self.result["code"] = code
@@ -152,6 +163,9 @@ def make_request(
path = b"/_matrix/client/r0/" + path
path = path.replace(b"//", b"/")
+ if not path.startswith(b"/"):
+ path = b"/" + path
+
if isinstance(content, text_type):
content = content.encode('utf8')
@@ -161,6 +175,7 @@ def make_request(
req = request(site, channel)
req.process = lambda: b""
req.content = BytesIO(content)
+ req.postpath = list(map(unquote, path[1:].split(b'/')))
if access_token:
req.requestHeaders.addRawHeader(
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 832e379a83..8664bc3d54 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -220,3 +220,28 @@ class MonthlyActiveUsersTestCase(HomeserverTestCase):
self.store.user_add_threepid(user2, "email", user2_email, now, now)
count = self.store.get_registered_reserved_users_count()
self.assertEquals(self.get_success(count), len(threepids))
+
+ def test_track_monthly_users_without_cap(self):
+ self.hs.config.limit_usage_by_mau = False
+ self.hs.config.mau_stats_only = True
+ self.hs.config.max_mau_value = 1 # should not matter
+
+ count = self.store.get_monthly_active_count()
+ self.assertEqual(0, self.get_success(count))
+
+ self.store.upsert_monthly_active_user("@user1:server")
+ self.store.upsert_monthly_active_user("@user2:server")
+ self.pump()
+
+ count = self.store.get_monthly_active_count()
+ self.assertEqual(2, self.get_success(count))
+
+ def test_no_users_when_not_tracking(self):
+ self.hs.config.limit_usage_by_mau = False
+ self.hs.config.mau_stats_only = False
+ self.store.upsert_monthly_active_user = Mock()
+
+ self.store.populate_monthly_active_users("@user:sever")
+ self.pump()
+
+ self.store.upsert_monthly_active_user.assert_not_called()
diff --git a/tests/test_federation.py b/tests/test_federation.py
index e1a34ccffd..1a5dc32c88 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -123,8 +123,8 @@ class MessageAcceptTests(unittest.TestCase):
"test.serv", lying_event, sent_to_us_directly=True
)
- # Step the reactor, so the database fetches come back
- self.reactor.advance(1)
+ # Step the reactor, so the database fetches come back
+ self.reactor.advance(1)
# on_receive_pdu should throw an error
failure = self.failureResultOf(d)
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 0afdeb0818..04f95c942f 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -171,6 +171,24 @@ class TestMauLimit(unittest.HomeserverTestCase):
self.assertEqual(e.code, 403)
self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+ def test_tracked_but_not_limited(self):
+ self.hs.config.max_mau_value = 1 # should not matter
+ self.hs.config.limit_usage_by_mau = False
+ self.hs.config.mau_stats_only = True
+
+ # Simply being able to create 2 users indicates that the
+ # limit was not reached.
+ token1 = self.create_user("kermit1")
+ self.do_sync_for_user(token1)
+ token2 = self.create_user("kermit2")
+ self.do_sync_for_user(token2)
+
+ # We do want to verify that the number of tracked users
+ # matches what we want though
+ count = self.store.get_monthly_active_count()
+ self.reactor.advance(100)
+ self.assertEqual(2, self.successResultOf(count))
+
def create_user(self, localpart):
request_data = json.dumps(
{
diff --git a/tests/test_server.py b/tests/test_server.py
index f0e6291b7e..634a8fbca5 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -27,6 +27,7 @@ from synapse.api.errors import Codes, SynapseError
from synapse.http.server import JsonResource
from synapse.http.site import SynapseSite, logger
from synapse.util import Clock
+from synapse.util.logcontext import make_deferred_yieldable
from tests import unittest
from tests.server import FakeTransport, make_request, render, setup_test_homeserver
@@ -95,7 +96,7 @@ class JsonResourceTests(unittest.TestCase):
d = Deferred()
d.addCallback(_throw)
self.reactor.callLater(1, d.callback, True)
- return d
+ return make_deferred_yieldable(d)
res = JsonResource(self.homeserver)
res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py
index 9ecc3ef14f..0968e86a7b 100644
--- a/tests/test_terms_auth.py
+++ b/tests/test_terms_auth.py
@@ -43,7 +43,7 @@ class TermsTestCase(unittest.HomeserverTestCase):
def test_ui_auth(self):
self.hs.config.user_consent_at_registration = True
self.hs.config.user_consent_policy_name = "My Cool Privacy Policy"
- self.hs.config.public_baseurl = "https://example.org"
+ self.hs.config.public_baseurl = "https://example.org/"
self.hs.config.user_consent_version = "1.0"
# Do a UI auth request
diff --git a/tests/unittest.py b/tests/unittest.py
index a9ce57da9a..092c930396 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import gc
import hashlib
import hmac
import logging
@@ -31,10 +31,12 @@ from synapse.http.server import JsonResource
from synapse.http.site import SynapseRequest
from synapse.server import HomeServer
from synapse.types import UserID, create_requester
-from synapse.util.logcontext import LoggingContextFilter
+from synapse.util.logcontext import LoggingContext, LoggingContextFilter
from tests.server import get_clock, make_request, render, setup_test_homeserver
-from tests.utils import default_config
+from tests.utils import default_config, setupdb
+
+setupdb()
# Set up putting Synapse's logs into Trial's.
rootLogger = logging.getLogger()
@@ -102,8 +104,16 @@ class TestCase(unittest.TestCase):
# traceback when a unit test exits leaving things on the reactor.
twisted.internet.base.DelayedCall.debug = True
- old_level = logging.getLogger().level
+ # if we're not starting in the sentinel logcontext, then to be honest
+ # all future bets are off.
+ if LoggingContext.current_context() is not LoggingContext.sentinel:
+ self.fail(
+ "Test starting with non-sentinel logging context %s" % (
+ LoggingContext.current_context(),
+ )
+ )
+ old_level = logging.getLogger().level
if old_level != level:
@around(self)
@@ -115,6 +125,16 @@ class TestCase(unittest.TestCase):
logging.getLogger().setLevel(level)
return orig()
+ @around(self)
+ def tearDown(orig):
+ ret = orig()
+ # force a GC to workaround problems with deferreds leaking logcontexts when
+ # they are GCed (see the logcontext docs)
+ gc.collect()
+ LoggingContext.set_current_context(LoggingContext.sentinel)
+
+ return ret
+
def assertObjectHasAttributes(self, attrs, obj):
"""Asserts that the given object has each of the attributes given, and
that the value of each matches according to assertEquals."""
diff --git a/tests/utils.py b/tests/utils.py
index 67ab916f30..52ab762010 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -134,6 +134,7 @@ def default_config(name):
config.hs_disabled_limit_type = ""
config.max_mau_value = 50
config.mau_trial_days = 0
+ config.mau_stats_only = False
config.mau_limits_reserved_threepids = []
config.admin_contact = None
config.rc_messages_per_second = 10000
diff --git a/tox.ini b/tox.ini
index 03ddaeb0b7..731094b5da 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@ deps =
mock
python-subunit
junitxml
+ coverage
# needed by some of the tests
lxml
@@ -27,11 +28,15 @@ deps =
setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code
+ COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
[testenv]
deps =
{[base]deps}
+whitelist_externals =
+ sh
+
setenv =
{[base]setenv}
@@ -39,7 +44,9 @@ passenv = *
commands =
/usr/bin/find "{toxinidir}" -name '*.pyc' -delete
- "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+ # Add this so that coverage will run on subprocesses
+ sh -c 'echo "import coverage; coverage.process_startup()" > {envsitepackagesdir}/../sitecustomize.py'
+ {envbindir}/coverage run "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
[testenv:py27]
@@ -70,7 +77,7 @@ usedevelop=true
usedevelop=true
deps =
{[base]deps}
- psycopg2
+ psycopg2
setenv =
{[base]setenv}
SYNAPSE_POSTGRES = 1
@@ -105,7 +112,7 @@ usedevelop=true
usedevelop=true
deps =
{[base]deps}
- psycopg2
+ psycopg2
setenv =
{[base]setenv}
SYNAPSE_POSTGRES = 1
@@ -135,3 +142,12 @@ deps = towncrier>=18.6.0rc1
commands =
python -m towncrier.check --compare-with=origin/develop
basepython = python3.6
+
+[testenv:codecov]
+skip_install = True
+deps =
+ coverage
+ codecov
+commands =
+ coverage combine
+ codecov -X gcov
\ No newline at end of file
|
