diff options
57 files changed, 573 insertions, 245 deletions
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index aa883ba505..1ead0d0030 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,5 +3,5 @@ <!-- Please read CONTRIBUTING.rst before submitting your pull request --> * [ ] Pull request is based on the develop branch -* [ ] Pull request includes a [changelog file](CONTRIBUTING.rst#changelog) -* [ ] Pull request includes a [sign off](CONTRIBUTING.rst#sign-off) +* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog) +* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a83d90153..d599aec74c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -65,4 +65,7 @@ Pierre Jaury <pierre at jaury.eu> * Docker packaging Serban Constantin <serban.constantin at gmail dot com> - * Small bug fix \ No newline at end of file + * Small bug fix + +Jason Robinson <jasonr at matrix.org> + * Minor fixes diff --git a/CHANGES.md b/CHANGES.md index 727275fa33..e403a65d17 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,72 @@ +Synapse 0.34.1 (2019-01-09) +=========================== + +Internal Changes +---------------- + +- Add better logging for unexpected errors while sending transactions ([\#4361](https://github.com/matrix-org/synapse/issues/4361), [\#4362](https://github.com/matrix-org/synapse/issues/4362)) + + +Synapse 0.34.1rc1 (2019-01-08) +============================== + +Features +-------- + +- Special-case a support user for use in verifying behaviour of a given server. The support user does not appear in user directory or monthly active user counts. ([\#4141](https://github.com/matrix-org/synapse/issues/4141), [\#4344](https://github.com/matrix-org/synapse/issues/4344)) +- Support for serving .well-known files ([\#4262](https://github.com/matrix-org/synapse/issues/4262)) +- Rework SAML2 authentication ([\#4265](https://github.com/matrix-org/synapse/issues/4265), [\#4267](https://github.com/matrix-org/synapse/issues/4267)) +- SAML2 authentication: Initialise user display name from SAML2 data ([\#4272](https://github.com/matrix-org/synapse/issues/4272)) +- Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values `email.enable_notifs`, `matrix-synapse-ldap3`, `postgres`, `resources.consent`, `saml2`, `url_preview`, and `test`. If you want to install all optional dependencies, you can use "all" instead. ([\#4298](https://github.com/matrix-org/synapse/issues/4298), [\#4325](https://github.com/matrix-org/synapse/issues/4325), [\#4327](https://github.com/matrix-org/synapse/issues/4327)) +- Add routes for reading account data. ([\#4303](https://github.com/matrix-org/synapse/issues/4303)) +- Add opt-in support for v2 rooms ([\#4307](https://github.com/matrix-org/synapse/issues/4307)) +- Add a script to generate a clean config file ([\#4315](https://github.com/matrix-org/synapse/issues/4315)) +- Return server data in /login response ([\#4319](https://github.com/matrix-org/synapse/issues/4319)) + + +Bugfixes +-------- + +- Fix contains_url check to be consistent with other instances in code-base and check that value is an instance of string. ([\#3405](https://github.com/matrix-org/synapse/issues/3405)) +- Fix CAS login when username is not valid in an MXID ([\#4264](https://github.com/matrix-org/synapse/issues/4264)) +- Send CORS headers for /media/config ([\#4279](https://github.com/matrix-org/synapse/issues/4279)) +- Add 'sandbox' to CSP for media reprository ([\#4284](https://github.com/matrix-org/synapse/issues/4284)) +- Make the new landing page prettier. ([\#4294](https://github.com/matrix-org/synapse/issues/4294)) +- Fix deleting E2E room keys when using old SQLite versions. ([\#4295](https://github.com/matrix-org/synapse/issues/4295)) +- The metric synapse_admin_mau:current previously did not update when config.mau_stats_only was set to True ([\#4305](https://github.com/matrix-org/synapse/issues/4305)) +- Fixed per-room account data filters ([\#4309](https://github.com/matrix-org/synapse/issues/4309)) +- Fix indentation in default config ([\#4313](https://github.com/matrix-org/synapse/issues/4313)) +- Fix synapse:latest docker upload ([\#4316](https://github.com/matrix-org/synapse/issues/4316)) +- Fix test_metric.py compatibility with prometheus_client 0.5. Contributed by Maarten de Vries <maarten@de-vri.es>. ([\#4317](https://github.com/matrix-org/synapse/issues/4317)) +- Avoid packaging _trial_temp directory in -py3 debian packages ([\#4326](https://github.com/matrix-org/synapse/issues/4326)) +- Check jinja version for consent resource ([\#4327](https://github.com/matrix-org/synapse/issues/4327)) +- fix NPE in /messages by checking if all events were filtered out ([\#4330](https://github.com/matrix-org/synapse/issues/4330)) +- Fix `python -m synapse.config` on Python 3. ([\#4356](https://github.com/matrix-org/synapse/issues/4356)) + + +Deprecations and Removals +------------------------- + +- Remove the deprecated v1/register API on Python 2. It was never ported to Python 3. ([\#4334](https://github.com/matrix-org/synapse/issues/4334)) + + +Internal Changes +---------------- + +- Getting URL previews of IP addresses no longer fails on Python 3. ([\#4215](https://github.com/matrix-org/synapse/issues/4215)) +- drop undocumented dependency on dateutil ([\#4266](https://github.com/matrix-org/synapse/issues/4266)) +- Update the example systemd config to use a virtualenv ([\#4273](https://github.com/matrix-org/synapse/issues/4273)) +- Update link to kernel DCO guide ([\#4274](https://github.com/matrix-org/synapse/issues/4274)) +- Make isort tox check print diff when it fails ([\#4283](https://github.com/matrix-org/synapse/issues/4283)) +- Log room_id in Unknown room errors ([\#4297](https://github.com/matrix-org/synapse/issues/4297)) +- Documentation improvements for coturn setup. Contributed by Krithin Sitaram. ([\#4333](https://github.com/matrix-org/synapse/issues/4333)) +- Update pull request template to use absolute links ([\#4341](https://github.com/matrix-org/synapse/issues/4341)) +- Update README to not lie about required restart when updating TLS certificates ([\#4343](https://github.com/matrix-org/synapse/issues/4343)) +- Update debian packaging for compatibility with transitional package ([\#4349](https://github.com/matrix-org/synapse/issues/4349)) +- Fix command hint to generate a config file when trying to start without a config file ([\#4353](https://github.com/matrix-org/synapse/issues/4353)) +- Add better logging for unexpected errors while sending transactions ([\#4358](https://github.com/matrix-org/synapse/issues/4358)) + + Synapse 0.34.0 (2018-12-20) =========================== diff --git a/README.rst b/README.rst index bae7da38b1..8bff55e78e 100644 --- a/README.rst +++ b/README.rst @@ -725,8 +725,8 @@ caveats, you will need to do the following: tell other servers how to find you. See `Setting up Federation`_. When updating the SSL certificate, just update the file pointed to by -``tls_certificate_path``: there is no need to restart synapse. (You may like to -use a symbolic link to help make this process atomic.) +``tls_certificate_path`` and then restart Synapse. (You may like to use a symbolic link +to help make this process atomic.) The most common mistake when setting up federation is not to tell Synapse about your SSL certificate. To check it, you can visit diff --git a/changelog.d/4141.feature b/changelog.d/4141.feature deleted file mode 100644 index 632d3547cb..0000000000 --- a/changelog.d/4141.feature +++ /dev/null @@ -1 +0,0 @@ -Special-case a support user for use in verifying behaviour of a given server. The support user does not appear in user directory or monthly active user counts. diff --git a/changelog.d/4215.misc b/changelog.d/4215.misc deleted file mode 100644 index bb90594836..0000000000 --- a/changelog.d/4215.misc +++ /dev/null @@ -1 +0,0 @@ -Getting URL previews of IP addresses no longer fails on Python 3. diff --git a/changelog.d/4262.feature b/changelog.d/4262.feature deleted file mode 100644 index 89cfdcab15..0000000000 --- a/changelog.d/4262.feature +++ /dev/null @@ -1 +0,0 @@ -Support for serving .well-known files diff --git a/changelog.d/4264.bugfix b/changelog.d/4264.bugfix deleted file mode 100644 index b914026932..0000000000 --- a/changelog.d/4264.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix CAS login when username is not valid in an MXID diff --git a/changelog.d/4265.feature b/changelog.d/4265.feature deleted file mode 100644 index da36986e2b..0000000000 --- a/changelog.d/4265.feature +++ /dev/null @@ -1 +0,0 @@ -Rework SAML2 authentication diff --git a/changelog.d/4266.misc b/changelog.d/4266.misc deleted file mode 100644 index 67fbde7484..0000000000 --- a/changelog.d/4266.misc +++ /dev/null @@ -1 +0,0 @@ -drop undocumented dependency on dateutil diff --git a/changelog.d/4267.feature b/changelog.d/4267.feature deleted file mode 100644 index da36986e2b..0000000000 --- a/changelog.d/4267.feature +++ /dev/null @@ -1 +0,0 @@ -Rework SAML2 authentication diff --git a/changelog.d/4272.feature b/changelog.d/4272.feature deleted file mode 100644 index 7a8f286957..0000000000 --- a/changelog.d/4272.feature +++ /dev/null @@ -1 +0,0 @@ -SAML2 authentication: Initialise user display name from SAML2 data diff --git a/changelog.d/4273.misc b/changelog.d/4273.misc deleted file mode 100644 index 2583372d26..0000000000 --- a/changelog.d/4273.misc +++ /dev/null @@ -1 +0,0 @@ -Update the example systemd config to use a virtualenv diff --git a/changelog.d/4274.misc b/changelog.d/4274.misc deleted file mode 100644 index c85fb53b57..0000000000 --- a/changelog.d/4274.misc +++ /dev/null @@ -1 +0,0 @@ -Update link to kernel DCO guide diff --git a/changelog.d/4279.bugfix b/changelog.d/4279.bugfix deleted file mode 100644 index 12de4f44c4..0000000000 --- a/changelog.d/4279.bugfix +++ /dev/null @@ -1 +0,0 @@ -Send CORS headers for /media/config diff --git a/changelog.d/4283.misc b/changelog.d/4283.misc deleted file mode 100644 index 21de5eb509..0000000000 --- a/changelog.d/4283.misc +++ /dev/null @@ -1 +0,0 @@ -Make isort tox check print diff when it fails diff --git a/changelog.d/4284.bugfix b/changelog.d/4284.bugfix deleted file mode 100644 index 4a9478fa28..0000000000 --- a/changelog.d/4284.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add 'sandbox' to CSP for media reprository diff --git a/changelog.d/4294.bugfix b/changelog.d/4294.bugfix deleted file mode 100644 index 98114869fc..0000000000 --- a/changelog.d/4294.bugfix +++ /dev/null @@ -1 +0,0 @@ -Make the new landing page prettier. diff --git a/changelog.d/4295.bugfix b/changelog.d/4295.bugfix deleted file mode 100644 index e1603cbcda..0000000000 --- a/changelog.d/4295.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix deleting E2E room keys when using old SQLite versions. diff --git a/changelog.d/4297.misc b/changelog.d/4297.misc deleted file mode 100644 index 63106e26f6..0000000000 --- a/changelog.d/4297.misc +++ /dev/null @@ -1 +0,0 @@ -Log room_id in Unknown room errors diff --git a/changelog.d/4298.feature b/changelog.d/4298.feature deleted file mode 100644 index 05ad70fe72..0000000000 --- a/changelog.d/4298.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead. diff --git a/changelog.d/4305.bugfix b/changelog.d/4305.bugfix deleted file mode 100644 index 499fb82077..0000000000 --- a/changelog.d/4305.bugfix +++ /dev/null @@ -1 +0,0 @@ -The metric synapse_admin_mau:current previously did not update when config.mau_stats_only was set to True diff --git a/changelog.d/4307.feature b/changelog.d/4307.feature deleted file mode 100644 index 314fc031f0..0000000000 --- a/changelog.d/4307.feature +++ /dev/null @@ -1 +0,0 @@ -Add opt-in support for v2 rooms diff --git a/changelog.d/4309.bugfix b/changelog.d/4309.bugfix deleted file mode 100644 index 93b3a4f30b..0000000000 --- a/changelog.d/4309.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed per-room account data filters diff --git a/changelog.d/4313.bugfix b/changelog.d/4313.bugfix deleted file mode 100644 index d10685dd62..0000000000 --- a/changelog.d/4313.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix indentation in default config diff --git a/changelog.d/4315.feature b/changelog.d/4315.feature deleted file mode 100644 index 23e82fd02d..0000000000 --- a/changelog.d/4315.feature +++ /dev/null @@ -1 +0,0 @@ -Add a script to generate a clean config file diff --git a/changelog.d/4316.bugfix b/changelog.d/4316.bugfix deleted file mode 100644 index bd152dc371..0000000000 --- a/changelog.d/4316.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix synapse:latest docker upload diff --git a/changelog.d/4317.bugfix b/changelog.d/4317.bugfix deleted file mode 100644 index 61bad5f2da..0000000000 --- a/changelog.d/4317.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix test_metric.py compatibility with prometheus_client 0.5. Contributed by Maarten de Vries <maarten@de-vri.es>. diff --git a/changelog.d/4319.feature b/changelog.d/4319.feature deleted file mode 100644 index 84221342bb..0000000000 --- a/changelog.d/4319.feature +++ /dev/null @@ -1 +0,0 @@ -Return server data in /login response \ No newline at end of file diff --git a/changelog.d/4333.misc b/changelog.d/4333.misc deleted file mode 100644 index 43f7139a48..0000000000 --- a/changelog.d/4333.misc +++ /dev/null @@ -1 +0,0 @@ -Documentation improvements for coturn setup. Contributed by Krithin Sitaram. diff --git a/changelog.d/4334.removal b/changelog.d/4334.removal deleted file mode 100644 index d4b8c56b7d..0000000000 --- a/changelog.d/4334.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the deprecated v1/register API on Python 2. It was never ported to Python 3. diff --git a/changelog.d/4368.misc b/changelog.d/4368.misc new file mode 100644 index 0000000000..020dacb547 --- /dev/null +++ b/changelog.d/4368.misc @@ -0,0 +1 @@ +Add better logging for unexpected errors while sending transactions diff --git a/changelog.d/4369.bugfix b/changelog.d/4369.bugfix new file mode 100644 index 0000000000..a78d557932 --- /dev/null +++ b/changelog.d/4369.bugfix @@ -0,0 +1 @@ +Prevent users with access tokens predating the introduction of device IDs from creating spurious entries in the user_ips table. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 2c1f0671b2..1e4ee43758 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -37,7 +37,7 @@ services: labels: - traefik.enable=true - traefik.frontend.rule=Host:my.matrix.Host - - traefik.port=8448 + - traefik.port=8008 db: image: docker.io/postgres:10-alpine diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 61ffb13192..83346c40f1 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -33,7 +33,8 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --extra-pip-arg="--no-cache-dir" \ - --extra-pip-arg="--compile" + --extra-pip-arg="--compile" \ + --extras="all" # we copy the tests to a temporary directory so that we can put them on the # PYTHONPATH without putting the uninstalled synapse on the pythonpath. @@ -41,8 +42,7 @@ tmpdir=`mktemp -d` trap "rm -r $tmpdir" EXIT cp -r tests "$tmpdir" -cd debian/matrix-synapse-py3 PYTHONPATH="$tmpdir" \ - ./opt/venvs/matrix-synapse/bin/python \ + debian/matrix-synapse-py3/opt/venvs/matrix-synapse/bin/python \ -B -m twisted.trial --reporter=text -j2 tests diff --git a/debian/changelog b/debian/changelog index 040c8e7cd3..921f7021d8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +matrix-synapse-py3 (0.34.1+1) stable; urgency=medium + + * Remove 'Breaks: matrix-synapse-ldap3'. (matrix-synapse-py3 includes + the matrix-synapse-ldap3 python files, which makes the + matrix-synapse-ldap3 debian package redundant but not broken. + + -- Synapse Packaging team <packages@matrix.org> Wed, 09 Jan 2019 15:30:00 +0000 + +matrix-synapse-py3 (0.34.1) stable; urgency=medium + + * New synapse release 0.34.1. + * Update Conflicts specifications to allow installation alongside our + matrix-synapse transitional package. + + -- Synapse Packaging team <packages@matrix.org> Wed, 09 Jan 2019 14:52:24 +0000 + matrix-synapse-py3 (0.34.0) stable; urgency=medium * New synapse release 0.34.0. diff --git a/debian/control b/debian/control index 552a81dcb0..634f04284a 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,7 @@ Maintainer: Synapse Packaging team <packages@matrix.org> Build-Depends: debhelper (>= 9), dh-systemd, - dh-virtualenv (>= 1.0), + dh-virtualenv (>= 1.1), lsb-release, python3-dev, python3, @@ -13,12 +13,15 @@ Build-Depends: python3-pip, python3-venv, tar, -Standards-Version: 3.9.5 +Standards-Version: 3.9.8 Homepage: https://github.com/matrix-org/synapse Package: matrix-synapse-py3 Architecture: amd64 -Conflicts: matrix-synapse +Provides: matrix-synapse +Breaks: + matrix-synapse (<< 0.34.0-0matrix2), + matrix-synapse (>= 0.34.0-1), Pre-Depends: dpkg (>= 1.16.1) Depends: adduser, diff --git a/docker/Dockerfile b/docker/Dockerfile index db44c02a92..4b739e7d02 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,9 +33,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \ COPY . /synapse RUN pip install --prefix="/install" --no-warn-script-location \ - lxml \ - psycopg2 \ - /synapse + /synapse[all] ### ### Stage 1: runtime diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index ea6b650af2..ab28e49291 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -11,6 +11,35 @@ # Get the distro we want to pull from as a dynamic build variable ARG distro="" + +### +### Stage 0: build a dh-virtualenv +### +FROM ${distro} as builder + +RUN apt-get update -qq -o Acquire::Languages=none +RUN env DEBIAN_FRONTEND=noninteractive apt-get install \ + -yqq --no-install-recommends \ + build-essential \ + ca-certificates \ + devscripts \ + equivs \ + wget + +# fetch and unpack the package +RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz +RUN tar xvf /dh-virtuenv-1.1.tar.gz + +# install its build deps +RUN cd dh-virtualenv-1.1/ \ + && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends" + +# build it +RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b + +### +### Stage 1 +### FROM ${distro} # Install the build dependencies @@ -21,15 +50,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \ debhelper \ devscripts \ dh-systemd \ - dh-virtualenv \ - equivs \ lsb-release \ python3-dev \ python3-pip \ python3-setuptools \ python3-venv \ - sqlite3 \ - wget + sqlite3 + +COPY --from=builder /dh-virtualenv_1.1-1_all.deb / +RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb WORKDIR /synapse/source ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] diff --git a/docker/build_debian.sh b/docker/build_debian.sh index cea5067fe9..6ed2b39898 100644 --- a/docker/build_debian.sh +++ b/docker/build_debian.sh @@ -6,20 +6,6 @@ set -ex DIST=`lsb_release -c -s` -# We need to build a newer dh_virtualenv on older OSes like Xenial. -if [ "$DIST" = 'xenial' ]; then - mkdir -p /tmp/dhvenv - cd /tmp/dhvenv - wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz - tar xvf 1.1.tar.gz - cd dh-virtualenv-1.1/ - env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io" - dpkg-buildpackage -us -uc -b - cd /tmp/dhvenv - apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb -fi - - # we get a read-only copy of the source: make a writeable copy cp -aT /synapse/source /synapse/build cd /synapse/build diff --git a/synapse/__init__.py b/synapse/__init__.py index 27241cb364..2935238fa2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.34.0" +__version__ = "0.34.1" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 48b903374d..0b464834ce 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -348,6 +348,24 @@ class IncompatibleRoomVersionError(SynapseError): ) +class RequestSendFailed(RuntimeError): + """Sending a HTTP request over federation failed due to not being able to + talk to the remote server for some reason. + + This exception is used to differentiate "expected" errors that arise due to + networking (e.g. DNS failures, connection timeouts etc), versus unexpected + errors (like programming errors). + """ + def __init__(self, inner_exception, can_retry): + super(RequestSendFailed, self).__init__( + "Failed to send request: %s: %s" % ( + type(inner_exception).__name__, inner_exception, + ) + ) + self.inner_exception = inner_exception + self.can_retry = can_retry + + def cs_error(msg, code=Codes.UNKNOWN, **kwargs): """ Utility method for constructing an error response for client-server interactions. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 677c0bdd4c..16ad654864 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from six import text_type + import jsonschema from canonicaljson import json from jsonschema import FormatChecker @@ -353,7 +355,7 @@ class Filter(object): sender = event.user_id room_id = None ev_type = "m.presence" - is_url = False + contains_url = False else: sender = event.get("sender", None) if not sender: @@ -368,13 +370,16 @@ class Filter(object): room_id = event.get("room_id", None) ev_type = event.get("type", None) - is_url = "url" in event.get("content", {}) + + content = event.get("content", {}) + # check if there is a string url field in the content for filtering purposes + contains_url = isinstance(content.get("url"), text_type) return self.check_fields( room_id, sender, ev_type, - is_url, + contains_url, ) def check_fields(self, room_id, sender, event_type, contains_url): diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index 233bf43fc8..b45adafdd3 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -19,15 +19,8 @@ from synapse import python_dependencies # noqa: E402 sys.dont_write_bytecode = True - try: python_dependencies.check_requirements() except python_dependencies.DependencyException as e: - message = "\n".join([ - "Missing Requirements: %s" % (", ".join(e.dependencies),), - "To install run:", - " pip install --upgrade --force %s" % (" ".join(e.dependencies),), - "", - ]) - sys.stderr.writelines(message) + sys.stderr.writelines(e.message) sys.exit(1) diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py index 79fe9c3dac..fca35b008c 100644 --- a/synapse/config/__main__.py +++ b/synapse/config/__main__.py @@ -16,7 +16,7 @@ from synapse.config._base import ConfigError if __name__ == "__main__": import sys - from homeserver import HomeServerConfig + from synapse.config.homeserver import HomeServerConfig action = sys.argv[1] diff --git a/synapse/config/server.py b/synapse/config/server.py index 120c2b81fc..fb57791098 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017 New Vector Ltd +# Copyright 2017-2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import logging import os.path from synapse.http.endpoint import parse_and_validate_server_name +from synapse.python_dependencies import DependencyException, check_requirements from ._base import Config, ConfigError @@ -204,6 +205,8 @@ class ServerConfig(Config): ] }) + _check_resource_config(self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: @@ -465,3 +468,36 @@ def _warn_if_webclient_configured(listeners): if name == 'webclient': logger.warning(NO_MORE_WEB_CLIENT_WARNING) return + + +KNOWN_RESOURCES = ( + 'client', + 'consent', + 'federation', + 'keys', + 'media', + 'metrics', + 'replication', + 'static', + 'webclient', +) + + +def _check_resource_config(listeners): + resource_names = set( + res_name + for listener in listeners + for res in listener.get("resources", []) + for res_name in res.get("names", []) + ) + + for resource in resource_names: + if resource not in KNOWN_RESOURCES: + raise ConfigError( + "Unknown listener resource '%s'" % (resource, ) + ) + if resource == "consent": + try: + check_requirements('resources.consent') + except DependencyException as e: + raise ConfigError(e.message) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 099ace28c1..fe787abaeb 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -22,7 +22,11 @@ from prometheus_client import Counter from twisted.internet import defer import synapse.metrics -from synapse.api.errors import FederationDeniedError, HttpResponseException +from synapse.api.errors import ( + FederationDeniedError, + HttpResponseException, + RequestSendFailed, +) from synapse.handlers.presence import format_user_presence_state, get_interested_remotes from synapse.metrics import ( LaterGauge, @@ -518,11 +522,21 @@ class TransactionQueue(object): ) except FederationDeniedError as e: logger.info(e) - except Exception as e: - logger.warn( - "TX [%s] Failed to send transaction: %s", + except HttpResponseException as e: + logger.warning( + "TX [%s] Received %d response to transaction: %s", + destination, e.code, e, + ) + except RequestSendFailed as e: + logger.warning("TX [%s] Failed to send transaction: %s", destination, e) + + for p, _ in pending_pdus: + logger.info("Failed to send event %s to %s", p.event_id, + destination) + except Exception: + logger.exception( + "TX [%s] Failed to send transaction", destination, - e, ) for p, _ in pending_pdus: logger.info("Failed to send event %s to %s", p.event_id, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 43f81bd607..9d257ecf31 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -235,6 +235,17 @@ class PaginationHandler(object): "room_key", next_key ) + if events: + if event_filter: + events = event_filter.filter(events) + + events = yield filter_events_for_client( + self.store, + user_id, + events, + is_peeking=(member_event_id is None), + ) + if not events: defer.returnValue({ "chunk": [], @@ -242,16 +253,6 @@ class PaginationHandler(object): "end": next_token.to_string(), }) - if event_filter: - events = event_filter.filter(events) - - events = yield filter_events_for_client( - self.store, - user_id, - events, - is_peeking=(member_event_id is None), - ) - state = None if event_filter and event_filter.lazy_load_members(): # TODO: remove redundant members diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 24b6110c20..f2a42f97a6 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -19,7 +19,7 @@ import random import sys from io import BytesIO -from six import PY3, string_types +from six import PY3, raise_from, string_types from six.moves import urllib import attr @@ -41,6 +41,7 @@ from synapse.api.errors import ( Codes, FederationDeniedError, HttpResponseException, + RequestSendFailed, SynapseError, ) from synapse.http.endpoint import matrix_federation_endpoint @@ -228,19 +229,18 @@ class MatrixFederationHttpClient(object): backoff_on_404 (bool): Back off if we get a 404 Returns: - Deferred: resolves with the http response object on success. - - Fails with ``HttpResponseException``: if we get an HTTP response - code >= 300. - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist - - (May also fail with plenty of other Exceptions for things like DNS - failures, connection failures, SSL failures.) + Deferred[twisted.web.client.Response]: resolves with the HTTP + response object on success. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ if timeout: _sec_timeout = timeout / 1000 @@ -335,23 +335,74 @@ class MatrixFederationHttpClient(object): reactor=self.hs.get_reactor(), ) - with Measure(self.clock, "outbound_request"): - response = yield make_deferred_yieldable( - request_deferred, + try: + with Measure(self.clock, "outbound_request"): + response = yield make_deferred_yieldable( + request_deferred, + ) + except DNSLookupError as e: + raise_from(RequestSendFailed(e, can_retry=retry_on_dns_fail), e) + except Exception as e: + raise_from(RequestSendFailed(e, can_retry=True), e) + + logger.info( + "{%s} [%s] Got response headers: %d %s", + request.txn_id, + request.destination, + response.code, + response.phrase.decode('ascii', errors='replace'), + ) + + if 200 <= response.code < 300: + pass + else: + # :'( + # Update transactions table? + d = treq.content(response) + d = timeout_deferred( + d, + timeout=_sec_timeout, + reactor=self.hs.get_reactor(), + ) + + try: + body = yield make_deferred_yieldable(d) + except Exception as e: + # Eh, we're already going to raise an exception so lets + # ignore if this fails. + logger.warn( + "{%s} [%s] Failed to get error response: %s %s: %s", + request.txn_id, + request.destination, + request.method, + url_str, + _flatten_response_never_received(e), + ) + body = None + + e = HttpResponseException( + response.code, response.phrase, body ) + # Retry if the error is a 429 (Too Many Requests), + # otherwise just raise a standard HttpResponseException + if response.code == 429: + raise_from(RequestSendFailed(e, can_retry=True), e) + else: + raise e + break - except Exception as e: + except RequestSendFailed as e: logger.warn( "{%s} [%s] Request failed: %s %s: %s", request.txn_id, request.destination, request.method, url_str, - _flatten_response_never_received(e), + _flatten_response_never_received(e.inner_exception), ) - if not retry_on_dns_fail and isinstance(e, DNSLookupError): + if not e.can_retry: raise if retries_left and not timeout: @@ -376,29 +427,16 @@ class MatrixFederationHttpClient(object): else: raise - logger.info( - "{%s} [%s] Got response headers: %d %s", - request.txn_id, - request.destination, - response.code, - response.phrase.decode('ascii', errors='replace'), - ) - - if 200 <= response.code < 300: - pass - else: - # :'( - # Update transactions table? - d = treq.content(response) - d = timeout_deferred( - d, - timeout=_sec_timeout, - reactor=self.hs.get_reactor(), - ) - body = yield make_deferred_yieldable(d) - raise HttpResponseException( - response.code, response.phrase, body - ) + except Exception as e: + logger.warn( + "{%s} [%s] Request failed: %s %s: %s", + request.txn_id, + request.destination, + request.method, + url_str, + _flatten_response_never_received(e), + ) + raise defer.returnValue(response) @@ -477,17 +515,18 @@ class MatrixFederationHttpClient(object): requests) Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. - - Fails with ``HttpResponseException`` if we get an HTTP response - code >= 300. - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist + Deferred[dict|list]: Succeeds when we get a 2xx HTTP response. The + result will be the decoded JSON body. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ request = MatrixFederationRequest( @@ -531,17 +570,18 @@ class MatrixFederationHttpClient(object): try the request anyway. args (dict): query params Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. - - Fails with ``HttpResponseException`` if we get an HTTP response - code >= 300. - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist + Deferred[dict|list]: Succeeds when we get a 2xx HTTP response. The + result will be the decoded JSON body. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ request = MatrixFederationRequest( @@ -586,17 +626,18 @@ class MatrixFederationHttpClient(object): ignore_backoff (bool): true to ignore the historical backoff data and try the request anyway. Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. - - Fails with ``HttpResponseException`` if we get an HTTP response - code >= 300. - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist + Deferred[dict|list]: Succeeds when we get a 2xx HTTP response. The + result will be the decoded JSON body. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ logger.debug("get_json args: %s", args) @@ -637,17 +678,18 @@ class MatrixFederationHttpClient(object): ignore_backoff (bool): true to ignore the historical backoff data and try the request anyway. Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. - - Fails with ``HttpResponseException`` if we get an HTTP response - code >= 300. - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist + Deferred[dict|list]: Succeeds when we get a 2xx HTTP response. The + result will be the decoded JSON body. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ request = MatrixFederationRequest( method="DELETE", @@ -680,18 +722,20 @@ class MatrixFederationHttpClient(object): args (dict): Optional dictionary used to create the query string. ignore_backoff (bool): true to ignore the historical backoff data and try the request anyway. - Returns: - Deferred: resolves with an (int,dict) tuple of the file length and - a dict of the response headers. - - Fails with ``HttpResponseException`` if we get an HTTP response code - >= 300 - - Fails with ``NotRetryingDestination`` if we are not yet ready - to retry this server. - Fails with ``FederationDeniedError`` if this destination - is not on our federation whitelist + Returns: + Deferred[tuple[int, dict]]: Resolves with an (int,dict) tuple of + the file length and a dict of the response headers. + + Raises: + HttpResponseException: If we get an HTTP response code >= 300 + (except 429). + NotRetryingDestination: If we are not yet ready to retry this + server. + FederationDeniedError: If this destination is not on our + federation whitelist + RequestSendFailed: If there were problems connecting to the + remote, due to e.g. DNS failures, connection timeouts etc. """ request = MatrixFederationRequest( method="GET", @@ -784,21 +828,21 @@ def check_content_type_is_json(headers): headers (twisted.web.http_headers.Headers): headers to check Raises: - RuntimeError if the + RequestSendFailed: if the Content-Type header is missing or isn't JSON """ c_type = headers.getRawHeaders(b"Content-Type") if c_type is None: - raise RuntimeError( + raise RequestSendFailed(RuntimeError( "No Content-Type header" - ) + ), can_retry=False) c_type = c_type[0].decode('ascii') # only the first header val, options = cgi.parse_header(c_type) if val != "application/json": - raise RuntimeError( + raise RequestSendFailed(RuntimeError( "Content-Type not application/json: was '%s'" % c_type - ) + ), can_retry=False) def encode_query_args(args): diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2c65ef5856..69c5f9fe2e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -65,9 +65,13 @@ REQUIREMENTS = [ ] CONDITIONAL_REQUIREMENTS = { - "email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"], + "email.enable_notifs": ["Jinja2>=2.9", "bleach>=1.4.2"], "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], "postgres": ["psycopg2>=2.6"], + + # ConsentResource uses select_autoescape, which arrived in jinja 2.9 + "resources.consent": ["Jinja2>=2.9"], + "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], "test": ["mock>=2.0"], @@ -84,18 +88,30 @@ def list_requirements(): class DependencyException(Exception): @property + def message(self): + return "\n".join([ + "Missing Requirements: %s" % (", ".join(self.dependencies),), + "To install run:", + " pip install --upgrade --force %s" % (" ".join(self.dependencies),), + "", + ]) + + @property def dependencies(self): for i in self.args[0]: yield '"' + i + '"' -def check_requirements(_get_distribution=get_distribution): - +def check_requirements(for_feature=None, _get_distribution=get_distribution): deps_needed = [] errors = [] - # Check the base dependencies exist -- they all must be installed. - for dependency in REQUIREMENTS: + if for_feature: + reqs = CONDITIONAL_REQUIREMENTS[for_feature] + else: + reqs = REQUIREMENTS + + for dependency in reqs: try: _get_distribution(dependency) except VersionConflict as e: @@ -108,23 +124,24 @@ def check_requirements(_get_distribution=get_distribution): deps_needed.append(dependency) errors.append("Needed %s but it was not installed" % (dependency,)) - # Check the optional dependencies are up to date. We allow them to not be - # installed. - OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) - - for dependency in OPTS: - try: - _get_distribution(dependency) - except VersionConflict: - deps_needed.append(dependency) - errors.append("Needed %s but it was not installed" % (dependency,)) - except DistributionNotFound: - # If it's not found, we don't care - pass + if not for_feature: + # Check the optional dependencies are up to date. We allow them to not be + # installed. + OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), []) + + for dependency in OPTS: + try: + _get_distribution(dependency) + except VersionConflict: + deps_needed.append(dependency) + errors.append("Needed %s but it was not installed" % (dependency,)) + except DistributionNotFound: + # If it's not found, we don't care + pass if deps_needed: for e in errors: - logging.exception(e) + logging.error(e) raise DependencyException(deps_needed) diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py index 371e9aa354..f171b8d626 100644 --- a/synapse/rest/client/v2_alpha/account_data.py +++ b/synapse/rest/client/v2_alpha/account_data.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import AuthError, SynapseError +from synapse.api.errors import AuthError, NotFoundError, SynapseError from synapse.http.servlet import RestServlet, parse_json_object_from_request from ._base import client_v2_patterns @@ -28,6 +28,7 @@ logger = logging.getLogger(__name__) class AccountDataServlet(RestServlet): """ PUT /user/{user_id}/account_data/{account_dataType} HTTP/1.1 + GET /user/{user_id}/account_data/{account_dataType} HTTP/1.1 """ PATTERNS = client_v2_patterns( "/user/(?P<user_id>[^/]*)/account_data/(?P<account_data_type>[^/]*)" @@ -57,10 +58,26 @@ class AccountDataServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_GET(self, request, user_id, account_data_type): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): + raise AuthError(403, "Cannot get account data for other users.") + + event = yield self.store.get_global_account_data_by_type_for_user( + account_data_type, user_id, + ) + + if event is None: + raise NotFoundError("Account data not found") + + defer.returnValue((200, event)) + class RoomAccountDataServlet(RestServlet): """ PUT /user/{user_id}/rooms/{room_id}/account_data/{account_dataType} HTTP/1.1 + GET /user/{user_id}/rooms/{room_id}/account_data/{account_dataType} HTTP/1.1 """ PATTERNS = client_v2_patterns( "/user/(?P<user_id>[^/]*)" @@ -99,6 +116,21 @@ class RoomAccountDataServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_GET(self, request, user_id, room_id, account_data_type): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): + raise AuthError(403, "Cannot get account data for other users.") + + event = yield self.store.get_account_data_for_room_and_type( + user_id, room_id, account_data_type, + ) + + if event is None: + raise NotFoundError("Room account data not found") + + defer.returnValue((200, event)) + def register_servlets(hs, http_server): AccountDataServlet(hs).register(http_server) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index e117836e9a..bdffa97805 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -30,6 +30,7 @@ from synapse.api.errors import ( FederationDeniedError, HttpResponseException, NotFoundError, + RequestSendFailed, SynapseError, ) from synapse.metrics.background_process_metrics import run_as_background_process @@ -372,10 +373,10 @@ class MediaRepository(object): "allow_remote": "false", } ) - except twisted.internet.error.DNSLookupError as e: - logger.warn("HTTP error fetching remote media %s/%s: %r", + except RequestSendFailed as e: + logger.warn("Request failed fetching remote media %s/%s: %r", server_name, media_id, e) - raise NotFoundError() + raise SynapseError(502, "Failed to fetch remote media") except HttpResponseException as e: logger.warn("HTTP error fetching remote media %s/%s: %s", diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1d3069b143..865b5e915a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -547,11 +547,19 @@ class SQLBaseStore(object): if lock: self.database_engine.lock_table(txn, table) + def _getwhere(key): + # If the value we're passing in is None (aka NULL), we need to use + # IS, not =, as NULL = NULL equals NULL (False). + if keyvalues[key] is None: + return "%s IS ?" % (key,) + else: + return "%s = ?" % (key,) + # First try to update. sql = "UPDATE %s SET %s WHERE %s" % ( table, ", ".join("%s = ?" % (k,) for k in values), - " AND ".join("%s = ?" % (k,) for k in keyvalues) + " AND ".join(_getwhere(k) for k in keyvalues) ) sqlargs = list(values.values()) + list(keyvalues.values()) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 10c3b9757f..c9e11c3135 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -114,6 +114,31 @@ class RegistrationWorkerStore(SQLBaseStore): return None + @cachedInlineCallbacks() + def is_support_user(self, user_id): + """Determines if the user is of type UserTypes.SUPPORT + + Args: + user_id (str): user id to test + + Returns: + Deferred[bool]: True if user is of type UserTypes.SUPPORT + """ + res = yield self.runInteraction( + "is_support_user", self.is_support_user_txn, user_id + ) + defer.returnValue(res) + + def is_support_user_txn(self, txn, user_id): + res = self._simple_select_one_onecol_txn( + txn=txn, + table="users", + keyvalues={"name": user_id}, + retcol="user_type", + allow_none=True, + ) + return True if res == UserTypes.SUPPORT else False + class RegistrationStore(RegistrationWorkerStore, background_updates.BackgroundUpdateStore): @@ -465,31 +490,6 @@ class RegistrationStore(RegistrationWorkerStore, defer.returnValue(res if res else False) - @cachedInlineCallbacks() - def is_support_user(self, user_id): - """Determines if the user is of type UserTypes.SUPPORT - - Args: - user_id (str): user id to test - - Returns: - Deferred[bool]: True if user is of type UserTypes.SUPPORT - """ - res = yield self.runInteraction( - "is_support_user", self.is_support_user_txn, user_id - ) - defer.returnValue(res) - - def is_support_user_txn(self, txn, user_id): - res = self._simple_select_one_onecol_txn( - txn=txn, - table="users", - keyvalues={"name": user_id}, - retcol="user_type", - allow_none=True, - ) - return True if res == UserTypes.SUPPORT else False - @defer.inlineCallbacks def user_add_threepid(self, user_id, medium, address, validated_at, added_at): yield self._simple_upsert("user_threepids", { diff --git a/synctl b/synctl index 7e79b05c39..816c898b36 100755 --- a/synctl +++ b/synctl @@ -156,7 +156,9 @@ def main(): write( "No config file found\n" "To generate a config file, run '%s -c %s --generate-config" - " --server-name=<server name>'\n" % (" ".join(SYNAPSE), options.configfile), + " --server-name=<server name> --report-stats=<yes/no>'\n" % ( + " ".join(SYNAPSE), options.configfile, + ), stream=sys.stderr, ) sys.exit(1) diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py index f3cb1423f0..b2e38276d8 100644 --- a/tests/http/test_fedclient.py +++ b/tests/http/test_fedclient.py @@ -20,6 +20,7 @@ from twisted.internet.error import ConnectingCancelledError, DNSLookupError from twisted.web.client import ResponseNeverReceived from twisted.web.http import HTTPChannel +from synapse.api.errors import RequestSendFailed from synapse.http.matrixfederationclient import ( MatrixFederationHttpClient, MatrixFederationRequest, @@ -49,7 +50,8 @@ class FederationClientTests(HomeserverTestCase): self.pump() f = self.failureResultOf(d) - self.assertIsInstance(f.value, DNSLookupError) + self.assertIsInstance(f.value, RequestSendFailed) + self.assertIsInstance(f.value.inner_exception, DNSLookupError) def test_client_never_connect(self): """ @@ -76,7 +78,11 @@ class FederationClientTests(HomeserverTestCase): self.reactor.advance(10.5) f = self.failureResultOf(d) - self.assertIsInstance(f.value, (ConnectingCancelledError, TimeoutError)) + self.assertIsInstance(f.value, RequestSendFailed) + self.assertIsInstance( + f.value.inner_exception, + (ConnectingCancelledError, TimeoutError), + ) def test_client_connect_no_response(self): """ @@ -107,7 +113,8 @@ class FederationClientTests(HomeserverTestCase): self.reactor.advance(10.5) f = self.failureResultOf(d) - self.assertIsInstance(f.value, ResponseNeverReceived) + self.assertIsInstance(f.value, RequestSendFailed) + self.assertIsInstance(f.value.inner_exception, ResponseNeverReceived) def test_client_gets_headers(self): """ diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 4577e9422b..858efe4992 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -62,6 +62,77 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase): r, ) + def test_insert_new_client_ip_none_device_id(self): + """ + An insert with a device ID of NULL will not create a new entry, but + update an existing entry in the user_ips table. + """ + self.reactor.advance(12345678) + + user_id = "@user:id" + + # Add & trigger the storage loop + self.get_success( + self.store.insert_client_ip( + user_id, "access_token", "ip", "user_agent", None + ) + ) + self.reactor.advance(200) + self.pump(0) + + result = self.get_success( + self.store._simple_select_list( + table="user_ips", + keyvalues={"user_id": user_id}, + retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"], + desc="get_user_ip_and_agents", + ) + ) + + self.assertEqual( + result, + [ + { + 'access_token': 'access_token', + 'ip': 'ip', + 'user_agent': 'user_agent', + 'device_id': None, + 'last_seen': 12345678000, + } + ], + ) + + # Add another & trigger the storage loop + self.get_success( + self.store.insert_client_ip( + user_id, "access_token", "ip", "user_agent", None + ) + ) + self.reactor.advance(10) + self.pump(0) + + result = self.get_success( + self.store._simple_select_list( + table="user_ips", + keyvalues={"user_id": user_id}, + retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"], + desc="get_user_ip_and_agents", + ) + ) + # Only one result, has been upserted. + self.assertEqual( + result, + [ + { + 'access_token': 'access_token', + 'ip': 'ip', + 'user_agent': 'user_agent', + 'device_id': None, + 'last_seen': 12345878000, + } + ], + ) + def test_disabled_monthly_active_user(self): self.hs.config.limit_usage_by_mau = False self.hs.config.max_mau_value = 50 |