From 95e02b856b2ac1409dc0d5575ee0b7be55105e9a Mon Sep 17 00:00:00 2001
From: Robert Swain <rob@matrix.org>
Date: Thu, 28 Sep 2017 12:12:47 +0200
Subject: docker: Initial Dockerfile and docker-compose.yaml

---
 Dockerfile                               | 58 ++++++++++++++++++++++++
 docker/README.md                         | 70 +++++++++++++++++++++++++++++
 docker/docker-compose.yaml               | 39 +++++++++++++++++
 docker/rootfs/etc/service/synapse/finish | 17 ++++++++
 docker/rootfs/etc/service/synapse/run    | 75 ++++++++++++++++++++++++++++++++
 5 files changed, 259 insertions(+)
 create mode 100644 Dockerfile
 create mode 100644 docker/README.md
 create mode 100644 docker/docker-compose.yaml
 create mode 100755 docker/rootfs/etc/service/synapse/finish
 create mode 100755 docker/rootfs/etc/service/synapse/run

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000..9b11a143f6
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,58 @@
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM phusion/baseimage:0.9.22
+
+COPY ./ /synapse/source/
+
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        build-essential \
+        libffi-dev \
+        libjpeg-dev \
+        libpq-dev \
+        libssl-dev \
+        libxslt1-dev \
+        python-pip \
+        python-setuptools \
+        python-virtualenv \
+        python2.7-dev \
+        sqlite3 \
+    && virtualenv -p python2.7 /synapse \
+    && . /synapse/bin/activate \
+    && pip install --upgrade pip \
+    && pip install --upgrade setuptools \
+    && pip install --upgrade psycopg2 \
+    && cd /synapse/source \
+    && pip install --upgrade ./ \
+    && cd / \
+    && rm -rf /synapse/source \
+    && apt-get autoremove -y \
+        build-essential \
+        libffi-dev \
+        libjpeg-dev \
+        libpq-dev \
+        libssl-dev \
+        libxslt1-dev \
+        python2.7-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY docker/rootfs/ /
+
+VOLUME /synapse/config/
+VOLUME /synapse/data/
+
+CMD ["/sbin/my_init"]
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000000..c9e6fd216d
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,70 @@
+# Synapse Docker
+
+## Build
+
+Build the docker image with the `docker build` command from the root of the synapse repository.
+
+```
+docker build -t matrix-org/synapse:v0.22.1 .
+```
+
+The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
+
+## Configure
+
+Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment.
+
+The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command.
+
+Docker container environment variables:
+* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten.
+* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`]
+* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking.
+* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
+* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`]
+* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`]
+* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`]
+
+```
+CONFIG_PATH=/my/magical/config/path/
+mkdir -p ${CONFIG_PATH}
+docker run \
+    --rm \
+    -e GENERATE_CONFIG=yes \
+    -e POSTGRES_PASSWORD=MyVerySecretPassword \
+    -e REPORT_STATS=yes \
+    -e SERVER_NAME=example.com \
+    -v ${CONFIG_PATH}:/synapse/config/ \
+    matrix-org/synapse:develop
+```
+
+This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`.
+
+## Run
+
+**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead.
+
+### Docker Compose
+
+A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition.
+
+From the `docker/` subdirectory of the synapse repository:
+```
+CONFIG_PATH=/my/magical/config/path/
+POSTGRES_PASSWORD=MyVerySecretPassword \
+docker-compose \
+    -p synapse \
+    up -d
+```
+
+### Docker
+
+Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database.
+
+```
+docker run \
+    -d \
+    --name synapse \
+    -v ${CONFIG_PATH}:/synapse/config/ \
+    matrix-org/synapse:v0.22.1
+```
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
new file mode 100644
index 0000000000..ff36081a9b
--- /dev/null
+++ b/docker/docker-compose.yaml
@@ -0,0 +1,39 @@
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3'
+
+services:
+    postgres:
+        image: postgres:9.6.5-alpine
+        environment:
+            POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+            POSTGRES_DB: synapse
+        expose:
+            - 5432
+        restart: unless-stopped
+        volumes:
+            - postgres-data:/var/lib/postgresql/data/
+
+    synapse:
+        image: matrix-org/synapse:develop
+        ports:
+            - 8008:8008
+            - 8448:8448
+        restart: unless-stopped
+        volumes:
+            - ${CONFIG_PATH}:/synapse/config/
+
+volumes:
+    postgres-data:
diff --git a/docker/rootfs/etc/service/synapse/finish b/docker/rootfs/etc/service/synapse/finish
new file mode 100755
index 0000000000..2aace581a1
--- /dev/null
+++ b/docker/rootfs/etc/service/synapse/finish
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kill -TERM 1
diff --git a/docker/rootfs/etc/service/synapse/run b/docker/rootfs/etc/service/synapse/run
new file mode 100755
index 0000000000..dd797d3ef9
--- /dev/null
+++ b/docker/rootfs/etc/service/synapse/run
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+: ${CONFIG_PATH:="/synapse/config"}
+: ${POSTGRES_DATABASE:="synapse"}
+: ${POSTGRES_HOST:="postgres"}
+: ${POSTGRES_USER:="postgres"}
+: ${REPORT_STATS:="yes"}
+: ${SERVER_NAME:="localhost"}
+
+DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml"
+HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml"
+SYNAPSE_COMMAND="python -m synapse.app.homeserver"
+
+. /synapse/bin/activate
+cd /synapse
+
+if [[ -n "${GENERATE_CONFIG}" ]]; then
+    ${SYNAPSE_COMMAND} \
+        --server-name ${SERVER_NAME} \
+        --config-path ${HOMESERVER_CONFIG_PATH} \
+        --generate-config \
+        --report-stats=${REPORT_STATS}
+
+    if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then
+        echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated."
+    else
+        echo "Generating ${DATABASE_CONFIG_PATH}..."
+        if [[ -n "${POSTGRES_PASSWORD}" ]]; then
+            (cat > ${DATABASE_CONFIG_PATH}) <<EOF
+database:
+  name: psycopg2
+  args:
+    host: ${POSTGRES_HOST}
+    user: ${POSTGRES_USER}
+    password: ${POSTGRES_PASSWORD}
+    database: ${POSTGRES_DATABASE}
+    cp_min: 5
+    cp_max: 10
+EOF
+        else
+            (cat > ${DATABASE_CONFIG_PATH}) <<EOF
+database:
+  name: "sqlite3"
+  args:
+    database: "/synapse/data/homeserver.db"
+EOF
+        fi
+        cat ${DATABASE_CONFIG_PATH} | grep -v password
+    fi
+
+    exit 0
+fi
+
+COMMAND="${SYNAPSE_COMMAND} --config-path ${HOMESERVER_CONFIG_PATH}"
+if [[ -r "${DATABASE_CONFIG_PATH}" ]]; then
+    COMMAND="${COMMAND} --config-path ${DATABASE_CONFIG_PATH}"
+fi
+
+exec ${COMMAND}
-- 
cgit 1.4.1


From 24d162814bc8c9ba05bfecac04e7218baebf2859 Mon Sep 17 00:00:00 2001
From: Robert Swain <rob@matrix.org>
Date: Fri, 29 Sep 2017 11:40:15 +0200
Subject: docker: s/matrix-org/matrixdotorg/g

---
 docker/README.md           | 8 ++++----
 docker/docker-compose.yaml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index c9e6fd216d..c15517d0e0 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -5,10 +5,10 @@
 Build the docker image with the `docker build` command from the root of the synapse repository.
 
 ```
-docker build -t matrix-org/synapse:v0.22.1 .
+docker build -t matrixdotorg/synapse:v0.22.1 .
 ```
 
-The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
+The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
 
 ## Configure
 
@@ -35,7 +35,7 @@ docker run \
     -e REPORT_STATS=yes \
     -e SERVER_NAME=example.com \
     -v ${CONFIG_PATH}:/synapse/config/ \
-    matrix-org/synapse:develop
+    matrixdotorg/synapse:v0.22.1
 ```
 
 This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`.
@@ -66,5 +66,5 @@ docker run \
     -d \
     --name synapse \
     -v ${CONFIG_PATH}:/synapse/config/ \
-    matrix-org/synapse:v0.22.1
+    matrixdotorg/synapse:v0.22.1
 ```
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index ff36081a9b..73cc29f8fd 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -27,7 +27,7 @@ services:
             - postgres-data:/var/lib/postgresql/data/
 
     synapse:
-        image: matrix-org/synapse:develop
+        image: matrixdotorg/synapse:v0.22.1
         ports:
             - 8008:8008
             - 8448:8448
-- 
cgit 1.4.1


From 053ecae4db7c51c7fa3d56f5ea2b1289d73fa10d Mon Sep 17 00:00:00 2001
From: Andrew Conrad <aconrad103@gmail.com>
Date: Tue, 14 Nov 2017 15:09:50 -0600
Subject: Mention SyTest in the README, after Development

Signed-off-by: Andrew Conrad <aconrad103@gmail.com>
---
 README.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/README.rst b/README.rst
index 9da8c7f7a8..cda0794e56 100644
--- a/README.rst
+++ b/README.rst
@@ -883,6 +883,17 @@ This should end with a 'PASSED' result::
 
     PASSED (successes=143)
 
+Running the Integration Tests
+=============================
+
+Synapse is accompanied by `SyTest <https://github.com/matrix-org/sytest>`_,
+a Matrix homeserver integration testing suite, which uses HTTP requests to
+access the API as a Matrix client would. It is able to run Synapse directly from
+the source tree, so installation of the server is not required.
+
+Testing with SyTest is recommended for verifying that changes related to the
+Client-Server API are functioning correctly. See the `installation instructions
+<https://github.com/matrix-org/sytest#installing>`_ for details.
 
 Building Internal API Documentation
 ===================================
-- 
cgit 1.4.1


From 4102468da98543ddb899857b67f5666ae4cfae92 Mon Sep 17 00:00:00 2001
From: Jeremy Cline <jeremy@jcline.org>
Date: Tue, 16 Jan 2018 16:13:28 -0500
Subject: Note that Synapse is available in Fedora

Signed-off-by: Jeremy Cline <jeremy@jcline.org>
---
 README.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.rst b/README.rst
index ef1eed4584..ef27b92ddd 100644
--- a/README.rst
+++ b/README.rst
@@ -354,6 +354,10 @@ https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one
 Fedora
 ------
 
+Synapse is in the Fedora repositories as ``matrix-synapse``::
+
+    sudo dnf install matrix-synapse
+
 Oleg Girko provides Fedora RPMs at
 https://obs.infoserver.lv/project/monitor/matrix-synapse
 
-- 
cgit 1.4.1


From 431476fbc4ef0c740e33e19ccc73996c2412e4f9 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 3 Feb 2018 20:18:36 +0100
Subject: Initial commit including a Dockerfile for synapse

---
 Dockerfile              | 16 ++++++++++++++++
 contrib/docker/start.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 Dockerfile
 create mode 100755 contrib/docker/start.py

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000..5f0433004f
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,16 @@
+FROM python:2-alpine
+
+RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers
+
+COPY synapse /usr/local/src/synapse
+COPY setup.py setup.cfg README.rst synctl /usr/local/src/
+
+RUN cd /usr/local/src \
+ && pip install --upgrade --process-dependency-links . \
+ && rm -rf setup.py setup.cfg synapse
+
+COPY contrib/docker /
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["/start.py"]
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
new file mode 100755
index 0000000000..4f63ea1ad5
--- /dev/null
+++ b/contrib/docker/start.py
@@ -0,0 +1,29 @@
+#!/usr/local/bin/python
+
+import jinja2
+import os
+import sys
+import socket
+
+convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+mode = sys.argv[1] if len(sys.argv) > 1 else None
+
+if "SYNAPSE_SERVER_NAME" not in os.environ:
+    print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
+    sys.exit(2)
+
+params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"),
+          "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"),
+          "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
+
+if mode == "generate":
+    params.append("--generate-config")
+
+# Parse the configuration file
+if not os.path.exists("/compiled"):
+    os.mkdir("/compiled")
+convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml")
+convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME"))
+
+# TODO, replace with a call to synapse.app.homeserver.run()
+os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params)
-- 
cgit 1.4.1


From d434ae33875b4c7d5ee04c45e454ce237ce578f8 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 3 Feb 2018 20:30:08 +0100
Subject: Add template config files for the Docker image

---
 contrib/docker/conf/homeserver.yaml | 181 ++++++++++++++++++++++++++++++++++++
 contrib/docker/conf/log.config      |  36 +++++++
 2 files changed, 217 insertions(+)
 create mode 100644 contrib/docker/conf/homeserver.yaml
 create mode 100644 contrib/docker/conf/log.config

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
new file mode 100644
index 0000000000..851c389c19
--- /dev/null
+++ b/contrib/docker/conf/homeserver.yaml
@@ -0,0 +1,181 @@
+# vim:ft=yaml
+
+## TLS ##
+
+tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
+tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
+tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh"
+no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }}
+tls_fingerprints: []
+
+## Server ##
+
+server_name: "{{ SYNAPSE_SERVER_NAME }}"
+pid_file: /homeserver.pid
+web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }}
+soft_file_limit: 0
+
+## Ports ##
+
+listeners:
+  {% if not SYNAPSE_NO_TLS %}
+  -
+    port: 8448
+    bind_addresses: ['0.0.0.0']
+    type: http
+    tls: true
+    x_forwarded: false
+    resources:
+      - names: [client, webclient]
+        compress: true
+      - names: [federation]  # Federation APIs
+        compress: false
+  {% endif %}
+
+  - port: 8008
+    tls: false
+    bind_addresses: ['0.0.0.0']
+    type: http
+    x_forwarded: false
+
+    resources:
+      - names: [client, webclient]
+        compress: true
+      - names: [federation]
+        compress: false
+
+## Database ##
+
+{% if SYNAPSE_DB_HOST %}
+
+{% else %}
+database:
+  name: "sqlite3"
+  args:
+    database: "/data/homeserver.db"
+{% endif %}
+
+## Performance ##
+
+event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}"
+verbose: 0
+log_file: "/data/homeserver.log"
+log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config"
+
+## Ratelimiting ##
+
+rc_messages_per_second: 0.2
+rc_message_burst_count: 10.0
+federation_rc_window_size: 1000
+federation_rc_sleep_limit: 10
+federation_rc_sleep_delay: 500
+federation_rc_reject_limit: 50
+federation_rc_concurrent: 3
+
+## Files ##
+
+media_store_path: "/data/media"
+uploads_path: "/data/uploads"
+max_upload_size: "10M"
+max_image_pixels: "32M"
+dynamic_thumbnails: false
+
+# List of thumbnail to precalculate when an image is uploaded.
+thumbnail_sizes:
+- width: 32
+  height: 32
+  method: crop
+- width: 96
+  height: 96
+  method: crop
+- width: 320
+  height: 240
+  method: scale
+- width: 640
+  height: 480
+  method: scale
+- width: 800
+  height: 600
+  method: scale
+
+url_preview_enabled: False
+max_spider_size: "10M"
+
+## Captcha ##
+
+recaptcha_public_key: "YOUR_PUBLIC_KEY"
+recaptcha_private_key: "YOUR_PRIVATE_KEY"
+enable_registration_captcha: False
+recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+
+## Turn ##
+
+turn_uris: []
+turn_shared_secret: "YOUR_SHARED_SECRET"
+turn_user_lifetime: "1h"
+turn_allow_guests: True
+
+## Registration ##
+
+enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }}
+registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}"
+bcrypt_rounds: 12
+allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }}
+
+# The list of identity servers trusted to verify third party
+# identifiers by this server.
+trusted_third_party_id_servers:
+    - matrix.org
+    - vector.im
+    - riot.im
+
+## Metrics ###
+
+enable_metrics: False
+report_stats: False
+
+## API Configuration ##
+
+room_invite_state_types:
+    - "m.room.join_rules"
+    - "m.room.canonical_alias"
+    - "m.room.avatar"
+    - "m.room.name"
+
+app_service_config_files: []
+macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
+expire_access_token: False
+
+## Signing Keys ##
+
+signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key"
+old_signing_keys: {}
+key_refresh_interval: "1d" # 1 Day.
+
+# The trusted servers to download signing keys from.
+perspectives:
+  servers:
+    "matrix.org":
+      verify_keys:
+        "ed25519:auto":
+          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+
+password_config:
+   enabled: true
+
+#email:
+#   enable_notifs: false
+#   smtp_host: "localhost"
+#   smtp_port: 25
+#   smtp_user: "exampleusername"
+#   smtp_pass: "examplepassword"
+#   require_transport_security: False
+#   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
+#   app_name: Matrix
+#   template_dir: res/templates
+#   notif_template_html: notif_mail.html
+#   notif_template_text: notif_mail.txt
+#   notif_for_new_users: True
+#   riot_base_url: "http://localhost/riot"
+
+enable_group_creation: true
diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config
new file mode 100644
index 0000000000..45e7eef953
--- /dev/null
+++ b/contrib/docker/conf/log.config
@@ -0,0 +1,36 @@
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  file:
+    class: logging.handlers.RotatingFileHandler
+    formatter: precise
+    filename: /data/homeserver.log
+    maxBytes: 104857600
+    backupCount: 10
+    filters: [context]
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+
+loggers:
+    synapse:
+        level: INFO
+
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: INFO
+
+root:
+    level: INFO
+    handlers: [file, console]
-- 
cgit 1.4.1


From 48bc22f89dadb8278cf2b8c940604534999d246f Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 10:58:07 +0100
Subject: Allow for a wheel cache and include missing files in the build

---
 .dockerignore           |  5 +++++
 Dockerfile              | 13 +++++++------
 contrib/docker/start.py |  1 +
 3 files changed, 13 insertions(+), 6 deletions(-)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000..f36f86fbb7
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+Dockerfile
+.travis.yml
+.gitignore
+demo/etc
+tox.ini
diff --git a/Dockerfile b/Dockerfile
index 5f0433004f..277246b697 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,15 +2,16 @@ FROM python:2-alpine
 
 RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers
 
-COPY synapse /usr/local/src/synapse
-COPY setup.py setup.cfg README.rst synctl /usr/local/src/
+COPY . /synapse
 
-RUN cd /usr/local/src \
- && pip install --upgrade --process-dependency-links . \
+# A wheel cache may be provided in ./cache for faster build
+RUN cd /synapse \
+ && pip install --upgrade pip setuptools \
+ && mkdir -p /synapse/cache \
+ && pip install -f /synapse/cache --upgrade --process-dependency-links . \
+ && mv /synapse/contrib/docker/* / \
  && rm -rf setup.py setup.cfg synapse
 
-COPY contrib/docker /
-
 VOLUME ["/data"]
 
 ENTRYPOINT ["/start.py"]
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 4f63ea1ad5..2c427ba1b7 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -22,6 +22,7 @@ if mode == "generate":
 # Parse the configuration file
 if not os.path.exists("/compiled"):
     os.mkdir("/compiled")
+
 convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml")
 convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME"))
 
-- 
cgit 1.4.1


From 6d1e28a8426da9e954a3edec25a8717376c583f3 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 11:14:06 +0100
Subject: Generate any missing keys before starting synapse

---
 contrib/docker/start.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 2c427ba1b7..e50d23be5f 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -3,7 +3,7 @@
 import jinja2
 import os
 import sys
-import socket
+import subprocess
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 mode = sys.argv[1] if len(sys.argv) > 1 else None
@@ -12,19 +12,20 @@ if "SYNAPSE_SERVER_NAME" not in os.environ:
     print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
     sys.exit(2)
 
-params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"),
-          "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"),
-          "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
-
-if mode == "generate":
-    params.append("--generate-config")
+args = ["python", "-m", "synapse.app.homeserver",
+        "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"),
+        "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"),
+        "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
 
 # Parse the configuration file
-if not os.path.exists("/compiled"):
-    os.mkdir("/compiled")
-
+if not os.path.exists("/compiled"): os.mkdir("/compiled")
 convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml")
 convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME"))
 
-# TODO, replace with a call to synapse.app.homeserver.run()
-os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params)
+# In generate mode, generate a configuration, missing keys, then exit
+if mode == "generate":
+    os.execv("/usr/local/bin/python", args + ["--generate-config"])
+# In normal mode, generate missing keys if any, then run synapse
+else:
+    subprocess.check_output(args + ["--generate-keys"])
+    os.execv("/usr/local/bin/python", args)
-- 
cgit 1.4.1


From f2bf0cda02fef358172033b28dab5f1805c31cad Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 11:40:20 +0100
Subject: Generate shared secrets if not defined in the environment

---
 contrib/docker/start.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index e50d23be5f..7057f85f61 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -5,10 +5,11 @@ import os
 import sys
 import subprocess
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
 mode = sys.argv[1] if len(sys.argv) > 1 else None
+environ = os.environ.copy()
 
-if "SYNAPSE_SERVER_NAME" not in os.environ:
+if "SYNAPSE_SERVER_NAME" not in environ:
     print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
     sys.exit(2)
 
@@ -17,10 +18,16 @@ args = ["python", "-m", "synapse.app.homeserver",
         "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"),
         "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
 
+# Generate any missing shared secret
+for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"):
+    if secret not in environ:
+        print("Generating a random secret for {}".format(secret))
+        environ[secret] = os.urandom(32).encode("hex")
+
 # Parse the configuration file
 if not os.path.exists("/compiled"): os.mkdir("/compiled")
-convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml")
-convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME"))
+convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
+convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ)
 
 # In generate mode, generate a configuration, missing keys, then exit
 if mode == "generate":
-- 
cgit 1.4.1


From 886c2d50197bb3558168ce45b6975777780c7aad Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 12:20:29 +0100
Subject: Support an external postgresql config in the Docker image

---
 contrib/docker/conf/homeserver.yaml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 851c389c19..1a685320c0 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -46,8 +46,17 @@ listeners:
 
 ## Database ##
 
-{% if SYNAPSE_DB_HOST %}
-
+{% if SYNAPSE_DB_PASSWORD %}
+database:
+  name: "psycopg2"
+  args:
+    user: "{{ SYNAPSE_DB_USER or "matrix" }}"
+    password: "{{ SYNAPSE_DB_PASSWORD }}"
+    database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}"
+    host: "{{ SYNAPSE_DB_HOST or "db" }}"
+    port: "{{ SYNAPSE_DB_PORT or "5432" }}"
+    cp_min: 5
+    cp_max: 10
 {% else %}
 database:
   name: "sqlite3"
-- 
cgit 1.4.1


From 042757feb25ba9f0cccfdaa6c8775957880c2d2e Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 12:28:42 +0100
Subject: Install the postgres dependencies

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 277246b697..881c25c243 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,12 @@
 FROM python:2-alpine
 
-RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers
+RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev
 
 COPY . /synapse
 
 # A wheel cache may be provided in ./cache for faster build
 RUN cd /synapse \
- && pip install --upgrade pip setuptools \
+ && pip install --upgrade pip setuptools psycopg2 \
  && mkdir -p /synapse/cache \
  && pip install -f /synapse/cache --upgrade --process-dependency-links . \
  && mv /synapse/contrib/docker/* / \
-- 
cgit 1.4.1


From 1ba2fe114c79dd0f2f2fb8868f8cb0a9f5893652 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 12:55:20 +0100
Subject: Provide an example docker compose file

---
 Dockerfile                        |  2 ++
 contrib/docker/docker-compose.yml | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 contrib/docker/docker-compose.yml

diff --git a/Dockerfile b/Dockerfile
index 881c25c243..25f3746303 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,4 +14,6 @@ RUN cd /synapse \
 
 VOLUME ["/data"]
 
+EXPOSE 8448
+
 ENTRYPOINT ["/start.py"]
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
new file mode 100644
index 0000000000..659bee2979
--- /dev/null
+++ b/contrib/docker/docker-compose.yml
@@ -0,0 +1,32 @@
+# This compose file is compatible with Copose itself, it might need some
+# adjustments to run properly with stack.
+
+version: '3'
+
+services:
+
+  synapse:
+    image: matrix/synapse
+    # See the readme for a full documentation of the environment settings
+    environment:
+      - SYNAPSE_SERVER_NAME=my.matrix.host
+      - SYNAPSE_DB_PASSWORD=changeme
+    volumes:
+      - ./files:/data
+    # One may either expose ports directly
+    ports:
+      - 8448:8448/tcp
+    # ... or use a reverse proxy, here is an example for traefik
+    labels:
+      - traefik.enable=true
+      - traefik.frontend.rule=Host:my.matrix.Host
+      - traefik.port=8448
+
+  db:
+    image: postgres:latest
+    # Change that password, of course!
+    environment:
+      - POSTGRES_USER=matrix
+      - POSTGRES_PASSWORD=changeme
+    volumes:
+      - ./schemas:/var/lib/postgres
-- 
cgit 1.4.1


From a207cccb059451682564cddf0e39c7c45b06cb72 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 15:04:26 +0100
Subject: Reuse environment variables of the postgres container

---
 contrib/docker/conf/homeserver.yaml | 12 ++++++------
 contrib/docker/docker-compose.yml   | 11 ++++++++---
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 1a685320c0..7450cc1228 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -46,15 +46,15 @@ listeners:
 
 ## Database ##
 
-{% if SYNAPSE_DB_PASSWORD %}
+{% if POSTGRES_PASSWORD %}
 database:
   name: "psycopg2"
   args:
-    user: "{{ SYNAPSE_DB_USER or "matrix" }}"
-    password: "{{ SYNAPSE_DB_PASSWORD }}"
-    database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}"
-    host: "{{ SYNAPSE_DB_HOST or "db" }}"
-    port: "{{ SYNAPSE_DB_PORT or "5432" }}"
+    user: "{{ POSTGRES_USER or "matrix" }}"
+    password: "{{ POSTGRES_PASSWORD }}"
+    database: "{{ POSTGRES_DB or "matrix" }}"
+    host: "{{ POSTGRES_HOST or "db" }}"
+    port: "{{ POSTGRES_PORT or "5432" }}"
     cp_min: 5
     cp_max: 10
 {% else %}
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 659bee2979..b8f9741f05 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,13 +6,18 @@ version: '3'
 services:
 
   synapse:
-    image: matrix/synapse
+    image: synapse
+    # Since snyapse does not retry to connect to the database, restart upon
+    # failure
+    restart: unless-stopped
     # See the readme for a full documentation of the environment settings
     environment:
       - SYNAPSE_SERVER_NAME=my.matrix.host
-      - SYNAPSE_DB_PASSWORD=changeme
+      - SYNAPSE_ENABLE_REGISTRATION=yes
     volumes:
       - ./files:/data
+    depends_on:
+      - db
     # One may either expose ports directly
     ports:
       - 8448:8448/tcp
@@ -29,4 +34,4 @@ services:
       - POSTGRES_USER=matrix
       - POSTGRES_PASSWORD=changeme
     volumes:
-      - ./schemas:/var/lib/postgres
+      - ./schemas:/var/lib/postgresql/data
-- 
cgit 1.4.1


From 84a9209ba7294243e5bbefede46a3dee7994da9b Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 15:08:43 +0100
Subject: Remove etc/service files from rob's branch

---
 contrib/docker/rob/docker-compose.yaml             | 39 -----------
 .../docker/rob/rootfs/etc/service/synapse/finish   | 17 -----
 contrib/docker/rob/rootfs/etc/service/synapse/run  | 75 ----------------------
 3 files changed, 131 deletions(-)
 delete mode 100644 contrib/docker/rob/docker-compose.yaml
 delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/finish
 delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/run

diff --git a/contrib/docker/rob/docker-compose.yaml b/contrib/docker/rob/docker-compose.yaml
deleted file mode 100644
index 73cc29f8fd..0000000000
--- a/contrib/docker/rob/docker-compose.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2017 Vector Creations Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: '3'
-
-services:
-    postgres:
-        image: postgres:9.6.5-alpine
-        environment:
-            POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-            POSTGRES_DB: synapse
-        expose:
-            - 5432
-        restart: unless-stopped
-        volumes:
-            - postgres-data:/var/lib/postgresql/data/
-
-    synapse:
-        image: matrixdotorg/synapse:v0.22.1
-        ports:
-            - 8008:8008
-            - 8448:8448
-        restart: unless-stopped
-        volumes:
-            - ${CONFIG_PATH}:/synapse/config/
-
-volumes:
-    postgres-data:
diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/finish b/contrib/docker/rob/rootfs/etc/service/synapse/finish
deleted file mode 100755
index 2aace581a1..0000000000
--- a/contrib/docker/rob/rootfs/etc/service/synapse/finish
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2017 Vector Creations Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-kill -TERM 1
diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/run b/contrib/docker/rob/rootfs/etc/service/synapse/run
deleted file mode 100755
index dd797d3ef9..0000000000
--- a/contrib/docker/rob/rootfs/etc/service/synapse/run
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2017 Vector Creations Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-: ${CONFIG_PATH:="/synapse/config"}
-: ${POSTGRES_DATABASE:="synapse"}
-: ${POSTGRES_HOST:="postgres"}
-: ${POSTGRES_USER:="postgres"}
-: ${REPORT_STATS:="yes"}
-: ${SERVER_NAME:="localhost"}
-
-DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml"
-HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml"
-SYNAPSE_COMMAND="python -m synapse.app.homeserver"
-
-. /synapse/bin/activate
-cd /synapse
-
-if [[ -n "${GENERATE_CONFIG}" ]]; then
-    ${SYNAPSE_COMMAND} \
-        --server-name ${SERVER_NAME} \
-        --config-path ${HOMESERVER_CONFIG_PATH} \
-        --generate-config \
-        --report-stats=${REPORT_STATS}
-
-    if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then
-        echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated."
-    else
-        echo "Generating ${DATABASE_CONFIG_PATH}..."
-        if [[ -n "${POSTGRES_PASSWORD}" ]]; then
-            (cat > ${DATABASE_CONFIG_PATH}) <<EOF
-database:
-  name: psycopg2
-  args:
-    host: ${POSTGRES_HOST}
-    user: ${POSTGRES_USER}
-    password: ${POSTGRES_PASSWORD}
-    database: ${POSTGRES_DATABASE}
-    cp_min: 5
-    cp_max: 10
-EOF
-        else
-            (cat > ${DATABASE_CONFIG_PATH}) <<EOF
-database:
-  name: "sqlite3"
-  args:
-    database: "/synapse/data/homeserver.db"
-EOF
-        fi
-        cat ${DATABASE_CONFIG_PATH} | grep -v password
-    fi
-
-    exit 0
-fi
-
-COMMAND="${SYNAPSE_COMMAND} --config-path ${HOMESERVER_CONFIG_PATH}"
-if [[ -r "${DATABASE_CONFIG_PATH}" ]]; then
-    COMMAND="${COMMAND} --config-path ${DATABASE_CONFIG_PATH}"
-fi
-
-exec ${COMMAND}
-- 
cgit 1.4.1


From 9a87b8aaf7c66380cfbdfdf35bb24f24eeef28f5 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 15:27:32 +0100
Subject: Update sumperdump Docker readme to match this image properties

---
 contrib/docker/README.md | 102 +++++++++++++++++++++++++++++------------------
 1 file changed, 64 insertions(+), 38 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index c15517d0e0..8ba5f79692 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -1,5 +1,12 @@
 # Synapse Docker
 
+This Docker image will run Synapse as a single process. It does not provide any
+database server or TURN server that you should run separately.
+
+If you run a Postgres server, you should simply have it in the same Compose
+project or set the proper environment variables and the image will automatically
+use that server.
+
 ## Build
 
 Build the docker image with the `docker build` command from the root of the synapse repository.
@@ -10,54 +17,29 @@ docker build -t matrixdotorg/synapse:v0.22.1 .
 
 The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
 
-## Configure
+You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project.
 
-Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment.
+## Run
 
-The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command.
+It is recommended that you use Docker Compose to run your containers, including
+this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
+with example labels for a reverse proxy and other artifacts.
 
-Docker container environment variables:
-* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten.
-* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking.
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`]
-* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`]
-* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`]
+Then, to run the server:
 
 ```
-CONFIG_PATH=/my/magical/config/path/
-mkdir -p ${CONFIG_PATH}
-docker run \
-    --rm \
-    -e GENERATE_CONFIG=yes \
-    -e POSTGRES_PASSWORD=MyVerySecretPassword \
-    -e REPORT_STATS=yes \
-    -e SERVER_NAME=example.com \
-    -v ${CONFIG_PATH}:/synapse/config/ \
-    matrixdotorg/synapse:v0.22.1
+docker-compose up -d
 ```
 
-This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`.
+In the case you specified a custom path for you configuration file and wish to
+generate a fresh ``homeserver.yaml``, simply run:
 
-## Run
-
-**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead.
-
-### Docker Compose
-
-A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition.
-
-From the `docker/` subdirectory of the synapse repository:
 ```
-CONFIG_PATH=/my/magical/config/path/
-POSTGRES_PASSWORD=MyVerySecretPassword \
-docker-compose \
-    -p synapse \
-    up -d
+docker-compose run synapse generate
 ```
 
-### Docker
+If you do not wish to use Compose, you may still run this image using plain
+Docker commands:
 
 Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database.
 
@@ -65,6 +47,50 @@ Note that the following is just a guideline and you may need to add parameters t
 docker run \
     -d \
     --name synapse \
-    -v ${CONFIG_PATH}:/synapse/config/ \
+    -v ${DATA_PATH}:/data \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
     matrixdotorg/synapse:v0.22.1
 ```
+
+
+## Volumes
+
+The image expects a single volue, located at ``/data``, that will hold:
+
+* temporary files during uploads;
+* uploaded media and thumbnais;
+* the SQLite database if you do not configure postgres.
+
+## Environment
+
+If you do not specify a custom path for the configuration file, a very generic
+file will be generated, based on the following environment settings.
+These are a good starting point for setting up your own deployment.
+
+Synapse specific settings:
+
+* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
+* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all
+  other options then).
+* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
+  you run your own TLS-capable reverse proxy).
+* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client.
+* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
+  the Synapse instance.
+* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
+* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
+* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous
+  statistics reporting back to the Matrix project which helps us to get funding.
+
+Shared secrets, these will be initialized to random values if not set:
+
+* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
+  registration is disable.
+* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon.
+
+Database specific values (will use SQLite if not set):
+
+* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`]
+* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
+* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
+* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-- 
cgit 1.4.1


From b8ab78b82c60ab9c897891cf9f53c9cc44873a25 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 15:41:54 +0100
Subject: Add the build cache/ folder to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 491047c352..d13017f400 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,6 +32,7 @@ demo/media_store.*
 demo/etc
 
 uploads
+cache
 
 .idea/
 media_store/
-- 
cgit 1.4.1


From f72c9c1fb650c7f3bac4d77d4e24fa0469d09ebb Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 16:18:40 +0100
Subject: Fix multiple typos

---
 MANIFEST.in                       |  1 +
 contrib/docker/README.md          | 19 ++++++++++++-------
 contrib/docker/docker-compose.yml |  7 ++++---
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index afb60e12ee..8c9a57a9ca 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -25,6 +25,7 @@ recursive-include synapse/static *.js
 exclude jenkins.sh
 exclude jenkins*.sh
 exclude jenkins*
+exclude Dockerfile
 recursive-exclude jenkins *.sh
 
 prune .github
diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 8ba5f79692..73e53e4306 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -23,7 +23,7 @@ You may have a local Python wheel cache available, in which case copy the releva
 
 It is recommended that you use Docker Compose to run your containers, including
 this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
-with example labels for a reverse proxy and other artifacts.
+including example labels for reverse proxying and other artifacts.
 
 Then, to run the server:
 
@@ -35,13 +35,13 @@ In the case you specified a custom path for you configuration file and wish to
 generate a fresh ``homeserver.yaml``, simply run:
 
 ```
-docker-compose run synapse generate
+docker-compose run --rm synapse generate
 ```
 
 If you do not wish to use Compose, you may still run this image using plain
-Docker commands:
-
-Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database.
+Docker commands. Note that the following is just a guideline and you may need
+to add parameters to the docker run command to account for the network situation
+with your postgres database.
 
 ```
 docker run \
@@ -55,7 +55,7 @@ docker run \
 
 ## Volumes
 
-The image expects a single volue, located at ``/data``, that will hold:
+The image expects a single volume, located at ``/data``, that will hold:
 
 * temporary files during uploads;
 * uploaded media and thumbnais;
@@ -63,10 +63,15 @@ The image expects a single volue, located at ``/data``, that will hold:
 
 ## Environment
 
-If you do not specify a custom path for the configuration file, a very generic
+Unless you specify a custom path for the configuration file, a very generic
 file will be generated, based on the following environment settings.
 These are a good starting point for setting up your own deployment.
 
+Global settings:
+
+* ``UID``, the user id Synapse will run as [default 991]
+* ``GID``, the group id Synapse will run as [default 991]
+
 Synapse specific settings:
 
 * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index b8f9741f05..727743effa 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -1,4 +1,4 @@
-# This compose file is compatible with Copose itself, it might need some
+# This compose file is compatible with Compose itself, it might need some
 # adjustments to run properly with stack.
 
 version: '3'
@@ -18,10 +18,11 @@ services:
       - ./files:/data
     depends_on:
       - db
-    # One may either expose ports directly
+    # In order to expose Synapse, remove one of the following, you might for
+    # instance expose the TLS port directly:
     ports:
       - 8448:8448/tcp
-    # ... or use a reverse proxy, here is an example for traefik
+    # ... or use a reverse proxy, here is an example for traefik:
     labels:
       - traefik.enable=true
       - traefik.frontend.rule=Host:my.matrix.Host
-- 
cgit 1.4.1


From e9021e16c49c4224782040449b43fb0015c9f05c Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sun, 4 Feb 2018 23:19:08 +0100
Subject: Run the server as an unprivileged user

---
 Dockerfile              |  2 +-
 contrib/docker/start.py | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 25f3746303..f687a4f2bb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
 FROM python:2-alpine
 
-RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev
+RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev
 
 COPY . /synapse
 
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 7057f85f61..8bc72bf428 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -9,14 +9,16 @@ convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(sr
 mode = sys.argv[1] if len(sys.argv) > 1 else None
 environ = os.environ.copy()
 
+# Check mandatory parameters and build the base start arguments
 if "SYNAPSE_SERVER_NAME" not in environ:
     print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
     sys.exit(2)
 
+permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
 args = ["python", "-m", "synapse.app.homeserver",
-        "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"),
-        "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"),
-        "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
+        "--server-name", environ.get("SYNAPSE_SERVER_NAME"),
+        "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"),
+        "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
 
 # Generate any missing shared secret
 for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"):
@@ -35,4 +37,5 @@ if mode == "generate":
 # In normal mode, generate missing keys if any, then run synapse
 else:
     subprocess.check_output(args + ["--generate-keys"])
-    os.execv("/usr/local/bin/python", args)
+    subprocess.check_output(["chown", "-R", permissions, "/data"])
+    os.execv("/sbin/su-exec", ["su-exec", permissions] + args)
-- 
cgit 1.4.1


From 8db84e9b2198ad4a80f6e9e66e2fc8c2d44fbeca Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 20:08:35 +0100
Subject: Remove docker related files from the python manifest

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 8c9a57a9ca..e2a6623a63 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -26,6 +26,7 @@ exclude jenkins.sh
 exclude jenkins*.sh
 exclude jenkins*
 exclude Dockerfile
+exclude .dockerignore
 recursive-exclude jenkins *.sh
 
 prune .github
-- 
cgit 1.4.1


From 81010a126e9bbcce018b104921ff5221a67f99cc Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 21:28:15 +0100
Subject: Add dynamic recaptcha configuration in the Docker image

---
 contrib/docker/README.md            | 4 ++++
 contrib/docker/conf/homeserver.yaml | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 73e53e4306..f4bc78908d 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -86,6 +86,10 @@ Synapse specific settings:
 * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
 * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
+* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
+  key in order to enable recaptcha upon registration
+* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
+  key in order to enable recaptcha upon registration
 
 Shared secrets, these will be initialized to random values if not set:
 
diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 7450cc1228..6f1de24aad 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -112,10 +112,17 @@ max_spider_size: "10M"
 
 ## Captcha ##
 
+{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %}
+recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}"
+recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}"
+enable_registration_captcha: True
+recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+{% else %}
 recaptcha_public_key: "YOUR_PUBLIC_KEY"
 recaptcha_private_key: "YOUR_PRIVATE_KEY"
 enable_registration_captcha: False
 recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+{% endif %}
 
 ## Turn ##
 
-- 
cgit 1.4.1


From cd51931b62aef63dacf4d79cdfa5de56da4eeda6 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 21:53:53 +0100
Subject: Add dynamic TURN configuration in the Docker image

---
 contrib/docker/README.md            | 7 +++++--
 contrib/docker/conf/homeserver.yaml | 9 +++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index f4bc78908d..0da7b56628 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -87,9 +87,12 @@ Synapse specific settings:
 * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration
+  key in order to enable recaptcha upon registration.
 * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration
+  key in order to enable recaptcha upon registration.
+* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
+  uris to enable TURN for this homeserver.
+* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
 
 Shared secrets, these will be initialized to random values if not set:
 
diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 6f1de24aad..6f8fb24e5f 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -126,10 +126,19 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
 
 ## Turn ##
 
+{% if SYNAPSE_TURN_URIS %}
+turn_uris:
+{% for uri in SYNAPSE_TURN_URIS.split(',') %}  - {{ uri }}
+{% endfor %}
+turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}"
+turn_user_lifetime: "1h"
+turn_allow_guests: True
+{% else %}
 turn_uris: []
 turn_shared_secret: "YOUR_SHARED_SECRET"
 turn_user_lifetime: "1h"
 turn_allow_guests: True
+{% endif %}
 
 ## Registration ##
 
-- 
cgit 1.4.1


From cf4ef60e287a6a61844ab260606721db2b7ee0cd Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 22:10:03 +0100
Subject: Document the cache factor environment variable for Docker

---
 contrib/docker/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 0da7b56628..8b1e0afee6 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -84,6 +84,7 @@ Synapse specific settings:
   the Synapse instance.
 * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
 * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
+* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
 * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-- 
cgit 1.4.1


From d8c7da5dca907bf65293b6b967200141cad69410 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 22:12:50 +0100
Subject: Fix a typo in the Docker README

---
 contrib/docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 8b1e0afee6..624dca747a 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -58,7 +58,7 @@ docker run \
 The image expects a single volume, located at ``/data``, that will hold:
 
 * temporary files during uploads;
-* uploaded media and thumbnais;
+* uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres.
 
 ## Environment
-- 
cgit 1.4.1


From f5364b47ec4b67a552976b9dbb9594d448744e30 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 22:14:40 +0100
Subject: Point to the 'latest' tag in the Docker documentation

---
 contrib/docker/README.md          | 2 +-
 contrib/docker/docker-compose.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 624dca747a..87354b9bc3 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -49,7 +49,7 @@ docker run \
     --name synapse \
     -v ${DATA_PATH}:/data \
     -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    matrixdotorg/synapse:v0.22.1
+    matrixdotorg/synapse:latest
 ```
 
 
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 727743effa..3fb156db47 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,7 +6,7 @@ version: '3'
 services:
 
   synapse:
-    image: synapse
+    image: matrixdotorg/synapse:latest
     # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
-- 
cgit 1.4.1


From 630573a9325f826e81f04650fd83e64d1c831035 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 22:57:22 +0100
Subject: Do not copy documentation files to the Docker root folder

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f687a4f2bb..2ae503ce79 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,7 +9,7 @@ RUN cd /synapse \
  && pip install --upgrade pip setuptools psycopg2 \
  && mkdir -p /synapse/cache \
  && pip install -f /synapse/cache --upgrade --process-dependency-links . \
- && mv /synapse/contrib/docker/* / \
+ && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \
  && rm -rf setup.py setup.cfg synapse
 
 VOLUME ["/data"]
-- 
cgit 1.4.1


From ee3b160a2ad375223b4304184304605a35f1b406 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 22:57:35 +0100
Subject: Only generate configuration files when necessary

---
 contrib/docker/start.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 8bc72bf428..d3364e4226 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -26,16 +26,18 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE
         print("Generating a random secret for {}".format(secret))
         environ[secret] = os.urandom(32).encode("hex")
 
-# Parse the configuration file
-if not os.path.exists("/compiled"): os.mkdir("/compiled")
-convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ)
-
 # In generate mode, generate a configuration, missing keys, then exit
 if mode == "generate":
     os.execv("/usr/local/bin/python", args + ["--generate-config"])
+
 # In normal mode, generate missing keys if any, then run synapse
 else:
+    # Parse the configuration file
+    if "SYNAPSE_CONFIG_PATH" not in environ:
+        if not os.path.exists("/compiled"): os.mkdir("/compiled")
+        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
+        convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ)
+    # Generate missing keys and start synapse
     subprocess.check_output(args + ["--generate-keys"])
     subprocess.check_output(["chown", "-R", permissions, "/data"])
     os.execv("/sbin/su-exec", ["su-exec", permissions] + args)
-- 
cgit 1.4.1


From 107a5c94418de55d52a0232115d5c3c7efc6d285 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 23:02:33 +0100
Subject: Add the non-tls port to the expose list

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 2ae503ce79..bec884ce9b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,6 +14,6 @@ RUN cd /synapse \
 
 VOLUME ["/data"]
 
-EXPOSE 8448
+EXPOSE 8008/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
-- 
cgit 1.4.1


From 1ffd9cb93617fe9bb2367d575786c0ff222cd415 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Mon, 5 Feb 2018 23:13:27 +0100
Subject: Support loading application service files from /data/appservices/

---
 contrib/docker/README.md            | 7 ++++++-
 contrib/docker/conf/homeserver.yaml | 9 ++++++++-
 contrib/docker/start.py             | 4 ++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 87354b9bc3..c1724fe269 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -59,7 +59,12 @@ The image expects a single volume, located at ``/data``, that will hold:
 
 * temporary files during uploads;
 * uploaded media and thumbnails;
-* the SQLite database if you do not configure postgres.
+* the SQLite database if you do not configure postgres;
+* the appservices configuration.
+
+In order to setup an application service, simply create an ``appservices``
+directory in the data volume and write the application service Yaml
+configuration file there. Multiple application services are supported.
 
 ## Environment
 
diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 6f8fb24e5f..e5d3f965e4 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -128,7 +128,7 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
 
 {% if SYNAPSE_TURN_URIS %}
 turn_uris:
-{% for uri in SYNAPSE_TURN_URIS.split(',') %}  - {{ uri }}
+{% for uri in SYNAPSE_TURN_URIS.split(',') %}    - "{{ uri }}"
 {% endfor %}
 turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}"
 turn_user_lifetime: "1h"
@@ -167,7 +167,14 @@ room_invite_state_types:
     - "m.room.avatar"
     - "m.room.name"
 
+{% if SYNAPSE_APPSERVICES %}
+app_service_config_files:
+{% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
+{% endfor %}
+{% else %}
 app_service_config_files: []
+{% endif %}
+
 macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
 expire_access_token: False
 
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index d3364e4226..8ade0f227d 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -4,6 +4,7 @@ import jinja2
 import os
 import sys
 import subprocess
+import glob
 
 convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
 mode = sys.argv[1] if len(sys.argv) > 1 else None
@@ -26,6 +27,9 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE
         print("Generating a random secret for {}".format(secret))
         environ[secret] = os.urandom(32).encode("hex")
 
+# Load appservices configurations
+environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
+
 # In generate mode, generate a configuration, missing keys, then exit
 if mode == "generate":
     os.execv("/usr/local/bin/python", args + ["--generate-config"])
-- 
cgit 1.4.1


From 63fd148724399d52f3435b1c172435d7cabcde4c Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 19:46:11 +0100
Subject: Make it clear that two modes are avaiable in the documentation,
 improve the compose file

---
 contrib/docker/README.md          | 17 ++++++++++-------
 contrib/docker/docker-compose.yml | 12 ++++++++++--
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index c1724fe269..197bad103e 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -12,7 +12,7 @@ use that server.
 Build the docker image with the `docker build` command from the root of the synapse repository.
 
 ```
-docker build -t matrixdotorg/synapse:v0.22.1 .
+docker build -t matrixdotorg/synapse .
 ```
 
 The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
@@ -76,12 +76,17 @@ Global settings:
 
 * ``UID``, the user id Synapse will run as [default 991]
 * ``GID``, the group id Synapse will run as [default 991]
+* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
 
-Synapse specific settings:
+If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
+then customize it manually. No other environment variable is required.
+
+Otherwise, a dynamic configuration file will be used. The following environment
+variables are available for configuration:
 
 * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all
-  other options then).
+* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous
+  statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
   you run your own TLS-capable reverse proxy).
 * ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client.
@@ -90,8 +95,6 @@ Synapse specific settings:
 * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
 * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
 * ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
-* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
   key in order to enable recaptcha upon registration.
 * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
@@ -100,7 +103,7 @@ Synapse specific settings:
   uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
 
-Shared secrets, these will be initialized to random values if not set:
+Shared secrets, that will be initialized to random values if not set:
 
 * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
   registration is disable.
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 3fb156db47..b07984ea34 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,7 +6,7 @@ version: '3'
 services:
 
   synapse:
-    image: matrixdotorg/synapse:latest
+    image: docker.io/matrixdotorg/synapse:latest
     # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
@@ -15,7 +15,12 @@ services:
       - SYNAPSE_SERVER_NAME=my.matrix.host
       - SYNAPSE_ENABLE_REGISTRATION=yes
     volumes:
+      # You may either store all the files in a local folder
       - ./files:/data
+      # .. or you may split this between different storage points
+      # - ./files:/data
+      # - /path/to/ssd:/data/uploads
+      # - /path/to/large_hdd:/data/media
     depends_on:
       - db
     # In order to expose Synapse, remove one of the following, you might for
@@ -29,10 +34,13 @@ services:
       - traefik.port=8448
 
   db:
-    image: postgres:latest
+    image: postgres:10-alpine
     # Change that password, of course!
     environment:
       - POSTGRES_USER=matrix
       - POSTGRES_PASSWORD=changeme
     volumes:
+      # You may store the database tables in a local folder..
       - ./schemas:/var/lib/postgresql/data
+      # .. or store them on some high performance storage for better results
+      # - /path/to/ssd/storage:/var/lib/postfesql/data
-- 
cgit 1.4.1


From 58df3a8c5dcf5de12bcf9190551dc8241d32b8a1 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 19:48:53 +0100
Subject: Add some documentation about high performance storage

---
 contrib/docker/README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 197bad103e..3710afb0cf 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -31,11 +31,11 @@ Then, to run the server:
 docker-compose up -d
 ```
 
-In the case you specified a custom path for you configuration file and wish to
+In the case you specify a custom path for you configuration file and wish to
 generate a fresh ``homeserver.yaml``, simply run:
 
 ```
-docker-compose run --rm synapse generate
+docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
 ```
 
 If you do not wish to use Compose, you may still run this image using plain
@@ -62,6 +62,11 @@ The image expects a single volume, located at ``/data``, that will hold:
 * the SQLite database if you do not configure postgres;
 * the appservices configuration.
 
+You are free to use separate volumes depending on storage endpoints at your
+disposal. For instance, ``/data/media`` coud be stored on a large but low
+performance hdd storage while other files could be stored on high performance
+endpoints.
+
 In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
-- 
cgit 1.4.1


From 084afbb6a06f2661bed503bf49b0291ce999c6c1 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 19:50:04 +0100
Subject: Rename the permissions variable to avoid confusion

---
 contrib/docker/start.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 8ade0f227d..13a10a11bb 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -15,7 +15,7 @@ if "SYNAPSE_SERVER_NAME" not in environ:
     print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
     sys.exit(2)
 
-permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
+ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
 args = ["python", "-m", "synapse.app.homeserver",
         "--server-name", environ.get("SYNAPSE_SERVER_NAME"),
         "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"),
@@ -43,5 +43,5 @@ else:
         convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ)
     # Generate missing keys and start synapse
     subprocess.check_output(args + ["--generate-keys"])
-    subprocess.check_output(["chown", "-R", permissions, "/data"])
-    os.execv("/sbin/su-exec", ["su-exec", permissions] + args)
+    subprocess.check_output(["chown", "-R", ownership, "/data"])
+    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)
-- 
cgit 1.4.1


From b8a4dceb3cee6b69d1b1b882cef1f96a3ff6249f Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 20:41:41 +0100
Subject: Refactor the start script to better handle mandatory parameters

---
 contrib/docker/README.md          |  2 +-
 contrib/docker/docker-compose.yml |  1 +
 contrib/docker/start.py           | 56 +++++++++++++++++++++++----------------
 3 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 3710afb0cf..0493d2ee6e 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -90,7 +90,7 @@ Otherwise, a dynamic configuration file will be used. The following environment
 variables are available for configuration:
 
 * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous
+* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
   you run your own TLS-capable reverse proxy).
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index b07984ea34..3d0b3c0ea4 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -13,6 +13,7 @@ services:
     # See the readme for a full documentation of the environment settings
     environment:
       - SYNAPSE_SERVER_NAME=my.matrix.host
+      - SYNAPSE_REPORT_STATS=no
       - SYNAPSE_ENABLE_REGISTRATION=yes
     volumes:
       # You may either store all the files in a local folder
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 13a10a11bb..32142bbe00 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -6,42 +6,52 @@ import sys
 import subprocess
 import glob
 
+# Utility functions
 convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
-mode = sys.argv[1] if len(sys.argv) > 1 else None
-environ = os.environ.copy()
 
-# Check mandatory parameters and build the base start arguments
-if "SYNAPSE_SERVER_NAME" not in environ:
-    print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.")
-    sys.exit(2)
-
-ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
-args = ["python", "-m", "synapse.app.homeserver",
-        "--server-name", environ.get("SYNAPSE_SERVER_NAME"),
-        "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"),
-        "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")]
+def check_arguments(environ, args):
+    for argument in args:
+        if argument not in environ:
+            print("Environment variable %s is mandatory, exiting." % argument)
+            sys.exit(2)
 
-# Generate any missing shared secret
-for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"):
-    if secret not in environ:
-        print("Generating a random secret for {}".format(secret))
-        environ[secret] = os.urandom(32).encode("hex")
+def generate_secrets(environ, secrets):
+    for secret in secrets:
+        if secret not in environ:
+            print("Generating a random secret for {}".format(secret))
+            environ[secret] = os.urandom(32).encode("hex")
 
-# Load appservices configurations
-environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
+# Prepare the configuration
+mode = sys.argv[1] if len(sys.argv) > 1 else None
+environ = os.environ.copy()
+ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
+args = ["python", "-m", "synapse.app.homeserver"]
 
 # In generate mode, generate a configuration, missing keys, then exit
 if mode == "generate":
-    os.execv("/usr/local/bin/python", args + ["--generate-config"])
+    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
+    args += [
+        "--server-name", environ["SYNAPSE_SERVER_NAME"],
+        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
+        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
+        "--generate-config"
+    ]
+    os.execv("/usr/local/bin/python", args)
 
 # In normal mode, generate missing keys if any, then run synapse
 else:
     # Parse the configuration file
-    if "SYNAPSE_CONFIG_PATH" not in environ:
+    if "SYNAPSE_CONFIG_PATH" in environ:
+        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
+    else:
+        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
+        generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"))
+        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
         convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-        convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ)
+        convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ)
+        subprocess.check_output(["chown", "-R", ownership, "/data"])
+        args += ["--config-path", "/compiled/homeserver.yaml"]
     # Generate missing keys and start synapse
     subprocess.check_output(args + ["--generate-keys"])
-    subprocess.check_output(["chown", "-R", ownership, "/data"])
     os.execv("/sbin/su-exec", ["su-exec", ownership] + args)
-- 
cgit 1.4.1


From e174c46a295ca6e06b217b5dcbcf995b890e6d07 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 20:42:57 +0100
Subject: Use 'synapse' as a default postgres user in Docker examples

---
 contrib/docker/conf/homeserver.yaml | 4 ++--
 contrib/docker/docker-compose.yml   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index e5d3f965e4..1ca1fe991f 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -50,9 +50,9 @@ listeners:
 database:
   name: "psycopg2"
   args:
-    user: "{{ POSTGRES_USER or "matrix" }}"
+    user: "{{ POSTGRES_USER or "synapse" }}"
     password: "{{ POSTGRES_PASSWORD }}"
-    database: "{{ POSTGRES_DB or "matrix" }}"
+    database: "{{ POSTGRES_DB or "synapse" }}"
     host: "{{ POSTGRES_HOST or "db" }}"
     port: "{{ POSTGRES_PORT or "5432" }}"
     cp_min: 5
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 3d0b3c0ea4..e447bf1212 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -38,7 +38,7 @@ services:
     image: postgres:10-alpine
     # Change that password, of course!
     environment:
-      - POSTGRES_USER=matrix
+      - POSTGRES_USER=synapse
       - POSTGRES_PASSWORD=changeme
     volumes:
       # You may store the database tables in a local folder..
-- 
cgit 1.4.1


From 914a59cb8c12b25d77ed9a81e4543c23d7e10b5e Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 20:43:45 +0100
Subject: Disable the Web client in the Docker image

---
 contrib/docker/README.md            | 1 -
 contrib/docker/conf/homeserver.yaml | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 0493d2ee6e..9f40dc0d58 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -94,7 +94,6 @@ variables are available for configuration:
   statistics reporting back to the Matrix project which helps us to get funding.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
   you run your own TLS-capable reverse proxy).
-* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client.
 * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
   the Synapse instance.
 * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 1ca1fe991f..19a2cbad29 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -12,7 +12,7 @@ tls_fingerprints: []
 
 server_name: "{{ SYNAPSE_SERVER_NAME }}"
 pid_file: /homeserver.pid
-web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }}
+web_client: False
 soft_file_limit: 0
 
 ## Ports ##
@@ -26,7 +26,7 @@ listeners:
     tls: true
     x_forwarded: false
     resources:
-      - names: [client, webclient]
+      - names: [client]
         compress: true
       - names: [federation]  # Federation APIs
         compress: false
@@ -39,7 +39,7 @@ listeners:
     x_forwarded: false
 
     resources:
-      - names: [client, webclient]
+      - names: [client]
         compress: true
       - names: [federation]
         compress: false
-- 
cgit 1.4.1


From a0af0054ec91e92a6843c121cd27e92ea63c1034 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 20:46:11 +0100
Subject: Honor the SYNAPSE_REPORT_STATS parameter in the Docker image

---
 contrib/docker/conf/homeserver.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 19a2cbad29..3b57f7174d 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -156,8 +156,13 @@ trusted_third_party_id_servers:
 
 ## Metrics ###
 
+{% if SYNAPSE_REPORT_STATS.lower() == "yes" %}
+enable_metrics: True
+report_stats: True
+{% else %}
 enable_metrics: False
 report_stats: False
+{% endif %}
 
 ## API Configuration ##
 
-- 
cgit 1.4.1


From ef1f8d4be6e970043b8283f5caa1ca764652ad56 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 20:53:12 +0100
Subject: Enable email server configuration from environment variables

---
 contrib/docker/README.md            |  7 +++++++
 contrib/docker/conf/homeserver.yaml | 33 +++++++++++++++++----------------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 9f40dc0d58..b74c72698c 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -119,3 +119,10 @@ Database specific values (will use SQLite if not set):
 * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
 * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
 * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
+
+Mail server specific values (will not send emails if not set):
+
+* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
+* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
+* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
+* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 3b57f7174d..198b8ddee7 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -146,6 +146,7 @@ enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }}
 registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}"
 bcrypt_rounds: 12
 allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }}
+enable_group_creation: true
 
 # The list of identity servers trusted to verify third party
 # identifiers by this server.
@@ -200,19 +201,19 @@ perspectives:
 password_config:
    enabled: true
 
-#email:
-#   enable_notifs: false
-#   smtp_host: "localhost"
-#   smtp_port: 25
-#   smtp_user: "exampleusername"
-#   smtp_pass: "examplepassword"
-#   require_transport_security: False
-#   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
-#   app_name: Matrix
-#   template_dir: res/templates
-#   notif_template_html: notif_mail.html
-#   notif_template_text: notif_mail.txt
-#   notif_for_new_users: True
-#   riot_base_url: "http://localhost/riot"
-
-enable_group_creation: true
+{% if SYNAPSE_SMTP_HOST %}
+email:
+   enable_notifs: false
+   smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
+   smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
+   smtp_user: "{{ SYNAPSE_SMTP_USER }}"
+   smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
+   require_transport_security: False
+   notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
+   app_name: Matrix
+   template_dir: res/templates
+   notif_template_html: notif_mail.html
+   notif_template_text: notif_mail.txt
+   notif_for_new_users: True
+   riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
+{% endif %}
-- 
cgit 1.4.1


From b9b668e4bb6ec3c0ae6c696a4b7abf6f537e0504 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 21:39:36 +0100
Subject: Update to Alpine 3.7 and switch to libressl

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index bec884ce9b..8085f3d354 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
-FROM python:2-alpine
+FROM docker.io/python:2-alpine3.7
 
-RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev
+RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev
 
 COPY . /synapse
 
-- 
cgit 1.4.1


From d8680c969bb2e8248436d7352ebf0f5cd1daea7b Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 21:55:35 +0100
Subject: Make it clear that the image has two modes of operation

---
 contrib/docker/README.md | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index b74c72698c..27e25afcef 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -21,23 +21,42 @@ You may have a local Python wheel cache available, in which case copy the releva
 
 ## Run
 
+This image is designed to run either with an automatically generated configuration
+file or with a custom configuration that requires manual edition.
+
+### Automated configuration
+
 It is recommended that you use Docker Compose to run your containers, including
 this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
 including example labels for reverse proxying and other artifacts.
 
-Then, to run the server:
+Read the section about environment variables and set at least mandatory variables,
+then run the server:
 
 ```
 docker-compose up -d
 ```
 
-In the case you specify a custom path for you configuration file and wish to
-generate a fresh ``homeserver.yaml``, simply run:
+### Manual configuration
+
+A sample ``docker-compose.yml`` is provided, including example labels for
+reverse proxying and other artifacts.
+
+Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
+to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
 
 ```
 docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
 ```
 
+Then, customize your configuration and run the server:
+
+```
+docker-compose up -d
+```
+
+### Without Compose
+
 If you do not wish to use Compose, you may still run this image using plain
 Docker commands. Note that the following is just a guideline and you may need
 to add parameters to the docker run command to account for the network situation
@@ -49,10 +68,10 @@ docker run \
     --name synapse \
     -v ${DATA_PATH}:/data \
     -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    matrixdotorg/synapse:latest
+    -e SYNAPSE_REPORT_STATS=yes \
+    docker.io/matrixdotorg/synapse:latest
 ```
 
-
 ## Volumes
 
 The image expects a single volume, located at ``/data``, that will hold:
-- 
cgit 1.4.1


From 48e2c641b80ac57d16701e4a27f348ea4f2c66cc Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 21:58:12 +0100
Subject: Specify the Docker registry in the build tag

---
 contrib/docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 27e25afcef..221d9c53b5 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -12,7 +12,7 @@ use that server.
 Build the docker image with the `docker build` command from the root of the synapse repository.
 
 ```
-docker build -t matrixdotorg/synapse .
+docker build -t docker.io/matrixdotorg/synapse .
 ```
 
 The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
-- 
cgit 1.4.1


From a03c382966a3219acd26851db9cc6558e5c53310 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Thu, 8 Feb 2018 22:00:43 +0100
Subject: Specify the Docker registry for the postgres image

---
 contrib/docker/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index e447bf1212..1d2aebbcd3 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -35,7 +35,7 @@ services:
       - traefik.port=8448
 
   db:
-    image: postgres:10-alpine
+    image: docker.io/postgres:10-alpine
     # Change that password, of course!
     environment:
       - POSTGRES_USER=synapse
-- 
cgit 1.4.1


From e511979fe6c4a03da3e9c1d16672e263f54ee2d3 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Fri, 9 Feb 2018 00:13:26 +0100
Subject: Make SYNAPSE_MACAROON_SECRET_KEY a mandatory option

---
 contrib/docker/README.md | 3 ++-
 contrib/docker/start.py  | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 221d9c53b5..25c358c847 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -111,6 +111,8 @@ variables are available for configuration:
 * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
 * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
+* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens
+  to the server, set this to a proper random key.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
   you run your own TLS-capable reverse proxy).
 * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
@@ -130,7 +132,6 @@ Shared secrets, that will be initialized to random values if not set:
 
 * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
   registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon.
 
 Database specific values (will use SQLite if not set):
 
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 32142bbe00..d4c1140b1d 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -44,8 +44,8 @@ else:
     if "SYNAPSE_CONFIG_PATH" in environ:
         args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
     else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
-        generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"))
+        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY"))
+        generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",))
         environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
         convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-- 
cgit 1.4.1


From ca70148c0569295a2b9ecdd1cd9cd85a203f20e7 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Fri, 9 Feb 2018 00:23:19 +0100
Subject: Fix the path to the log config file

---
 contrib/docker/conf/homeserver.yaml | 2 +-
 contrib/docker/start.py             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml
index 198b8ddee7..6bc25bb45f 100644
--- a/contrib/docker/conf/homeserver.yaml
+++ b/contrib/docker/conf/homeserver.yaml
@@ -69,7 +69,7 @@ database:
 event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}"
 verbose: 0
 log_file: "/data/homeserver.log"
-log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config"
+log_config: "/compiled/log.config"
 
 ## Ratelimiting ##
 
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index d4c1140b1d..75c30b8ac0 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -49,7 +49,7 @@ else:
         environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
         convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-        convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ)
+        convert("/conf/log.config", "/compiled/log.config", environ)
         subprocess.check_output(["chown", "-R", ownership, "/data"])
         args += ["--config-path", "/compiled/homeserver.yaml"]
     # Generate missing keys and start synapse
-- 
cgit 1.4.1


From 6f0b1f85f9f34401219eab4b4977a63c698ce987 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 10 Feb 2018 00:05:03 +0100
Subject: Generate macaroon and registration secrets, then store the results to
 the data dir

---
 contrib/docker/docker-compose.yml |  2 +-
 contrib/docker/start.py           | 19 ++++++++++++++-----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 1d2aebbcd3..9e32dd87de 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,7 +6,7 @@ version: '3'
 services:
 
   synapse:
-    image: docker.io/matrixdotorg/synapse:latest
+    image: synapse #docker.io/matrixdotorg/synapse:latest
     # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
diff --git a/contrib/docker/start.py b/contrib/docker/start.py
index 75c30b8ac0..90e8b9c51a 100755
--- a/contrib/docker/start.py
+++ b/contrib/docker/start.py
@@ -16,10 +16,16 @@ def check_arguments(environ, args):
             sys.exit(2)
 
 def generate_secrets(environ, secrets):
-    for secret in secrets:
+    for name, secret in secrets.items():
         if secret not in environ:
-            print("Generating a random secret for {}".format(secret))
-            environ[secret] = os.urandom(32).encode("hex")
+            filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
+            if os.path.exists(filename):
+                with open(filename) as handle: value = handle.read()
+            else:
+                print("Generating a random secret for {}".format(name))
+                value = os.urandom(32).encode("hex")
+                with open(filename, "w") as handle: handle.write(value)
+            environ[secret] = value
 
 # Prepare the configuration
 mode = sys.argv[1] if len(sys.argv) > 1 else None
@@ -44,8 +50,11 @@ else:
     if "SYNAPSE_CONFIG_PATH" in environ:
         args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
     else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY"))
-        generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",))
+        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
+        generate_secrets(environ, {
+            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
+            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
+        })
         environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
         convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-- 
cgit 1.4.1


From b815aa0e2db8f50116b1443b559ca13fe6ad1750 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 10 Feb 2018 21:59:58 +0100
Subject: Remove an accidentally committed test configuration

---
 contrib/docker/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 9e32dd87de..1d2aebbcd3 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,7 +6,7 @@ version: '3'
 services:
 
   synapse:
-    image: synapse #docker.io/matrixdotorg/synapse:latest
+    image: docker.io/matrixdotorg/synapse:latest
     # Since snyapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
-- 
cgit 1.4.1


From 07f1b7181997dca91b67dca7561ce4c532caf253 Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 10 Feb 2018 23:57:36 +0100
Subject: Explicitely provide the postgres password to synapse in the Compose
 example

---
 contrib/docker/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 1d2aebbcd3..46e72601d3 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -15,6 +15,7 @@ services:
       - SYNAPSE_SERVER_NAME=my.matrix.host
       - SYNAPSE_REPORT_STATS=no
       - SYNAPSE_ENABLE_REGISTRATION=yes
+      - POSTGRES_PASSWORD=changeme
     volumes:
       # You may either store all the files in a local folder
       - ./files:/data
-- 
cgit 1.4.1


From f44b7c022f6bc8b30cb8c446e0922b26b8b8eb5a Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Sat, 10 Feb 2018 23:57:51 +0100
Subject: Disable logging to file and rely on the console when using Docker

---
 contrib/docker/conf/log.config | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config
index 45e7eef953..b5c907c4f9 100644
--- a/contrib/docker/conf/log.config
+++ b/contrib/docker/conf/log.config
@@ -10,13 +10,6 @@ filters:
     request: ""
 
 handlers:
-  file:
-    class: logging.handlers.RotatingFileHandler
-    formatter: precise
-    filename: /data/homeserver.log
-    maxBytes: 104857600
-    backupCount: 10
-    filters: [context]
   console:
     class: logging.StreamHandler
     formatter: precise
@@ -33,4 +26,4 @@ loggers:
 
 root:
     level: INFO
-    handlers: [file, console]
+    handlers: [console]
-- 
cgit 1.4.1


From 923d9300ede819aa45da546fafc240f40263e7c5 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Sat, 17 Feb 2018 21:53:46 -0700
Subject: Add a blurb explaining the main synapse worker

Signed-off-by: Travis Ralston <travpc@gmail.com>
---
 docs/workers.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/workers.rst b/docs/workers.rst
index dee04bbf3e..a5e084c22a 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -115,6 +115,18 @@ To manipulate a specific worker, you pass the -w option to synctl::
     synctl -w $CONFIG/workers/synchrotron.yaml restart
 
 
+After setting up your workers, you'll need to create a worker configuration for
+the main synapse process. That worker configuration should look like this:::
+
+    worker_app: synapse.app.homeserver
+    daemonize: true
+
+Be sure to keep this particular configuration limited as synapse may refuse to
+start if the regular ``worker_*`` options are given. The ``homeserver.yaml``
+configuration will be used to set up the main synapse process.
+
+**You must have a worker configuration for the main synapse process!**
+
 Available worker applications
 -----------------------------
 
-- 
cgit 1.4.1


From af7ed8e1ef3c8f9f8c247aa77f193c63a4e286a5 Mon Sep 17 00:00:00 2001
From: dklug <dklug@ucmerced.edu>
Date: Fri, 2 Mar 2018 22:01:27 -0800
Subject: Return 401 for invalid access_token on logout

Signed-off-by: Duncan Klug <dklug@ucmerced.edu>
---
 synapse/rest/client/v1/logout.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v1/logout.py b/synapse/rest/client/v1/logout.py
index ca49955935..e092158cb7 100644
--- a/synapse/rest/client/v1/logout.py
+++ b/synapse/rest/client/v1/logout.py
@@ -44,7 +44,10 @@ class LogoutRestServlet(ClientV1RestServlet):
             requester = yield self.auth.get_user_by_req(request)
         except AuthError:
             # this implies the access token has already been deleted.
-            pass
+            defer.returnValue((401, {
+                "errcode": "M_UNKNOWN_TOKEN",
+                "error": "Access Token unknown or expired"
+            }))
         else:
             if requester.device_id is None:
                 # the acccess token wasn't associated with a device.
-- 
cgit 1.4.1


From 47ce527f459e0a28a45a2299db799ea18d632021 Mon Sep 17 00:00:00 2001
From: Jonas Platte <jplatte+git@posteo.de>
Date: Tue, 13 Mar 2018 14:10:07 +0100
Subject: Add room_id to the response of `rooms/{roomId}/join`

Fixes #2349
---
 synapse/rest/client/v1/room.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index f8999d64d7..6dc31bf9ae 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -655,7 +655,12 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
             content=event_content,
         )
 
-        defer.returnValue((200, {}))
+        return_value = {}
+
+        if membership_action == "join":
+            return_value["room_id"] = room_id
+
+        defer.returnValue((200, return_value))
 
     def _has_3pid_invite_keys(self, content):
         for key in {"id_server", "medium", "address"}:
-- 
cgit 1.4.1


From 91ea0202e6f4a519e332a6c456aedfe4b7d627c9 Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Wed, 14 Mar 2018 16:45:37 +0100
Subject: move handling of auto_join_rooms to RegisterHandler

Currently the handling of auto_join_rooms only works when a user
registers itself via public register api. Registrations via
registration_shared_secret and ModuleApi do not work

This auto_joins the users in the registration handler which enables
the auto join feature for all 3 registration paths.

This is related to issue #2725

Signed-Off-by: Matthias Kesler <krombel@krombel.de>
---
 synapse/handlers/register.py             | 36 ++++++++++++++++++++++++++++++--
 synapse/rest/client/v2_alpha/register.py | 32 ----------------------------
 tests/rest/client/v1/test_events.py      |  1 +
 3 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ed5939880a..88b76278d6 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -23,7 +23,7 @@ from synapse.api.errors import (
 )
 from synapse.http.client import CaptchaServerHttpClient
 from synapse import types
-from synapse.types import UserID
+from synapse.types import UserID, create_requester, RoomID, RoomAlias
 from synapse.util.async import run_on_reactor
 from synapse.util.threepids import check_3pid_allowed
 from ._base import BaseHandler
@@ -201,10 +201,17 @@ class RegistrationHandler(BaseHandler):
                     token = None
                     attempts += 1
 
+        # auto-join the user to any rooms we're supposed to dump them into
+        fake_requester = create_requester(user_id)
+        for r in self.hs.config.auto_join_rooms:
+            try:
+                yield self._join_user_to_room(fake_requester, r)
+            except Exception as e:
+                logger.error("Failed to join new user to %r: %r", r, e)
+
         # We used to generate default identicons here, but nowadays
         # we want clients to generate their own as part of their branding
         # rather than there being consistent matrix-wide ones, so we don't.
-
         defer.returnValue((user_id, token))
 
     @defer.inlineCallbacks
@@ -477,3 +484,28 @@ class RegistrationHandler(BaseHandler):
         )
 
         defer.returnValue((user_id, access_token))
+
+    @defer.inlineCallbacks
+    def _join_user_to_room(self, requester, room_identifier):
+        room_id = None
+        room_member_handler = self.hs.get_room_member_handler()
+        if RoomID.is_valid(room_identifier):
+            room_id = room_identifier
+        elif RoomAlias.is_valid(room_identifier):
+            room_alias = RoomAlias.from_string(room_identifier)
+            room_id, remote_room_hosts = (
+                yield room_member_handler.lookup_room_alias(room_alias)
+            )
+            room_id = room_id.to_string()
+        else:
+            raise SynapseError(400, "%s was not legal room ID or room alias" % (
+                room_identifier,
+            ))
+
+        yield room_member_handler.update_membership(
+            requester=requester,
+            target=requester.user,
+            room_id=room_id,
+            remote_room_hosts=remote_room_hosts,
+            action="join",
+        )
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 0ba62bddc1..f317c919dc 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -20,7 +20,6 @@ import synapse
 import synapse.types
 from synapse.api.auth import get_access_token_from_request, has_access_token
 from synapse.api.constants import LoginType
-from synapse.types import RoomID, RoomAlias
 from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
 from synapse.http.servlet import (
     RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string
@@ -405,14 +404,6 @@ class RegisterRestServlet(RestServlet):
                 generate_token=False,
             )
 
-            # auto-join the user to any rooms we're supposed to dump them into
-            fake_requester = synapse.types.create_requester(registered_user_id)
-            for r in self.hs.config.auto_join_rooms:
-                try:
-                    yield self._join_user_to_room(fake_requester, r)
-                except Exception as e:
-                    logger.error("Failed to join new user to %r: %r", r, e)
-
             # remember that we've now registered that user account, and with
             #  what user ID (since the user may not have specified)
             self.auth_handler.set_session_data(
@@ -445,29 +436,6 @@ class RegisterRestServlet(RestServlet):
     def on_OPTIONS(self, _):
         return 200, {}
 
-    @defer.inlineCallbacks
-    def _join_user_to_room(self, requester, room_identifier):
-        room_id = None
-        if RoomID.is_valid(room_identifier):
-            room_id = room_identifier
-        elif RoomAlias.is_valid(room_identifier):
-            room_alias = RoomAlias.from_string(room_identifier)
-            room_id, remote_room_hosts = (
-                yield self.room_member_handler.lookup_room_alias(room_alias)
-            )
-            room_id = room_id.to_string()
-        else:
-            raise SynapseError(400, "%s was not legal room ID or room alias" % (
-                room_identifier,
-            ))
-
-        yield self.room_member_handler.update_membership(
-            requester=requester,
-            target=requester.user,
-            room_id=room_id,
-            action="join",
-        )
-
     @defer.inlineCallbacks
     def _do_appservice_registration(self, username, as_token, body):
         user_id = yield self.registration_handler.appservice_register(
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 2b89c0a3c7..a8d09600bd 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -123,6 +123,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
         self.ratelimiter.send_message.return_value = (True, 0)
         hs.config.enable_registration_captcha = False
         hs.config.enable_registration = True
+        hs.config.auto_join_rooms = []
 
         hs.get_handlers().federation_handler = Mock()
 
-- 
cgit 1.4.1


From cb2c7c0669c7f4d4cc41d65a7f2328589fa04fc5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 14 Mar 2018 16:09:20 +0000
Subject: Update CHANGES.rst

WIP, need to add most recent PRs
---
 CHANGES.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index a7ed49e105..0bcd5a7177 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -8,6 +8,97 @@ This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
 
 
+Changes in synapse v0.27.0-rc1 (2018-03-14)
+===========================================
+
+Features:
+ * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
+ * Add shiney new purge API. New implementation is:-
+  * Faster 
+  * Supports clearing by timestamp
+  * Supports deleting of local events
+  * Is transactional 
+(PR #2858,2867,2882, 2946,2962,2943)
+
+ * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
+ * Add /room/{id}/event/{id} to synapse (PR #2766)
+ * Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live!
+ * Add federation_domain_whitelist option (PR #2820,2821)
+
+
+Changes:
+
+ * Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are:-
+   * Event Creation
+   * Push Actions
+
+With more to follow
+
+ (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856)
+
+ * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
+ * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
+ * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
+ * Better logging when login can't find a 3pid (PR #2744)
+ * add ?ts massaging for ASes (PR #2754)
+ * Remove 'verbosity'/'log_file' from generated cfg (PR #2755)
+ * Make indentation of generated log config consistent (PR #2762)
+ * Remove dead code related to default thumbnails (PR #2764)
+ * Update http request metrics before calling servlet (PR #2770)
+ * Do bcrypt hashing in a background thread (PR #2773)
+ * When using synctl with workers, don't start the main synapse automatically (PR #2774)
+ * Make Counter render floats (PR #2778)
+ * Store state groups separately from events (PR #2784)
+ * Reorganise request and block metrics (PR #2785)
+ * Metrics for number of RDATA commands received (PR #2786)
+ * Metrics for events processed in appservice and fed sender (PR #2787)
+ * Optimise LoggingContext creation and copying (PR #2792)
+ * Track db txn time in millisecs (PR #2793)
+ * Track DB scheduling delay per-request (PR #2794, #2795)
+ * Sanity checking for user ids (PR #2797)
+ * better exception logging in callbackmetrics (PR #2809)
+ * Fix bugs in block metrics (PR #2810)
+ * Add some comments about the reactor tick time metric (PR #2816)
+ * Use a connection pool for the SimpleHttpClient (PR #2817)
+ * Remove unused/bitrotted MemoryDataStore (PR #2828)
+ * Make it possible to run tests against postgres (PR #2829)
+ * Factor out get_db_conn to HomeServer base class (PR #2830)
+ * Logging and metrics for the http pusher (PR #2833)
+ * Improve exception handling in persist_event (PR #2834)
+ * montoring metrics for number of cache evictions (PR #2844)
+ * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
+ * Remove ability for AS users to call /events and /sync (PR #2948)
+ * use bcrypt.checkpw (PR #2949) Thanks to @krombel!
+ * Factor run_in_background out from preserve_fn (PR #2961)
+ * Add a metric which increments when a request is received (PR #2965)
+ * Improve caching for read_marker API (PR #2927)
+ 
+
+Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270)
+
+
+
+Bug fixes:
+
+ * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv!
+ * Check missing fields in event_from_pdu_json (PR #2745)
+ * Fix templating error with unban permission message (PR #2761) Thanks to @turt2live!
+ * Fix flaky test_rooms UTs (PR #2765)
+ * Fix publicised groups GET API (singular) over federation (PR #2772)
+ * Fix a logcontext leak in persist_events (PR #2790)
+ * Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796)
+ * fix SQL when searching all users (PR #2803)
+ * Fix server 500 on public rooms call when no rooms exist (PR #2827)
+ * Fix SQL for user search (PR #2831)
+ * Fix sql error in quarantine_media (PR #2837)
+ * Handle url_previews with no content-type (PR #2845)
+ * Add missing yield during 3pid signature checks (PR #2933)
+ * Fix race in sync when joining room (PR #2944)
+ * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
+ * Fix scary-looking dns resolution errors (PR #2838)
+
+
+
 Changes in synapse v0.26.0 (2018-01-05)
 =======================================
 
-- 
cgit 1.4.1


From a492b17fe2cbbb10687d9a7cbe9eda67b0562656 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 14 Mar 2018 16:18:09 +0000
Subject: Update CHANGES.rst

clean formatting
---
 CHANGES.rst | 128 ++++++++++++++++++++++++++++--------------------------------
 1 file changed, 60 insertions(+), 68 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 0bcd5a7177..4d1b4ba13c 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,66 +12,58 @@ Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
 
 Features:
- * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
- * Add shiney new purge API. New implementation is:-
+* Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
+* Add shiney new purge API. New implementation is:-
   * Faster 
   * Supports clearing by timestamp
   * Supports deleting of local events
-  * Is transactional 
-(PR #2858,2867,2882, 2946,2962,2943)
+  * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
 
- * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
- * Add /room/{id}/event/{id} to synapse (PR #2766)
- * Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live!
- * Add federation_domain_whitelist option (PR #2820,2821)
+* Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
+* Add /room/{id}/event/{id} to synapse (PR #2766)
+* Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live!
+* Add federation_domain_whitelist option (PR #2820,2821)
 
 
 Changes:
 
- * Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are:-
-   * Event Creation
-   * Push Actions
-
-With more to follow
-
- (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856)
-
- * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
- * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
- * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
- * Better logging when login can't find a 3pid (PR #2744)
- * add ?ts massaging for ASes (PR #2754)
- * Remove 'verbosity'/'log_file' from generated cfg (PR #2755)
- * Make indentation of generated log config consistent (PR #2762)
- * Remove dead code related to default thumbnails (PR #2764)
- * Update http request metrics before calling servlet (PR #2770)
- * Do bcrypt hashing in a background thread (PR #2773)
- * When using synctl with workers, don't start the main synapse automatically (PR #2774)
- * Make Counter render floats (PR #2778)
- * Store state groups separately from events (PR #2784)
- * Reorganise request and block metrics (PR #2785)
- * Metrics for number of RDATA commands received (PR #2786)
- * Metrics for events processed in appservice and fed sender (PR #2787)
- * Optimise LoggingContext creation and copying (PR #2792)
- * Track db txn time in millisecs (PR #2793)
- * Track DB scheduling delay per-request (PR #2794, #2795)
- * Sanity checking for user ids (PR #2797)
- * better exception logging in callbackmetrics (PR #2809)
- * Fix bugs in block metrics (PR #2810)
- * Add some comments about the reactor tick time metric (PR #2816)
- * Use a connection pool for the SimpleHttpClient (PR #2817)
- * Remove unused/bitrotted MemoryDataStore (PR #2828)
- * Make it possible to run tests against postgres (PR #2829)
- * Factor out get_db_conn to HomeServer base class (PR #2830)
- * Logging and metrics for the http pusher (PR #2833)
- * Improve exception handling in persist_event (PR #2834)
- * montoring metrics for number of cache evictions (PR #2844)
- * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
- * Remove ability for AS users to call /events and /sync (PR #2948)
- * use bcrypt.checkpw (PR #2949) Thanks to @krombel!
- * Factor run_in_background out from preserve_fn (PR #2961)
- * Add a metric which increments when a request is received (PR #2965)
- * Improve caching for read_marker API (PR #2927)
+* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are XXXXXX, (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856)
+* Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
+* Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
+* Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
+* Better logging when login can't find a 3pid (PR #2744)
+* add ?ts massaging for ASes (PR #2754)
+* Remove 'verbosity'/'log_file' from generated cfg (PR #2755)
+* Make indentation of generated log config consistent (PR #2762)
+* Remove dead code related to default thumbnails (PR #2764)
+* Update http request metrics before calling servlet (PR #2770)
+* Do bcrypt hashing in a background thread (PR #2773)
+* When using synctl with workers, don't start the main synapse automatically (PR #2774)
+* Make Counter render floats (PR #2778)
+* Store state groups separately from events (PR #2784)
+* Reorganise request and block metrics (PR #2785)
+* Metrics for number of RDATA commands received (PR #2786)
+* Metrics for events processed in appservice and fed sender (PR #2787)
+* Optimise LoggingContext creation and copying (PR #2792)
+* Track db txn time in millisecs (PR #2793)
+* Track DB scheduling delay per-request (PR #2794, #2795)
+* Sanity checking for user ids (PR #2797)
+* better exception logging in callbackmetrics (PR #2809)
+* Fix bugs in block metrics (PR #2810)
+* Add some comments about the reactor tick time metric (PR #2816)
+* Use a connection pool for the SimpleHttpClient (PR #2817)
+* Remove unused/bitrotted MemoryDataStore (PR #2828)
+* Make it possible to run tests against postgres (PR #2829)
+* Factor out get_db_conn to HomeServer base class (PR #2830)
+* Logging and metrics for the http pusher (PR #2833)
+* Improve exception handling in persist_event (PR #2834)
+* montoring metrics for number of cache evictions (PR #2844)
+* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
+* Remove ability for AS users to call /events and /sync (PR #2948)
+* use bcrypt.checkpw (PR #2949) Thanks to @krombel!
+* Factor run_in_background out from preserve_fn (PR #2961)
+* Add a metric which increments when a request is received (PR #2965)
+* Improve caching for read_marker API (PR #2927)
  
 
 Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270)
@@ -80,22 +72,22 @@ Synapse 0.27.0 begins the process of rationalising metric names. To enable a gra
 
 Bug fixes:
 
- * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv!
- * Check missing fields in event_from_pdu_json (PR #2745)
- * Fix templating error with unban permission message (PR #2761) Thanks to @turt2live!
- * Fix flaky test_rooms UTs (PR #2765)
- * Fix publicised groups GET API (singular) over federation (PR #2772)
- * Fix a logcontext leak in persist_events (PR #2790)
- * Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796)
- * fix SQL when searching all users (PR #2803)
- * Fix server 500 on public rooms call when no rooms exist (PR #2827)
- * Fix SQL for user search (PR #2831)
- * Fix sql error in quarantine_media (PR #2837)
- * Handle url_previews with no content-type (PR #2845)
- * Add missing yield during 3pid signature checks (PR #2933)
- * Fix race in sync when joining room (PR #2944)
- * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
- * Fix scary-looking dns resolution errors (PR #2838)
+* synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv!
+* Check missing fields in event_from_pdu_json (PR #2745)
+* Fix templating error with unban permission message (PR #2761) Thanks to @turt2live!
+* Fix flaky test_rooms UTs (PR #2765)
+* Fix publicised groups GET API (singular) over federation (PR #2772)
+* Fix a logcontext leak in persist_events (PR #2790)
+* Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796)
+* fix SQL when searching all users (PR #2803)
+* Fix server 500 on public rooms call when no rooms exist (PR #2827)
+* Fix SQL for user search (PR #2831)
+* Fix sql error in quarantine_media (PR #2837)
+* Handle url_previews with no content-type (PR #2845)
+* Add missing yield during 3pid signature checks (PR #2933)
+* Fix race in sync when joining room (PR #2944)
+* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
+* Fix scary-looking dns resolution errors (PR #2838)
 
 
 
-- 
cgit 1.4.1


From fb647164f2a008cdba56577ad0f41c4b4c1147d1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 14 Mar 2018 16:20:36 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 4d1b4ba13c..53a97a7697 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -14,10 +14,10 @@ Changes in synapse v0.27.0-rc1 (2018-03-14)
 Features:
 * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
 * Add shiney new purge API. New implementation is:-
-  * Faster 
-  * Supports clearing by timestamp
-  * Supports deleting of local events
-  * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
+ * Faster 
+ * Supports clearing by timestamp
+ * Supports deleting of local events
+ * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
 
 * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
 * Add /room/{id}/event/{id} to synapse (PR #2766)
-- 
cgit 1.4.1


From c33c1ceddd5da8195b38059dce31255209075ba2 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Wed, 14 Mar 2018 11:09:08 -0600
Subject: OCD: Make the event_creator routes regex a code block

All the others are code blocks, so this one should be to (currently it is a blockquote).

Signed-off-by: Travis Ralston <travpc@gmail.com>
---
 docs/workers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/workers.rst b/docs/workers.rst
index dee04bbf3e..80f8d2181a 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -230,7 +230,7 @@ file. For example::
 ``synapse.app.event_creator``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Handles non-state event creation. It can handle REST endpoints matching:
+Handles non-state event creation. It can handle REST endpoints matching::
 
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
 
-- 
cgit 1.4.1


From 5ccb57d3ff6e3affd32cc796f625b2378483d871 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 14 Mar 2018 17:12:58 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 53a97a7697..beac2117d3 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,16 +1,17 @@
 Unreleased
 ==========
 
+
+
+Changes in synapse v0.27.0-rc1 (2018-03-14)
+===========================================
+
 synctl no longer starts the main synapse when using ``-a`` option with workers.
 A new worker file should be added with ``worker_app: synapse.app.homeserver``.
 
 This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
 
-
-Changes in synapse v0.27.0-rc1 (2018-03-14)
-===========================================
-
 Features:
 * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
 * Add shiney new purge API. New implementation is:-
@@ -27,7 +28,7 @@ Features:
 
 Changes:
 
-* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are XXXXXX, (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856)
+* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more [here](https://github.com/matrix-org/synapse/blob/master/docs/workers.rst), (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995)
 * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
 * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
 * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
@@ -64,12 +65,9 @@ Changes:
 * Factor run_in_background out from preserve_fn (PR #2961)
 * Add a metric which increments when a request is received (PR #2965)
 * Improve caching for read_marker API (PR #2927)
+* Add Measure block for persist_events (PR #2975)
  
 
-Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270)
-
-
-
 Bug fixes:
 
 * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv!
-- 
cgit 1.4.1


From 10fdcf561d614235ba8f6ef3f980ac16ca0d7df7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erikj@jki.re>
Date: Wed, 14 Mar 2018 17:30:17 +0000
Subject: Fix up rst formatting

---
 CHANGES.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index beac2117d3..42fe461b24 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -13,12 +13,14 @@ This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
 
 Features:
+
 * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
-* Add shiney new purge API. New implementation is:-
- * Faster 
- * Supports clearing by timestamp
- * Supports deleting of local events
- * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
+* Add shiney new purge API. New implementation is:
+
+  * Faster 
+  * Supports clearing by timestamp
+  * Supports deleting of local events
+  * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
 
 * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
 * Add /room/{id}/event/{id} to synapse (PR #2766)
@@ -28,7 +30,7 @@ Features:
 
 Changes:
 
-* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more [here](https://github.com/matrix-org/synapse/blob/master/docs/workers.rst), (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995)
+* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more at `docs/workers.rst <docs/metrics-howto.rst>`_ (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995)
 * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
 * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
 * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
-- 
cgit 1.4.1


From 2059b8573f997e3210e0fa995d03f9168f8aeba6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erikj@jki.re>
Date: Wed, 14 Mar 2018 18:11:21 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 86 +++++++++++++++++--------------------------------------------
 1 file changed, 24 insertions(+), 62 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 42fe461b24..38cd3bec65 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -14,80 +14,42 @@ reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#bloc
 
 Features:
 
-* Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767)
-* Add shiney new purge API. New implementation is:
-
-  * Faster 
-  * Supports clearing by timestamp
-  * Supports deleting of local events
-  * Is transactional (PR #2858,2867,2882, 2946,2962,2943)
-
-* Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813)
-* Add /room/{id}/event/{id} to synapse (PR #2766)
-* Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live!
-* Add federation_domain_whitelist option (PR #2820,2821)
+* Add ability for ASes to override message send time (PR #2754)
+* Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767)
+* Add purge API features, see `docs/admin_api/purge_history_api.rst <docs/admin_api/purge_history_api.rst>`_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943)
+* Add support for whitelisting 3PIDs that users can register. (PR #2813)
+* Add ``/room/{id}/event/{id}`` API (PR #2766)
+* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live!
+* Add ``federation_domain_whitelist`` option (PR #2820, #2821)
 
 
 Changes:
 
-* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more at `docs/workers.rst <docs/metrics-howto.rst>`_ (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995)
-* Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
-* Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
-* Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz!
-* Better logging when login can't find a 3pid (PR #2744)
-* add ?ts massaging for ASes (PR #2754)
-* Remove 'verbosity'/'log_file' from generated cfg (PR #2755)
-* Make indentation of generated log config consistent (PR #2762)
-* Remove dead code related to default thumbnails (PR #2764)
-* Update http request metrics before calling servlet (PR #2770)
-* Do bcrypt hashing in a background thread (PR #2773)
+* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst <docs/metrics-howto.rst>`_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784)
+* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
+* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
+* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz!
+* Remove ``verbosity``/``log_file`` from generated config (PR #2755)
+* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838)
 * When using synctl with workers, don't start the main synapse automatically (PR #2774)
-* Make Counter render floats (PR #2778)
-* Store state groups separately from events (PR #2784)
-* Reorganise request and block metrics (PR #2785)
-* Metrics for number of RDATA commands received (PR #2786)
-* Metrics for events processed in appservice and fed sender (PR #2787)
-* Optimise LoggingContext creation and copying (PR #2792)
-* Track db txn time in millisecs (PR #2793)
-* Track DB scheduling delay per-request (PR #2794, #2795)
-* Sanity checking for user ids (PR #2797)
-* better exception logging in callbackmetrics (PR #2809)
-* Fix bugs in block metrics (PR #2810)
-* Add some comments about the reactor tick time metric (PR #2816)
-* Use a connection pool for the SimpleHttpClient (PR #2817)
-* Remove unused/bitrotted MemoryDataStore (PR #2828)
-* Make it possible to run tests against postgres (PR #2829)
-* Factor out get_db_conn to HomeServer base class (PR #2830)
-* Logging and metrics for the http pusher (PR #2833)
-* Improve exception handling in persist_event (PR #2834)
-* montoring metrics for number of cache evictions (PR #2844)
+* Minor performance improvements (PR #2773, #2792)
+* Use a connection pool for non-federation outbound connections (PR #2817)
+* Make it possible to run unit tests against postgres (PR #2829)
 * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
 * Remove ability for AS users to call /events and /sync (PR #2948)
-* use bcrypt.checkpw (PR #2949) Thanks to @krombel!
-* Factor run_in_background out from preserve_fn (PR #2961)
-* Add a metric which increments when a request is received (PR #2965)
-* Improve caching for read_marker API (PR #2927)
-* Add Measure block for persist_events (PR #2975)
- 
+* Use bcrypt.checkpw (PR #2949) Thanks to @krombel!
 
 Bug fixes:
 
-* synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv!
-* Check missing fields in event_from_pdu_json (PR #2745)
-* Fix templating error with unban permission message (PR #2761) Thanks to @turt2live!
-* Fix flaky test_rooms UTs (PR #2765)
+* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv!
+* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live!
 * Fix publicised groups GET API (singular) over federation (PR #2772)
-* Fix a logcontext leak in persist_events (PR #2790)
-* Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796)
-* fix SQL when searching all users (PR #2803)
-* Fix server 500 on public rooms call when no rooms exist (PR #2827)
-* Fix SQL for user search (PR #2831)
-* Fix sql error in quarantine_media (PR #2837)
-* Handle url_previews with no content-type (PR #2845)
-* Add missing yield during 3pid signature checks (PR #2933)
-* Fix race in sync when joining room (PR #2944)
+* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831)
+* Fix error on ``/publicRooms`` when no rooms exist (PR #2827)
+* Fix bug in quarantine_media (PR #2837)
+* Fix url_previews when no Content-Type is returned from URL (PR #2845)
+* Fix rare race in sync API when joining room (PR #2944)
 * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
-* Fix scary-looking dns resolution errors (PR #2838)
 
 
 
-- 
cgit 1.4.1


From 7d26591048ac5ab6263ddb24a840c58edfd85f50 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 15 Mar 2018 10:33:24 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 38cd3bec65..a50d3ea5ba 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -6,12 +6,13 @@ Unreleased
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
 
-synctl no longer starts the main synapse when using ``-a`` option with workers.
-A new worker file should be added with ``worker_app: synapse.app.homeserver``.
+The common case for running Synapse is not to run separate workers, but for those that do, synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``.
 
 This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
 
+Note, old metrics will be removed from the 0.28.0 release.
+
 Features:
 
 * Add ability for ASes to override message send time (PR #2754)
-- 
cgit 1.4.1


From 7367a4a8239a04047fa648694c08f3d1c63dd3ee Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 15 Mar 2018 10:33:52 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index a50d3ea5ba..48791cec33 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -6,7 +6,7 @@ Unreleased
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
 
-The common case for running Synapse is not to run separate workers, but for those that do, synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``.
+The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``.
 
 This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
-- 
cgit 1.4.1


From b29d1abab6ca42f693e3b83ab31440fee95d00f2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 15 Mar 2018 10:34:15 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 48791cec33..931ecb4d31 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,8 +1,3 @@
-Unreleased
-==========
-
-
-
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
 
-- 
cgit 1.4.1


From 068c21ab102b22aeac7fc9aa1f4d2d229a807872 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <github@rvanderhoff.org.uk>
Date: Thu, 15 Mar 2018 10:36:31 +0000
Subject: CHANGES.rst: reword metric deprecation

---
 CHANGES.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 931ecb4d31..e25327f28f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,8 +5,7 @@ The common case for running Synapse is not to run separate workers, but for thos
 
 This release also begins the process of renaming a number of the metrics
 reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
-
-Note, old metrics will be removed from the 0.28.0 release.
+Note that the v0.28.0 release will remove the deprecated metric names.
 
 Features:
 
-- 
cgit 1.4.1


From 0ad5125814dc18a79423740ac54f96e16a427758 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <github@rvanderhoff.org.uk>
Date: Thu, 15 Mar 2018 11:05:42 +0000
Subject: Update purge_history_api.rst

clarify that `purge_history` will not purge state
---
 docs/admin_api/purge_history_api.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.rst
index ea2922da5c..2da833c827 100644
--- a/docs/admin_api/purge_history_api.rst
+++ b/docs/admin_api/purge_history_api.rst
@@ -16,9 +16,11 @@ including an ``access_token`` of a server admin.
 
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
-deleted, and room state data before the cutoff is always removed).
+deleted.)
 
-To delete local events as well, set ``delete_local_events`` in the body:
+Room state data (such as joins, leaves, topic) is always preserved.
+
+To delete local message events as well, set ``delete_local_events`` in the body:
 
 .. code:: json
 
-- 
cgit 1.4.1


From 5ea624b0f5f6ad11b1640729b96272f8e7c94d46 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <github@rvanderhoff.org.uk>
Date: Thu, 15 Mar 2018 11:48:35 +0000
Subject: CONTRIBUTING.rst: fix CI info

---
 CONTRIBUTING.rst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 2a88647ca3..c6ee16efc7 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -30,8 +30,12 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Jenkins for continuous integration (http://matrix.org/jenkins), and
-typically all pull requests get automatically tested Jenkins: if your change breaks the build, Jenkins will yell about it in #matrix-dev:matrix.org so please lurk there and keep an eye open.
+We use `Jenkins <http://matrix.org/jenkins>`_ and 
+`Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
+integration. All pull requests to synapse get automatically tested by Travis; 
+the Jenkins builds require an adminstrator to start them. If your change 
+breaks the build, this will be shown in github, so please keep an eye on the 
+pull request for feedback.
 
 Code style
 ~~~~~~~~~~
@@ -115,4 +119,4 @@ can't be accepted. Git makes this trivial - just use the -s flag when you do
 Conclusion
 ~~~~~~~~~~
 
-That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!
\ No newline at end of file
+That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!
-- 
cgit 1.4.1


From ddb00efc1ddec646d02e8def6053003f04d077d7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 14:41:30 +0000
Subject: Bump version number

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index ef8853bd24..d72d88b47d 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.26.0"
+__version__ = "0.27.0-rc1"
-- 
cgit 1.4.1


From 2cc9f76bc3cfa012dcdfe614bdda7e689b8b5e65 Mon Sep 17 00:00:00 2001
From: NotAFile <nota@notafile.com>
Date: Thu, 15 Mar 2018 16:11:17 +0100
Subject: replace old style error catching with 'as' keyword

This is both easier to read and compatible with python3 (not that that
matters)

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/app/synctl.py      | 4 ++--
 synapse/handlers/device.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py
index 0f0ddfa78a..b0e1b5e66a 100755
--- a/synapse/app/synctl.py
+++ b/synapse/app/synctl.py
@@ -38,7 +38,7 @@ def pid_running(pid):
     try:
         os.kill(pid, 0)
         return True
-    except OSError, err:
+    except OSError as err:
         if err.errno == errno.EPERM:
             return True
         return False
@@ -98,7 +98,7 @@ def stop(pidfile, app):
         try:
             os.kill(pid, signal.SIGTERM)
             write("stopped %s" % (app,), colour=GREEN)
-        except OSError, err:
+        except OSError as err:
             if err.errno == errno.ESRCH:
                 write("%s not running" % (app,), colour=YELLOW)
             elif err.errno == errno.EPERM:
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 40f3d24678..f7457a7082 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -155,7 +155,7 @@ class DeviceHandler(BaseHandler):
 
         try:
             yield self.store.delete_device(user_id, device_id)
-        except errors.StoreError, e:
+        except errors.StoreError as e:
             if e.code == 404:
                 # no match
                 pass
@@ -204,7 +204,7 @@ class DeviceHandler(BaseHandler):
 
         try:
             yield self.store.delete_devices(user_id, device_ids)
-        except errors.StoreError, e:
+        except errors.StoreError as e:
             if e.code == 404:
                 # no match
                 pass
@@ -243,7 +243,7 @@ class DeviceHandler(BaseHandler):
                 new_display_name=content.get("display_name")
             )
             yield self.notify_device_update(user_id, [device_id])
-        except errors.StoreError, e:
+        except errors.StoreError as e:
             if e.code == 404:
                 raise errors.NotFoundError()
             else:
-- 
cgit 1.4.1


From 7c7706f42b56dd61f5eb17679aa12247f7058ed5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 15:40:13 +0000
Subject: Fix bug where state cache used lots of memory

The state cache bases its size on the sum of the size of entries. The
size of the entry is calculated once on insertion, so it is important
that the size of entries does not change.

The DictionaryCache modified the entries size, which caused the state
cache to incorrectly think it was smaller than it actually was.
---
 synapse/util/caches/dictionary_cache.py | 6 +++++-
 synapse/util/caches/lrucache.py         | 8 ++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py
index d4105822b3..1709e8b429 100644
--- a/synapse/util/caches/dictionary_cache.py
+++ b/synapse/util/caches/dictionary_cache.py
@@ -132,9 +132,13 @@ class DictionaryCache(object):
                 self._update_or_insert(key, value, known_absent)
 
     def _update_or_insert(self, key, value, known_absent):
-        entry = self.cache.setdefault(key, DictionaryEntry(False, set(), {}))
+        # We pop and reinsert as we need to tell the cache the size may have
+        # changed
+
+        entry = self.cache.pop(key, DictionaryEntry(False, set(), {}))
         entry.value.update(value)
         entry.known_absent.update(known_absent)
+        self.cache[key] = entry
 
     def _insert(self, key, value, known_absent):
         self.cache[key] = DictionaryEntry(True, known_absent, value)
diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py
index f088dd430e..a4bf8fa6ae 100644
--- a/synapse/util/caches/lrucache.py
+++ b/synapse/util/caches/lrucache.py
@@ -154,14 +154,14 @@ class LruCache(object):
         def cache_set(key, value, callbacks=[]):
             node = cache.get(key, None)
             if node is not None:
-                if value != node.value:
+                if node.callbacks and value != node.value:
                     for cb in node.callbacks:
                         cb()
                     node.callbacks.clear()
 
-                    if size_callback:
-                        cached_cache_len[0] -= size_callback(node.value)
-                        cached_cache_len[0] += size_callback(value)
+                if size_callback:
+                    cached_cache_len[0] -= size_callback(node.value)
+                    cached_cache_len[0] += size_callback(value)
 
                 node.callbacks.update(callbacks)
 
-- 
cgit 1.4.1


From 9cf519769b70311d06b43a97e47cfb5a06e6ade4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 17:48:42 +0000
Subject: Use .iter* to avoid copies in StateHandler

---
 synapse/state.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index 932f602508..18386556ae 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -132,7 +132,7 @@ class StateHandler(object):
 
         state_map = yield self.store.get_events(state.values(), get_prev_content=False)
         state = {
-            key: state_map[e_id] for key, e_id in state.items() if e_id in state_map
+            key: state_map[e_id] for key, e_id in state.iteritems() if e_id in state_map
         }
 
         defer.returnValue(state)
@@ -378,7 +378,7 @@ class StateHandler(object):
             new_state = resolve_events_with_state_map(state_set_ids, state_map)
 
         new_state = {
-            key: state_map[ev_id] for key, ev_id in new_state.items()
+            key: state_map[ev_id] for key, ev_id in new_state.iteritems()
         }
 
         return new_state
@@ -458,15 +458,15 @@ class StateResolutionHandler(object):
             # build a map from state key to the event_ids which set that state.
             # dict[(str, str), set[str])
             state = {}
-            for st in state_groups_ids.values():
-                for key, e_id in st.items():
+            for st in state_groups_ids.itervalues():
+                for key, e_id in st.iteritems():
                     state.setdefault(key, set()).add(e_id)
 
             # build a map from state key to the event_ids which set that state,
             # including only those where there are state keys in conflict.
             conflicted_state = {
                 k: list(v)
-                for k, v in state.items()
+                for k, v in state.iteritems()
                 if len(v) > 1
             }
 
@@ -480,7 +480,7 @@ class StateResolutionHandler(object):
                     )
             else:
                 new_state = {
-                    key: e_ids.pop() for key, e_ids in state.items()
+                    key: e_ids.pop() for key, e_ids in state.iteritems()
                 }
 
             # if the new state matches any of the input state groups, we can
@@ -488,8 +488,8 @@ class StateResolutionHandler(object):
             # which will be used as a cache key for future resolutions, but
             # not get persisted.
             state_group = None
-            new_state_event_ids = frozenset(new_state.values())
-            for sg, events in state_groups_ids.items():
+            new_state_event_ids = frozenset(new_state.iteritems())
+            for sg, events in state_groups_ids.iteritems():
                 if new_state_event_ids == frozenset(e_id for e_id in events):
                     state_group = sg
                     break
@@ -702,7 +702,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_
 
     auth_events = {
         key: state_map[ev_id]
-        for key, ev_id in auth_event_ids.items()
+        for key, ev_id in auth_event_ids.iteritems()
         if ev_id in state_map
     }
 
@@ -740,7 +740,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.items():
+    for key, events in conflicted_state.iteritems():
         if key[0] == EventTypes.JoinRules:
             logger.debug("Resolving conflicted join rules %r", events)
             resolved_state[key] = _resolve_auth_events(
@@ -750,7 +750,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.items():
+    for key, events in conflicted_state.iteritems():
         if key[0] == EventTypes.Member:
             logger.debug("Resolving conflicted member lists %r", events)
             resolved_state[key] = _resolve_auth_events(
@@ -760,7 +760,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.items():
+    for key, events in conflicted_state.iteritems():
         if key not in resolved_state:
             logger.debug("Resolving conflicted state %r:%r", key, events)
             resolved_state[key] = _resolve_normal_events(
-- 
cgit 1.4.1


From 5a6e54264d3a9f8b9b1f2e99db94e31ea3c21d24 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 9 Mar 2018 13:56:26 +0000
Subject: Make 'unexpected logging context' into warnings

I think we've now fixed enough of these that the rest can be logged at
warning.
---
 synapse/util/logcontext.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index d660ec785b..fdd4a93d07 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -163,7 +163,7 @@ class LoggingContext(object):
         current = self.set_current_context(self.previous_context)
         if current is not self:
             if current is self.sentinel:
-                logger.debug("Expected logging context %s has been lost", self)
+                logger.warn("Expected logging context %s has been lost", self)
             else:
                 logger.warn(
                     "Current logging context %s is not expected context %s",
@@ -278,7 +278,7 @@ class PreserveLoggingContext(object):
         context = LoggingContext.set_current_context(self.current_context)
 
         if context != self.new_context:
-            logger.debug(
+            logger.warn(
                 "Unexpected logging context: %s is not %s",
                 context, self.new_context,
             )
-- 
cgit 1.4.1


From 926ba76e23ea9d55638baff541cdfaeb9e01ac47 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 23:43:31 +0000
Subject: Replace ujson with simplejson

---
 synapse/api/filtering.py                         |  2 +-
 synapse/handlers/e2e_keys.py                     |  2 +-
 synapse/handlers/message.py                      |  6 +++---
 synapse/http/server.py                           |  5 ++---
 synapse/replication/tcp/commands.py              |  2 +-
 synapse/rest/client/v1/room.py                   |  2 +-
 synapse/rest/client/v2_alpha/sync.py             |  2 +-
 synapse/rest/media/v1/preview_url_resource.py    |  2 +-
 synapse/storage/account_data.py                  |  2 +-
 synapse/storage/background_updates.py            |  2 +-
 synapse/storage/deviceinbox.py                   | 12 ++++++------
 synapse/storage/devices.py                       |  2 +-
 synapse/storage/end_to_end_keys.py               |  2 +-
 synapse/storage/event_push_actions.py            |  2 +-
 synapse/storage/events.py                        |  4 ++--
 synapse/storage/receipts.py                      |  2 +-
 synapse/storage/room.py                          |  2 +-
 synapse/storage/roommember.py                    |  2 +-
 synapse/storage/schema/delta/25/fts.py           |  4 ++--
 synapse/storage/schema/delta/27/ts.py            |  2 +-
 synapse/storage/schema/delta/31/search_update.py |  4 ++--
 synapse/storage/schema/delta/33/event_fields.py  |  4 ++--
 synapse/storage/search.py                        |  2 +-
 synapse/storage/tags.py                          |  2 +-
 synapse/storage/transactions.py                  |  2 +-
 25 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 83206348e5..db43219d24 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -17,7 +17,7 @@ from synapse.storage.presence import UserPresenceState
 from synapse.types import UserID, RoomID
 from twisted.internet import defer
 
-import ujson as json
+import simplejson as json
 import jsonschema
 from jsonschema import FormatChecker
 
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 668a90e495..ce2c87e400 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import ujson as json
+import simplejson as json
 import logging
 
 from canonicaljson import encode_canonical_json
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 21f1717dd2..d7413833ed 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -35,7 +35,7 @@ from canonicaljson import encode_canonical_json
 
 import logging
 import random
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -561,8 +561,8 @@ class MessageHandler(BaseHandler):
 
         # Ensure that we can round trip before trying to persist in db
         try:
-            dump = ujson.dumps(unfreeze(event.content))
-            ujson.loads(dump)
+            dump = simplejson.dumps(unfreeze(event.content))
+            simplejson.loads(dump)
         except Exception:
             logger.exception("Failed to encode content: %r", event.content)
             raise
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 25466cd292..f1e9002e4d 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -36,7 +36,7 @@ from twisted.web.util import redirectTo
 import collections
 import logging
 import urllib
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -370,8 +370,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
         if canonical_json or synapse.events.USE_FROZEN_DICTS:
             json_bytes = encode_canonical_json(json_object)
         else:
-            # ujson doesn't like frozen_dicts.
-            json_bytes = ujson.dumps(json_object, ensure_ascii=False)
+            json_bytes = simplejson.dumps(json_object)
 
     return respond_with_json_bytes(
         request, code, json_bytes,
diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 171227cce2..9633404f73 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -19,7 +19,7 @@ allowed to be sent by which side.
 """
 
 import logging
-import ujson as json
+import simplejson as json
 
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 75b735b47d..80989731fa 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -29,7 +29,7 @@ from synapse.http.servlet import (
 
 import logging
 import urllib
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index a0a8e4b8e4..eb91c0b293 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit
 import itertools
 import logging
 
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 40d2e664eb..a413cb6226 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -35,7 +35,7 @@ import os
 import re
 import fnmatch
 import cgi
-import ujson as json
+import simplejson as json
 import urlparse
 import itertools
 import datetime
diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py
index 56a0bde549..40a2ad8d05 100644
--- a/synapse/storage/account_data.py
+++ b/synapse/storage/account_data.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 
 from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
 
-import ujson as json
+import simplejson as json
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index 11a1b942f1..8f3bff311a 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -19,7 +19,7 @@ from . import engines
 
 from twisted.internet import defer
 
-import ujson as json
+import simplejson as json
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py
index 548e795daf..a879e5bfc1 100644
--- a/synapse/storage/deviceinbox.py
+++ b/synapse/storage/deviceinbox.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 import logging
-import ujson
+import simplejson
 
 from twisted.internet import defer
 
@@ -85,7 +85,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
             )
             rows = []
             for destination, edu in remote_messages_by_destination.items():
-                edu_json = ujson.dumps(edu)
+                edu_json = simplejson.dumps(edu)
                 rows.append((destination, stream_id, now_ms, edu_json))
             txn.executemany(sql, rows)
 
@@ -177,7 +177,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
                     " WHERE user_id = ?"
                 )
                 txn.execute(sql, (user_id,))
-                message_json = ujson.dumps(messages_by_device["*"])
+                message_json = simplejson.dumps(messages_by_device["*"])
                 for row in txn:
                     # Add the message for all devices for this user on this
                     # server.
@@ -199,7 +199,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
                     # Only insert into the local inbox if the device exists on
                     # this server
                     device = row[0]
-                    message_json = ujson.dumps(messages_by_device[device])
+                    message_json = simplejson.dumps(messages_by_device[device])
                     messages_json_for_user[device] = message_json
 
             if messages_json_for_user:
@@ -253,7 +253,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
             messages = []
             for row in txn:
                 stream_pos = row[0]
-                messages.append(ujson.loads(row[1]))
+                messages.append(simplejson.loads(row[1]))
             if len(messages) < limit:
                 stream_pos = current_stream_id
             return (messages, stream_pos)
@@ -389,7 +389,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
             messages = []
             for row in txn:
                 stream_pos = row[0]
-                messages.append(ujson.loads(row[1]))
+                messages.append(simplejson.loads(row[1]))
             if len(messages) < limit:
                 stream_pos = current_stream_id
             return (messages, stream_pos)
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index bd2effdf34..712106b83a 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import ujson as json
+import simplejson as json
 
 from twisted.internet import defer
 
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index 2cebb203c6..ff8538ddf8 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 from synapse.util.caches.descriptors import cached
 
 from canonicaljson import encode_canonical_json
-import ujson as json
+import simplejson as json
 
 from ._base import SQLBaseStore
 
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 8efe2fd4bb..575d710d5d 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -21,7 +21,7 @@ from synapse.types import RoomStreamToken
 from .stream import lower_bound
 
 import logging
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index d08f7571d7..bcca563293 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,7 +38,7 @@ from functools import wraps
 import synapse.metrics
 
 import logging
-import ujson as json
+import simplejson as json
 
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
@@ -56,7 +56,7 @@ event_counter = metrics.register_counter(
 
 def encode_json(json_object):
     if USE_FROZEN_DICTS:
-        # ujson doesn't like frozen_dicts
+        # simplejson doesn't like frozen_dicts
         return encode_canonical_json(json_object)
     else:
         return json.dumps(json_object, ensure_ascii=False)
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 12b3cc7f5f..2c3aa33693 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -20,7 +20,7 @@ from synapse.util.caches.stream_change_cache import StreamChangeCache
 from twisted.internet import defer
 
 import logging
-import ujson as json
+import simplejson as json
 
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 23688430b7..2051d8506d 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -23,7 +23,7 @@ from .engines import PostgresEngine, Sqlite3Engine
 
 import collections
 import logging
-import ujson as json
+import simplejson as json
 import re
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 3e77fd3901..c1ca299285 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -27,7 +27,7 @@ from synapse.api.constants import Membership, EventTypes
 from synapse.types import get_domain_from_id
 
 import logging
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py
index 4269ac69ad..e7351c3ae6 100644
--- a/synapse/storage/schema/delta/25/fts.py
+++ b/synapse/storage/schema/delta/25/fts.py
@@ -17,7 +17,7 @@ import logging
 from synapse.storage.prepare_database import get_statements
 from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +66,7 @@ def run_create(cur, database_engine, *args, **kwargs):
             "max_stream_id_exclusive": max_stream_id + 1,
             "rows_inserted": 0,
         }
-        progress_json = ujson.dumps(progress)
+        progress_json = simplejson.dumps(progress)
 
         sql = (
             "INSERT into background_updates (update_name, progress_json)"
diff --git a/synapse/storage/schema/delta/27/ts.py b/synapse/storage/schema/delta/27/ts.py
index 71b12a2731..c0176c41ee 100644
--- a/synapse/storage/schema/delta/27/ts.py
+++ b/synapse/storage/schema/delta/27/ts.py
@@ -16,7 +16,7 @@ import logging
 
 from synapse.storage.prepare_database import get_statements
 
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py
index 470ae0c005..fe6b7d196d 100644
--- a/synapse/storage/schema/delta/31/search_update.py
+++ b/synapse/storage/schema/delta/31/search_update.py
@@ -16,7 +16,7 @@ from synapse.storage.engines import PostgresEngine
 from synapse.storage.prepare_database import get_statements
 
 import logging
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +49,7 @@ def run_create(cur, database_engine, *args, **kwargs):
             "rows_inserted": 0,
             "have_added_indexes": False,
         }
-        progress_json = ujson.dumps(progress)
+        progress_json = simplejson.dumps(progress)
 
         sql = (
             "INSERT into background_updates (update_name, progress_json)"
diff --git a/synapse/storage/schema/delta/33/event_fields.py b/synapse/storage/schema/delta/33/event_fields.py
index 83066cccc9..1e002f9db2 100644
--- a/synapse/storage/schema/delta/33/event_fields.py
+++ b/synapse/storage/schema/delta/33/event_fields.py
@@ -15,7 +15,7 @@
 from synapse.storage.prepare_database import get_statements
 
 import logging
-import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,7 @@ def run_create(cur, database_engine, *args, **kwargs):
             "max_stream_id_exclusive": max_stream_id + 1,
             "rows_inserted": 0,
         }
-        progress_json = ujson.dumps(progress)
+        progress_json = simplejson.dumps(progress)
 
         sql = (
             "INSERT into background_updates (update_name, progress_json)"
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 479b04c636..c19e4ea449 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -21,7 +21,7 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
 import logging
 import re
-import ujson as json
+import simplejson as json
 
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py
index bff73f3f04..982a500520 100644
--- a/synapse/storage/tags.py
+++ b/synapse/storage/tags.py
@@ -17,7 +17,7 @@ from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached
 from twisted.internet import defer
 
-import ujson as json
+import simplejson as json
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 8f61f7ffae..f825264ea9 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -23,7 +23,7 @@ from canonicaljson import encode_canonical_json
 from collections import namedtuple
 
 import logging
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
-- 
cgit 1.4.1


From ba48755d56ba134590f7b5dcb5da44371639fc4c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 23:57:26 +0000
Subject: Bump version and changelog

---
 CHANGES.rst         | 8 ++++++++
 synapse/__init__.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index dcf9adc95c..38ba4249b8 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,11 @@
+Changes in synapse v0.26.1 (2018-03-15)
+=======================================
+
+Bug fixes:
+
+* Fix bug where an invalid event caused server to stop functioning correctly,
+  due to parsing and serializing bugs in ujson library.
+
 Changes in synapse v0.26.0 (2018-01-05)
 =======================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index ef8853bd24..80efad3a65 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.26.0"
+__version__ = "0.26.1"
-- 
cgit 1.4.1


From 5b631ff41a3230d8b6ed662e5aeb406c86989f9f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 16 Mar 2018 00:07:08 +0000
Subject: Remove wrong comment

---
 synapse/storage/events.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bcca563293..73658a9927 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -56,7 +56,6 @@ event_counter = metrics.register_counter(
 
 def encode_json(json_object):
     if USE_FROZEN_DICTS:
-        # simplejson doesn't like frozen_dicts
         return encode_canonical_json(json_object)
     else:
         return json.dumps(json_object, ensure_ascii=False)
-- 
cgit 1.4.1


From 0092cf38ae95173519ffcc403b20b4fd2dec5c8d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 16 Mar 2018 00:11:58 +0000
Subject: Newline

---
 CHANGES.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 38ba4249b8..3b32928238 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -6,6 +6,7 @@ Bug fixes:
 * Fix bug where an invalid event caused server to stop functioning correctly,
   due to parsing and serializing bugs in ujson library.
 
+
 Changes in synapse v0.26.0 (2018-01-05)
 =======================================
 
-- 
cgit 1.4.1


From a8ce159be43560e9aea8f3be65110eea49d1f50e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 23:38:43 +0000
Subject: Replace some ujson with simplejson to make it work

---
 synapse/http/server.py               | 3 ++-
 synapse/rest/client/v2_alpha/sync.py | 2 +-
 synapse/storage/events.py            | 2 +-
 synapse/storage/events_worker.py     | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 4b567215c8..3c7a0ef97a 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -38,6 +38,7 @@ import collections
 import logging
 import urllib
 import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -462,7 +463,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
             json_bytes = encode_canonical_json(json_object)
         else:
             # ujson doesn't like frozen_dicts.
-            json_bytes = ujson.dumps(json_object, ensure_ascii=False)
+            json_bytes = simplejson.dumps(json_object)
 
     return respond_with_json_bytes(
         request, code, json_bytes,
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index a0a8e4b8e4..eb91c0b293 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit
 import itertools
 import logging
 
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 3890878170..9fc65229fd 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,7 +38,7 @@ from functools import wraps
 import synapse.metrics
 
 import logging
-import ujson as json
+import simplejson as json
 
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 86c3b48ad4..2e23dd78ba 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -28,7 +28,7 @@ from synapse.api.errors import SynapseError
 from collections import namedtuple
 
 import logging
-import ujson as json
+import simplejson as json
 
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
-- 
cgit 1.4.1


From 38f952b9bc96cf72d96bab3510e8f428a8247542 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 19 Mar 2018 09:27:36 +0000
Subject: spell out not to massively increase bcrypt rounds

---
 synapse/config/registration.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index 336959094b..c5384b3ad4 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -77,7 +77,9 @@ class RegistrationConfig(Config):
 
         # Set the number of bcrypt rounds used to generate password hash.
         # Larger numbers increase the work factor needed to generate the hash.
-        # The default number of rounds is 12.
+        # The default number is 12 (which equates to 2^12 rounds).
+        # N.B. that increasing this will exponentially increase the time required
+        # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
         bcrypt_rounds: 12
 
         # Allows users to register as guests without a password/email/etc, and
-- 
cgit 1.4.1


From 9a0d783c113ae74c55e409d33219cd77f3662b9f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 19 Mar 2018 11:35:53 +0000
Subject: Add comments

---
 synapse/util/caches/lrucache.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py
index a4bf8fa6ae..1c5a982094 100644
--- a/synapse/util/caches/lrucache.py
+++ b/synapse/util/caches/lrucache.py
@@ -154,11 +154,18 @@ class LruCache(object):
         def cache_set(key, value, callbacks=[]):
             node = cache.get(key, None)
             if node is not None:
+                # We sometimes store large objects, e.g. dicts, which cause
+                # the inequality check to take a long time. So let's only do
+                # the check if we have some callbacks to call.
                 if node.callbacks and value != node.value:
                     for cb in node.callbacks:
                         cb()
                     node.callbacks.clear()
 
+                # We don't bother to protect this by value != node.value as
+                # generally size_callback will be cheap compared with equality
+                # checks. (For example, taking the size of two dicts is quicker
+                # than comparing them for equality.)
                 if size_callback:
                     cached_cache_len[0] -= size_callback(node.value)
                     cached_cache_len[0] += size_callback(value)
-- 
cgit 1.4.1


From 1a3aa957ca1164268c9bbbd564491c9d46fbb7d2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 19 Mar 2018 15:11:00 +0000
Subject: Update CHANGES.rst

---
 CHANGES.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index e25327f28f..55358287a0 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,10 @@
+Changes in synapse v0.27.0-rc2 (2018-03-19)
+===========================================
+Bugs:
+
+* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005)
+
+
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
 
-- 
cgit 1.4.1


From c384705ee877ba40e1a736b68ebe07434662cbaa Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 19 Mar 2018 15:11:58 +0000
Subject: Update __init__.py

bump version
---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index d72d88b47d..71b409ba8b 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.0-rc1"
+__version__ = "0.27.0-rc2"
-- 
cgit 1.4.1


From 610accbb7f58d7a07007dedaa054c8ad7a9f5851 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 19 Mar 2018 16:06:02 +0000
Subject: Fix replication after switch to simplejson

Turns out that simplejson serialises namedtuple's as dictionaries rather
than tuples by default.
---
 synapse/replication/tcp/commands.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 9633404f73..1b4b657210 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -107,7 +107,7 @@ class RdataCommand(Command):
         return " ".join((
             self.stream_name,
             str(self.token) if self.token is not None else "batch",
-            json.dumps(self.row),
+            json.dumps(self.row, namedtuple_as_object=False),
         ))
 
 
@@ -301,7 +301,9 @@ class InvalidateCacheCommand(Command):
         return cls(cache_func, json.loads(keys_json))
 
     def to_line(self):
-        return " ".join((self.cache_func, json.dumps(self.keys)))
+        return " ".join((
+            self.cache_func, json.dumps(self.keys, namedtuple_as_object=False)
+        ))
 
 
 class UserIpCommand(Command):
-- 
cgit 1.4.1


From 9aa5a0af514b3f2719dd01aa221039c5ba1c0c80 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 20 Mar 2018 09:58:13 +0000
Subject: Explicitly use simplejson

---
 synapse/replication/tcp/commands.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 1b4b657210..0005ad5879 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -19,7 +19,7 @@ allowed to be sent by which side.
 """
 
 import logging
-import simplejson as json
+import simplejson
 
 
 logger = logging.getLogger(__name__)
@@ -100,14 +100,14 @@ class RdataCommand(Command):
         return cls(
             stream_name,
             None if token == "batch" else int(token),
-            json.loads(row_json)
+            simplejson.loads(row_json)
         )
 
     def to_line(self):
         return " ".join((
             self.stream_name,
             str(self.token) if self.token is not None else "batch",
-            json.dumps(self.row, namedtuple_as_object=False),
+            simplejson.dumps(self.row, namedtuple_as_object=False),
         ))
 
 
@@ -298,11 +298,11 @@ class InvalidateCacheCommand(Command):
     def from_line(cls, line):
         cache_func, keys_json = line.split(" ", 1)
 
-        return cls(cache_func, json.loads(keys_json))
+        return cls(cache_func, simplejson.loads(keys_json))
 
     def to_line(self):
         return " ".join((
-            self.cache_func, json.dumps(self.keys, namedtuple_as_object=False)
+            self.cache_func, simplejson.dumps(self.keys, namedtuple_as_object=False)
         ))
 
 
@@ -327,14 +327,14 @@ class UserIpCommand(Command):
     def from_line(cls, line):
         user_id, jsn = line.split(" ", 1)
 
-        access_token, ip, user_agent, device_id, last_seen = json.loads(jsn)
+        access_token, ip, user_agent, device_id, last_seen = simplejson.loads(jsn)
 
         return cls(
             user_id, access_token, ip, user_agent, device_id, last_seen
         )
 
     def to_line(self):
-        return self.user_id + " " + json.dumps((
+        return self.user_id + " " + simplejson.dumps((
             self.access_token, self.ip, self.user_agent, self.device_id,
             self.last_seen,
         ))
-- 
cgit 1.4.1


From f5aa027c2f9d836df7b98da6c1c5fee6b22d0a3d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 20 Mar 2018 15:06:22 +0000
Subject: Update CHANGES.rst

rearrange ordering of releases to match chronology
---
 CHANGES.rst | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 788150bcfc..bac78d3c1c 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,10 +1,21 @@
 Changes in synapse v0.27.0-rc2 (2018-03-19)
 ===========================================
-Bugs:
+
+Pulls in v0.26.1
+
+Bug fixes:
 
 * Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005)
+
+
+Changes in synapse v0.26.1 (2018-03-15)
+=======================================
+
+Bug fixes:
+
 * Fix bug where an invalid event caused server to stop functioning correctly,
-  due to parsing and serializing bugs in ujson library. (pulled in from 0.26.1)
+  due to parsing and serializing bugs in ujson library (PR #3008)
+
 
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
@@ -55,16 +66,6 @@ Bug fixes:
 * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
 
 
-Changes in synapse v0.26.1 (2018-03-15)
-=======================================
-
-Bug fixes:
-
-* Fix bug where an invalid event caused server to stop functioning correctly,
-  due to parsing and serializing bugs in ujson library.
-
-
-
 Changes in synapse v0.26.0 (2018-01-05)
 =======================================
 
-- 
cgit 1.4.1


From 1c41b05c8c98f0b9157c791b5b8ebf5f9fe85acf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 21 Mar 2018 17:46:26 +0000
Subject: Add Cache-Control headers to all JSON APIs

It is especially important that sync requests don't get cached, as if a
sync returns the same token given then the client will call sync with
the same parameters again. If the previous response was cached it will
get reused, resulting in the client tight looping making the same
request and never making any progress.

In general, clients will expect to get up to date data when requesting
APIs, and so its safer to do a blanket no cache policy than only
whitelisting APIs that we know will break things if they get cached.
---
 synapse/http/server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 1551db239d..f19c068ef6 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -488,6 +488,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
     request.setHeader(b"Content-Type", b"application/json")
     request.setHeader(b"Server", version_string)
     request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
+    request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate")
 
     if send_cors:
         set_cors_headers(request)
-- 
cgit 1.4.1


From fde8e8f09fb0e62d4670e33b75319b5bd57f484f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 22 Mar 2018 11:42:16 +0000
Subject: Fix s/iteriterms/itervalues

---
 synapse/state.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/state.py b/synapse/state.py
index 18386556ae..a7f20350f1 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -488,7 +488,7 @@ class StateResolutionHandler(object):
             # which will be used as a cache key for future resolutions, but
             # not get persisted.
             state_group = None
-            new_state_event_ids = frozenset(new_state.iteritems())
+            new_state_event_ids = frozenset(new_state.itervalues())
             for sg, events in state_groups_ids.iteritems():
                 if new_state_event_ids == frozenset(e_id for e_id in events):
                     state_group = sg
-- 
cgit 1.4.1


From 8cbbfaefc1fc597cbbef52a10dbfb8ecd4d8a8cd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 23 Mar 2018 10:32:50 +0000
Subject: 404 correctly on missing paths via NoResource

fixes https://github.com/matrix-org/synapse/issues/2043 and https://github.com/matrix-org/synapse/issues/2029
---
 synapse/app/appservice.py        | 4 ++--
 synapse/app/client_reader.py     | 4 ++--
 synapse/app/event_creator.py     | 4 ++--
 synapse/app/federation_reader.py | 4 ++--
 synapse/app/federation_sender.py | 4 ++--
 synapse/app/frontend_proxy.py    | 4 ++--
 synapse/app/homeserver.py        | 4 ++--
 synapse/app/media_repository.py  | 4 ++--
 synapse/app/pusher.py            | 4 ++--
 synapse/app/synchrotron.py       | 4 ++--
 synapse/app/user_dir.py          | 4 ++--
 synapse/util/httpresourcetree.py | 4 ++--
 12 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index c6fe4516d1..f2540023a7 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -36,7 +36,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.appservice")
 
@@ -64,7 +64,7 @@ class AppserviceServer(HomeServer):
                 if name == "metrics":
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 0a8ce9bc66..267d34c881 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -44,7 +44,7 @@ from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.client_reader")
 
@@ -88,7 +88,7 @@ class ClientReaderServer(HomeServer):
                         "/_matrix/client/api/v1": resource,
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 172e989b54..b915d12d53 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -52,7 +52,7 @@ from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.event_creator")
 
@@ -104,7 +104,7 @@ class EventCreatorServer(HomeServer):
                         "/_matrix/client/api/v1": resource,
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 20d157911b..c1dc66dd17 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -41,7 +41,7 @@ from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.federation_reader")
 
@@ -77,7 +77,7 @@ class FederationReaderServer(HomeServer):
                         FEDERATION_PREFIX: TransportLayerServer(self),
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index f760826d27..0cc3331519 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -42,7 +42,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.federation_sender")
 
@@ -91,7 +91,7 @@ class FederationSenderServer(HomeServer):
                 if name == "metrics":
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index 816c080d18..de889357c3 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -44,7 +44,7 @@ from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.frontend_proxy")
 
@@ -142,7 +142,7 @@ class FrontendProxyServer(HomeServer):
                         "/_matrix/client/api/v1": resource,
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index e477c7ced6..c00afbba28 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -56,7 +56,7 @@ from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 from twisted.application import service
 from twisted.internet import defer, reactor
-from twisted.web.resource import EncodingResourceWrapper, Resource
+from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
 
@@ -126,7 +126,7 @@ class SynapseHomeServer(HomeServer):
         if WEB_CLIENT_PREFIX in resources:
             root_resource = RootRedirect(WEB_CLIENT_PREFIX)
         else:
-            root_resource = Resource()
+            root_resource = NoResource()
 
         root_resource = create_resource_tree(resources, root_resource)
 
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index 84c5791b3b..fc8282bbc1 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -43,7 +43,7 @@ from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.media_repository")
 
@@ -84,7 +84,7 @@ class MediaRepositoryServer(HomeServer):
                         ),
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 98a4a7c62c..d5c3a85195 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -37,7 +37,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.pusher")
 
@@ -94,7 +94,7 @@ class PusherServer(HomeServer):
                 if name == "metrics":
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index abe91dcfbd..508b66613d 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -56,7 +56,7 @@ from synapse.util.manhole import manhole
 from synapse.util.stringutils import random_string
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.synchrotron")
 
@@ -269,7 +269,7 @@ class SynchrotronServer(HomeServer):
                         "/_matrix/client/api/v1": resource,
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 494ccb702c..5f845e80d1 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -43,7 +43,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import reactor
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.user_dir")
 
@@ -116,7 +116,7 @@ class UserDirectoryServer(HomeServer):
                         "/_matrix/client/api/v1": resource,
                     })
 
-        root_resource = create_resource_tree(resources, Resource())
+        root_resource = create_resource_tree(resources, NoResource())
 
         _base.listen_tcp(
             bind_addresses,
diff --git a/synapse/util/httpresourcetree.py b/synapse/util/httpresourcetree.py
index 45be47159a..d747849553 100644
--- a/synapse/util/httpresourcetree.py
+++ b/synapse/util/httpresourcetree.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.web.resource import Resource
+from twisted.web.resource import NoResource
 
 import logging
 
@@ -45,7 +45,7 @@ def create_resource_tree(desired_tree, root_resource):
         for path_seg in full_path.split('/')[1:-1]:
             if path_seg not in last_resource.listNames():
                 # resource doesn't exist, so make a "dummy resource"
-                child_resource = Resource()
+                child_resource = NoResource()
                 last_resource.putChild(path_seg, child_resource)
                 res_id = _resource_id(last_resource, path_seg)
                 resource_mappings[res_id] = child_resource
-- 
cgit 1.4.1


From 8efe773ef1dc09c295fe494befbef568257095db Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 23 Mar 2018 21:38:42 +0000
Subject: fix typo

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index bac78d3c1c..00c8a2b77d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -39,7 +39,7 @@ Features:
 
 Changes:
 
-* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst <docs/metrics-howto.rst>`_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784)
+* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst <docs/workers.rst>`_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784)
 * Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
 * Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
 * No longer require a specific version of saml2 (PR #2695) Thanks to @okurz!
-- 
cgit 1.4.1


From a052aa42e7d500f32b034bbbf1e8ad11fed02aa9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 12:02:20 +0100
Subject: Linearize calls to _generate_user_id

---
 synapse/handlers/register.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ed5939880a..d07a8801de 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -24,7 +24,7 @@ from synapse.api.errors import (
 from synapse.http.client import CaptchaServerHttpClient
 from synapse import types
 from synapse.types import UserID
-from synapse.util.async import run_on_reactor
+from synapse.util.async import run_on_reactor, Linearizer
 from synapse.util.threepids import check_3pid_allowed
 from ._base import BaseHandler
 
@@ -46,6 +46,8 @@ class RegistrationHandler(BaseHandler):
 
         self.macaroon_gen = hs.get_macaroon_generator()
 
+        self._generate_user_id_linearizer = Linearizer(name="_generate_user_id_linearizer")
+
     @defer.inlineCallbacks
     def check_username(self, localpart, guest_access_token=None,
                        assigned_user_id=None):
@@ -345,9 +347,10 @@ class RegistrationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _generate_user_id(self, reseed=False):
         if reseed or self._next_generated_user_id is None:
-            self._next_generated_user_id = (
-                yield self.store.find_next_generated_user_id_localpart()
-            )
+            with (yield self._generate_user_id_linearizer.queue(())):
+                self._next_generated_user_id = (
+                    yield self.store.find_next_generated_user_id_localpart()
+                )
 
         id = self._next_generated_user_id
         self._next_generated_user_id += 1
-- 
cgit 1.4.1


From 8d6dc106d188308f687db1f696cad7ad1b9ae83d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 12:02:44 +0100
Subject: Don't use _cursor_to_dict in find_next_generated_user_id_localpart

---
 synapse/storage/registration.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index d809b2ba46..6b557ca0cf 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -460,14 +460,12 @@ class RegistrationStore(RegistrationWorkerStore,
         """
         def _find_next_generated_user_id(txn):
             txn.execute("SELECT name FROM users")
-            rows = self.cursor_to_dict(txn)
 
             regex = re.compile("^@(\d+):")
 
             found = set()
 
-            for r in rows:
-                user_id = r["name"]
+            for user_id, in txn:
                 match = regex.search(user_id)
                 if match:
                     found.add(int(match.group(1)))
-- 
cgit 1.4.1


From 44cd6e135806387552bab5cf7e8be256d53632b6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 12:06:48 +0100
Subject: PEP8

---
 synapse/handlers/register.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index d07a8801de..f1bf81ed7d 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -46,7 +46,9 @@ class RegistrationHandler(BaseHandler):
 
         self.macaroon_gen = hs.get_macaroon_generator()
 
-        self._generate_user_id_linearizer = Linearizer(name="_generate_user_id_linearizer")
+        self._generate_user_id_linearizer = Linearizer(
+            name="_generate_user_id_linearizer",
+        )
 
     @defer.inlineCallbacks
     def check_username(self, localpart, guest_access_token=None,
-- 
cgit 1.4.1


From fecb45e0c38912f72655d53faa953e7713511e7f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 13:31:49 +0100
Subject: Remove last usage of ujson

---
 synapse/python_dependencies.py  | 1 -
 synapse/storage/group_server.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 91179ce532..40eedb63cb 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,7 +34,6 @@ REQUIREMENTS = {
     "bcrypt": ["bcrypt>=3.1.0"],
     "pillow": ["PIL"],
     "pydenticon": ["pydenticon"],
-    "ujson": ["ujson"],
     "blist": ["blist"],
     "pysaml2>=3.0.0": ["saml2>=3.0.0"],
     "pymacaroons-pynacl": ["pymacaroons"],
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8fde1aab8e..d03858234b 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError
 
 from ._base import SQLBaseStore
 
-import ujson as json
+import simplejson as json
 
 
 # The category ID for the "default" category. We don't store as null in the
-- 
cgit 1.4.1


From 51406dab96cb3a4de2fad02a31984cfcb613f809 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 26 Mar 2018 14:48:19 +0100
Subject: version bump

---
 CHANGES.rst         | 6 ++++++
 synapse/__init__.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index bac78d3c1c..92c19b13e9 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,9 @@
+Changes in synapse v0.27.0 (2018-03-26)
+=======================================
+
+No changes since v0.27.0-rc2
+
+
 Changes in synapse v0.27.0-rc2 (2018-03-19)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 71b409ba8b..70a0297052 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.0-rc2"
+__version__ = "0.27.0"
-- 
cgit 1.4.1


From 91878620027d17b478ca678482f501d3cfb2edaf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 16:19:38 +0100
Subject: Bump version and changelog

---
 CHANGES.rst         | 6 ++++++
 synapse/__init__.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 92c19b13e9..e5e8e31a1a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,9 @@
+Changes in synapse v0.27.1 (2018-03-19)
+=======================================
+
+Meta release as v0.27.0 temporarily pointed to the wrong commit
+
+
 Changes in synapse v0.27.0 (2018-03-26)
 =======================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 70a0297052..2cb6ef4652 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.0"
+__version__ = "0.27.1"
-- 
cgit 1.4.1


From 01f72e2fc7379db7d02d5bc1eae96167aa320389 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 16:21:26 +0100
Subject: Fix date

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index e5e8e31a1a..ddef8c205b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.27.1 (2018-03-19)
+Changes in synapse v0.27.1 (2018-03-26)
 =======================================
 
 Meta release as v0.27.0 temporarily pointed to the wrong commit
-- 
cgit 1.4.1


From a9d7d98d3f87cc20d94c9cc4ee9285e6a2dddb2c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Mar 2018 16:36:53 +0100
Subject: Bum version and changelog

---
 CHANGES.rst         | 8 ++++++++
 synapse/__init__.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index ddef8c205b..1372de4246 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,11 @@
+Changes in synapse v0.27.2 (2018-03-26)
+=======================================
+
+Bug fixes:
+
+* Fix bug which broke TCP replication between workers (PR #3015)
+
+
 Changes in synapse v0.27.1 (2018-03-26)
 =======================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 2cb6ef4652..a9d5198aba 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.1"
+__version__ = "0.27.2"
-- 
cgit 1.4.1


From 3f49e131d9a31d5bf198f2aa6b102562b2e747d0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 10:30:03 +0100
Subject: Add counter metrics for calculating state delta

This will allow us to measure how often we calculate state deltas in
event persistence that we would have been able to calculate at the same
time we calculated the state for the event.
---
 synapse/storage/events.py | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 85ce6bea1a..f3a362db20 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -52,6 +52,15 @@ persist_event_counter = metrics.register_counter("persisted_events")
 event_counter = metrics.register_counter(
     "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
 )
+state_delta_counter = metrics.register_counter(
+    "state_delta",
+)
+state_delta_single_event_counter = metrics.register_counter(
+    "state_delta_single_event",
+)
+state_delta_reuse_delta_counter = metrics.register_counter(
+    "state_delta_reuse_delta",
+)
 
 
 def encode_json(json_object):
@@ -368,7 +377,8 @@ class EventsStore(EventsWorkerStore):
                                 room_id, ev_ctx_rm, latest_event_ids
                             )
 
-                            if new_latest_event_ids == set(latest_event_ids):
+                            latest_event_ids = set(latest_event_ids)
+                            if new_latest_event_ids == latest_event_ids:
                                 # No change in extremities, so no change in state
                                 continue
 
@@ -389,6 +399,25 @@ class EventsStore(EventsWorkerStore):
                                 if all_single_prev_not_state:
                                     continue
 
+                            state_delta_counter.inc()
+                            if len(new_latest_event_ids) == 1:
+                                state_delta_single_event_counter.inc()
+
+                                # This is a fairly handwavey check to see if we could
+                                # have guessed what the delta would have been when
+                                # processing one of these events.
+                                # What we're interested in is if the latest extremities
+                                # were the same when we created the event as they are
+                                # now. We guess this by looking at the prev events and
+                                # checking if they match up, as when this server creates
+                                # a new event it will use the extremities as the prev
+                                # events.
+                                for ev, _ in ev_ctx_rm:
+                                    prev_event_ids = set(e for e, _ in ev.prev_events)
+                                    if latest_event_ids == prev_event_ids:
+                                        state_delta_reuse_delta_counter.inc()
+                                    break
+
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
                             )
-- 
cgit 1.4.1


From 3e0c0660b31827f9d13c1fb5153a77e70813d5d3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:01:34 +0100
Subject: Also do check inside linearizer

---
 synapse/handlers/register.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index f1bf81ed7d..dd03705279 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -350,9 +350,10 @@ class RegistrationHandler(BaseHandler):
     def _generate_user_id(self, reseed=False):
         if reseed or self._next_generated_user_id is None:
             with (yield self._generate_user_id_linearizer.queue(())):
-                self._next_generated_user_id = (
-                    yield self.store.find_next_generated_user_id_localpart()
-                )
+                if reseed or self._next_generated_user_id is None:
+                    self._next_generated_user_id = (
+                        yield self.store.find_next_generated_user_id_localpart()
+                    )
 
         id = self._next_generated_user_id
         self._next_generated_user_id += 1
-- 
cgit 1.4.1


From e70287cff3adeeeff2a7a1e95fc5845feb365710 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:13:38 +0100
Subject: Comment

---
 synapse/storage/events.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f3a362db20..ee8b3c46d3 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -52,12 +52,19 @@ persist_event_counter = metrics.register_counter("persisted_events")
 event_counter = metrics.register_counter(
     "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
 )
+
+# The number of times we are recalculating the current state
 state_delta_counter = metrics.register_counter(
     "state_delta",
 )
+# The number of times we are recalculating state when there is only a
+# single forward extremity
 state_delta_single_event_counter = metrics.register_counter(
     "state_delta_single_event",
 )
+# The number of times we are reculating state when we could have resonably
+# calculated the delta when we calculated the state for an event we were
+# persisting.
 state_delta_reuse_delta_counter = metrics.register_counter(
     "state_delta_reuse_delta",
 )
-- 
cgit 1.4.1


From 152c2ac19e6af373a9e79a8f4da80d9e77a5396f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:13:46 +0100
Subject: Fix indent

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ee8b3c46d3..95b36d669b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -423,7 +423,7 @@ class EventsStore(EventsWorkerStore):
                                     prev_event_ids = set(e for e, _ in ev.prev_events)
                                     if latest_event_ids == prev_event_ids:
                                         state_delta_reuse_delta_counter.inc()
-                                    break
+                                        break
 
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
-- 
cgit 1.4.1


From 800cfd5774d9726a00c53f2d84ce0286ab64c17b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Mar 2018 13:30:39 +0100
Subject: Comment

---
 synapse/storage/events.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 95b36d669b..f3d65f4338 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -415,10 +415,11 @@ class EventsStore(EventsWorkerStore):
                                 # processing one of these events.
                                 # What we're interested in is if the latest extremities
                                 # were the same when we created the event as they are
-                                # now. We guess this by looking at the prev events and
-                                # checking if they match up, as when this server creates
-                                # a new event it will use the extremities as the prev
-                                # events.
+                                # now. When this server creates a new event (as opposed
+                                # to receiving it over federation) it will use the
+                                # forward extremities as the prev_events, so we can
+                                # guess this by looking at the prev_events and checking
+                                # if they match the current forward extremities.
                                 for ev, _ in ev_ctx_rm:
                                     prev_event_ids = set(e for e, _ in ev.prev_events)
                                     if latest_event_ids == prev_event_ids:
-- 
cgit 1.4.1


From c2a5cf2fe32d2cd582711669b7c0ce74682e1c05 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 27 Mar 2018 17:07:31 +0100
Subject: factor out exception handling for keys/claim and keys/query

this stuff is badly c&p'ed
---
 synapse/handlers/e2e_keys.py | 53 +++++++++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 80b359b2e7..41521e6990 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -134,23 +135,8 @@ class E2eKeysHandler(object):
                     if user_id in destination_query:
                         results[user_id] = keys
 
-            except CodeMessageException as e:
-                failures[destination] = {
-                    "status": e.code, "message": e.message
-                }
-            except NotRetryingDestination as e:
-                failures[destination] = {
-                    "status": 503, "message": "Not ready for retry",
-                }
-            except FederationDeniedError as e:
-                failures[destination] = {
-                    "status": 403, "message": "Federation Denied",
-                }
             except Exception as e:
-                # include ConnectionRefused and other errors
-                failures[destination] = {
-                    "status": 503, "message": e.message
-                }
+                failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(defer.gatherResults([
             preserve_fn(do_remote_query)(destination)
@@ -252,19 +238,8 @@ class E2eKeysHandler(object):
                 for user_id, keys in remote_result["one_time_keys"].items():
                     if user_id in device_keys:
                         json_result[user_id] = keys
-            except CodeMessageException as e:
-                failures[destination] = {
-                    "status": e.code, "message": e.message
-                }
-            except NotRetryingDestination as e:
-                failures[destination] = {
-                    "status": 503, "message": "Not ready for retry",
-                }
             except Exception as e:
-                # include ConnectionRefused and other errors
-                failures[destination] = {
-                    "status": 503, "message": e.message
-                }
+                failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(defer.gatherResults([
             preserve_fn(claim_client_keys)(destination)
@@ -362,6 +337,28 @@ class E2eKeysHandler(object):
         )
 
 
+def _exception_to_failure(e):
+    if isinstance(e, CodeMessageException):
+        return {
+            "status": e.code, "message": e.message,
+        }
+
+    if isinstance(e, NotRetryingDestination):
+        return {
+            "status": 503, "message": "Not ready for retry",
+        }
+
+    if isinstance(e, FederationDeniedError):
+        return {
+            "status": 403, "message": "Federation Denied",
+        }
+
+    # include ConnectionRefused and other errors
+    return {
+        "status": 503, "message": e.message,
+    }
+
+
 def _one_time_keys_match(old_key_json, new_key):
     old_key = json.loads(old_key_json)
 
-- 
cgit 1.4.1


From a134c572a6697fa6443525493e3fc13f74452d34 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 27 Mar 2018 17:15:06 +0100
Subject: Stringify exceptions for keys/{query,claim}

Make sure we stringify any exceptions we return from keys/query and keys/claim,
to avoid a 'not JSON serializable' error later

Fixes #3010
---
 synapse/handlers/e2e_keys.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 41521e6990..325c0c4a9f 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -354,8 +354,11 @@ def _exception_to_failure(e):
         }
 
     # include ConnectionRefused and other errors
+    #
+    # Note that some Exceptions (notably twisted's ResponseFailed etc) don't
+    # give a string for e.message, which simplejson then fails to serialize.
     return {
-        "status": 503, "message": e.message,
+        "status": 503, "message": str(e.message),
     }
 
 
-- 
cgit 1.4.1


From ef520d8d0e152a24fb6660fdd2def214b6e9caae Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Tue, 27 Mar 2018 14:12:22 +0100
Subject: Include coarse CPU and Memory use in stats callbacks.

This requires the psutil module, and is still opt-in based on the report_stats
config option.
---
 UPGRADE.rst               | 12 ++++++++++++
 synapse/app/homeserver.py | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/UPGRADE.rst b/UPGRADE.rst
index 2efe7ea60f..f6bb1070b1 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -48,6 +48,18 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
+Upgrading to $NEXT_VERSION
+====================
+
+This release expands the anonymous usage stats sent if the opt-in
+``report_stats`` configuration is set to ``true``. We now capture RSS memory 
+and cpu use at a very coarse level. This requires administrators to install
+the optional ``psutil`` python module.
+
+We would appreciate it if you could assist by ensuring this module is available
+and ``report_stats`` is enabled. This will let us see if performance changes to
+synapse are having an impact to the general community.
+
 Upgrading to v0.15.0
 ====================
 
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index c00afbba28..313be42ded 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -401,6 +401,7 @@ def run(hs):
     start_time = clock.time()
 
     stats = {}
+    stats_process = None
 
     @defer.inlineCallbacks
     def phone_stats_home():
@@ -427,6 +428,10 @@ def run(hs):
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
+        if stats_process is not None:
+            with stats_process.oneshot():
+                stats["memory_rss"] = stats_process.memory_info().rss
+                stats["cpu_average"] = int(stats_process.cpu_info(interval=None))
 
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
@@ -438,6 +443,21 @@ def run(hs):
             logger.warn("Error reporting stats: %s", e)
 
     if hs.config.report_stats:
+        try:
+            import psutil
+            stats_process = psutil.Process()
+            # Ensure we can fetch both, and make the initial request for cpu_percent
+            # so the next request will use this as the initial point.
+            stats_process.memory_info().rss
+            stats_process.cpu_percent(interval=None)
+        except (ImportError, AttributeError):
+            logger.warn(
+                    "report_stats enabled but psutil is not installed or incorrect version."
+                    " Disabling reporting of memory/cpu stats."
+                    " Ensuring psutil is available will help matrix track performance changes across releases."
+            )
+            stats_process = None
+
         logger.info("Scheduling stats reporting for 3 hour intervals")
         clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
 
-- 
cgit 1.4.1


From 9187e0762f0b4f028d15fac4502e458f513d6642 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 10:02:32 +0100
Subject: count_daily_users failed if db was sqlite due to type failure -
 presumably this prevcented all sqlite homeservers reporting home

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index de00cae447..b97e5e5ff4 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -260,7 +260,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 ) u
             """
 
-            txn.execute(sql, (yesterday,))
+            txn.execute(sql, (str(yesterday),))
             count, = txn.fetchone()
             return count
 
-- 
cgit 1.4.1


From a32d2548d986f7075e8310184ce0b70c69513a02 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 10:39:13 +0100
Subject: query and call for r30 stats

---
 synapse/app/homeserver.py   |  2 ++
 synapse/storage/__init__.py | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index c00afbba28..8bce9f1ace 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -425,6 +425,8 @@ def run(hs):
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
+        stats["r30_users"] = yield hs.get_datastore().count_r30_users()
+
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b97e5e5ff4..10f99c3cd5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -267,6 +267,42 @@ class DataStore(RoomMemberStore, RoomStore,
         ret = yield self.runInteraction("count_users", _count_users)
         defer.returnValue(ret)
 
+    @defer.inlineCallbacks
+    def count_r30_users(self):
+        """
+        Counts the number of 30 day retained users, defined as:-
+         * Users who have created their accounts more than 30 days
+         * Where last seen at most 30 days ago
+         * Where account creation and last_seen are > 30 days
+        """
+        def _count_r30_users(txn):
+            thirty_days_in_secs = 86400 * 30
+            now = int(self._clock.time_msec())
+            thirty_days_ago_in_secs = now - thirty_days_in_secs
+
+            sql = """
+                SELECT COALESCE(count(*), 0) FROM (
+                    SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
+                    FROM users, user_ips
+                    WHERE users.name = user_ips.user_id
+                    AND appservice_id is NULL
+                    AND users.creation_ts < ?
+                    AND user_ips.last_seen/1000 > ?
+                    AND (user_ips.last_seen/1000) - users.creation_ts > ?
+                    GROUP BY users.name, users.creation_ts
+                ) u
+            """
+
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs,
+                              thirty_days_in_secs))
+
+            count, = txn.fetchone()
+            return count
+
+        ret = yield self.runInteraction("count_r30_users", _count_r30_users)
+        defer.returnValue(ret)
+
     def get_users(self):
         """Function to reterive a list of users in users table.
 
-- 
cgit 1.4.1


From a9cb1a35c85f62bb0114dabd62d118c80d66e415 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 10:57:27 +0100
Subject: fix tests/storage/test_user_directory.py

---
 tests/storage/test_user_directory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 0891308f25..88add45217 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -62,7 +62,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase):
         self.assertFalse(r["limited"])
         self.assertEqual(1, len(r["results"]))
         self.assertDictEqual(r["results"][0], {
-            "user_id": BOB,
+            "d.user_id": BOB,
             "display_name": "bob",
             "avatar_url": None,
         })
-- 
cgit 1.4.1


From 01ccc9e6f25a87d7906d7907afd9e8527228215b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 28 Mar 2018 11:03:52 +0100
Subject: Measure time it takes to calculate state group ID

---
 synapse/state.py | 55 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index a7f20350f1..26093c8434 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -483,33 +483,34 @@ class StateResolutionHandler(object):
                     key: e_ids.pop() for key, e_ids in state.iteritems()
                 }
 
-            # if the new state matches any of the input state groups, we can
-            # use that state group again. Otherwise we will generate a state_id
-            # which will be used as a cache key for future resolutions, but
-            # not get persisted.
-            state_group = None
-            new_state_event_ids = frozenset(new_state.itervalues())
-            for sg, events in state_groups_ids.iteritems():
-                if new_state_event_ids == frozenset(e_id for e_id in events):
-                    state_group = sg
-                    break
-
-            # TODO: We want to create a state group for this set of events, to
-            # increase cache hits, but we need to make sure that it doesn't
-            # end up as a prev_group without being added to the database
-
-            prev_group = None
-            delta_ids = None
-            for old_group, old_ids in state_groups_ids.iteritems():
-                if not set(new_state) - set(old_ids):
-                    n_delta_ids = {
-                        k: v
-                        for k, v in new_state.iteritems()
-                        if old_ids.get(k) != v
-                    }
-                    if not delta_ids or len(n_delta_ids) < len(delta_ids):
-                        prev_group = old_group
-                        delta_ids = n_delta_ids
+            with Measure(self.clock, "state.create_group_ids"):
+                # if the new state matches any of the input state groups, we can
+                # use that state group again. Otherwise we will generate a state_id
+                # which will be used as a cache key for future resolutions, but
+                # not get persisted.
+                state_group = None
+                new_state_event_ids = frozenset(new_state.itervalues())
+                for sg, events in state_groups_ids.iteritems():
+                    if new_state_event_ids == frozenset(e_id for e_id in events):
+                        state_group = sg
+                        break
+
+                # TODO: We want to create a state group for this set of events, to
+                # increase cache hits, but we need to make sure that it doesn't
+                # end up as a prev_group without being added to the database
+
+                prev_group = None
+                delta_ids = None
+                for old_group, old_ids in state_groups_ids.iteritems():
+                    if not set(new_state) - set(old_ids):
+                        n_delta_ids = {
+                            k: v
+                            for k, v in new_state.iteritems()
+                            if old_ids.get(k) != v
+                        }
+                        if not delta_ids or len(n_delta_ids) < len(delta_ids):
+                            prev_group = old_group
+                            delta_ids = n_delta_ids
 
             cache = _StateCacheEntry(
                 state=new_state,
-- 
cgit 1.4.1


From 545001b9e4b1d6710145d3efe2117fbdf823fb38 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 11:19:45 +0100
Subject: Fix search_user_dir multiple sqlite versions do different things

---
 synapse/storage/user_directory.py    | 4 ++--
 tests/storage/test_user_directory.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index dfdcbb3181..d6e289ffbe 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -667,7 +667,7 @@ class UserDirectoryStore(SQLBaseStore):
             # The array of numbers are the weights for the various part of the
             # search: (domain, _, display name, localpart)
             sql = """
-                SELECT d.user_id, display_name, avatar_url
+                SELECT d.user_id AS user_id, display_name, avatar_url
                 FROM user_directory_search
                 INNER JOIN user_directory AS d USING (user_id)
                 %s
@@ -702,7 +702,7 @@ class UserDirectoryStore(SQLBaseStore):
             search_query = _parse_query_sqlite(search_term)
 
             sql = """
-                SELECT d.user_id, display_name, avatar_url
+                SELECT d.user_id AS user_id, display_name, avatar_url
                 FROM user_directory_search
                 INNER JOIN user_directory AS d USING (user_id)
                 %s
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 88add45217..0891308f25 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -62,7 +62,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase):
         self.assertFalse(r["limited"])
         self.assertEqual(1, len(r["results"]))
         self.assertDictEqual(r["results"][0], {
-            "d.user_id": BOB,
+            "user_id": BOB,
             "display_name": "bob",
             "avatar_url": None,
         })
-- 
cgit 1.4.1


From 0f890f477eb2ed03b8fd48710d1960210f44a334 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 11:49:57 +0100
Subject: No need to cast in count_daily_users

---
 synapse/storage/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 10f99c3cd5..ba43b2d8ec 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -250,7 +250,7 @@ class DataStore(RoomMemberStore, RoomStore,
         Counts the number of users who used this homeserver in the last 24 hours.
         """
         def _count_users(txn):
-            yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24),
+            yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24)
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
@@ -260,7 +260,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 ) u
             """
 
-            txn.execute(sql, (str(yesterday),))
+            txn.execute(sql, (yesterday,))
             count, = txn.fetchone()
             return count
 
-- 
cgit 1.4.1


From 788e69098c93f2433ef907015666c624bb39318f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 12:03:13 +0100
Subject: Add user_ips last seen index

---
 synapse/storage/client_ips.py                           |  7 +++++++
 .../schema/delta/48/add_user_ips_last_seen_index.sql    | 17 +++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql

diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index a03d1d6104..7b44dae0fc 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -48,6 +48,13 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
             columns=["user_id", "device_id", "last_seen"],
         )
 
+        self.register_background_index_update(
+            "user_ips_last_seen_index",
+            index_name="user_ips_last_seen",
+            table="user_ips",
+            columns=["user_id", "last_seen"],
+        )
+
         # (user_id, access_token, ip) -> (user_agent, device_id, last_seen)
         self._batch_row_update = {}
 
diff --git a/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql
new file mode 100644
index 0000000000..9248b0b24a
--- /dev/null
+++ b/synapse/storage/schema/delta/48/add_user_ips_last_seen_index.sql
@@ -0,0 +1,17 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT into background_updates (update_name, progress_json)
+    VALUES ('user_ips_last_seen_index', '{}');
-- 
cgit 1.4.1


From 4ceaa7433a324afab23c4a445cabe3da965e5846 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 28 Mar 2018 12:08:09 +0100
Subject: As daemonizing will make a new process, defer call to init.

---
 synapse/app/homeserver.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 313be42ded..0737945ede 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -401,7 +401,7 @@ def run(hs):
     start_time = clock.time()
 
     stats = {}
-    stats_process = None
+    stats_process = []
 
     @defer.inlineCallbacks
     def phone_stats_home():
@@ -428,10 +428,13 @@ def run(hs):
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
-        if stats_process is not None:
-            with stats_process.oneshot():
-                stats["memory_rss"] = stats_process.memory_info().rss
-                stats["cpu_average"] = int(stats_process.cpu_info(interval=None))
+        if len(stats_process) > 0:
+            stats["memory_rss"] = 0
+            stats["cpu_average"] = 0
+            for process in stats_process:
+                with process.oneshot():
+                    stats["memory_rss"] += process.memory_info().rss
+                    stats["cpu_average"] += int(process.cpu_percent(interval=None))
 
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
@@ -442,25 +445,32 @@ def run(hs):
         except Exception as e:
             logger.warn("Error reporting stats: %s", e)
 
-    if hs.config.report_stats:
+    def performance_stats_init():
         try:
             import psutil
-            stats_process = psutil.Process()
+            process = psutil.Process()
             # Ensure we can fetch both, and make the initial request for cpu_percent
             # so the next request will use this as the initial point.
-            stats_process.memory_info().rss
-            stats_process.cpu_percent(interval=None)
+            process.memory_info().rss
+            process.cpu_percent(interval=None)
+            logger.info("report_stats can use psutil")
+            stats_process.append(process)
         except (ImportError, AttributeError):
             logger.warn(
-                    "report_stats enabled but psutil is not installed or incorrect version."
-                    " Disabling reporting of memory/cpu stats."
-                    " Ensuring psutil is available will help matrix track performance changes across releases."
+                "report_stats enabled but psutil is not installed or incorrect version."
+                " Disabling reporting of memory/cpu stats."
+                " Ensuring psutil is available will help matrix track performance changes"
+                " across releases."
             )
-            stats_process = None
 
+    if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
         clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
 
+        # We need to defer this init for the cases that we daemonize
+        # otherwise the process ID we get is that of the non-daemon process
+        clock.call_later(15, performance_stats_init)
+
         # We wait 5 minutes to send the first set of stats as the server can
         # be quite busy the first few minutes
         clock.call_later(5 * 60, phone_stats_home)
-- 
cgit 1.4.1


From 792d340572026becf48fe73421f0b73cf575fe46 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 12:25:02 +0100
Subject: rename stat to future proof

---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 8bce9f1ace..286f4dcf7b 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -425,7 +425,7 @@ def run(hs):
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
-        stats["r30_users"] = yield hs.get_datastore().count_r30_users()
+        stats["r30_users_all"] = yield hs.get_datastore().count_r30_users()
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
-- 
cgit 1.4.1


From 79452edeee94a09a826ee2b41a08811b823a3ad6 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:03:37 +0100
Subject: Add joinability for groups

Adds API to set the 'joinable' flag, and corresponding flag in the
table.
---
 synapse/federation/transport/client.py | 17 +++++++++++++++++
 synapse/federation/transport/server.py | 20 ++++++++++++++++++++
 synapse/groups/groups_server.py        | 19 +++++++++++++++++++
 synapse/handlers/groups_local.py       |  3 +++
 synapse/rest/client/v2_alpha/groups.py | 28 ++++++++++++++++++++++++++++
 synapse/storage/group_server.py        | 13 +++++++++++++
 synapse/storage/prepare_database.py    |  3 ++-
 7 files changed, 102 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 5488e82985..46a797b4ba 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -856,6 +857,22 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    @log_function
+    def set_group_joinable(self, destination, group_id, requester_user_id,
+                                  content):
+        """Sets whether a group is joinable without an invite or knock
+        """
+        path = PREFIX + "/groups/%s/joinable" % (group_id,)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            args={"requester_user_id": requester_user_id},
+            data=content,
+            ignore_backoff=True,
+        )
+
+
     @log_function
     def delete_group_summary_user(self, destination, group_id, requester_user_id,
                                   user_id, role_id):
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a66a6b0692..107deb4e1e 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -1124,6 +1125,24 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
         defer.returnValue((200, resp))
 
 
+class FederationGroupsJoinableServlet(BaseFederationServlet):
+    """Sets whether a group is joinable without an invite or knock
+    """
+    PATH = "/groups/(?P<group_id>[^/]*)/joinable$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id):
+        requester_user_id = parse_string_from_args(query, "requester_user_id")
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = yield self.handler.set_group_joinable(
+            group_id, requester_user_id, content
+        )
+
+        defer.returnValue((200, new_content))
+
+
 FEDERATION_SERVLET_CLASSES = (
     FederationSendServlet,
     FederationPullServlet,
@@ -1172,6 +1191,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsSummaryUsersServlet,
     FederationGroupsAddRoomsServlet,
     FederationGroupsAddRoomsConfigServlet,
+    FederationGroupsJoinableServlet,
 )
 
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 0b995aed70..25cbfb1691 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -205,6 +206,24 @@ class GroupsServerHandler(object):
 
         defer.returnValue({})
 
+    @defer.inlineCallbacks
+    def set_group_joinable(self, group_id, requester_user_id, content):
+        """Sets whether a group is joinable without an invite or knock
+        """
+        yield self.check_group_is_ours(
+            group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
+        )
+
+        is_joinable = content.get('joinable')
+        if is_joinable is None:
+            raise SynapseError(
+                400, "No value specified for 'joinable'"
+            )
+
+        yield self.store.set_group_joinable(group_id, is_joinable=is_joinable)
+
+        defer.returnValue({})
+
     @defer.inlineCallbacks
     def get_group_categories(self, group_id, requester_user_id):
         """Get all categories in a group (as seen by user)
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index e4d0cc8b02..c9671b9046 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -90,6 +91,8 @@ class GroupsLocalHandler(object):
     get_group_role = _create_rerouter("get_group_role")
     get_group_roles = _create_rerouter("get_group_roles")
 
+    set_group_joinable = _create_rerouter("set_group_joinable")
+
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
         """Get the group summary for a group.
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index f762dbfa9a..dc8247d172 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -401,6 +402,32 @@ class GroupInvitedUsersServlet(RestServlet):
         defer.returnValue((200, result))
 
 
+class GroupJoinableServlet(RestServlet):
+    """Set whether a group is joinable without an invite
+    """
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/joinable$")
+
+    def __init__(self, hs):
+        super(GroupJoinableServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.groups_handler = hs.get_groups_local_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        requester_user_id = requester.user.to_string()
+
+        content = parse_json_object_from_request(request)
+
+        result = yield self.groups_handler.set_group_joinable(
+            group_id,
+            requester_user_id,
+            content,
+        )
+
+        defer.returnValue((200, result))
+
+
 class GroupCreateServlet(RestServlet):
     """Create a group
     """
@@ -738,6 +765,7 @@ def register_servlets(hs, http_server):
     GroupInvitedUsersServlet(hs).register(http_server)
     GroupUsersServlet(hs).register(http_server)
     GroupRoomServlet(hs).register(http_server)
+    GroupJoinableServlet(hs).register(http_server)
     GroupCreateServlet(hs).register(http_server)
     GroupAdminRoomsServlet(hs).register(http_server)
     GroupAdminRoomsConfigServlet(hs).register(http_server)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8fde1aab8e..96553d4fb1 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,6 +30,18 @@ _DEFAULT_ROLE_ID = ""
 
 
 class GroupServerStore(SQLBaseStore):
+    def set_group_joinable(self, group_id, is_joinable):
+        return self._simple_update_one(
+            table="groups",
+            keyvalues={
+                "group_id": group_id,
+            },
+            updatevalues={
+                "is_joinable": is_joinable,
+            },
+            desc="set_group_joinable",
+        )
+
     def get_group(self, group_id):
         return self._simple_select_one(
             table="groups",
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index c845a0cec5..04411a665f 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 47
+SCHEMA_VERSION = 48
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
-- 
cgit 1.4.1


From 352e1ff9ed945fd7f2655bf47d591184fc980afb Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:07:57 +0100
Subject: Add schema delta file

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/groups_joinable.sql

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
new file mode 100644
index 0000000000..fb9c7a8d1c
--- /dev/null
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -0,0 +1,16 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT 0;
-- 
cgit 1.4.1


From a1642708331ef64e38f4d2708cee9eefbc3d391e Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:23:00 +0100
Subject: Make column definition that works on both dbs

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index fb9c7a8d1c..39c8fed46c 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT 0;
+ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT (CAST(0 AS BOOLEAN));
-- 
cgit 1.4.1


From 33f6195d9ae91520aee9d108d60245b5265ac714 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 28 Mar 2018 14:25:25 +0100
Subject: Handle review comments

---
 synapse/app/homeserver.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 0737945ede..b935beb974 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -401,6 +401,9 @@ def run(hs):
     start_time = clock.time()
 
     stats = {}
+
+    # Contains the list of processes we will be monitoring
+    # currently either 0 or 1
     stats_process = []
 
     @defer.inlineCallbacks
@@ -428,13 +431,13 @@ def run(hs):
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
+
         if len(stats_process) > 0:
             stats["memory_rss"] = 0
             stats["cpu_average"] = 0
             for process in stats_process:
-                with process.oneshot():
-                    stats["memory_rss"] += process.memory_info().rss
-                    stats["cpu_average"] += int(process.cpu_percent(interval=None))
+                stats["memory_rss"] += process.memory_info().rss
+                stats["cpu_average"] += int(process.cpu_percent(interval=None))
 
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
@@ -459,8 +462,8 @@ def run(hs):
             logger.warn(
                 "report_stats enabled but psutil is not installed or incorrect version."
                 " Disabling reporting of memory/cpu stats."
-                " Ensuring psutil is available will help matrix track performance changes"
-                " across releases."
+                " Ensuring psutil is available will help matrix.org track performance"
+                " changes across releases."
             )
 
     if hs.config.report_stats:
@@ -469,7 +472,7 @@ def run(hs):
 
         # We need to defer this init for the cases that we daemonize
         # otherwise the process ID we get is that of the non-daemon process
-        clock.call_later(15, performance_stats_init)
+        clock.call_later(0, performance_stats_init)
 
         # We wait 5 minutes to send the first set of stats as the server can
         # be quite busy the first few minutes
-- 
cgit 1.4.1


From 32260baa410e1ae8200f636861a57bf2039e2cf0 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:29:42 +0100
Subject: pep8

---
 synapse/federation/transport/client.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 46a797b4ba..5a6b63350b 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -859,7 +859,7 @@ class TransportLayerClient(object):
 
     @log_function
     def set_group_joinable(self, destination, group_id, requester_user_id,
-                                  content):
+                           content):
         """Sets whether a group is joinable without an invite or knock
         """
         path = PREFIX + "/groups/%s/joinable" % (group_id,)
@@ -872,7 +872,6 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
-
     @log_function
     def delete_group_summary_user(self, destination, group_id, requester_user_id,
                                   user_id, role_id):
-- 
cgit 1.4.1


From 86932be2cb1837688d154ff78fb6418f78483133 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 14:36:53 +0100
Subject: Support multi client R30 for psql

---
 synapse/app/homeserver.py   |  4 +++-
 synapse/storage/__init__.py | 36 ++++++++++++++++++++++++++++--------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 286f4dcf7b..35e2b00f1b 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -425,7 +425,9 @@ def run(hs):
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
-        stats["r30_users_all"] = yield hs.get_datastore().count_r30_users()
+        r30_results = yield hs.get_datastore().count_r30_users()
+        for name, count in r30_results.items():
+            stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index ba43b2d8ec..b651973c79 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,6 +280,15 @@ class DataStore(RoomMemberStore, RoomStore,
             now = int(self._clock.time_msec())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
 
+            # Are these filters sufficiently robust?
+            filters = {
+                "ALL": "",
+                "IOS": "^(Vector|Riot|Riot\.im)\/.* iOS",
+                "ANDROID": "^(Dalvik|Riot|Riot\.im)\/.* Android",
+                "ELECTRON": "Electron",
+                "WEB": "(Gecko|Mozilla)",
+            }
+
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
                     SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
@@ -289,16 +298,27 @@ class DataStore(RoomMemberStore, RoomStore,
                     AND users.creation_ts < ?
                     AND user_ips.last_seen/1000 > ?
                     AND (user_ips.last_seen/1000) - users.creation_ts > ?
-                    GROUP BY users.name, users.creation_ts
-                ) u
             """
 
-            txn.execute(sql, (thirty_days_ago_in_secs,
-                              thirty_days_ago_in_secs,
-                              thirty_days_in_secs))
-
-            count, = txn.fetchone()
-            return count
+            if isinstance(self.database_engine, PostgresEngine):
+                sql = sql + "AND user_ips.user_agent ~ ? "
+            sql = sql + "GROUP BY users.name, users.creation_ts ) u"
+
+            results = {}
+            if isinstance(self.database_engine, PostgresEngine):
+                for filter_name, user_agent_filter in filters.items():
+                    txn.execute(sql, (thirty_days_ago_in_secs,
+                                      thirty_days_ago_in_secs,
+                                      thirty_days_in_secs,
+                                      user_agent_filter))
+                    results[filter_name], = txn.fetchone()
+
+            else:
+                txn.execute(sql, (thirty_days_ago_in_secs,
+                                  thirty_days_ago_in_secs,
+                                  thirty_days_in_secs))
+                results["ALL"], = txn.fetchone()
+            return results
 
         ret = yield self.runInteraction("count_r30_users", _count_r30_users)
         defer.returnValue(ret)
-- 
cgit 1.4.1


From 4262aba17b643bc82c5cce92298dac0a27b2727c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 14:40:03 +0100
Subject: bump schema version

---
 synapse/storage/prepare_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index c845a0cec5..68675e15d2 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 47
+SCHEMA_VERSION = 48
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
-- 
cgit 1.4.1


From a838444a70195588de55a514524c4af720099177 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:50:30 +0100
Subject: Grr. Copy the definition from is_admin

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 39c8fed46c..9e106e909c 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOLEAN NOT NULL DEFAULT (CAST(0 AS BOOLEAN));
+ALTER TABLE groups ADD COLUMN is_joinable BOOL DEFAULT 0 NOT NULL;
-- 
cgit 1.4.1


From 929b34963d320f571512453dac980ef235914956 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 14:53:55 +0100
Subject: OK, smallint it is then

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 9e106e909c..ace7d0a723 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,4 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable BOOL DEFAULT 0 NOT NULL;
+ALTER TABLE groups ADD COLUMN is_joinable SMALLINT DEFAULT 0 NOT NULL;
-- 
cgit 1.4.1


From 241e4e86873d5880f564791e3768247fa55c3fa8 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 16:25:53 +0100
Subject: remove twisted deferral cruft

---
 synapse/storage/__init__.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b651973c79..b2b85e266d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -244,7 +244,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return [UserPresenceState(**row) for row in rows]
 
-    @defer.inlineCallbacks
     def count_daily_users(self):
         """
         Counts the number of users who used this homeserver in the last 24 hours.
@@ -264,10 +263,9 @@ class DataStore(RoomMemberStore, RoomStore,
             count, = txn.fetchone()
             return count
 
-        ret = yield self.runInteraction("count_users", _count_users)
-        defer.returnValue(ret)
+        return self.runInteraction("count_users", _count_users)
+
 
-    @defer.inlineCallbacks
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
@@ -320,8 +318,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 results["ALL"], = txn.fetchone()
             return results
 
-        ret = yield self.runInteraction("count_r30_users", _count_r30_users)
-        defer.returnValue(ret)
+        return self.runInteraction("count_r30_users", _count_r30_users)
 
     def get_users(self):
         """Function to reterive a list of users in users table.
-- 
cgit 1.4.1


From c5de6987c210cce906cf279d85cbd98cd14bfc52 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 16:44:11 +0100
Subject: This should probably be a PUT

---
 synapse/rest/client/v2_alpha/groups.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index dc8247d172..aa94130e57 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -413,7 +413,7 @@ class GroupJoinableServlet(RestServlet):
         self.groups_handler = hs.get_groups_local_handler()
 
     @defer.inlineCallbacks
-    def on_POST(self, request, group_id):
+    def on_PUT(self, request, group_id):
         requester = yield self.auth.get_user_by_req(request)
         requester_user_id = requester.user.to_string()
 
-- 
cgit 1.4.1


From 16aeb415478f32185679ba5ff6c89d2cd3242861 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 28 Mar 2018 16:47:56 +0100
Subject: Update README.rst

update docker hub url
---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index ef27b92ddd..8812cc1b4f 100644
--- a/README.rst
+++ b/README.rst
@@ -157,8 +157,8 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-Alternatively, Silvio Fricke has contributed a Dockerfile to automate the
-above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/.
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate the
+above in Docker at https://hub.docker.com/r/avhost/docker-matrix/tags/
 
 Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
 tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
-- 
cgit 1.4.1


From dc7c020b33dc9606089fa66fdec2dacb7f807f6d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 28 Mar 2018 17:25:15 +0100
Subject:  fix pep8 errors

---
 synapse/storage/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b2b85e266d..70c6171404 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 from synapse.storage.devices import DeviceStore
 from .appservice import (
     ApplicationServiceStore, ApplicationServiceTransactionStore
@@ -265,7 +263,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return self.runInteraction("count_users", _count_users)
 
-
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
-- 
cgit 1.4.1


From 9ee44a372d4fcf6a461b610230a285610613e8ac Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 29 Mar 2018 16:45:34 +0100
Subject: Remove need for sqlite specific query

---
 synapse/storage/__init__.py | 87 +++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 30 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 70c6171404..0b4693041f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -269,50 +269,77 @@ class DataStore(RoomMemberStore, RoomStore,
          * Users who have created their accounts more than 30 days
          * Where last seen at most 30 days ago
          * Where account creation and last_seen are > 30 days
+
+         Returns counts globaly for a given user as well as breaking
+         by platform
         """
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time_msec())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
 
-            # Are these filters sufficiently robust?
-            filters = {
-                "ALL": "",
-                "IOS": "^(Vector|Riot|Riot\.im)\/.* iOS",
-                "ANDROID": "^(Dalvik|Riot|Riot\.im)\/.* Android",
-                "ELECTRON": "Electron",
-                "WEB": "(Gecko|Mozilla)",
-            }
+            sql = """
+                SELECT platform, COALESCE(count(*), 0) FROM (
+                     SELECT users.name, platform, users.creation_ts * 1000, MAX(uip.last_seen)
+                     FROM users
+                     INNER JOIN (
+                         SELECT
+                         user_id,
+                         last_seen,
+                         CASE
+                             WHEN user_agent LIKE '%Android%' THEN 'android'
+                             WHEN user_agent LIKE '%iOS%' THEN 'ios'
+                             WHEN user_agent LIKE '%Electron%' THEN 'electron'
+                             WHEN user_agent LIKE '%Mozilla%' THEN 'web'
+                             WHEN user_agent LIKE '%Gecko%' THEN 'web'
+                             ELSE 'unknown'
+                         END
+                         AS platform
+                         FROM user_ips
+                     ) uip
+                     ON users.name = uip.user_id
+                     AND users.appservice_id is NULL
+                     AND users.creation_ts < ?
+                     AND uip.last_seen/1000 > ?
+                     AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
+                     GROUP BY users.name, platform, users.creation_ts
+                ) u GROUP BY platform
+            """
+
+            results = {}
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs))
+            rows = txn.fetchall()
+            for row in rows:
+                if row[0] is 'unknown':
+                    pass
+                results[row[0]] = row[1]
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
-                    SELECT users.name, users.creation_ts * 1000, MAX(user_ips.last_seen)
-                    FROM users, user_ips
-                    WHERE users.name = user_ips.user_id
+                    SELECT users.name, users.creation_ts * 1000, MAX(uip.last_seen)
+                    FROM users
+                    INNER JOIN (
+                        SELECT
+                        user_id,
+                        last_seen
+                        FROM user_ips
+                    ) uip
+                    ON users.name = uip.user_id
                     AND appservice_id is NULL
                     AND users.creation_ts < ?
-                    AND user_ips.last_seen/1000 > ?
-                    AND (user_ips.last_seen/1000) - users.creation_ts > ?
+                    AND uip.last_seen/1000 > ?
+                    AND (uip.last_seen/1000) - users.creation_ts > 86400 * 30
+                    GROUP BY users.name, users.creation_ts
+                ) u
             """
 
-            if isinstance(self.database_engine, PostgresEngine):
-                sql = sql + "AND user_ips.user_agent ~ ? "
-            sql = sql + "GROUP BY users.name, users.creation_ts ) u"
+            txn.execute(sql, (thirty_days_ago_in_secs,
+                              thirty_days_ago_in_secs))
+
+            count, = txn.fetchone()
+            results['all'] = count
 
-            results = {}
-            if isinstance(self.database_engine, PostgresEngine):
-                for filter_name, user_agent_filter in filters.items():
-                    txn.execute(sql, (thirty_days_ago_in_secs,
-                                      thirty_days_ago_in_secs,
-                                      thirty_days_in_secs,
-                                      user_agent_filter))
-                    results[filter_name], = txn.fetchone()
-
-            else:
-                txn.execute(sql, (thirty_days_ago_in_secs,
-                                  thirty_days_ago_in_secs,
-                                  thirty_days_in_secs))
-                results["ALL"], = txn.fetchone()
             return results
 
         return self.runInteraction("count_r30_users", _count_r30_users)
-- 
cgit 1.4.1


From b4e37c6f50b91dd0ea90c773185884659e3a738a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 29 Mar 2018 17:27:39 +0100
Subject: pep8

---
 synapse/storage/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 0b4693041f..f68e436df0 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,7 +280,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-                     SELECT users.name, platform, users.creation_ts * 1000, MAX(uip.last_seen)
+                     SELECT users.name, platform, users.creation_ts * 1000,
+                                                            MAX(uip.last_seen)
                      FROM users
                      INNER JOIN (
                          SELECT
@@ -317,7 +318,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT COALESCE(count(*), 0) FROM (
-                    SELECT users.name, users.creation_ts * 1000, MAX(uip.last_seen)
+                    SELECT users.name, users.creation_ts * 1000,
+                                                        MAX(uip.last_seen)
                     FROM users
                     INNER JOIN (
                         SELECT
-- 
cgit 1.4.1


From fcfe7f6ad3a2a9c285ac96008395fc47e096ff4b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 22:45:52 +0100
Subject: Use simplejson throughout

Let's use simplejson rather than json, for consistency.
---
 synapse/api/errors.py                                    | 3 ++-
 synapse/handlers/identity.py                             | 8 +++++---
 synapse/storage/schema/delta/14/upgrade_appservice_db.py | 3 ++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index aa15f73f36..bee59e80dd 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -15,9 +15,10 @@
 
 """Contains exceptions and error codes."""
 
-import json
 import logging
 
+import simplejson as json
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 9efcdff1d6..91a0898860 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -15,6 +15,11 @@
 # limitations under the License.
 
 """Utilities for interacting with Identity Servers"""
+
+import logging
+
+import simplejson as json
+
 from twisted.internet import defer
 
 from synapse.api.errors import (
@@ -24,9 +29,6 @@ from ._base import BaseHandler
 from synapse.util.async import run_on_reactor
 from synapse.api.errors import SynapseError, Codes
 
-import json
-import logging
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
index 8755bb2e49..4d725b92fe 100644
--- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py
+++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import logging
 
+import simplejson as json
+
 logger = logging.getLogger(__name__)
 
 
-- 
cgit 1.4.1


From 05630758f25d958bf60fde4df5f80a89e4a9a0ac Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 22:57:28 +0100
Subject: Use static JSONEncoders

using json.dumps with custom options requires us to create a new JSONEncoder on
each call. It's more efficient to create one upfront and reuse it.
---
 synapse/handlers/message.py         |  4 ++--
 synapse/replication/tcp/commands.py |  8 +++++---
 synapse/storage/events.py           | 23 ++++++++---------------
 synapse/util/frozenutils.py         | 19 +++++++++++++++++++
 4 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 5a8ddc253e..6de6e13b7b 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -27,7 +27,7 @@ from synapse.types import (
 from synapse.util.async import run_on_reactor, ReadWriteLock, Limiter
 from synapse.util.logcontext import preserve_fn, run_in_background
 from synapse.util.metrics import measure_func
-from synapse.util.frozenutils import unfreeze
+from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.stringutils import random_string
 from synapse.visibility import filter_events_for_client
 from synapse.replication.http.send_event import send_event_to_master
@@ -678,7 +678,7 @@ class EventCreationHandler(object):
 
         # Ensure that we can round trip before trying to persist in db
         try:
-            dump = simplejson.dumps(unfreeze(event.content))
+            dump = frozendict_json_encoder.encode(event.content)
             simplejson.loads(dump)
         except Exception:
             logger.exception("Failed to encode content: %r", event.content)
diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 0005ad5879..34bcf903a3 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -24,6 +24,8 @@ import simplejson
 
 logger = logging.getLogger(__name__)
 
+_json_encoder = simplejson.JSONEncoder(namedtuple_as_object=False)
+
 
 class Command(object):
     """The base command class.
@@ -107,7 +109,7 @@ class RdataCommand(Command):
         return " ".join((
             self.stream_name,
             str(self.token) if self.token is not None else "batch",
-            simplejson.dumps(self.row, namedtuple_as_object=False),
+            _json_encoder.dumps(self.row),
         ))
 
 
@@ -302,7 +304,7 @@ class InvalidateCacheCommand(Command):
 
     def to_line(self):
         return " ".join((
-            self.cache_func, simplejson.dumps(self.keys, namedtuple_as_object=False)
+            self.cache_func, _json_encoder.encode(self.keys),
         ))
 
 
@@ -334,7 +336,7 @@ class UserIpCommand(Command):
         )
 
     def to_line(self):
-        return self.user_id + " " + simplejson.dumps((
+        return self.user_id + " " + _json_encoder.encode((
             self.access_token, self.ip, self.user_agent, self.device_id,
             self.last_seen,
         ))
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f3d65f4338..ece5e6c41f 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -14,15 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.events_worker import EventsWorkerStore
+from collections import OrderedDict, deque, namedtuple
+from functools import wraps
+import logging
 
+import simplejson as json
 from twisted.internet import defer
 
-from synapse.events import USE_FROZEN_DICTS
 
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.util.async import ObservableDeferred
+from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import (
-    PreserveLoggingContext, make_deferred_yieldable
+    PreserveLoggingContext, make_deferred_yieldable,
 )
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
@@ -30,16 +34,8 @@ from synapse.api.constants import EventTypes
 from synapse.api.errors import SynapseError
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.types import get_domain_from_id
-
-from canonicaljson import encode_canonical_json
-from collections import deque, namedtuple, OrderedDict
-from functools import wraps
-
 import synapse.metrics
 
-import logging
-import simplejson as json
-
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
 from synapse.events.snapshot import EventContext   # noqa: F401
@@ -71,10 +67,7 @@ state_delta_reuse_delta_counter = metrics.register_counter(
 
 
 def encode_json(json_object):
-    if USE_FROZEN_DICTS:
-        return encode_canonical_json(json_object)
-    else:
-        return json.dumps(json_object, ensure_ascii=False)
+    return frozendict_json_encoder.encode(json_object)
 
 
 class _EventPeristenceQueue(object):
diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py
index 6322f0f55c..f497b51f4a 100644
--- a/synapse/util/frozenutils.py
+++ b/synapse/util/frozenutils.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from frozendict import frozendict
+import simplejson as json
 
 
 def freeze(o):
@@ -49,3 +50,21 @@ def unfreeze(o):
         pass
 
     return o
+
+
+def _handle_frozendict(obj):
+    """Helper for EventEncoder. Makes frozendicts serializable by returning
+    the underlying dict
+    """
+    if type(obj) is frozendict:
+        # fishing the protected dict out of the object is a bit nasty,
+        # but we don't really want the overhead of copying the dict.
+        return obj._dict
+    raise TypeError('Object of type %s is not JSON serializable' %
+                    obj.__class__.__name__)
+
+
+# A JSONEncoder which is capable of encoding frozendics without barfing
+frozendict_json_encoder = json.JSONEncoder(
+    default=_handle_frozendict,
+)
-- 
cgit 1.4.1


From 2fe3f848b92ee9493a724935167fad84678a7eb2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 29 Mar 2018 23:05:33 +0100
Subject: Remove uses of events.content

---
 synapse/storage/room.py       | 7 ++++---
 synapse/storage/roommember.py | 6 ++++--
 synapse/storage/search.py     | 6 ++++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 908551d6d9..740c036975 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -594,7 +594,8 @@ class RoomStore(RoomWorkerStore, SearchStore):
 
         while next_token:
             sql = """
-                SELECT stream_ordering, content FROM events
+                SELECT stream_ordering, json FROM events
+                JOIN event_json USING (event_id)
                 WHERE room_id = ?
                     AND stream_ordering < ?
                     AND contains_url = ? AND outlier = ?
@@ -606,8 +607,8 @@ class RoomStore(RoomWorkerStore, SearchStore):
             next_token = None
             for stream_ordering, content_json in txn:
                 next_token = stream_ordering
-                content = json.loads(content_json)
-
+                event_json = json.loads(content_json)
+                content = event_json["content"]
                 content_url = content.get("url")
                 thumbnail_url = content.get("info", {}).get("thumbnail_url")
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index d662d1cfc0..6a861943a2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -645,8 +645,9 @@ class RoomMemberStore(RoomMemberWorkerStore):
 
         def add_membership_profile_txn(txn):
             sql = ("""
-                SELECT stream_ordering, event_id, events.room_id, content
+                SELECT stream_ordering, event_id, events.room_id, event_json.json
                 FROM events
+                INNER JOIN event_json USING (event_id)
                 INNER JOIN room_memberships USING (event_id)
                 WHERE ? <= stream_ordering AND stream_ordering < ?
                 AND type = 'm.room.member'
@@ -667,7 +668,8 @@ class RoomMemberStore(RoomMemberWorkerStore):
                 event_id = row["event_id"]
                 room_id = row["room_id"]
                 try:
-                    content = json.loads(row["content"])
+                    event_json = json.loads(row["json"])
+                    content = event_json['content']
                 except Exception:
                     continue
 
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 984643b057..426cbe6e1a 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -75,8 +75,9 @@ class SearchStore(BackgroundUpdateStore):
 
         def reindex_search_txn(txn):
             sql = (
-                "SELECT stream_ordering, event_id, room_id, type, content, "
+                "SELECT stream_ordering, event_id, room_id, type, json, "
                 " origin_server_ts FROM events"
+                " JOIN event_json USING (event_id)"
                 " WHERE ? <= stream_ordering AND stream_ordering < ?"
                 " AND (%s)"
                 " ORDER BY stream_ordering DESC"
@@ -104,7 +105,8 @@ class SearchStore(BackgroundUpdateStore):
                     stream_ordering = row["stream_ordering"]
                     origin_server_ts = row["origin_server_ts"]
                     try:
-                        content = json.loads(row["content"])
+                        event_json = json.loads(row["json"])
+                        content = event_json["content"]
                     except Exception:
                         continue
 
-- 
cgit 1.4.1


From 11597ddea5c43fdd2c6593b6bf4619a7bbdf3122 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Fri, 30 Mar 2018 23:59:02 +0200
Subject: improve mxid check performance ~4x

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/types.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/types.py b/synapse/types.py
index 7cb24cecb2..f1f41ccf90 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -12,11 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import string
 
 from synapse.api.errors import SynapseError
 
 from collections import namedtuple
+import re
 
 
 class Requester(namedtuple("Requester", [
@@ -214,7 +214,8 @@ class GroupID(DomainSpecificString):
         return group_id
 
 
-mxid_localpart_allowed_characters = set("_-./=" + string.ascii_lowercase + string.digits)
+# A regex that matches any valid mxid characters
+MXID_LOCALPART_REGEX = re.compile("^[_\-./=a-z0-9]*$")
 
 
 def contains_invalid_mxid_characters(localpart):
@@ -226,7 +227,7 @@ def contains_invalid_mxid_characters(localpart):
     Returns:
         bool: True if there are any naughty characters
     """
-    return any(c not in mxid_localpart_allowed_characters for c in localpart)
+    return not MXID_LOCALPART_REGEX.match(localpart)
 
 
 class StreamToken(
-- 
cgit 1.4.1


From 3ee4ad09eb9bcd0214da83b66214afa3ddb08116 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 3 Apr 2018 15:09:48 +0100
Subject: Fix json encoding bug in replication

json encoders have an encode method, not a dumps method.
---
 synapse/replication/tcp/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 34bcf903a3..12aac3cc6b 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -109,7 +109,7 @@ class RdataCommand(Command):
         return " ".join((
             self.stream_name,
             str(self.token) if self.token is not None else "batch",
-            _json_encoder.dumps(self.row),
+            _json_encoder.encode(self.row),
         ))
 
 
-- 
cgit 1.4.1


From eb8d8d6f57c7f6017548aa95409bb8cc346a5ae0 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Tue, 3 Apr 2018 15:40:43 +0100
Subject: Use join_policy API instead of joinable

The API is now under
 /groups/$group_id/setting/m.join_policy

and expects a JSON blob of the shape

```json
{
  "m.join_policy": {
    "type": "invite"
  }
}
```

where "invite" could alternatively be "open".
---
 synapse/federation/transport/client.py             |  4 +--
 synapse/federation/transport/server.py             |  8 ++---
 synapse/groups/groups_server.py                    | 41 ++++++++++++++++++----
 synapse/handlers/groups_local.py                   |  2 +-
 synapse/rest/client/v2_alpha/groups.py             | 12 +++----
 synapse/storage/group_server.py                    |  6 ++--
 .../storage/schema/delta/48/groups_joinable.sql    |  8 ++++-
 7 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 5a6b63350b..0f7f656824 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -860,9 +860,9 @@ class TransportLayerClient(object):
     @log_function
     def set_group_joinable(self, destination, group_id, requester_user_id,
                            content):
-        """Sets whether a group is joinable without an invite or knock
+        """Sets the join policy for a group
         """
-        path = PREFIX + "/groups/%s/joinable" % (group_id,)
+        path = PREFIX + "/groups/%s/setting/m.join_policy" % (group_id,)
 
         return self.client.post_json(
             destination=destination,
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 107deb4e1e..a52d3948f4 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1125,10 +1125,10 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
         defer.returnValue((200, resp))
 
 
-class FederationGroupsJoinableServlet(BaseFederationServlet):
+class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
     """Sets whether a group is joinable without an invite or knock
     """
-    PATH = "/groups/(?P<group_id>[^/]*)/joinable$"
+    PATH = "/groups/(?P<group_id>[^/]*)/setting/m.join_policy$"
 
     @defer.inlineCallbacks
     def on_POST(self, origin, content, query, group_id):
@@ -1136,7 +1136,7 @@ class FederationGroupsJoinableServlet(BaseFederationServlet):
         if get_domain_from_id(requester_user_id) != origin:
             raise SynapseError(403, "requester_user_id doesn't match origin")
 
-        new_content = yield self.handler.set_group_joinable(
+        new_content = yield self.handler.set_group_join_policy(
             group_id, requester_user_id, content
         )
 
@@ -1191,7 +1191,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsSummaryUsersServlet,
     FederationGroupsAddRoomsServlet,
     FederationGroupsAddRoomsConfigServlet,
-    FederationGroupsJoinableServlet,
+    FederationGroupsSettingJoinPolicyServlet,
 )
 
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 25cbfb1691..70781e1854 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -207,20 +207,24 @@ class GroupsServerHandler(object):
         defer.returnValue({})
 
     @defer.inlineCallbacks
-    def set_group_joinable(self, group_id, requester_user_id, content):
-        """Sets whether a group is joinable without an invite or knock
+    def set_group_join_policy(self, group_id, requester_user_id, content):
+        """Sets the group join policy.
+
+        Currently supported policies are:
+         - "invite": an invite must be received and accepted in order to join.
+         - "open": anyone can join.
         """
         yield self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
 
-        is_joinable = content.get('joinable')
-        if is_joinable is None:
+        join_policy = _parse_join_policy_from_contents(content)
+        if join_policy is None:
             raise SynapseError(
-                400, "No value specified for 'joinable'"
+                400, "No value specified for 'm.join_policy'"
             )
 
-        yield self.store.set_group_joinable(group_id, is_joinable=is_joinable)
+        yield self.store.set_group_join_policy(group_id, join_policy=join_policy)
 
         defer.returnValue({})
 
@@ -854,6 +858,31 @@ class GroupsServerHandler(object):
         })
 
 
+def _parse_join_policy_from_contents(content):
+    """Given a content for a request, return the specified join policy or None
+    """
+
+    join_policy_dict = content.get("m.join_policy")
+    if join_policy_dict:
+        return _parse_join_policy_dict(join_policy_dict)
+    else:
+        return None
+
+
+def _parse_join_policy_dict(join_policy_dict):
+    """Given a dict for the "m.join_policy" config return the join policy specified
+    """
+    join_policy_type = join_policy_dict.get("type")
+    if not join_policy_type:
+        return True
+
+    if join_policy_type not in ("invite", "open"):
+        raise SynapseError(
+            400, "Synapse only supports 'invite'/'open' join rule"
+        )
+    return join_policy_type
+
+
 def _parse_visibility_from_contents(content):
     """Given a content for a request parse out whether the entity should be
     public or not
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index c9671b9046..5f7b0ff305 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -91,7 +91,7 @@ class GroupsLocalHandler(object):
     get_group_role = _create_rerouter("get_group_role")
     get_group_roles = _create_rerouter("get_group_roles")
 
-    set_group_joinable = _create_rerouter("set_group_joinable")
+    set_group_join_policy = _create_rerouter("set_group_join_policy")
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index aa94130e57..8faaa1d6a0 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -402,13 +402,13 @@ class GroupInvitedUsersServlet(RestServlet):
         defer.returnValue((200, result))
 
 
-class GroupJoinableServlet(RestServlet):
-    """Set whether a group is joinable without an invite
+class GroupSettingJoinPolicyServlet(RestServlet):
+    """Set group join policy
     """
-    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/joinable$")
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/setting/m.join_policy$")
 
     def __init__(self, hs):
-        super(GroupJoinableServlet, self).__init__()
+        super(GroupSettingJoinPolicyServlet, self).__init__()
         self.auth = hs.get_auth()
         self.groups_handler = hs.get_groups_local_handler()
 
@@ -419,7 +419,7 @@ class GroupJoinableServlet(RestServlet):
 
         content = parse_json_object_from_request(request)
 
-        result = yield self.groups_handler.set_group_joinable(
+        result = yield self.groups_handler.set_group_join_policy(
             group_id,
             requester_user_id,
             content,
@@ -765,7 +765,7 @@ def register_servlets(hs, http_server):
     GroupInvitedUsersServlet(hs).register(http_server)
     GroupUsersServlet(hs).register(http_server)
     GroupRoomServlet(hs).register(http_server)
-    GroupJoinableServlet(hs).register(http_server)
+    GroupSettingJoinPolicyServlet(hs).register(http_server)
     GroupCreateServlet(hs).register(http_server)
     GroupAdminRoomsServlet(hs).register(http_server)
     GroupAdminRoomsConfigServlet(hs).register(http_server)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 96553d4fb1..db66ea1eb0 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -30,16 +30,16 @@ _DEFAULT_ROLE_ID = ""
 
 
 class GroupServerStore(SQLBaseStore):
-    def set_group_joinable(self, group_id, is_joinable):
+    def set_group_join_policy(self, group_id, join_policy):
         return self._simple_update_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
             },
             updatevalues={
-                "is_joinable": is_joinable,
+                "join_policy": join_policy,
             },
-            desc="set_group_joinable",
+            desc="set_group_join_policy",
         )
 
     def get_group(self, group_id):
diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index ace7d0a723..ab3b00286d 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -13,4 +13,10 @@
  * limitations under the License.
  */
 
-ALTER TABLE groups ADD COLUMN is_joinable SMALLINT DEFAULT 0 NOT NULL;
+/* 
+ * This isn't a real ENUM because sqlite doesn't support it
+ * and we use a default of NULL for inserted rows and interpret
+ * NULL at the python store level as necessary so that existing
+ * rows are given the correct default policy.
+ */
+ALTER TABLE groups ADD COLUMN join_policy TEXT DEFAULT NULL;
-- 
cgit 1.4.1


From a9a74101a4925bd208db682952b5dadf4b157a8d Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 4 Apr 2018 08:58:53 +0100
Subject: Document the behaviour of ResponseCache

it looks like everything that uses ResponseCache expects to have to
`make_deferred_yieldable` its results. It's debatable whether that is the best
approach, but let's document it for now to avoid further confusion.
---
 synapse/util/caches/response_cache.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 00af539880..4ecd91deb5 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -31,6 +31,18 @@ class ResponseCache(object):
         self.timeout_sec = timeout_ms / 1000.
 
     def get(self, key):
+        """Look up the given key.
+
+        Returns a deferred which doesn't follow the synapse logcontext rules,
+        so you'll probably want to make_deferred_yieldable it.
+
+        Args:
+            key (str):
+
+        Returns:
+            twisted.internet.defer.Deferred|None: None if there is no entry
+            for this key; otherwise a deferred result.
+        """
         result = self.pending_result_cache.get(key)
         if result is not None:
             return result.observe()
@@ -38,6 +50,26 @@ class ResponseCache(object):
             return None
 
     def set(self, key, deferred):
+        """Set the entry for the given key to the given deferred.
+
+        *deferred* should run its callbacks in the sentinel logcontext (ie,
+        you should wrap normal synapse deferreds with
+        logcontext.run_in_background).
+
+        Returns a new Deferred which also doesn't follow the synapse logcontext
+        rules, so you will want to make_deferred_yieldable it
+
+        (TODO: before using this more widely, it might make sense to refactor
+        it and get() so that they do the necessary wrapping rather than having
+        to do it everywhere ResponseCache is used.)
+
+        Args:
+            key (str):
+            deferred (twisted.internet.defer.Deferred):
+
+        Returns:
+            twisted.internet.defer.Deferred
+        """
         result = ObservableDeferred(deferred, consumeErrors=True)
         self.pending_result_cache[key] = result
 
-- 
cgit 1.4.1


From 72251d1b979db0bc96e5d95ac70b8e1cd78cde7c Mon Sep 17 00:00:00 2001
From: Silke <silke@slxh.eu>
Date: Tue, 20 Mar 2018 10:40:16 +0100
Subject: Remove address resolution of hosts in SRV records

Signed-off-by: Silke Hofstra <silke@slxh.eu>
---
 synapse/http/endpoint.py | 103 ++++-------------------------------------------
 tests/test_dns.py        |  29 +------------
 2 files changed, 10 insertions(+), 122 deletions(-)

diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index 87639b9151..00572c2897 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -12,8 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import socket
-
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet import defer, reactor
 from twisted.internet.error import ConnectError
@@ -33,7 +31,7 @@ SERVER_CACHE = {}
 
 # our record of an individual server which can be tried to reach a destination.
 #
-# "host" is actually a dotted-quad or ipv6 address string. Except when there's
+# "host" is the hostname acquired from the SRV record. Except when there's
 # no SRV record, in which case it is the original hostname.
 _Server = collections.namedtuple(
     "_Server", "priority weight host port expires"
@@ -297,20 +295,13 @@ def resolve_service(service_name, dns_client=client, cache=SERVER_CACHE, clock=t
 
             payload = answer.payload
 
-            hosts = yield _get_hosts_for_srv_record(
-                dns_client, str(payload.target)
-            )
-
-            for (ip, ttl) in hosts:
-                host_ttl = min(answer.ttl, ttl)
-
-                servers.append(_Server(
-                    host=ip,
-                    port=int(payload.port),
-                    priority=int(payload.priority),
-                    weight=int(payload.weight),
-                    expires=int(clock.time()) + host_ttl,
-                ))
+            servers.append(_Server(
+                host=str(payload.target),
+                port=int(payload.port),
+                priority=int(payload.priority),
+                weight=int(payload.weight),
+                expires=int(clock.time()) + answer.ttl,
+            ))
 
         servers.sort()
         cache[service_name] = list(servers)
@@ -328,81 +319,3 @@ def resolve_service(service_name, dns_client=client, cache=SERVER_CACHE, clock=t
             raise e
 
     defer.returnValue(servers)
-
-
-@defer.inlineCallbacks
-def _get_hosts_for_srv_record(dns_client, host):
-    """Look up each of the hosts in a SRV record
-
-    Args:
-        dns_client (twisted.names.dns.IResolver):
-        host (basestring): host to look up
-
-    Returns:
-        Deferred[list[(str, int)]]: a list of (host, ttl) pairs
-
-    """
-    ip4_servers = []
-    ip6_servers = []
-
-    def cb(res):
-        # lookupAddress and lookupIP6Address return a three-tuple
-        # giving the answer, authority, and additional sections of the
-        # response.
-        #
-        # we only care about the answers.
-
-        return res[0]
-
-    def eb(res, record_type):
-        if res.check(DNSNameError):
-            return []
-        logger.warn("Error looking up %s for %s: %s", record_type, host, res)
-        return res
-
-    # no logcontexts here, so we can safely fire these off and gatherResults
-    d1 = dns_client.lookupAddress(host).addCallbacks(
-        cb, eb, errbackArgs=("A", ))
-    d2 = dns_client.lookupIPV6Address(host).addCallbacks(
-        cb, eb, errbackArgs=("AAAA", ))
-    results = yield defer.DeferredList(
-        [d1, d2], consumeErrors=True)
-
-    # if all of the lookups failed, raise an exception rather than blowing out
-    # the cache with an empty result.
-    if results and all(s == defer.FAILURE for (s, _) in results):
-        defer.returnValue(results[0][1])
-
-    for (success, result) in results:
-        if success == defer.FAILURE:
-            continue
-
-        for answer in result:
-            if not answer.payload:
-                continue
-
-            try:
-                if answer.type == dns.A:
-                    ip = answer.payload.dottedQuad()
-                    ip4_servers.append((ip, answer.ttl))
-                elif answer.type == dns.AAAA:
-                    ip = socket.inet_ntop(
-                        socket.AF_INET6, answer.payload.address,
-                    )
-                    ip6_servers.append((ip, answer.ttl))
-                else:
-                    # the most likely candidate here is a CNAME record.
-                    # rfc2782 says srvs may not point to aliases.
-                    logger.warn(
-                        "Ignoring unexpected DNS record type %s for %s",
-                        answer.type, host,
-                    )
-                    continue
-            except Exception as e:
-                logger.warn("Ignoring invalid DNS response for %s: %s",
-                            host, e)
-                continue
-
-    # keep the ipv4 results before the ipv6 results, mostly to match historical
-    # behaviour.
-    defer.returnValue(ip4_servers + ip6_servers)
diff --git a/tests/test_dns.py b/tests/test_dns.py
index d08b0f4333..af607d626f 100644
--- a/tests/test_dns.py
+++ b/tests/test_dns.py
@@ -33,8 +33,6 @@ class DnsTestCase(unittest.TestCase):
 
         service_name = "test_service.example.com"
         host_name = "example.com"
-        ip_address = "127.0.0.1"
-        ip6_address = "::1"
 
         answer_srv = dns.RRHeader(
             type=dns.SRV,
@@ -43,29 +41,9 @@ class DnsTestCase(unittest.TestCase):
             )
         )
 
-        answer_a = dns.RRHeader(
-            type=dns.A,
-            payload=dns.Record_A(
-                address=ip_address,
-            )
-        )
-
-        answer_aaaa = dns.RRHeader(
-            type=dns.AAAA,
-            payload=dns.Record_AAAA(
-                address=ip6_address,
-            )
-        )
-
         dns_client_mock.lookupService.return_value = defer.succeed(
             ([answer_srv], None, None),
         )
-        dns_client_mock.lookupAddress.return_value = defer.succeed(
-            ([answer_a], None, None),
-        )
-        dns_client_mock.lookupIPV6Address.return_value = defer.succeed(
-            ([answer_aaaa], None, None),
-        )
 
         cache = {}
 
@@ -74,13 +52,10 @@ class DnsTestCase(unittest.TestCase):
         )
 
         dns_client_mock.lookupService.assert_called_once_with(service_name)
-        dns_client_mock.lookupAddress.assert_called_once_with(host_name)
-        dns_client_mock.lookupIPV6Address.assert_called_once_with(host_name)
 
-        self.assertEquals(len(servers), 2)
+        self.assertEquals(len(servers), 1)
         self.assertEquals(servers, cache[service_name])
-        self.assertEquals(servers[0].host, ip_address)
-        self.assertEquals(servers[1].host, ip6_address)
+        self.assertEquals(servers[0].host, host_name)
 
     @defer.inlineCallbacks
     def test_from_cache_expired_and_dns_fail(self):
-- 
cgit 1.4.1


From f92963f5db236c1afb2a489a44c9afdae7d61edc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 4 Apr 2018 12:08:29 +0100
Subject: Revert "improve mxid check performance"

---
 synapse/types.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/types.py b/synapse/types.py
index f1f41ccf90..7cb24cecb2 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -12,11 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import string
 
 from synapse.api.errors import SynapseError
 
 from collections import namedtuple
-import re
 
 
 class Requester(namedtuple("Requester", [
@@ -214,8 +214,7 @@ class GroupID(DomainSpecificString):
         return group_id
 
 
-# A regex that matches any valid mxid characters
-MXID_LOCALPART_REGEX = re.compile("^[_\-./=a-z0-9]*$")
+mxid_localpart_allowed_characters = set("_-./=" + string.ascii_lowercase + string.digits)
 
 
 def contains_invalid_mxid_characters(localpart):
@@ -227,7 +226,7 @@ def contains_invalid_mxid_characters(localpart):
     Returns:
         bool: True if there are any naughty characters
     """
-    return not MXID_LOCALPART_REGEX.match(localpart)
+    return any(c not in mxid_localpart_allowed_characters for c in localpart)
 
 
 class StreamToken(
-- 
cgit 1.4.1


From 616835187702a0c6f16042e3efb452e1ee3e7826 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 3 Apr 2018 20:41:21 +0200
Subject: Add b prefixes to some strings that are bytes in py3

This has no effect on python2

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/api/auth.py                | 10 +++++-----
 synapse/app/frontend_proxy.py      |  2 +-
 synapse/http/server.py             |  4 ++--
 synapse/http/site.py               |  6 +++---
 synapse/rest/client/v1/register.py |  4 ++--
 tests/utils.py                     |  2 +-
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index ac0a3655a5..f17fda6315 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -204,8 +204,8 @@ class Auth(object):
 
             ip_addr = self.hs.get_ip_from_request(request)
             user_agent = request.requestHeaders.getRawHeaders(
-                "User-Agent",
-                default=[""]
+                b"User-Agent",
+                default=[b""]
             )[0]
             if user and access_token and ip_addr:
                 self.store.insert_client_ip(
@@ -672,7 +672,7 @@ def has_access_token(request):
         bool: False if no access_token was given, True otherwise.
     """
     query_params = request.args.get("access_token")
-    auth_headers = request.requestHeaders.getRawHeaders("Authorization")
+    auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
     return bool(query_params) or bool(auth_headers)
 
 
@@ -692,8 +692,8 @@ def get_access_token_from_request(request, token_not_found_http_status=401):
         AuthError: If there isn't an access_token in the request.
     """
 
-    auth_headers = request.requestHeaders.getRawHeaders("Authorization")
-    query_params = request.args.get("access_token")
+    auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
+    query_params = request.args.get(b"access_token")
     if auth_headers:
         # Try the get the access_token from a "Authorization: Bearer"
         # header
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index de889357c3..b349e3e3ce 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -90,7 +90,7 @@ class KeyUploadServlet(RestServlet):
             # They're actually trying to upload something, proxy to main synapse.
             # Pass through the auth headers, if any, in case the access token
             # is there.
-            auth_headers = request.requestHeaders.getRawHeaders("Authorization", [])
+            auth_headers = request.requestHeaders.getRawHeaders(b"Authorization", [])
             headers = {
                 "Authorization": auth_headers,
             }
diff --git a/synapse/http/server.py b/synapse/http/server.py
index f19c068ef6..d979e76639 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -324,7 +324,7 @@ class JsonResource(HttpServer, resource.Resource):
                 register_paths, so will return (possibly via Deferred) either
                 None, or a tuple of (http code, response body).
         """
-        if request.method == "OPTIONS":
+        if request.method == b"OPTIONS":
             return _options_handler, {}
 
         # Loop through all the registered callbacks to check if the method
@@ -536,7 +536,7 @@ def finish_request(request):
 
 def _request_user_agent_is_curl(request):
     user_agents = request.requestHeaders.getRawHeaders(
-        "User-Agent", default=[]
+        b"User-Agent", default=[]
     )
     for user_agent in user_agents:
         if "curl" in user_agent:
diff --git a/synapse/http/site.py b/synapse/http/site.py
index e422c8dfae..c8b46e1af2 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -20,7 +20,7 @@ import logging
 import re
 import time
 
-ACCESS_TOKEN_RE = re.compile(r'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
+ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
 
 
 class SynapseRequest(Request):
@@ -43,12 +43,12 @@ class SynapseRequest(Request):
 
     def get_redacted_uri(self):
         return ACCESS_TOKEN_RE.sub(
-            r'\1<redacted>\3',
+            br'\1<redacted>\3',
             self.uri
         )
 
     def get_user_agent(self):
-        return self.requestHeaders.getRawHeaders("User-Agent", [None])[-1]
+        return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]
 
     def started_processing(self):
         self.site.access_logger.info(
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index 5c5fa8f7ab..8a82097178 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -348,9 +348,9 @@ class RegisterRestServlet(ClientV1RestServlet):
         admin = register_json.get("admin", None)
 
         # Its important to check as we use null bytes as HMAC field separators
-        if "\x00" in user:
+        if b"\x00" in user:
             raise SynapseError(400, "Invalid user")
-        if "\x00" in password:
+        if b"\x00" in password:
             raise SynapseError(400, "Invalid password")
 
         # str() because otherwise hmac complains that 'unicode' does not
diff --git a/tests/utils.py b/tests/utils.py
index 8efd3a3475..f15317d27b 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -212,7 +212,7 @@ class MockHttpResource(HttpServer):
 
         headers = {}
         if federation_auth:
-            headers["Authorization"] = ["X-Matrix origin=test,key=,sig="]
+            headers[b"Authorization"] = ["X-Matrix origin=test,key=,sig="]
         mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
 
         # return the right path if the event requires it
-- 
cgit 1.4.1


From 301b339494f473fddd04cad9a9b107615e9dfa8d Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Wed, 4 Apr 2018 08:45:51 -0600
Subject: Move the mention of the main synapse worker higher up

Signed-off-by: Travis Ralston <travpc@gmail.com>
---
 docs/workers.rst | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/docs/workers.rst b/docs/workers.rst
index a5e084c22a..bf8dd1ee48 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -55,7 +55,12 @@ synapse process.)
 
 You then create a set of configs for the various worker processes.  These
 should be worker configuration files, and should be stored in a dedicated
-subdirectory, to allow synctl to manipulate them.
+subdirectory, to allow synctl to manipulate them. An additional configuration
+for the master synapse process will need to be created because the process will
+not be started automatically. That configuration should look like this::
+
+    worker_app: synapse.app.homeserver
+    daemonize: true
 
 Each worker configuration file inherits the configuration of the main homeserver
 configuration file.  You can then override configuration specific to that worker,
@@ -115,18 +120,6 @@ To manipulate a specific worker, you pass the -w option to synctl::
     synctl -w $CONFIG/workers/synchrotron.yaml restart
 
 
-After setting up your workers, you'll need to create a worker configuration for
-the main synapse process. That worker configuration should look like this:::
-
-    worker_app: synapse.app.homeserver
-    daemonize: true
-
-Be sure to keep this particular configuration limited as synapse may refuse to
-start if the regular ``worker_*`` options are given. The ``homeserver.yaml``
-configuration will be used to set up the main synapse process.
-
-**You must have a worker configuration for the main synapse process!**
-
 Available worker applications
 -----------------------------
 
-- 
cgit 1.4.1


From 204fc985204f0c24574ad2bf9fa9518d4fa7552d Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Wed, 4 Apr 2018 08:46:17 -0600
Subject: Document the additional routes for the event_creator worker

Fixes https://github.com/matrix-org/synapse/issues/3018

Signed-off-by: Travis Ralston <travpc@gmail.com>
---
 docs/workers.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/workers.rst b/docs/workers.rst
index bf8dd1ee48..c3868d6e41 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -235,9 +235,11 @@ file. For example::
 ``synapse.app.event_creator``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Handles non-state event creation. It can handle REST endpoints matching:
+Handles some event creation. It can handle REST endpoints matching:
 
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$
+    ^/_matrix/client/(api/v1|r0|unstable)/join/
 
 It will create events locally and then send them on to the main synapse
 instance to be persisted and handled.
-- 
cgit 1.4.1


From e4570c53dd35f00103e2353884d1dd446fc4c0f4 Mon Sep 17 00:00:00 2001
From: Jan Christian Grünhage <jcgruenhage@matrix.org>
Date: Wed, 4 Apr 2018 16:46:58 +0100
Subject: phone home cache size configurations

---
 synapse/app/homeserver.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b935beb974..464799ac90 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -48,6 +48,7 @@ from synapse.server import HomeServer
 from synapse.storage import are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
 from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
+from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
@@ -431,6 +432,8 @@ def run(hs):
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
         stats["daily_sent_messages"] = daily_sent_messages
+        stats["cache_factor"] = CACHE_SIZE_FACTOR
+        stats["event_cache_size"] = hs.config.event_cache_size
 
         if len(stats_process) > 0:
             stats["memory_rss"] = 0
-- 
cgit 1.4.1


From 518f6de0881378b1fa356e21256436491d43c93c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 4 Apr 2018 19:46:28 +0100
Subject: Remove redundant metrics which were deprecated in 0.27.0.

---
 CHANGES.rst             |  9 +++++++++
 UPGRADE.rst             |  9 ++++++++-
 docs/metrics-howto.rst  | 11 +++++++++++
 synapse/http/server.py  | 26 --------------------------
 synapse/util/metrics.py | 25 -------------------------
 5 files changed, 28 insertions(+), 52 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 38372381ac..5fbad54427 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,12 @@
+Changes in synapse v0.28.0 (2018-xx-xx)
+=======================================
+
+As previously advised, this release removes a number of redundant Prometheus
+metrics. Administrators may need to update their dashboards and alerting rules
+to use the updated metric names, if they have not already done so. See
+`docs/metrics-howto.rst <docs/metrics-howto.rst#deprecated-metrics-removed-in-0-28-0>`_
+for more details.
+
 Changes in synapse v0.27.2 (2018-03-26)
 =======================================
 
diff --git a/UPGRADE.rst b/UPGRADE.rst
index f6bb1070b1..39a16b1c0c 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -52,7 +52,7 @@ Upgrading to $NEXT_VERSION
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory 
+``report_stats`` configuration is set to ``true``. We now capture RSS memory
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 
@@ -60,6 +60,13 @@ We would appreciate it if you could assist by ensuring this module is available
 and ``report_stats`` is enabled. This will let us see if performance changes to
 synapse are having an impact to the general community.
 
+This release also removes a number of redundant Prometheus metrics.
+Administrators may need to update their dashboards and alerting rules to use
+the updated metric names, if they have not already done so. See
+`docs/metrics-howto.rst <docs/metrics-howto.rst#deprecated-metrics-removed-in-0-28-0>`_
+for more details.
+
+
 Upgrading to v0.15.0
 ====================
 
diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst
index 8acc479bc3..5e2d7c52ec 100644
--- a/docs/metrics-howto.rst
+++ b/docs/metrics-howto.rst
@@ -34,6 +34,17 @@ How to monitor Synapse metrics using Prometheus
    Restart prometheus.
 
 
+Deprecated metrics removed in 0.28.0
+------------------------------------
+
+Synapse 0.28.0 removes all of the metrics deprecated by 0.27.0, which are those
+listed under "Old name" below. This has been done to reduce the bandwidth used
+by gathering metrics and the storage requirements for the Prometheus server, as
+well as reducing CPU overhead for both Synapse and Prometheus.
+
+Administrators should update any alerts or monitoring dashboards to use the
+"New name" listed below.
+
 Block and response metrics renamed for 0.27.0
 ---------------------------------------------
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index f19c068ef6..02c7e46f08 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -47,17 +47,6 @@ metrics = synapse.metrics.get_metrics_for(__name__)
 response_count = metrics.register_counter(
     "response_count",
     labels=["method", "servlet", "tag"],
-    alternative_names=(
-        # the following are all deprecated aliases for the same metric
-        metrics.name_prefix + x for x in (
-            "_requests",
-            "_response_time:count",
-            "_response_ru_utime:count",
-            "_response_ru_stime:count",
-            "_response_db_txn_count:count",
-            "_response_db_txn_duration:count",
-        )
-    )
 )
 
 requests_counter = metrics.register_counter(
@@ -73,39 +62,24 @@ outgoing_responses_counter = metrics.register_counter(
 response_timer = metrics.register_counter(
     "response_time_seconds",
     labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_time:total",
-    ),
 )
 
 response_ru_utime = metrics.register_counter(
     "response_ru_utime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_utime:total",
-    ),
 )
 
 response_ru_stime = metrics.register_counter(
     "response_ru_stime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_stime:total",
-    ),
 )
 
 response_db_txn_count = metrics.register_counter(
     "response_db_txn_count", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_count:total",
-    ),
 )
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
 response_db_txn_duration = metrics.register_counter(
     "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_duration:total",
-    ),
 )
 
 # seconds spent waiting for a db connection, when processing this request
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index e4b5687a4b..c3d8237e8f 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -31,53 +31,28 @@ metrics = synapse.metrics.get_metrics_for(__name__)
 block_counter = metrics.register_counter(
     "block_count",
     labels=["block_name"],
-    alternative_names=(
-        # the following are all deprecated aliases for the same metric
-        metrics.name_prefix + x for x in (
-            "_block_timer:count",
-            "_block_ru_utime:count",
-            "_block_ru_stime:count",
-            "_block_db_txn_count:count",
-            "_block_db_txn_duration:count",
-        )
-    )
 )
 
 block_timer = metrics.register_counter(
     "block_time_seconds",
     labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_timer:total",
-    ),
 )
 
 block_ru_utime = metrics.register_counter(
     "block_ru_utime_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_ru_utime:total",
-    ),
 )
 
 block_ru_stime = metrics.register_counter(
     "block_ru_stime_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_ru_stime:total",
-    ),
 )
 
 block_db_txn_count = metrics.register_counter(
     "block_db_txn_count", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_db_txn_count:total",
-    ),
 )
 
 # seconds spent waiting for db txns, excluding scheduling time, in this block
 block_db_txn_duration = metrics.register_counter(
     "block_db_txn_duration_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_db_txn_duration:total",
-    ),
 )
 
 # seconds spent waiting for a db connection, in this block
-- 
cgit 1.4.1


From 0e5f479fc05ef9257c1bfce033c8fb91e6244ffe Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 5 Apr 2018 12:16:46 +0100
Subject: Review comments

Use iteritems over item to loop over dict
formatting
---
 synapse/app/homeserver.py   | 2 +-
 synapse/storage/__init__.py | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 35e2b00f1b..777e9c529a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -426,7 +426,7 @@ def run(hs):
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
         r30_results = yield hs.get_datastore().count_r30_users()
-        for name, count in r30_results.items():
+        for name, count in r30_results.iteritems():
             stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index f68e436df0..4800584b59 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -280,8 +280,9 @@ class DataStore(RoomMemberStore, RoomStore,
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-                     SELECT users.name, platform, users.creation_ts * 1000,
-                                                            MAX(uip.last_seen)
+                     SELECT
+                        users.name, platform, users.creation_ts * 1000,
+                        MAX(uip.last_seen)
                      FROM users
                      INNER JOIN (
                          SELECT
@@ -310,8 +311,8 @@ class DataStore(RoomMemberStore, RoomStore,
             results = {}
             txn.execute(sql, (thirty_days_ago_in_secs,
                               thirty_days_ago_in_secs))
-            rows = txn.fetchall()
-            for row in rows:
+
+            for row in txn:
                 if row[0] is 'unknown':
                     pass
                 results[row[0]] = row[1]
-- 
cgit 1.4.1


From b214a04ffc1200535f1d9c6ec45717cd266f36e5 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 13:29:16 +0100
Subject: Document set_group_join_policy

---
 synapse/storage/group_server.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index db66ea1eb0..ab4f710f7d 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -31,6 +31,12 @@ _DEFAULT_ROLE_ID = ""
 
 class GroupServerStore(SQLBaseStore):
     def set_group_join_policy(self, group_id, join_policy):
+        """Set the join policy of a group.
+
+        join_policy can be one of:
+         * "invite"
+         * "open"
+        """
         return self._simple_update_one(
             table="groups",
             keyvalues={
-- 
cgit 1.4.1


From 700e5e719875dd7008791f52828bb3cd92d6ce21 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 14:01:17 +0100
Subject: Use DEFAULT join_policy of "invite" in db

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index ab3b00286d..53add94367 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -19,4 +19,4 @@
  * NULL at the python store level as necessary so that existing
  * rows are given the correct default policy.
  */
-ALTER TABLE groups ADD COLUMN join_policy TEXT DEFAULT NULL;
+ALTER TABLE groups ADD COLUMN join_policy TEXT NON NULL DEFAULT 'invite';
-- 
cgit 1.4.1


From 104c0bc1d5d1f2a487c50d63b22caa477b091976 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 14:07:16 +0100
Subject: Use "/settings/" (plural)

---
 synapse/federation/transport/client.py | 2 +-
 synapse/federation/transport/server.py | 2 +-
 synapse/rest/client/v2_alpha/groups.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 0f7f656824..1fe162d55b 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -862,7 +862,7 @@ class TransportLayerClient(object):
                            content):
         """Sets the join policy for a group
         """
-        path = PREFIX + "/groups/%s/setting/m.join_policy" % (group_id,)
+        path = PREFIX + "/groups/%s/settings/m.join_policy" % (group_id,)
 
         return self.client.post_json(
             destination=destination,
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a52d3948f4..3658ca75f3 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1128,7 +1128,7 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
 class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
     """Sets whether a group is joinable without an invite or knock
     """
-    PATH = "/groups/(?P<group_id>[^/]*)/setting/m.join_policy$"
+    PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy$"
 
     @defer.inlineCallbacks
     def on_POST(self, origin, content, query, group_id):
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py
index 8faaa1d6a0..3bb1ec2af6 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/v2_alpha/groups.py
@@ -405,7 +405,7 @@ class GroupInvitedUsersServlet(RestServlet):
 class GroupSettingJoinPolicyServlet(RestServlet):
     """Set group join policy
     """
-    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/setting/m.join_policy$")
+    PATTERNS = client_v2_patterns("/groups/(?P<group_id>[^/]*)/settings/m.join_policy$")
 
     def __init__(self, hs):
         super(GroupSettingJoinPolicyServlet, self).__init__()
-- 
cgit 1.4.1


From 917380e89d2d323be1a6ea03e53a31ed335c80df Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 14:32:12 +0100
Subject: NON NULL -> NOT NULL

---
 synapse/storage/schema/delta/48/groups_joinable.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/delta/48/groups_joinable.sql
index 53add94367..ce26eaf0c9 100644
--- a/synapse/storage/schema/delta/48/groups_joinable.sql
+++ b/synapse/storage/schema/delta/48/groups_joinable.sql
@@ -19,4 +19,4 @@
  * NULL at the python store level as necessary so that existing
  * rows are given the correct default policy.
  */
-ALTER TABLE groups ADD COLUMN join_policy TEXT NON NULL DEFAULT 'invite';
+ALTER TABLE groups ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'invite';
-- 
cgit 1.4.1


From 01afc563c39006c21bb7752831cd62c146edc135 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 5 Apr 2018 16:24:04 +0100
Subject: Fix overzealous cache invalidation

Fixes an issue where a cache invalidation would invalidate *all* pending
entries, rather than just the entry that we intended to invalidate.
---
 synapse/util/caches/descriptors.py    | 64 +++++++++++++++++++++--------------
 tests/util/caches/test_descriptors.py | 46 +++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 26 deletions(-)

diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index bf3a66eae4..68285a7594 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,12 +40,11 @@ _CacheSentinel = object()
 
 class CacheEntry(object):
     __slots__ = [
-        "deferred", "sequence", "callbacks", "invalidated"
+        "deferred", "callbacks", "invalidated"
     ]
 
-    def __init__(self, deferred, sequence, callbacks):
+    def __init__(self, deferred, callbacks):
         self.deferred = deferred
-        self.sequence = sequence
         self.callbacks = set(callbacks)
         self.invalidated = False
 
@@ -62,7 +62,6 @@ class Cache(object):
         "max_entries",
         "name",
         "keylen",
-        "sequence",
         "thread",
         "metrics",
         "_pending_deferred_cache",
@@ -80,7 +79,6 @@ class Cache(object):
 
         self.name = name
         self.keylen = keylen
-        self.sequence = 0
         self.thread = None
         self.metrics = register_cache(name, self.cache)
 
@@ -113,11 +111,10 @@ class Cache(object):
         callbacks = [callback] if callback else []
         val = self._pending_deferred_cache.get(key, _CacheSentinel)
         if val is not _CacheSentinel:
-            if val.sequence == self.sequence:
-                val.callbacks.update(callbacks)
-                if update_metrics:
-                    self.metrics.inc_hits()
-                return val.deferred
+            val.callbacks.update(callbacks)
+            if update_metrics:
+                self.metrics.inc_hits()
+            return val.deferred
 
         val = self.cache.get(key, _CacheSentinel, callbacks=callbacks)
         if val is not _CacheSentinel:
@@ -137,12 +134,9 @@ class Cache(object):
         self.check_thread()
         entry = CacheEntry(
             deferred=value,
-            sequence=self.sequence,
             callbacks=callbacks,
         )
 
-        entry.callbacks.update(callbacks)
-
         existing_entry = self._pending_deferred_cache.pop(key, None)
         if existing_entry:
             existing_entry.invalidate()
@@ -150,13 +144,25 @@ class Cache(object):
         self._pending_deferred_cache[key] = entry
 
         def shuffle(result):
-            if self.sequence == entry.sequence:
-                existing_entry = self._pending_deferred_cache.pop(key, None)
-                if existing_entry is entry:
-                    self.cache.set(key, result, entry.callbacks)
-                else:
-                    entry.invalidate()
+            existing_entry = self._pending_deferred_cache.pop(key, None)
+            if existing_entry is entry:
+                self.cache.set(key, result, entry.callbacks)
             else:
+                # oops, the _pending_deferred_cache has been updated since
+                # we started our query, so we are out of date.
+                #
+                # Better put back whatever we took out. (We do it this way
+                # round, rather than peeking into the _pending_deferred_cache
+                # and then removing on a match, to make the common case faster)
+                if existing_entry is not None:
+                    self._pending_deferred_cache[key] = existing_entry
+
+                # we're not going to put this entry into the cache, so need
+                # to make sure that the invalidation callbacks are called.
+                # That was probably done when _pending_deferred_cache was
+                # updated, but it's possible that `set` was called without
+                # `invalidate` being previously called, in which case it may
+                # not have been. Either way, let's double-check now.
                 entry.invalidate()
             return result
 
@@ -168,25 +174,29 @@ class Cache(object):
 
     def invalidate(self, key):
         self.check_thread()
+        self.cache.pop(key, None)
 
-        # Increment the sequence number so that any SELECT statements that
-        # raced with the INSERT don't update the cache (SYN-369)
-        self.sequence += 1
+        # if we have a pending lookup for this key, remove it from the
+        # _pending_deferred_cache, which will (a) stop it being returned
+        # for future queries and (b) stop it being persisted as a proper entry
+        # in self.cache.
         entry = self._pending_deferred_cache.pop(key, None)
+
+        # run the invalidation callbacks now, rather than waiting for the
+        # deferred to resolve.
         if entry:
             entry.invalidate()
 
-        self.cache.pop(key, None)
-
     def invalidate_many(self, key):
         self.check_thread()
         if not isinstance(key, tuple):
             raise TypeError(
                 "The cache key must be a tuple not %r" % (type(key),)
             )
-        self.sequence += 1
         self.cache.del_multi(key)
 
+        # if we have a pending lookup for this key, remove it from the
+        # _pending_deferred_cache, as above
         entry_dict = self._pending_deferred_cache.pop(key, None)
         if entry_dict is not None:
             for entry in iterate_tree_cache_entry(entry_dict):
@@ -194,8 +204,10 @@ class Cache(object):
 
     def invalidate_all(self):
         self.check_thread()
-        self.sequence += 1
         self.cache.clear()
+        for entry in self._pending_deferred_cache.itervalues():
+            entry.invalidate()
+        self._pending_deferred_cache.clear()
 
 
 class _CacheDescriptorBase(object):
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index 3f14ab503f..2516fe40f4 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from functools import partial
 import logging
 
 import mock
@@ -25,6 +27,50 @@ from tests import unittest
 logger = logging.getLogger(__name__)
 
 
+class CacheTestCase(unittest.TestCase):
+    def test_invalidate_all(self):
+        cache = descriptors.Cache("testcache")
+
+        callback_record = [False, False]
+
+        def record_callback(idx):
+            callback_record[idx] = True
+
+        # add a couple of pending entries
+        d1 = defer.Deferred()
+        cache.set("key1", d1, partial(record_callback, 0))
+
+        d2 = defer.Deferred()
+        cache.set("key2", d2, partial(record_callback, 1))
+
+        # lookup should return the deferreds
+        self.assertIs(cache.get("key1"), d1)
+        self.assertIs(cache.get("key2"), d2)
+
+        # let one of the lookups complete
+        d2.callback("result2")
+        self.assertEqual(cache.get("key2"), "result2")
+
+        # now do the invalidation
+        cache.invalidate_all()
+
+        # lookup should return none
+        self.assertIsNone(cache.get("key1", None))
+        self.assertIsNone(cache.get("key2", None))
+
+        # both callbacks should have been callbacked
+        self.assertTrue(
+            callback_record[0], "Invalidation callback for key1 not called",
+        )
+        self.assertTrue(
+            callback_record[1], "Invalidation callback for key2 not called",
+        )
+
+        # letting the other lookup complete should do nothing
+        d1.callback("result1")
+        self.assertIsNone(cache.get("key1", None))
+
+
 class DescriptorTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_cache(self):
-- 
cgit 1.4.1


From c7ede92d0b0e3d9a1c9a6cea8ecccbb87d1fb0d9 Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Thu, 5 Apr 2018 17:26:21 +0200
Subject: make prometheus config compliant to v0.28

---
 contrib/prometheus/consoles/synapse.html | 20 ++++++++++----------
 contrib/prometheus/synapse-v1.rules      |  6 +++---
 contrib/prometheus/synapse-v2.rules      | 12 ++++++------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/contrib/prometheus/consoles/synapse.html b/contrib/prometheus/consoles/synapse.html
index e23d8a1fce..69aa87f85e 100644
--- a/contrib/prometheus/consoles/synapse.html
+++ b/contrib/prometheus/consoles/synapse.html
@@ -202,11 +202,11 @@ new PromConsole.Graph({
 <h1>Requests</h1>
 
 <h3>Requests by Servlet</h3>
-<div id="synapse_http_server_requests_servlet"></div>
+<div id="synapse_http_server_request_count_servlet"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_http_server_requests_servlet"),
-  expr: "rate(synapse_http_server_requests:servlet[2m])",
+  node: document.querySelector("#synapse_http_server_request_count_servlet"),
+  expr: "rate(synapse_http_server_request_count:servlet[2m])",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -215,11 +215,11 @@ new PromConsole.Graph({
 })
 </script>
 <h4>&nbsp;(without <tt>EventStreamRestServlet</tt> or <tt>SyncRestServlet</tt>)</h4>
-<div id="synapse_http_server_requests_servlet_minus_events"></div>
+<div id="synapse_http_server_request_count_servlet_minus_events"></div>
 <script>
 new PromConsole.Graph({
-  node: document.querySelector("#synapse_http_server_requests_servlet_minus_events"),
-  expr: "rate(synapse_http_server_requests:servlet{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
+  node: document.querySelector("#synapse_http_server_request_count_servlet_minus_events"),
+  expr: "rate(synapse_http_server_request_count:servlet{servlet!=\"EventStreamRestServlet\", servlet!=\"SyncRestServlet\"}[2m])",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -233,7 +233,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_time_avg"),
-  expr: "rate(synapse_http_server_response_time:total[2m]) / rate(synapse_http_server_response_time:count[2m]) / 1000",
+  expr: "rate(synapse_http_server_response_time_seconds[2m]) / rate(synapse_http_server_response_count[2m]) / 1000",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -276,7 +276,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_ru_utime"),
-  expr: "rate(synapse_http_server_response_ru_utime:total[2m])",
+  expr: "rate(synapse_http_server_response_ru_utime_seconds[2m])",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -291,7 +291,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_response_db_txn_duration"),
-  expr: "rate(synapse_http_server_response_db_txn_duration:total[2m])",
+  expr: "rate(synapse_http_server_response_db_txn_duration_seconds[2m])",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
@@ -306,7 +306,7 @@ new PromConsole.Graph({
 <script>
 new PromConsole.Graph({
   node: document.querySelector("#synapse_http_server_send_time_avg"),
-  expr: "rate(synapse_http_server_response_time:total{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_time:count{servlet='RoomSendEventRestServlet'}[2m]) / 1000",
+  expr: "rate(synapse_http_server_response_time_second{servlet='RoomSendEventRestServlet'}[2m]) / rate(synapse_http_server_response_count{servlet='RoomSendEventRestServlet'}[2m]) / 1000",
   name: "[[servlet]]",
   yAxisFormatter: PromConsole.NumberFormatter.humanize,
   yHoverFormatter: PromConsole.NumberFormatter.humanize,
diff --git a/contrib/prometheus/synapse-v1.rules b/contrib/prometheus/synapse-v1.rules
index b6f84174b0..4c900ba537 100644
--- a/contrib/prometheus/synapse-v1.rules
+++ b/contrib/prometheus/synapse-v1.rules
@@ -1,10 +1,10 @@
 synapse_federation_transaction_queue_pendingEdus:total = sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)
 synapse_federation_transaction_queue_pendingPdus:total = sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)
 
-synapse_http_server_requests:method{servlet=""} = sum(synapse_http_server_requests) by (method)
-synapse_http_server_requests:servlet{method=""} = sum(synapse_http_server_requests) by (servlet)
+synapse_http_server_request_count:method{servlet=""} = sum(synapse_http_server_request_count) by (method)
+synapse_http_server_request_count:servlet{method=""} = sum(synapse_http_server_request_count) by (servlet)
 
-synapse_http_server_requests:total{servlet=""} = sum(synapse_http_server_requests:by_method) by (servlet)
+synapse_http_server_request_count:total{servlet=""} = sum(synapse_http_server_request_count:by_method) by (servlet)
 
 synapse_cache:hit_ratio_5m = rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])
 synapse_cache:hit_ratio_30s = rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])
diff --git a/contrib/prometheus/synapse-v2.rules b/contrib/prometheus/synapse-v2.rules
index 07e37a885e..6ccca2daaf 100644
--- a/contrib/prometheus/synapse-v2.rules
+++ b/contrib/prometheus/synapse-v2.rules
@@ -5,19 +5,19 @@ groups:
     expr: "sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)"
   - record: "synapse_federation_transaction_queue_pendingPdus:total"
     expr:   "sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)"
-  - record: 'synapse_http_server_requests:method'
+  - record: 'synapse_http_server_request_count:method'
     labels:
       servlet: ""
-    expr: "sum(synapse_http_server_requests) by (method)"
-  - record: 'synapse_http_server_requests:servlet'
+    expr: "sum(synapse_http_server_request_count) by (method)"
+  - record: 'synapse_http_server_request_count:servlet'
     labels:
       method: ""
-    expr: 'sum(synapse_http_server_requests) by (servlet)'
+    expr: 'sum(synapse_http_server_request_count) by (servlet)'
 
-  - record: 'synapse_http_server_requests:total'
+  - record: 'synapse_http_server_request_count:total'
     labels:
       servlet: ""
-    expr: 'sum(synapse_http_server_requests:by_method) by (servlet)'
+    expr: 'sum(synapse_http_server_request_count:by_method) by (servlet)'
 
   - record: 'synapse_cache:hit_ratio_5m'
     expr: 'rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])'
-- 
cgit 1.4.1


From 15e8ed874ff13f30b79e6a47cf814b5a2a7dfc9a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 6 Apr 2018 09:28:36 +0100
Subject: more verbosity in synctl

---
 synapse/app/synctl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py
index b0e1b5e66a..712dfa870e 100755
--- a/synapse/app/synctl.py
+++ b/synapse/app/synctl.py
@@ -252,6 +252,7 @@ def main():
             for running_pid in running_pids:
                 while pid_running(running_pid):
                     time.sleep(0.2)
+            write("All processes exited; now restarting...")
 
     if action == "start" or action == "restart":
         if start_stop_synapse:
-- 
cgit 1.4.1


From 1d71f484d4ec00fd41e3ef195622d0d5dba6d372 Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Fri, 6 Apr 2018 12:54:09 +0200
Subject: use PUT instead of POST for federating groups/m.join_policy

---
 synapse/federation/transport/client.py | 2 +-
 synapse/federation/transport/server.py | 2 +-
 synapse/http/matrixfederationclient.py | 6 +++++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 1fe162d55b..3beab47832 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -864,7 +864,7 @@ class TransportLayerClient(object):
         """
         path = PREFIX + "/groups/%s/settings/m.join_policy" % (group_id,)
 
-        return self.client.post_json(
+        return self.client.put_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 3658ca75f3..b98e30459c 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1131,7 +1131,7 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
     PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy$"
 
     @defer.inlineCallbacks
-    def on_POST(self, origin, content, query, group_id):
+    def on_PUT(self, origin, content, query, group_id):
         requester_user_id = parse_string_from_args(query, "requester_user_id")
         if get_domain_from_id(requester_user_id) != origin:
             raise SynapseError(403, "requester_user_id doesn't match origin")
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 9145405cb0..60a29081e8 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -286,7 +286,8 @@ class MatrixFederationHttpClient(object):
         headers_dict[b"Authorization"] = auth_headers
 
     @defer.inlineCallbacks
-    def put_json(self, destination, path, data={}, json_data_callback=None,
+    def put_json(self, destination, path, args={}, data={},
+                 json_data_callback=None,
                  long_retries=False, timeout=None,
                  ignore_backoff=False,
                  backoff_on_404=False):
@@ -296,6 +297,7 @@ class MatrixFederationHttpClient(object):
             destination (str): The remote server to send the HTTP request
                 to.
             path (str): The HTTP path.
+            args (dict): query params
             data (dict): A dict containing the data that will be used as
                 the request body. This will be encoded as JSON.
             json_data_callback (callable): A callable returning the dict to
@@ -342,6 +344,7 @@ class MatrixFederationHttpClient(object):
             path,
             body_callback=body_callback,
             headers_dict={"Content-Type": ["application/json"]},
+            query_bytes=encode_query_args(args),
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
@@ -373,6 +376,7 @@ class MatrixFederationHttpClient(object):
                 giving up. None indicates no timeout.
             ignore_backoff (bool): true to ignore the historical backoff data and
                 try the request anyway.
+            args (dict): query params
         Returns:
             Deferred: Succeeds when we get a 2xx HTTP response. The result
             will be the decoded JSON body.
-- 
cgit 1.4.1


From 7b824f147590c6519ca8889e6d406928de7f1e6e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 6 Apr 2018 13:20:05 +0100
Subject: Add response size metrics

---
 synapse/http/server.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 02c7e46f08..ac75206ef5 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -87,6 +87,11 @@ response_db_sched_duration = metrics.register_counter(
     "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"]
 )
 
+# size in bytes of the response written
+response_size = metrics.register_counter(
+    "response_size", labels=["method", "servlet", "tag"]
+)
+
 _next_request_id = 0
 
 
@@ -400,6 +405,8 @@ class RequestMetrics(object):
             context.db_sched_duration_ms / 1000., request.method, self.name, tag
         )
 
+        response_size.inc_by(request.sentLength, request.method, self.name, tag)
+
 
 class RootRedirect(resource.Resource):
     """Redirects the root '/' path to another path."""
-- 
cgit 1.4.1


From e01ba5bda315ed80f2a809e2f2e917eaf7488fac Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 6 Apr 2018 13:45:10 +0100
Subject: Port script: avoid nasty errors when setting up

We really shouldn't spit out "Failed to create port table", it looks scary.
---
 scripts/synapse_port_db | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index d46581e4e1..7fb8be3abd 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -491,7 +492,7 @@ class Porter(object):
 
             def create_port_table(txn):
                 txn.execute(
-                    "CREATE TABLE port_from_sqlite3 ("
+                    "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
                     " table_name varchar(100) NOT NULL UNIQUE,"
                     " forward_rowid bigint NOT NULL,"
                     " backward_rowid bigint NOT NULL"
@@ -517,14 +518,11 @@ class Porter(object):
                     "alter_table", alter_table
                 )
             except Exception as e:
-                logger.info("Failed to create port table: %s", e)
+                pass
 
-            try:
-                yield self.postgres_store.runInteraction(
-                    "create_port_table", create_port_table
-                )
-            except Exception as e:
-                logger.info("Failed to create port table: %s", e)
+            yield self.postgres_store.runInteraction(
+                "create_port_table", create_port_table
+            )
 
             self.progress.set_state("Setting up")
 
-- 
cgit 1.4.1


From 01579384ccd26d112df5169522fd16234e8316c7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 6 Apr 2018 13:47:11 +0100
Subject: Port script: clean up a bit

Improve logging and comments. Group all the stuff to do with inspecting tables
together rather than creating the port tables in the middle.
---
 scripts/synapse_port_db | 54 ++++++++++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 7fb8be3abd..dc1a10bf28 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -251,6 +251,12 @@ class Porter(object):
     @defer.inlineCallbacks
     def handle_table(self, table, postgres_size, table_size, forward_chunk,
                      backward_chunk):
+        logger.info(
+            "Table %s: %i/%i (rows %i-%i) already ported",
+            table, postgres_size, table_size,
+            backward_chunk+1, forward_chunk-1,
+        )
+
         if not table_size:
             return
 
@@ -468,28 +474,7 @@ class Porter(object):
             self.progress.set_state("Preparing PostgreSQL")
             self.setup_db(postgres_config, postgres_engine)
 
-            # Step 2. Get tables.
-            self.progress.set_state("Fetching tables")
-            sqlite_tables = yield self.sqlite_store._simple_select_onecol(
-                table="sqlite_master",
-                keyvalues={
-                    "type": "table",
-                },
-                retcol="name",
-            )
-
-            postgres_tables = yield self.postgres_store._simple_select_onecol(
-                table="information_schema.tables",
-                keyvalues={},
-                retcol="distinct table_name",
-            )
-
-            tables = set(sqlite_tables) & set(postgres_tables)
-
-            self.progress.set_state("Creating tables")
-
-            logger.info("Found %d tables", len(tables))
-
+            self.progress.set_state("Creating port tables")
             def create_port_table(txn):
                 txn.execute(
                     "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
@@ -524,9 +509,27 @@ class Porter(object):
                 "create_port_table", create_port_table
             )
 
-            self.progress.set_state("Setting up")
+            # Step 2. Get tables.
+            self.progress.set_state("Fetching tables")
+            sqlite_tables = yield self.sqlite_store._simple_select_onecol(
+                table="sqlite_master",
+                keyvalues={
+                    "type": "table",
+                },
+                retcol="name",
+            )
+
+            postgres_tables = yield self.postgres_store._simple_select_onecol(
+                table="information_schema.tables",
+                keyvalues={},
+                retcol="distinct table_name",
+            )
+
+            tables = set(sqlite_tables) & set(postgres_tables)
+            logger.info("Found %d tables", len(tables))
 
-            # Set up tables.
+            # Step 3. Figure out what still needs copying
+            self.progress.set_state("Checking on port progress")
             setup_res = yield defer.gatherResults(
                 [
                     self.setup_table(table)
@@ -537,7 +540,8 @@ class Porter(object):
                 consumeErrors=True,
             )
 
-            # Process tables.
+            # Step 4. Do the copying.
+            self.progress.set_state("Copying to postgres")
             yield defer.gatherResults(
                 [
                     self.handle_table(*res)
-- 
cgit 1.4.1


From 6a9777ba028af1d9803d13e879eaf62773c5bd83 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 6 Apr 2018 13:48:40 +0100
Subject: Port script: Set up state_group_id_seq

Fixes https://github.com/matrix-org/synapse/issues/3050.
---
 scripts/synapse_port_db | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index dc1a10bf28..7b23a44854 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -550,6 +550,9 @@ class Porter(object):
                 consumeErrors=True,
             )
 
+            # Step 5. Do final post-processing
+            yield self._setup_state_group_id_seq()
+
             self.progress.done()
         except:
             global end_error_exec_info
@@ -709,6 +712,16 @@ class Porter(object):
 
         defer.returnValue((done, remaining + done))
 
+    def _setup_state_group_id_seq(self):
+        def r(txn):
+            txn.execute("SELECT MAX(id) FROM state_groups")
+            next_id = txn.fetchone()[0]+1
+            txn.execute(
+                "ALTER SEQUENCE state_group_id_seq RESTART WITH %s",
+                (next_id,),
+            )
+        return self.postgres_store.runInteraction("setup_state_group_id_seq", r)
+
 
 ##############################################
 ###### The following is simply UI stuff ######
-- 
cgit 1.4.1


From b370fe61c0aeccac7745ad404dad925ec217ba6d Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 17:18:02 +0100
Subject: Implement group join API

---
 synapse/federation/transport/client.py | 13 ++++++++++
 synapse/federation/transport/server.py | 18 ++++++++++++++
 synapse/groups/groups_server.py        | 45 ++++++++++++++++++++++++++++++++++
 synapse/handlers/groups_local.py       | 40 +++++++++++++++++++++++++++++-
 synapse/storage/group_server.py        | 12 ++++++---
 5 files changed, 124 insertions(+), 4 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 3beab47832..370f7ba78b 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -614,6 +614,19 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    @log_function
+    def join_group(self, destination, group_id, user_id, content):
+        """Attempts to join a group
+        """
+        path = PREFIX + "/groups/%s/users/%s/join" % (group_id, user_id)
+
+        return self.client.post_json(
+            destination=destination,
+            path=path,
+            data=content,
+            ignore_backoff=True,
+        )
+
     @log_function
     def invite_to_group(self, destination, group_id, user_id, requester_user_id, content):
         """Invite a user to a group
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index b98e30459c..4c94d5a36c 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -803,6 +803,23 @@ class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
+class FederationGroupsJoinServlet(BaseFederationServlet):
+    """Attempt to join a group
+    """
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join$"
+
+    @defer.inlineCallbacks
+    def on_POST(self, origin, content, query, group_id, user_id):
+        if get_domain_from_id(user_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        new_content = yield self.handler.join_group(
+            group_id, user_id, content,
+        )
+
+        defer.returnValue((200, new_content))
+
+
 class FederationGroupsRemoveUserServlet(BaseFederationServlet):
     """Leave or kick a user from the group
     """
@@ -1182,6 +1199,7 @@ GROUP_SERVER_SERVLET_CLASSES = (
     FederationGroupsInvitedUsersServlet,
     FederationGroupsInviteServlet,
     FederationGroupsAcceptInviteServlet,
+    FederationGroupsJoinServlet,
     FederationGroupsRemoveUserServlet,
     FederationGroupsSummaryRoomsServlet,
     FederationGroupsCategoriesServlet,
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 70781e1854..77b6273e39 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -723,6 +723,51 @@ class GroupsServerHandler(object):
             "attestation": local_attestation,
         })
 
+    @defer.inlineCallbacks
+    def join_group(self, group_id, requester_user_id, content):
+        """User tries to join the group.
+
+        This will error if the group requires an invite/knock to join
+        """
+
+        yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+
+        group_info = yield self.store.get_group(
+            group_id,
+        )
+        if not group_info['is_joinable']:
+            raise SynapseError(403, "Group is not publicly joinable")
+
+        if not self.hs.is_mine_id(requester_user_id):
+            local_attestation = self.attestations.create_attestation(
+                group_id, requester_user_id,
+            )
+            remote_attestation = content["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                user_id=requester_user_id,
+                group_id=group_id,
+            )
+        else:
+            local_attestation = None
+            remote_attestation = None
+
+        is_public = _parse_visibility_from_contents(content)
+
+        yield self.store.add_user_to_group(
+            group_id, requester_user_id,
+            is_admin=False,
+            is_public=is_public,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+        )
+
+        defer.returnValue({
+            "state": "join",
+            "attestation": local_attestation,
+        })
+
     @defer.inlineCallbacks
     def knock(self, group_id, requester_user_id, content):
         """A user requests becoming a member of the group
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 5f7b0ff305..977993e7d4 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -229,7 +229,45 @@ class GroupsLocalHandler(object):
     def join_group(self, group_id, user_id, content):
         """Request to join a group
         """
-        raise NotImplementedError()  # TODO
+        if self.is_mine_id(group_id):
+            yield self.groups_server_handler.join_group(
+                group_id, user_id, content
+            )
+            local_attestation = None
+            remote_attestation = None
+        else:
+            local_attestation = self.attestations.create_attestation(group_id, user_id)
+            content["attestation"] = local_attestation
+
+            res = yield self.transport_client.join_group(
+                get_domain_from_id(group_id), group_id, user_id, content,
+            )
+
+            remote_attestation = res["attestation"]
+
+            yield self.attestations.verify_attestation(
+                remote_attestation,
+                group_id=group_id,
+                user_id=user_id,
+                server_name=get_domain_from_id(group_id),
+            )
+
+        # TODO: Check that the group is public and we're being added publically
+        is_publicised = content.get("publicise", False)
+
+        token = yield self.store.register_user_group_membership(
+            group_id, user_id,
+            membership="join",
+            is_admin=False,
+            local_attestation=local_attestation,
+            remote_attestation=remote_attestation,
+            is_publicised=is_publicised,
+        )
+        self.notifier.on_new_event(
+            "groups_key", token, users=[user_id],
+        )
+
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index db316a27ec..16c11a056f 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -48,19 +48,25 @@ class GroupServerStore(SQLBaseStore):
             desc="set_group_join_policy",
         )
 
+    @defer.inlineCallbacks
     def get_group(self, group_id):
-        return self._simple_select_one(
+        ret = yield self._simple_select_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
             },
             retcols=(
-                "name", "short_description", "long_description", "avatar_url", "is_public"
+                "name", "short_description", "long_description", "avatar_url", "is_public", "is_joinable",
             ),
             allow_none=True,
-            desc="is_user_in_group",
+            desc="get_group",
         )
 
+        if ret and 'is_joinable' in ret:
+            ret['is_joinable'] = bool(ret['is_joinable'])
+
+        defer.returnValue(ret)
+
     def get_users_in_group(self, group_id, include_private=False):
         # TODO: Pagination
 
-- 
cgit 1.4.1


From edb45aae38b7bdeae1cbf18b435be5f8610c2d48 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 28 Mar 2018 17:18:50 +0100
Subject: pep8

---
 synapse/storage/group_server.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 16c11a056f..5fbe0ada4e 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -56,7 +56,8 @@ class GroupServerStore(SQLBaseStore):
                 "group_id": group_id,
             },
             retcols=(
-                "name", "short_description", "long_description", "avatar_url", "is_public", "is_joinable",
+                "name", "short_description", "long_description",
+                "avatar_url", "is_public", "is_joinable",
             ),
             allow_none=True,
             desc="get_group",
-- 
cgit 1.4.1


From 6eb3aa94b6befcc89f6fee426053b3feee316bb9 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Tue, 3 Apr 2018 11:48:17 +0100
Subject: Factor out add_user from accept_invite and join_group

---
 synapse/groups/groups_server.py | 70 +++++++++++++++++------------------------
 1 file changed, 29 insertions(+), 41 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 77b6273e39..2c02da4725 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -678,30 +678,21 @@ class GroupsServerHandler(object):
             raise SynapseError(502, "Unknown state returned by HS")
 
     @defer.inlineCallbacks
-    def accept_invite(self, group_id, requester_user_id, content):
-        """User tries to accept an invite to the group.
+    def add_user(self, group_id, user_id, content):
+        """Add a user to a group based on a content dict.
 
-        This is different from them asking to join, and so should error if no
-        invite exists (and they're not a member of the group)
+        See accept_invite, join_group.
         """
-
-        yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
-
-        is_invited = yield self.store.is_user_invited_to_local_group(
-            group_id, requester_user_id,
-        )
-        if not is_invited:
-            raise SynapseError(403, "User not invited to group")
-
-        if not self.hs.is_mine_id(requester_user_id):
+        if not self.hs.is_mine_id(user_id):
             local_attestation = self.attestations.create_attestation(
-                group_id, requester_user_id,
+                group_id, user_id,
             )
+
             remote_attestation = content["attestation"]
 
             yield self.attestations.verify_attestation(
                 remote_attestation,
-                user_id=requester_user_id,
+                user_id=user_id,
                 group_id=group_id,
             )
         else:
@@ -711,13 +702,33 @@ class GroupsServerHandler(object):
         is_public = _parse_visibility_from_contents(content)
 
         yield self.store.add_user_to_group(
-            group_id, requester_user_id,
+            group_id, user_id,
             is_admin=False,
             is_public=is_public,
             local_attestation=local_attestation,
             remote_attestation=remote_attestation,
         )
 
+        defer.returnValue(local_attestation)
+
+    @defer.inlineCallbacks
+    def accept_invite(self, group_id, requester_user_id, content):
+        """User tries to accept an invite to the group.
+
+        This is different from them asking to join, and so should error if no
+        invite exists (and they're not a member of the group)
+        """
+
+        yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+
+        is_invited = yield self.store.is_user_invited_to_local_group(
+            group_id, requester_user_id,
+        )
+        if not is_invited:
+            raise SynapseError(403, "User not invited to group")
+
+        local_attestation = yield self.add_user(group_id, requester_user_id, content)
+
         defer.returnValue({
             "state": "join",
             "attestation": local_attestation,
@@ -738,30 +749,7 @@ class GroupsServerHandler(object):
         if not group_info['is_joinable']:
             raise SynapseError(403, "Group is not publicly joinable")
 
-        if not self.hs.is_mine_id(requester_user_id):
-            local_attestation = self.attestations.create_attestation(
-                group_id, requester_user_id,
-            )
-            remote_attestation = content["attestation"]
-
-            yield self.attestations.verify_attestation(
-                remote_attestation,
-                user_id=requester_user_id,
-                group_id=group_id,
-            )
-        else:
-            local_attestation = None
-            remote_attestation = None
-
-        is_public = _parse_visibility_from_contents(content)
-
-        yield self.store.add_user_to_group(
-            group_id, requester_user_id,
-            is_admin=False,
-            is_public=is_public,
-            local_attestation=local_attestation,
-            remote_attestation=remote_attestation,
-        )
+        local_attestation = yield self.add_user(group_id, requester_user_id, content)
 
         defer.returnValue({
             "state": "join",
-- 
cgit 1.4.1


From f8d1917fce3b08b613f76131d6a82a7576379251 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 16:31:24 +0100
Subject: Fix federation client `set_group_joinable` typo

---
 synapse/federation/transport/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 370f7ba78b..67ff7da51b 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -871,7 +871,7 @@ class TransportLayerClient(object):
         )
 
     @log_function
-    def set_group_joinable(self, destination, group_id, requester_user_id,
+    def set_group_join_policy(self, destination, group_id, requester_user_id,
                            content):
         """Sets the join policy for a group
         """
-- 
cgit 1.4.1


From ae85c7804e733aa1adaed06a9de51445a084858e Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 16:31:57 +0100
Subject: is_joinable -> join_rule

---
 synapse/groups/groups_server.py | 2 +-
 synapse/storage/group_server.py | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 2c02da4725..507ec232c5 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -746,7 +746,7 @@ class GroupsServerHandler(object):
         group_info = yield self.store.get_group(
             group_id,
         )
-        if not group_info['is_joinable']:
+        if group_info['join_policy'] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
         local_attestation = yield self.add_user(group_id, requester_user_id, content)
diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 5fbe0ada4e..d81609dd16 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -57,15 +57,12 @@ class GroupServerStore(SQLBaseStore):
             },
             retcols=(
                 "name", "short_description", "long_description",
-                "avatar_url", "is_public", "is_joinable",
+                "avatar_url", "is_public", "join_rule",
             ),
             allow_none=True,
             desc="get_group",
         )
 
-        if ret and 'is_joinable' in ret:
-            ret['is_joinable'] = bool(ret['is_joinable'])
-
         defer.returnValue(ret)
 
     def get_users_in_group(self, group_id, include_private=False):
-- 
cgit 1.4.1


From 87c864b6984f6876f224e0d4ac2e2b872d2723ce Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 16:49:22 +0100
Subject: join_rule -> join_policy

---
 synapse/storage/group_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index d81609dd16..8bd8ca8124 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -57,7 +57,7 @@ class GroupServerStore(SQLBaseStore):
             },
             retcols=(
                 "name", "short_description", "long_description",
-                "avatar_url", "is_public", "join_rule",
+                "avatar_url", "is_public", "join_policy",
             ),
             allow_none=True,
             desc="get_group",
-- 
cgit 1.4.1


From cd087a265db8037ce2fe158ddc0e9b35a612d4bb Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 17:14:27 +0100
Subject: Don't use redundant inlineCallbacks

---
 synapse/storage/group_server.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py
index 8bd8ca8124..da05ccb027 100644
--- a/synapse/storage/group_server.py
+++ b/synapse/storage/group_server.py
@@ -48,9 +48,8 @@ class GroupServerStore(SQLBaseStore):
             desc="set_group_join_policy",
         )
 
-    @defer.inlineCallbacks
     def get_group(self, group_id):
-        ret = yield self._simple_select_one(
+        return self._simple_select_one(
             table="groups",
             keyvalues={
                 "group_id": group_id,
@@ -63,8 +62,6 @@ class GroupServerStore(SQLBaseStore):
             desc="get_group",
         )
 
-        defer.returnValue(ret)
-
     def get_users_in_group(self, group_id, include_private=False):
         # TODO: Pagination
 
-- 
cgit 1.4.1


From 6850f8aea3482443dff1c330fb63f7ca8dde0025 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 17:16:31 +0100
Subject: Get group_info from existing call to check_group_is_ours

---
 synapse/groups/groups_server.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 507ec232c5..1b516011d8 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -741,11 +741,7 @@ class GroupsServerHandler(object):
         This will error if the group requires an invite/knock to join
         """
 
-        yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
-
-        group_info = yield self.store.get_group(
-            group_id,
-        )
+        group_info = yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
         if group_info['join_policy'] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
-- 
cgit 1.4.1


From 112c2253e2846b48b75f4171d9dd94bfe6ecc8a1 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Thu, 5 Apr 2018 17:32:20 +0100
Subject: pep8

---
 synapse/federation/transport/client.py | 2 +-
 synapse/groups/groups_server.py        | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 67ff7da51b..50a967a7ec 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -872,7 +872,7 @@ class TransportLayerClient(object):
 
     @log_function
     def set_group_join_policy(self, destination, group_id, requester_user_id,
-                           content):
+                              content):
         """Sets the join policy for a group
         """
         path = PREFIX + "/groups/%s/settings/m.join_policy" % (group_id,)
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 1b516011d8..15d31549de 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -741,7 +741,9 @@ class GroupsServerHandler(object):
         This will error if the group requires an invite/knock to join
         """
 
-        group_info = yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
+        group_info = yield self.check_group_is_ours(
+            group_id, requester_user_id, and_exists=True
+        )
         if group_info['join_policy'] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
-- 
cgit 1.4.1


From b4478e586fe52392a439c94f525c3d885454f0a8 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Fri, 6 Apr 2018 11:37:49 +0100
Subject: add_user -> _add_user

---
 synapse/groups/groups_server.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 15d31549de..72dfed7dbb 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -678,7 +678,7 @@ class GroupsServerHandler(object):
             raise SynapseError(502, "Unknown state returned by HS")
 
     @defer.inlineCallbacks
-    def add_user(self, group_id, user_id, content):
+    def _add_user(self, group_id, user_id, content):
         """Add a user to a group based on a content dict.
 
         See accept_invite, join_group.
@@ -727,7 +727,7 @@ class GroupsServerHandler(object):
         if not is_invited:
             raise SynapseError(403, "User not invited to group")
 
-        local_attestation = yield self.add_user(group_id, requester_user_id, content)
+        local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
         defer.returnValue({
             "state": "join",
@@ -747,7 +747,7 @@ class GroupsServerHandler(object):
         if group_info['join_policy'] != "open":
             raise SynapseError(403, "Group is not publicly joinable")
 
-        local_attestation = yield self.add_user(group_id, requester_user_id, content)
+        local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
         defer.returnValue({
             "state": "join",
-- 
cgit 1.4.1


From 6bd1b7053e41350fcc1b451e40da82257e2120b5 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Fri, 6 Apr 2018 11:44:18 +0100
Subject: By default, join policy is "invite"

---
 synapse/groups/groups_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 72dfed7dbb..8310dea031 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -905,7 +905,7 @@ def _parse_join_policy_dict(join_policy_dict):
     """
     join_policy_type = join_policy_dict.get("type")
     if not join_policy_type:
-        return True
+        return "invite"
 
     if join_policy_type not in ("invite", "open"):
         raise SynapseError(
-- 
cgit 1.4.1


From 7945435587704c5abd075140bdd46c11db93c255 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Fri, 6 Apr 2018 14:06:32 +0100
Subject: When exposing group state, return is_openly_joinable

as opposed to join_policy, which is really only pertinent to the
synapse implementation of the group server.

By doing this we keep the group server concept extensible by
allowing arbitrarily complex rules for deciding whether a group
is openly joinable.
---
 synapse/groups/groups_server.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 8310dea031..290eec7127 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -407,6 +407,11 @@ class GroupsServerHandler(object):
         group_description = yield self.store.get_group(group_id)
 
         if group_description:
+            join_policy = group_description['join_policy']
+            del group_description['join_policy']
+
+            group_description['is_openly_joinable'] = join_policy == "open"
+
             defer.returnValue(group_description)
         else:
             raise SynapseError(404, "Unknown group")
-- 
cgit 1.4.1


From db2fd801f722ae8341b36314fb8929a80fd53996 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Fri, 6 Apr 2018 15:51:15 +0100
Subject: Explicitly grab individual columns from group object

---
 synapse/groups/groups_server.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 290eec7127..ad937c1721 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -404,13 +404,15 @@ class GroupsServerHandler(object):
 
         yield self.check_group_is_ours(group_id, requester_user_id)
 
-        group_description = yield self.store.get_group(group_id)
-
-        if group_description:
-            join_policy = group_description['join_policy']
-            del group_description['join_policy']
+        group = yield self.store.get_group(group_id)
 
-            group_description['is_openly_joinable'] = join_policy == "open"
+        if group:
+            cols = [
+                "name", "short_description", "long_description",
+                "avatar_url", "is_public",
+            ]
+            group_description = { key: group[key] for key in cols }
+            group_description["is_openly_joinable"] = group['join_policy'] == "open"
 
             defer.returnValue(group_description)
         else:
-- 
cgit 1.4.1


From 020a5013544b1f7046faa97b83eb2acc613334b1 Mon Sep 17 00:00:00 2001
From: Luke Barnard <luke@matrix.org>
Date: Fri, 6 Apr 2018 16:02:06 +0100
Subject: de-lint, quote consistency

---
 synapse/groups/groups_server.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index ad937c1721..2d95b04e0c 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -411,8 +411,8 @@ class GroupsServerHandler(object):
                 "name", "short_description", "long_description",
                 "avatar_url", "is_public",
             ]
-            group_description = { key: group[key] for key in cols }
-            group_description["is_openly_joinable"] = group['join_policy'] == "open"
+            group_description = {key: group[key] for key in cols}
+            group_description["is_openly_joinable"] = group["join_policy"] == "open"
 
             defer.returnValue(group_description)
         else:
-- 
cgit 1.4.1


From 4f40d058ccc030107bc33716401f8c0f4a7230d9 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Fri, 6 Apr 2018 22:57:06 +0200
Subject: Replace old-style raise with six.reraise

The old style raise is invalid syntax in python3. As noted in the docs,
this adds one more frame in the traceback, but I think this is
acceptable:

    <ipython-input-7-bcc5cba3de3f> in <module>()
         16     except:
         17         pass
    ---> 18     six.reraise(*x)

    /usr/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
        691             if value.__traceback__ is not tb:
        692                 raise value.with_traceback(tb)
    --> 693             raise value
        694         finally:
        695             value = None

    <ipython-input-7-bcc5cba3de3f> in <module>()
          9
         10 try:
    ---> 11     x()
         12 except:
         13     x = sys.exc_info()

Also note that this uses six, which is not formally a dependency yet,
but is included indirectly since most packages depend on it.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/rest/media/v1/media_storage.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index 83471b3173..7f263db239 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -16,6 +16,8 @@
 from twisted.internet import defer, threads
 from twisted.protocols.basic import FileSender
 
+import six
+
 from ._base import Responder
 
 from synapse.util.file_consumer import BackgroundFileConsumer
@@ -119,7 +121,7 @@ class MediaStorage(object):
                 os.remove(fname)
             except Exception:
                 pass
-            raise t, v, tb
+            six.reraise(t, v, tb)
 
         if not finished_called:
             raise Exception("Finished callback not called")
-- 
cgit 1.4.1


From b0500d37744a0eddc5df155b77a4d4c96eba7a70 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 27 Mar 2018 19:15:58 +0200
Subject: use python3-compatible prints

---
 synapse/config/_base.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index fa105bce72..dd591d8d32 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -288,22 +288,22 @@ class Config(object):
                     )
                     obj.invoke_all("generate_files", config)
                     config_file.write(config_bytes)
-                print (
+                print((
                     "A config file has been generated in %r for server name"
                     " %r with corresponding SSL keys and self-signed"
                     " certificates. Please review this file and customise it"
                     " to your needs."
-                ) % (config_path, server_name)
-                print (
+                ) % (config_path, server_name))
+                print(
                     "If this server name is incorrect, you will need to"
                     " regenerate the SSL certificates"
                 )
                 return
             else:
-                print (
+                print((
                     "Config file %r already exists. Generating any missing key"
                     " files."
-                ) % (config_path,)
+                ) % (config_path,))
                 generate_keys = True
 
         parser = argparse.ArgumentParser(
-- 
cgit 1.4.1


From e54c202b81707ef02b2e640489a69c2b266bebff Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 7 Apr 2018 00:37:36 +0200
Subject: Replace some type checks with six type checks

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 contrib/graph/graph3.py        | 4 +++-
 scripts/synapse_port_db        | 4 +++-
 synapse/appservice/__init__.py | 4 +++-
 synapse/config/_base.py        | 6 ++++--
 synapse/config/appservice.py   | 8 +++++---
 5 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py
index 88d92c89d7..7d3b4d7eb6 100644
--- a/contrib/graph/graph3.py
+++ b/contrib/graph/graph3.py
@@ -22,6 +22,8 @@ import argparse
 from synapse.events import FrozenEvent
 from synapse.util.frozenutils import unfreeze
 
+from six import string_types
+
 
 def make_graph(file_name, room_id, file_prefix, limit):
     print "Reading lines"
@@ -58,7 +60,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
                 value = "<null>"
-            elif isinstance(value, basestring):
+            elif isinstance(value, string_types):
                 pass
             else:
                 value = json.dumps(value)
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 7b23a44854..b9b828c154 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -30,6 +30,8 @@ import time
 import traceback
 import yaml
 
+from six import string_types
+
 
 logger = logging.getLogger("synapse_port_db")
 
@@ -574,7 +576,7 @@ class Porter(object):
         def conv(j, col):
             if j in bool_cols:
                 return bool(col)
-            elif isinstance(col, basestring) and "\0" in col:
+            elif isinstance(col, string_types) and "\0" in col:
                 logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
                 raise BadValueException();
             return col
diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py
index d5a7a5ce2f..5fdb579723 100644
--- a/synapse/appservice/__init__.py
+++ b/synapse/appservice/__init__.py
@@ -21,6 +21,8 @@ from twisted.internet import defer
 import logging
 import re
 
+from six import string_types
+
 logger = logging.getLogger(__name__)
 
 
@@ -146,7 +148,7 @@ class ApplicationService(object):
                         )
 
                 regex = regex_obj.get("regex")
-                if isinstance(regex, basestring):
+                if isinstance(regex, string_types):
                     regex_obj["regex"] = re.compile(regex)  # Pre-compile regex
                 else:
                     raise ValueError(
diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index fa105bce72..f79bd3dfc7 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -19,6 +19,8 @@ import os
 import yaml
 from textwrap import dedent
 
+from six import integer_types
+
 
 class ConfigError(Exception):
     pass
@@ -49,7 +51,7 @@ Missing mandatory `server_name` config option.
 class Config(object):
     @staticmethod
     def parse_size(value):
-        if isinstance(value, int) or isinstance(value, long):
+        if isinstance(value, integer_types):
             return value
         sizes = {"K": 1024, "M": 1024 * 1024}
         size = 1
@@ -61,7 +63,7 @@ class Config(object):
 
     @staticmethod
     def parse_duration(value):
-        if isinstance(value, int) or isinstance(value, long):
+        if isinstance(value, integer_types):
             return value
         second = 1000
         minute = 60 * second
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index aba0aec6e8..9a2359b6fd 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -21,6 +21,8 @@ import urllib
 import yaml
 import logging
 
+from six import string_types
+
 logger = logging.getLogger(__name__)
 
 
@@ -89,14 +91,14 @@ def _load_appservice(hostname, as_info, config_filename):
         "id", "as_token", "hs_token", "sender_localpart"
     ]
     for field in required_string_fields:
-        if not isinstance(as_info.get(field), basestring):
+        if not isinstance(as_info.get(field), string_types):
             raise KeyError("Required string field: '%s' (%s)" % (
                 field, config_filename,
             ))
 
     # 'url' must either be a string or explicitly null, not missing
     # to avoid accidentally turning off push for ASes.
-    if (not isinstance(as_info.get("url"), basestring) and
+    if (not isinstance(as_info.get("url"), string_types) and
             as_info.get("url", "") is not None):
         raise KeyError(
             "Required string field or explicit null: 'url' (%s)" % (config_filename,)
@@ -128,7 +130,7 @@ def _load_appservice(hostname, as_info, config_filename):
                         "Expected namespace entry in %s to be an object,"
                         " but got %s", ns, regex_obj
                     )
-                if not isinstance(regex_obj.get("regex"), basestring):
+                if not isinstance(regex_obj.get("regex"), string_types):
                     raise ValueError(
                         "Missing/bad type 'regex' key in %s", regex_obj
                     )
-- 
cgit 1.4.1


From 145d14656b19d64a6deca8facca02508ecc751fe Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 14 Feb 2018 13:52:03 +0000
Subject: Handle exceptions in get_hosts_for_room when sending events over
 federation

---
 synapse/federation/transaction_queue.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index a141ec9953..12e8df9cc6 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -184,17 +184,22 @@ class TransactionQueue(object):
                     if not is_mine and send_on_behalf_of is None:
                         continue
 
-                    # Get the state from before the event.
-                    # We need to make sure that this is the state from before
-                    # the event and not from after it.
-                    # Otherwise if the last member on a server in a room is
-                    # banned then it won't receive the event because it won't
-                    # be in the room after the ban.
-                    destinations = yield self.state.get_current_hosts_in_room(
-                        event.room_id, latest_event_ids=[
-                            prev_id for prev_id, _ in event.prev_events
-                        ],
-                    )
+                    try:
+                        # Get the state from before the event.
+                        # We need to make sure that this is the state from before
+                        # the event and not from after it.
+                        # Otherwise if the last member on a server in a room is
+                        # banned then it won't receive the event because it won't
+                        # be in the room after the ban.
+                        destinations = yield self.state.get_current_hosts_in_room(
+                            event.room_id, latest_event_ids=[
+                                prev_id for prev_id, _ in event.prev_events
+                            ],
+                        )
+                    except Exception:
+                        logger.exception("Failed to calculate hosts in room")
+                        continue
+
                     destinations = set(destinations)
 
                     if send_on_behalf_of is not None:
-- 
cgit 1.4.1


From 11974f37871f41a5827f243a3fc2057703f04d72 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 Apr 2018 11:45:22 +0100
Subject: Send federation events concurrently

---
 synapse/federation/transaction_queue.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 12e8df9cc6..43daf673c0 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -169,7 +169,7 @@ class TransactionQueue(object):
             while True:
                 last_token = yield self.store.get_federation_out_pos("events")
                 next_token, events = yield self.store.get_all_new_events_stream(
-                    last_token, self._last_poked_id, limit=20,
+                    last_token, self._last_poked_id, limit=100,
                 )
 
                 logger.debug("Handling %s -> %s", last_token, next_token)
@@ -177,12 +177,13 @@ class TransactionQueue(object):
                 if not events and next_token >= self._last_poked_id:
                     break
 
-                for event in events:
+                @defer.inlineCallbacks
+                def handle_event(event):
                     # Only send events for this server.
                     send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of()
                     is_mine = self.is_mine_id(event.event_id)
                     if not is_mine and send_on_behalf_of is None:
-                        continue
+                        return
 
                     try:
                         # Get the state from before the event.
@@ -198,7 +199,7 @@ class TransactionQueue(object):
                         )
                     except Exception:
                         logger.exception("Failed to calculate hosts in room")
-                        continue
+                        return
 
                     destinations = set(destinations)
 
@@ -212,6 +213,19 @@ class TransactionQueue(object):
 
                     self._send_pdu(event, destinations)
 
+                def handle_room_events(events):
+                    for event in events:
+                        return handle_event(event)
+
+                events_by_room = {}
+                for event in events:
+                    events_by_room.setdefault(event.room_id, []).append(event)
+
+                yield logcontext.make_deferred_yieldable(defer.gatherResults(
+                    [handle_room_events(evs) for evs in events_by_room.itervalues()],
+                    consumeErrors=True
+                ))
+
                 events_processed_counter.inc_by(len(events))
 
                 yield self.store.update_federation_out_pos(
-- 
cgit 1.4.1


From 56b0589865d0a8cc09e0149578faf71daa73da91 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 Apr 2018 12:04:18 +0100
Subject: Use create_and_send_nonmember_event everywhere

---
 synapse/rest/client/v1/room.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index d06cbdc35e..2ad0e5943b 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -165,17 +165,12 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
                 content=content,
             )
         else:
-            event, context = yield self.event_creation_hander.create_event(
+            event = yield self.event_creation_hander.create_and_send_nonmember_event(
                 requester,
                 event_dict,
-                token_id=requester.access_token_id,
                 txn_id=txn_id,
             )
 
-            yield self.event_creation_hander.send_nonmember_event(
-                requester, event, context,
-            )
-
         ret = {}
         if event:
             ret = {"event_id": event.event_id}
-- 
cgit 1.4.1


From e5082494ebb4b880643021eb94c848b15ae16f2c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 Apr 2018 12:07:39 +0100
Subject: Limit concurrent event sends for a room

---
 synapse/handlers/message.py | 99 +++++++++++++++++++++++----------------------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 6de6e13b7b..1814b205da 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -454,40 +454,39 @@ class EventCreationHandler(object):
         """
         builder = self.event_builder_factory.new(event_dict)
 
-        with (yield self.limiter.queue(builder.room_id)):
-            self.validator.validate_new(builder)
-
-            if builder.type == EventTypes.Member:
-                membership = builder.content.get("membership", None)
-                target = UserID.from_string(builder.state_key)
-
-                if membership in {Membership.JOIN, Membership.INVITE}:
-                    # If event doesn't include a display name, add one.
-                    profile = self.profile_handler
-                    content = builder.content
-
-                    try:
-                        if "displayname" not in content:
-                            content["displayname"] = yield profile.get_displayname(target)
-                        if "avatar_url" not in content:
-                            content["avatar_url"] = yield profile.get_avatar_url(target)
-                    except Exception as e:
-                        logger.info(
-                            "Failed to get profile information for %r: %s",
-                            target, e
-                        )
+        self.validator.validate_new(builder)
+
+        if builder.type == EventTypes.Member:
+            membership = builder.content.get("membership", None)
+            target = UserID.from_string(builder.state_key)
+
+            if membership in {Membership.JOIN, Membership.INVITE}:
+                # If event doesn't include a display name, add one.
+                profile = self.profile_handler
+                content = builder.content
+
+                try:
+                    if "displayname" not in content:
+                        content["displayname"] = yield profile.get_displayname(target)
+                    if "avatar_url" not in content:
+                        content["avatar_url"] = yield profile.get_avatar_url(target)
+                except Exception as e:
+                    logger.info(
+                        "Failed to get profile information for %r: %s",
+                        target, e
+                    )
 
-            if token_id is not None:
-                builder.internal_metadata.token_id = token_id
+        if token_id is not None:
+            builder.internal_metadata.token_id = token_id
 
-            if txn_id is not None:
-                builder.internal_metadata.txn_id = txn_id
+        if txn_id is not None:
+            builder.internal_metadata.txn_id = txn_id
 
-            event, context = yield self.create_new_client_event(
-                builder=builder,
-                requester=requester,
-                prev_event_ids=prev_event_ids,
-            )
+        event, context = yield self.create_new_client_event(
+            builder=builder,
+            requester=requester,
+            prev_event_ids=prev_event_ids,
+        )
 
         defer.returnValue((event, context))
 
@@ -557,27 +556,29 @@ class EventCreationHandler(object):
 
         See self.create_event and self.send_nonmember_event.
         """
-        event, context = yield self.create_event(
-            requester,
-            event_dict,
-            token_id=requester.access_token_id,
-            txn_id=txn_id
-        )
 
-        spam_error = self.spam_checker.check_event_for_spam(event)
-        if spam_error:
-            if not isinstance(spam_error, basestring):
-                spam_error = "Spam is not permitted here"
-            raise SynapseError(
-                403, spam_error, Codes.FORBIDDEN
+        with (yield self.limiter.queue(event_dict["room_id"])):
+            event, context = yield self.create_event(
+                requester,
+                event_dict,
+                token_id=requester.access_token_id,
+                txn_id=txn_id
             )
 
-        yield self.send_nonmember_event(
-            requester,
-            event,
-            context,
-            ratelimit=ratelimit,
-        )
+            spam_error = self.spam_checker.check_event_for_spam(event)
+            if spam_error:
+                if not isinstance(spam_error, basestring):
+                    spam_error = "Spam is not permitted here"
+                raise SynapseError(
+                    403, spam_error, Codes.FORBIDDEN
+                )
+
+            yield self.send_nonmember_event(
+                requester,
+                event,
+                context,
+                ratelimit=ratelimit,
+            )
         defer.returnValue(event)
 
     @measure_func("create_new_client_event")
-- 
cgit 1.4.1


From f3ef60662fd32672f980e3c6e31aa1b25e8ed808 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 9 Apr 2018 12:56:22 +0100
Subject: Return a 404 rather than a 500 on rejoining empty rooms

Filter ourselves out of the server list before checking for an empty remote
host list, to fix 500 error

Fixes #2141
---
 synapse/handlers/room_member.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 9977be8831..c45142d38d 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -852,6 +852,14 @@ class RoomMemberMasterHandler(RoomMemberHandler):
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         """Implements RoomMemberHandler._remote_join
         """
+        # filter ourselves out of remote_room_hosts: do_invite_join ignores it
+        # and if it is the only entry we'd like to return a 404 rather than a
+        # 500.
+
+        remote_room_hosts = [
+            host for host in remote_room_hosts if host != self.hs.hostname
+        ]
+
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
-- 
cgit 1.4.1


From 13decdbf96981782616a3ee1826fce1213a1bc89 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 9 Apr 2018 12:58:37 +0100
Subject: Revert "Merge pull request #3066 from
 matrix-org/rav/remove_redundant_metrics"

We aren't ready to release this yet, so I'm reverting it for now.

This reverts commit d1679a4ed7947b0814e0f2af9b888a16c588f1a1, reversing
changes made to e089100c6231541c446e37e157dec8feed02d283.
---
 CHANGES.rst             |  9 ---------
 UPGRADE.rst             |  9 +--------
 docs/metrics-howto.rst  | 11 -----------
 synapse/http/server.py  | 26 ++++++++++++++++++++++++++
 synapse/util/metrics.py | 25 +++++++++++++++++++++++++
 5 files changed, 52 insertions(+), 28 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 5fbad54427..38372381ac 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,12 +1,3 @@
-Changes in synapse v0.28.0 (2018-xx-xx)
-=======================================
-
-As previously advised, this release removes a number of redundant Prometheus
-metrics. Administrators may need to update their dashboards and alerting rules
-to use the updated metric names, if they have not already done so. See
-`docs/metrics-howto.rst <docs/metrics-howto.rst#deprecated-metrics-removed-in-0-28-0>`_
-for more details.
-
 Changes in synapse v0.27.2 (2018-03-26)
 =======================================
 
diff --git a/UPGRADE.rst b/UPGRADE.rst
index 39a16b1c0c..f6bb1070b1 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -52,7 +52,7 @@ Upgrading to $NEXT_VERSION
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory
+``report_stats`` configuration is set to ``true``. We now capture RSS memory 
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 
@@ -60,13 +60,6 @@ We would appreciate it if you could assist by ensuring this module is available
 and ``report_stats`` is enabled. This will let us see if performance changes to
 synapse are having an impact to the general community.
 
-This release also removes a number of redundant Prometheus metrics.
-Administrators may need to update their dashboards and alerting rules to use
-the updated metric names, if they have not already done so. See
-`docs/metrics-howto.rst <docs/metrics-howto.rst#deprecated-metrics-removed-in-0-28-0>`_
-for more details.
-
-
 Upgrading to v0.15.0
 ====================
 
diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst
index 5e2d7c52ec..8acc479bc3 100644
--- a/docs/metrics-howto.rst
+++ b/docs/metrics-howto.rst
@@ -34,17 +34,6 @@ How to monitor Synapse metrics using Prometheus
    Restart prometheus.
 
 
-Deprecated metrics removed in 0.28.0
-------------------------------------
-
-Synapse 0.28.0 removes all of the metrics deprecated by 0.27.0, which are those
-listed under "Old name" below. This has been done to reduce the bandwidth used
-by gathering metrics and the storage requirements for the Prometheus server, as
-well as reducing CPU overhead for both Synapse and Prometheus.
-
-Administrators should update any alerts or monitoring dashboards to use the
-"New name" listed below.
-
 Block and response metrics renamed for 0.27.0
 ---------------------------------------------
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index ac75206ef5..64e083ebfc 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -47,6 +47,17 @@ metrics = synapse.metrics.get_metrics_for(__name__)
 response_count = metrics.register_counter(
     "response_count",
     labels=["method", "servlet", "tag"],
+    alternative_names=(
+        # the following are all deprecated aliases for the same metric
+        metrics.name_prefix + x for x in (
+            "_requests",
+            "_response_time:count",
+            "_response_ru_utime:count",
+            "_response_ru_stime:count",
+            "_response_db_txn_count:count",
+            "_response_db_txn_duration:count",
+        )
+    )
 )
 
 requests_counter = metrics.register_counter(
@@ -62,24 +73,39 @@ outgoing_responses_counter = metrics.register_counter(
 response_timer = metrics.register_counter(
     "response_time_seconds",
     labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_time:total",
+    ),
 )
 
 response_ru_utime = metrics.register_counter(
     "response_ru_utime_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_ru_utime:total",
+    ),
 )
 
 response_ru_stime = metrics.register_counter(
     "response_ru_stime_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_ru_stime:total",
+    ),
 )
 
 response_db_txn_count = metrics.register_counter(
     "response_db_txn_count", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_db_txn_count:total",
+    ),
 )
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
 response_db_txn_duration = metrics.register_counter(
     "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_db_txn_duration:total",
+    ),
 )
 
 # seconds spent waiting for a db connection, when processing this request
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index c3d8237e8f..e4b5687a4b 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -31,28 +31,53 @@ metrics = synapse.metrics.get_metrics_for(__name__)
 block_counter = metrics.register_counter(
     "block_count",
     labels=["block_name"],
+    alternative_names=(
+        # the following are all deprecated aliases for the same metric
+        metrics.name_prefix + x for x in (
+            "_block_timer:count",
+            "_block_ru_utime:count",
+            "_block_ru_stime:count",
+            "_block_db_txn_count:count",
+            "_block_db_txn_duration:count",
+        )
+    )
 )
 
 block_timer = metrics.register_counter(
     "block_time_seconds",
     labels=["block_name"],
+    alternative_names=(
+        metrics.name_prefix + "_block_timer:total",
+    ),
 )
 
 block_ru_utime = metrics.register_counter(
     "block_ru_utime_seconds", labels=["block_name"],
+    alternative_names=(
+        metrics.name_prefix + "_block_ru_utime:total",
+    ),
 )
 
 block_ru_stime = metrics.register_counter(
     "block_ru_stime_seconds", labels=["block_name"],
+    alternative_names=(
+        metrics.name_prefix + "_block_ru_stime:total",
+    ),
 )
 
 block_db_txn_count = metrics.register_counter(
     "block_db_txn_count", labels=["block_name"],
+    alternative_names=(
+        metrics.name_prefix + "_block_db_txn_count:total",
+    ),
 )
 
 # seconds spent waiting for db txns, excluding scheduling time, in this block
 block_db_txn_duration = metrics.register_counter(
     "block_db_txn_duration_seconds", labels=["block_name"],
+    alternative_names=(
+        metrics.name_prefix + "_block_db_txn_duration:total",
+    ),
 )
 
 # seconds spent waiting for a db connection, in this block
-- 
cgit 1.4.1


From 687f3451bde8e954af7c51dc9cea52e53de84e32 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 9 Apr 2018 13:44:05 +0100
Subject: 0.27.3

---
 CHANGES.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 1372de4246..9f974e6875 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,49 @@
+Changes in synapse v0.27.3 (2018-04-09)
+=======================================
+
+Notable changes include API support for joinability of groups. Also new metrics 
+and phone home stats. Phone home stats include better visibility of system usage
+so we can tweak synpase to work better for all users rather than our own experience
+with matrix.org. Also, recording 'r30' stat which is the measure we use to track 
+overal growth of the Matrix ecosystem. It is defined as:-
+ "Counts the number of native 30 day retained users, defined as:-
+         * Users who have created their accounts more than 30 days
+         * Where last seen at most 30 days ago
+         * Where account creation and last_seen are > 30 days"
+
+
+Features
+* Add joinability for groups (PR #3045)
+* Implement group join API (PR #3046)
+* Add counter metrics for calculating state delta (PR #3033)
+* R30 stats (PR #3041)
+* Measure time it takes to calculate state group ID (PR #3043)
+* Add basic performance statistics to phone home (PR #3044)
+* Add response size metrics (PR #3071)
+* phone home cache size configurations (PR #3063)
+
+Changes
+* Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live!
+* Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile!
+* Use .iter* to avoid copies in StateHandler (PR #3006)
+* Linearize calls to _generate_user_id (PR #3029)
+* Remove last usage of ujson (PR #3030)
+* Use simplejson throughout (PR #3048)
+* Use static JSONEncoders (PR #3049)
+* Remove uses of events.content (PR #3060)
+* Improve database cache performance (PR #3068)
+
+Bug fixes
+* Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte!
+* Fix replication after switch to simplejson (PR #3015)
+* Fix replication after switch to simplejson (PR #3015)
+* 404 correctly on missing paths via NoResource (PR #3022)
+* Fix error when claiming e2e keys from offline servers (PR #3034)
+* fix tests/storage/test_user_directory.py (PR #3042)
+* use PUT instead of POST for federating groups/m.join_policy (PR #3070) Thanks to @krombel!
+* postgres port script: fix state_groups_pkey error (PR #3072)
+
+
 Changes in synapse v0.27.2 (2018-03-26)
 =======================================
 
-- 
cgit 1.4.1


From 072fb594465126a523126b7607d19fb2f190e01a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 13:49:25 +0100
Subject: bump version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index a9d5198aba..1fa2de5268 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.2"
+__version__ = "0.27.3"
-- 
cgit 1.4.1


From c115deed12746f08ccc6ef7294cc9649c99aa1b7 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 9 Apr 2018 15:54:32 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index df469a9930..23ee926f36 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,7 +12,8 @@ overal growth of the Matrix ecosystem. It is defined as:-
          * Where account creation and last_seen are > 30 days"
 
 
-Features
+Features:
+
 * Add joinability for groups (PR #3045)
 * Implement group join API (PR #3046)
 * Add counter metrics for calculating state delta (PR #3033)
@@ -22,7 +23,8 @@ Features
 * Add response size metrics (PR #3071)
 * phone home cache size configurations (PR #3063)
 
-Changes
+Changes:
+
 * Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live!
 * Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile!
 * Use .iter* to avoid copies in StateHandler (PR #3006)
@@ -33,7 +35,8 @@ Changes
 * Remove uses of events.content (PR #3060)
 * Improve database cache performance (PR #3068)
 
-Bug fixes
+Bug fixes:
+
 * Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte!
 * Fix replication after switch to simplejson (PR #3015)
 * Fix replication after switch to simplejson (PR #3015)
-- 
cgit 1.4.1


From 21d5a2a08e053bf69082dd0c7142e9b299cee0b0 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 9 Apr 2018 15:55:41 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 23ee926f36..e8ff469a13 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -6,7 +6,8 @@ and phone home stats. Phone home stats include better visibility of system usage
 so we can tweak synpase to work better for all users rather than our own experience
 with matrix.org. Also, recording 'r30' stat which is the measure we use to track 
 overal growth of the Matrix ecosystem. It is defined as:-
- "Counts the number of native 30 day retained users, defined as:-
+
+Counts the number of native 30 day retained users, defined as:-
          * Users who have created their accounts more than 30 days
          * Where last seen at most 30 days ago
          * Where account creation and last_seen are > 30 days"
-- 
cgit 1.4.1


From d9ae2bc826910c5e70766fb70f0d488faf350fe7 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 16:00:31 +0100
Subject: bump version to release candidate

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 1fa2de5268..354415e5c6 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.3"
+__version__ = "0.27.3-rc1"
-- 
cgit 1.4.1


From b151eb14a2c680467ac0ff6198a016cba3156fd2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 9 Apr 2018 16:01:59 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index e8ff469a13..d80756dc7c 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.27.3 (2018-04-09)
+Changes in synapse v0.27.3-rc1 (2018-04-09)
 =======================================
 
 Notable changes include API support for joinability of groups. Also new metrics 
-- 
cgit 1.4.1


From 6e025a97b4639e5869ecf7a9762aa559bfdf85f6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 9 Apr 2018 16:02:48 +0100
Subject: Handle all events in a room correctly

---
 synapse/federation/transaction_queue.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 43daf673c0..d8e5c08a3c 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -213,9 +213,10 @@ class TransactionQueue(object):
 
                     self._send_pdu(event, destinations)
 
+                @defer.inlineCallbacks
                 def handle_room_events(events):
                     for event in events:
-                        return handle_event(event)
+                        yield handle_event(event)
 
                 events_by_room = {}
                 for event in events:
-- 
cgit 1.4.1


From 5232d3bfb1168a00c136c62359bad4f0fbec0ae2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 17:25:57 +0100
Subject: version bump v0.27.3-rc2

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 354415e5c6..7f6090baf8 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.3-rc1"
+__version__ = "0.27.3-rc2"
-- 
cgit 1.4.1


From 64bc2162ef1c51cf5d5baa3cd3b303216b55df8c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 17:50:36 +0100
Subject: Fix psycopg2 interpolation

---
 synapse/storage/__init__.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 4800584b59..01c580b66e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -289,11 +289,11 @@ class DataStore(RoomMemberStore, RoomStore,
                          user_id,
                          last_seen,
                          CASE
-                             WHEN user_agent LIKE '%Android%' THEN 'android'
-                             WHEN user_agent LIKE '%iOS%' THEN 'ios'
-                             WHEN user_agent LIKE '%Electron%' THEN 'electron'
-                             WHEN user_agent LIKE '%Mozilla%' THEN 'web'
-                             WHEN user_agent LIKE '%Gecko%' THEN 'web'
+                             WHEN user_agent LIKE '%%Android%%' THEN 'android'
+                             WHEN user_agent LIKE '%%iOS%%' THEN 'ios'
+                             WHEN user_agent LIKE '%%Electron%%' THEN 'electron'
+                             WHEN user_agent LIKE '%%Mozilla%%' THEN 'web'
+                             WHEN user_agent LIKE '%%Gecko%%' THEN 'web'
                              ELSE 'unknown'
                          END
                          AS platform
-- 
cgit 1.4.1


From 9a311adfea940984f6a007a97afec39f23cb1be2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 17:52:08 +0100
Subject: v0.27.3-rc2

---
 CHANGES.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index d80756dc7c..684c5c7883 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,8 @@
+Changes in synapse v0.27.3-rc2 (2018-04-09)
+==========================================
+v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates
+the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
+
 Changes in synapse v0.27.3-rc1 (2018-04-09)
 =======================================
 
-- 
cgit 1.4.1


From 87770300d590eb105694e1cbc24aa798f788ca52 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:38:59 +0100
Subject: Fix msec to sec

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 01c580b66e..e736a857ea 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -276,7 +276,7 @@ class DataStore(RoomMemberStore, RoomStore,
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time_msec())
-            thirty_days_ago_in_secs = now - thirty_days_in_secs
+            thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs
 
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
-- 
cgit 1.4.1


From 61b439c90437527afdfa6b46e5ab88de6b074792 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:43:48 +0100
Subject: Fix msec to sec, again

---
 synapse/storage/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e736a857ea..417b9aa2a8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -275,9 +275,9 @@ class DataStore(RoomMemberStore, RoomStore,
         """
         def _count_r30_users(txn):
             thirty_days_in_secs = 86400 * 30
-            now = int(self._clock.time_msec())
-            thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs
-
+            now = int(self._clock.time())
+            thirty_days_ago_in_secs = now - thirty_days_in_secs
+            print str(thirty_days_ago_in_secs)
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
                      SELECT
-- 
cgit 1.4.1


From 41e06118954234878311cea3e7a55a88e6eec784 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 9 Apr 2018 18:44:20 +0100
Subject: remove errant print

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 417b9aa2a8..eacd49d6a5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -277,7 +277,7 @@ class DataStore(RoomMemberStore, RoomStore,
             thirty_days_in_secs = 86400 * 30
             now = int(self._clock.time())
             thirty_days_ago_in_secs = now - thirty_days_in_secs
-            print str(thirty_days_ago_in_secs)
+
             sql = """
                 SELECT platform, COALESCE(count(*), 0) FROM (
                      SELECT
-- 
cgit 1.4.1


From 9fbe70a7dc3afabfdac176ba1f4be32dd44602aa Mon Sep 17 00:00:00 2001
From: Vincent Breitmoser <look@my.amazin.horse>
Date: Sat, 6 Jan 2018 18:11:02 +0100
Subject: Use sortedcontainers instead of blist

This commit drop-in replaces blist with SortedContainers. They are
written in pure python so work with pypy, but perform as good as
native implementations, at least in a couple benchmarks:

http://www.grantjenks.com/docs/sortedcontainers/performance.html
---
 synapse/federation/send_queue.py           | 14 +++++++-------
 synapse/python_dependencies.py             |  2 +-
 synapse/util/caches/stream_change_cache.py |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 93e5acebc1..945832283f 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -35,7 +35,7 @@ from synapse.storage.presence import UserPresenceState
 from synapse.util.metrics import Measure
 import synapse.metrics
 
-from blist import sorteddict
+from sortedcontainers import SortedDict
 from collections import namedtuple
 
 import logging
@@ -56,19 +56,19 @@ class FederationRemoteSendQueue(object):
         self.is_mine_id = hs.is_mine_id
 
         self.presence_map = {}  # Pending presence map user_id -> UserPresenceState
-        self.presence_changed = sorteddict()  # Stream position -> user_id
+        self.presence_changed = SortedDict()  # Stream position -> user_id
 
         self.keyed_edu = {}  # (destination, key) -> EDU
-        self.keyed_edu_changed = sorteddict()  # stream position -> (destination, key)
+        self.keyed_edu_changed = SortedDict()  # stream position -> (destination, key)
 
-        self.edus = sorteddict()  # stream position -> Edu
+        self.edus = SortedDict()  # stream position -> Edu
 
-        self.failures = sorteddict()  # stream position -> (destination, Failure)
+        self.failures = SortedDict()  # stream position -> (destination, Failure)
 
-        self.device_messages = sorteddict()  # stream position -> destination
+        self.device_messages = SortedDict()  # stream position -> destination
 
         self.pos = 1
-        self.pos_time = sorteddict()
+        self.pos_time = SortedDict()
 
         # EVERYTHING IS SAD. In particular, python only makes new scopes when
         # we make a new function, so we need to make a new function so the inner
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 40eedb63cb..f9596bddaf 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,8 +34,8 @@ REQUIREMENTS = {
     "bcrypt": ["bcrypt>=3.1.0"],
     "pillow": ["PIL"],
     "pydenticon": ["pydenticon"],
-    "blist": ["blist"],
     "pysaml2>=3.0.0": ["saml2>=3.0.0"],
+    "sortedcontainers": ["sortedcontainers"],
     "pymacaroons-pynacl": ["pymacaroons"],
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 941d873ab8..2ff46090a6 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -16,7 +16,7 @@
 from synapse.util.caches import register_cache, CACHE_SIZE_FACTOR
 
 
-from blist import sorteddict
+from sortedcontainers import SortedDict
 import logging
 
 
@@ -35,7 +35,7 @@ class StreamChangeCache(object):
     def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
         self._max_size = int(max_size * CACHE_SIZE_FACTOR)
         self._entity_to_key = {}
-        self._cache = sorteddict()
+        self._cache = SortedDict()
         self._earliest_known_stream_pos = current_stream_pos
         self.name = name
         self.metrics = register_cache(self.name, self._cache)
-- 
cgit 1.4.1


From 89de9349815b79f55bfff53b7c8c6d43f8c30336 Mon Sep 17 00:00:00 2001
From: Vincent Breitmoser <look@my.amazin.horse>
Date: Sat, 6 Jan 2018 18:13:56 +0100
Subject: Use psycopg2cffi module instead of psycopg2 if running on pypy

The psycopg2 package isn't available for PyPy.  This commit adds a check
if the runtime is PyPy, and if it is uses psycopg2cffi module in favor
of psycopg2. This is almost a drop-in replacement, except for one place
where an additional cast to string is required.
---
 synapse/storage/_base.py            | 2 +-
 synapse/storage/engines/__init__.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2fbebd4907..2262776ab2 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -376,7 +376,7 @@ class SQLBaseStore(object):
         Returns:
             A list of dicts where the key is the column header.
         """
-        col_headers = list(intern(column[0]) for column in cursor.description)
+        col_headers = list(intern(str(column[0])) for column in cursor.description)
         results = list(
             dict(zip(col_headers, row)) for row in cursor
         )
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 338b495611..be7f6d6ac3 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -18,6 +18,7 @@ from .postgres import PostgresEngine
 from .sqlite3 import Sqlite3Engine
 
 import importlib
+import platform
 
 
 SUPPORTED_MODULE = {
@@ -31,7 +32,12 @@ def create_engine(database_config):
     engine_class = SUPPORTED_MODULE.get(name, None)
 
     if engine_class:
-        module = importlib.import_module(name)
+        needs_pypy_hack = (name == "psycopg2" and
+                platform.python_implementation() == "PyPy")
+        if needs_pypy_hack:
+            module = importlib.import_module("psycopg2cffi")
+        else:
+            module = importlib.import_module(name)
         return engine_class(module, database_config)
 
     raise RuntimeError(
-- 
cgit 1.4.1


From d1e56cfcd11bcd509d8fa3954c00e06a84bddd87 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 10 Apr 2018 00:21:51 +0100
Subject: Fix pep8 error on psycopg2cffi hack

---
 synapse/storage/engines/__init__.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index be7f6d6ac3..8c868ece75 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -32,12 +32,11 @@ def create_engine(database_config):
     engine_class = SUPPORTED_MODULE.get(name, None)
 
     if engine_class:
-        needs_pypy_hack = (name == "psycopg2" and
-                platform.python_implementation() == "PyPy")
-        if needs_pypy_hack:
-            module = importlib.import_module("psycopg2cffi")
-        else:
-            module = importlib.import_module(name)
+        # pypy requires psycopg2cffi rather than psycopg2
+        if (name == "psycopg2" and
+                platform.python_implementation() == "PyPy"):
+            name = "psycopg2cffi"
+        module = importlib.import_module(name)
         return engine_class(module, database_config)
 
     raise RuntimeError(
-- 
cgit 1.4.1


From f4284d943aee616f3100298634a113e51c8ab27e Mon Sep 17 00:00:00 2001
From: Vincent Breitmoser <look@my.amazin.horse>
Date: Sat, 6 Jan 2018 18:14:51 +0100
Subject: In DomainSpecificString, override __repr__ in addition to __str__

For some reason, string interpolation on a DomainSpecificString object
like "%r" % (domainSpecificStringObj) fails under PyPy, because the
default __repr__ implementation wants to iterate over the object. I'm
not sure why that happens, but overriding __repr__ instead of __str__
fixes this problem, and is arguably the more appropriate thing to do
anyways.
---
 synapse/types.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/types.py b/synapse/types.py
index 7cb24cecb2..cc7c182a78 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -169,7 +169,7 @@ class DomainSpecificString(
         except Exception:
             return False
 
-    __str__ = to_string
+    __repr__ = to_string
 
 
 class UserID(DomainSpecificString):
-- 
cgit 1.4.1


From 6d7f0f8dd3685b7d2c58a43662eb793defc186ff Mon Sep 17 00:00:00 2001
From: Vincent Breitmoser <look@my.amazin.horse>
Date: Mon, 8 Jan 2018 01:53:32 +0100
Subject: Don't disable GC when running on PyPy

PyPy's incminimark GC can't be triggered manually. From what I observed
there are no obvious issues with just letting it run normally. And
unlike CPython, it actually returns unused RAM to the system.

Signed-off-by: Vincent Breitmoser <look@my.amazin.horse>
---
 synapse/metrics/__init__.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 50d99d7a5c..2ed82b04a5 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -17,6 +17,7 @@ import logging
 import functools
 import time
 import gc
+import platform
 
 from twisted.internet import reactor
 
@@ -30,6 +31,7 @@ from .process_collector import register_process_collector
 logger = logging.getLogger(__name__)
 
 
+running_on_pypy = platform.python_implementation() == 'PyPy'
 all_metrics = []
 all_collectors = []
 
@@ -174,6 +176,9 @@ def runUntilCurrentTimer(func):
         tick_time.inc_by(end - start)
         pending_calls_metric.inc_by(num_pending)
 
+        if running_on_pypy:
+            return ret
+
         # Check if we need to do a manual GC (since its been disabled), and do
         # one if necessary.
         threshold = gc.get_threshold()
@@ -206,6 +211,7 @@ try:
 
     # We manually run the GC each reactor tick so that we can get some metrics
     # about time spent doing GC,
-    gc.disable()
+    if not running_on_pypy:
+        gc.disable()
 except AttributeError:
     pass
-- 
cgit 1.4.1


From dab87b84a330467abc037489ac2034b94ddadb63 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 11:16:08 +0100
Subject: URL quote path segments over federation

---
 synapse/federation/transport/client.py | 128 ++++++++++++++++++++-------------
 1 file changed, 80 insertions(+), 48 deletions(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 50a967a7ec..1976e5d2fb 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -21,6 +21,7 @@ from synapse.api.urls import FEDERATION_PREFIX as PREFIX
 from synapse.util.logutils import log_function
 
 import logging
+import urllib
 
 
 logger = logging.getLogger(__name__)
@@ -50,7 +51,7 @@ class TransportLayerClient(object):
         logger.debug("get_room_state dest=%s, room=%s",
                      destination, room_id)
 
-        path = PREFIX + "/state/%s/" % room_id
+        path = _create_path(PREFIX, "/state/%s/", room_id)
         return self.client.get_json(
             destination, path=path, args={"event_id": event_id},
         )
@@ -72,7 +73,7 @@ class TransportLayerClient(object):
         logger.debug("get_room_state_ids dest=%s, room=%s",
                      destination, room_id)
 
-        path = PREFIX + "/state_ids/%s/" % room_id
+        path = _create_path(PREFIX, "/state_ids/%s/", room_id)
         return self.client.get_json(
             destination, path=path, args={"event_id": event_id},
         )
@@ -94,7 +95,7 @@ class TransportLayerClient(object):
         logger.debug("get_pdu dest=%s, event_id=%s",
                      destination, event_id)
 
-        path = PREFIX + "/event/%s/" % (event_id, )
+        path = _create_path(PREFIX, "/event/%s/", event_id)
         return self.client.get_json(destination, path=path, timeout=timeout)
 
     @log_function
@@ -120,7 +121,7 @@ class TransportLayerClient(object):
             # TODO: raise?
             return
 
-        path = PREFIX + "/backfill/%s/" % (room_id,)
+        path = _create_path(PREFIX, "/backfill/%s/", room_id)
 
         args = {
             "v": event_tuples,
@@ -158,9 +159,11 @@ class TransportLayerClient(object):
         # generated by the json_data_callback.
         json_data = transaction.get_dict()
 
+        path = _create_path(PREFIX, "/send/%s/", transaction.transaction_id)
+
         response = yield self.client.put_json(
             transaction.destination,
-            path=PREFIX + "/send/%s/" % transaction.transaction_id,
+            path=path,
             data=json_data,
             json_data_callback=json_data_callback,
             long_retries=True,
@@ -178,7 +181,7 @@ class TransportLayerClient(object):
     @log_function
     def make_query(self, destination, query_type, args, retry_on_dns_fail,
                    ignore_backoff=False):
-        path = PREFIX + "/query/%s" % query_type
+        path = _create_path(PREFIX, "/query/%s", query_type)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -223,7 +226,7 @@ class TransportLayerClient(object):
                 "make_membership_event called with membership='%s', must be one of %s" %
                 (membership, ",".join(valid_memberships))
             )
-        path = PREFIX + "/make_%s/%s/%s" % (membership, room_id, user_id)
+        path = _create_path(PREFIX, "/make_%s/%s/%s", membership, room_id, user_id)
 
         ignore_backoff = False
         retry_on_dns_fail = False
@@ -249,7 +252,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_join(self, destination, room_id, event_id, content):
-        path = PREFIX + "/send_join/%s/%s" % (room_id, event_id)
+        path = _create_path(PREFIX, "/send_join/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -262,7 +265,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_leave(self, destination, room_id, event_id, content):
-        path = PREFIX + "/send_leave/%s/%s" % (room_id, event_id)
+        path = _create_path(PREFIX, "/send_leave/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -281,7 +284,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_invite(self, destination, room_id, event_id, content):
-        path = PREFIX + "/invite/%s/%s" % (room_id, event_id)
+        path = _create_path(PREFIX, "/invite/%s/%s", room_id, event_id)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -323,7 +326,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def exchange_third_party_invite(self, destination, room_id, event_dict):
-        path = PREFIX + "/exchange_third_party_invite/%s" % (room_id,)
+        path = _create_path(PREFIX, "/exchange_third_party_invite/%s", room_id,)
 
         response = yield self.client.put_json(
             destination=destination,
@@ -336,7 +339,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def get_event_auth(self, destination, room_id, event_id):
-        path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id)
+        path = _create_path(PREFIX, "/event_auth/%s/%s", room_id, event_id)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -348,7 +351,7 @@ class TransportLayerClient(object):
     @defer.inlineCallbacks
     @log_function
     def send_query_auth(self, destination, room_id, event_id, content):
-        path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id)
+        path = _create_path(PREFIX, "/query_auth/%s/%s", room_id, event_id)
 
         content = yield self.client.post_json(
             destination=destination,
@@ -410,7 +413,7 @@ class TransportLayerClient(object):
         Returns:
             A dict containg the device keys.
         """
-        path = PREFIX + "/user/devices/" + user_id
+        path = _create_path(PREFIX, "/user/devices/%s", user_id)
 
         content = yield self.client.get_json(
             destination=destination,
@@ -460,7 +463,7 @@ class TransportLayerClient(object):
     @log_function
     def get_missing_events(self, destination, room_id, earliest_events,
                            latest_events, limit, min_depth, timeout):
-        path = PREFIX + "/get_missing_events/%s" % (room_id,)
+        path = _create_path(PREFIX, "/get_missing_events/%s", room_id,)
 
         content = yield self.client.post_json(
             destination=destination,
@@ -480,7 +483,7 @@ class TransportLayerClient(object):
     def get_group_profile(self, destination, group_id, requester_user_id):
         """Get a group profile
         """
-        path = PREFIX + "/groups/%s/profile" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/profile", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -499,7 +502,7 @@ class TransportLayerClient(object):
             requester_user_id (str)
             content (dict): The new profile of the group
         """
-        path = PREFIX + "/groups/%s/profile" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/profile", group_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -513,7 +516,7 @@ class TransportLayerClient(object):
     def get_group_summary(self, destination, group_id, requester_user_id):
         """Get a group summary
         """
-        path = PREFIX + "/groups/%s/summary" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/summary", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -526,7 +529,7 @@ class TransportLayerClient(object):
     def get_rooms_in_group(self, destination, group_id, requester_user_id):
         """Get all rooms in a group
         """
-        path = PREFIX + "/groups/%s/rooms" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/rooms", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -539,7 +542,7 @@ class TransportLayerClient(object):
                           content):
         """Add a room to a group
         """
-        path = PREFIX + "/groups/%s/room/%s" % (group_id, room_id,)
+        path = _create_path(PREFIX, "/groups/%s/room/%s", group_id, room_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -553,7 +556,10 @@ class TransportLayerClient(object):
                              config_key, content):
         """Update room in group
         """
-        path = PREFIX + "/groups/%s/room/%s/config/%s" % (group_id, room_id, config_key,)
+        path = _create_path(
+            PREFIX, "/groups/%s/room/%s/config/%s",
+            group_id, room_id, config_key,
+        )
 
         return self.client.post_json(
             destination=destination,
@@ -566,7 +572,7 @@ class TransportLayerClient(object):
     def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
         """Remove a room from a group
         """
-        path = PREFIX + "/groups/%s/room/%s" % (group_id, room_id,)
+        path = _create_path(PREFIX, "/groups/%s/room/%s", group_id, room_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -579,7 +585,7 @@ class TransportLayerClient(object):
     def get_users_in_group(self, destination, group_id, requester_user_id):
         """Get users in a group
         """
-        path = PREFIX + "/groups/%s/users" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/users", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -592,7 +598,7 @@ class TransportLayerClient(object):
     def get_invited_users_in_group(self, destination, group_id, requester_user_id):
         """Get users that have been invited to a group
         """
-        path = PREFIX + "/groups/%s/invited_users" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/invited_users", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -605,7 +611,10 @@ class TransportLayerClient(object):
     def accept_group_invite(self, destination, group_id, user_id, content):
         """Accept a group invite
         """
-        path = PREFIX + "/groups/%s/users/%s/accept_invite" % (group_id, user_id)
+        path = _create_path(
+            PREFIX, "/groups/%s/users/%s/accept_invite",
+            group_id, user_id,
+        )
 
         return self.client.post_json(
             destination=destination,
@@ -618,7 +627,7 @@ class TransportLayerClient(object):
     def join_group(self, destination, group_id, user_id, content):
         """Attempts to join a group
         """
-        path = PREFIX + "/groups/%s/users/%s/join" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/join", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -631,7 +640,7 @@ class TransportLayerClient(object):
     def invite_to_group(self, destination, group_id, user_id, requester_user_id, content):
         """Invite a user to a group
         """
-        path = PREFIX + "/groups/%s/users/%s/invite" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/invite", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -647,7 +656,7 @@ class TransportLayerClient(object):
         invited.
         """
 
-        path = PREFIX + "/groups/local/%s/users/%s/invite" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/local/%s/users/%s/invite", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -661,7 +670,7 @@ class TransportLayerClient(object):
                                user_id, content):
         """Remove a user fron a group
         """
-        path = PREFIX + "/groups/%s/users/%s/remove" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/users/%s/remove", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -678,7 +687,7 @@ class TransportLayerClient(object):
         kicked from the group.
         """
 
-        path = PREFIX + "/groups/local/%s/users/%s/remove" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/local/%s/users/%s/remove", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -693,7 +702,7 @@ class TransportLayerClient(object):
         the attestations
         """
 
-        path = PREFIX + "/groups/%s/renew_attestation/%s" % (group_id, user_id)
+        path = _create_path(PREFIX, "/groups/%s/renew_attestation/%s", group_id, user_id)
 
         return self.client.post_json(
             destination=destination,
@@ -708,11 +717,12 @@ class TransportLayerClient(object):
         """Update a room entry in a group summary
         """
         if category_id:
-            path = PREFIX + "/groups/%s/summary/categories/%s/rooms/%s" % (
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/categories/%s/rooms/%s",
                 group_id, category_id, room_id,
             )
         else:
-            path = PREFIX + "/groups/%s/summary/rooms/%s" % (group_id, room_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/rooms/%s", group_id, room_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -728,11 +738,12 @@ class TransportLayerClient(object):
         """Delete a room entry in a group summary
         """
         if category_id:
-            path = PREFIX + "/groups/%s/summary/categories/%s/rooms/%s" % (
+            path = _create_path(
+                PREFIX + "/groups/%s/summary/categories/%s/rooms/%s",
                 group_id, category_id, room_id,
             )
         else:
-            path = PREFIX + "/groups/%s/summary/rooms/%s" % (group_id, room_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/rooms/%s", group_id, room_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -745,7 +756,7 @@ class TransportLayerClient(object):
     def get_group_categories(self, destination, group_id, requester_user_id):
         """Get all categories in a group
         """
-        path = PREFIX + "/groups/%s/categories" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -758,7 +769,7 @@ class TransportLayerClient(object):
     def get_group_category(self, destination, group_id, requester_user_id, category_id):
         """Get category info in a group
         """
-        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -772,7 +783,7 @@ class TransportLayerClient(object):
                               content):
         """Update a category in a group
         """
-        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -787,7 +798,7 @@ class TransportLayerClient(object):
                               category_id):
         """Delete a category in a group
         """
-        path = PREFIX + "/groups/%s/categories/%s" % (group_id, category_id,)
+        path = _create_path(PREFIX, "/groups/%s/categories/%s", group_id, category_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -800,7 +811,7 @@ class TransportLayerClient(object):
     def get_group_roles(self, destination, group_id, requester_user_id):
         """Get all roles in a group
         """
-        path = PREFIX + "/groups/%s/roles" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles", group_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -813,7 +824,7 @@ class TransportLayerClient(object):
     def get_group_role(self, destination, group_id, requester_user_id, role_id):
         """Get a roles info
         """
-        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.get_json(
             destination=destination,
@@ -827,7 +838,7 @@ class TransportLayerClient(object):
                           content):
         """Update a role in a group
         """
-        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -841,7 +852,7 @@ class TransportLayerClient(object):
     def delete_group_role(self, destination, group_id, requester_user_id, role_id):
         """Delete a role in a group
         """
-        path = PREFIX + "/groups/%s/roles/%s" % (group_id, role_id,)
+        path = _create_path(PREFIX, "/groups/%s/roles/%s", group_id, role_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -856,11 +867,12 @@ class TransportLayerClient(object):
         """Update a users entry in a group
         """
         if role_id:
-            path = PREFIX + "/groups/%s/summary/roles/%s/users/%s" % (
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/roles/%s/users/%s",
                 group_id, role_id, user_id,
             )
         else:
-            path = PREFIX + "/groups/%s/summary/users/%s" % (group_id, user_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/users/%s", group_id, user_id,)
 
         return self.client.post_json(
             destination=destination,
@@ -875,7 +887,7 @@ class TransportLayerClient(object):
                               content):
         """Sets the join policy for a group
         """
-        path = PREFIX + "/groups/%s/settings/m.join_policy" % (group_id,)
+        path = _create_path(PREFIX, "/groups/%s/settings/m.join_policy", group_id,)
 
         return self.client.put_json(
             destination=destination,
@@ -891,11 +903,12 @@ class TransportLayerClient(object):
         """Delete a users entry in a group
         """
         if role_id:
-            path = PREFIX + "/groups/%s/summary/roles/%s/users/%s" % (
+            path = _create_path(
+                PREFIX, "/groups/%s/summary/roles/%s/users/%s",
                 group_id, role_id, user_id,
             )
         else:
-            path = PREFIX + "/groups/%s/summary/users/%s" % (group_id, user_id,)
+            path = _create_path(PREFIX, "/groups/%s/summary/users/%s", group_id, user_id,)
 
         return self.client.delete_json(
             destination=destination,
@@ -918,3 +931,22 @@ class TransportLayerClient(object):
             data=content,
             ignore_backoff=True,
         )
+
+
+def _create_path(prefix, path, *args):
+    """Creates a path from the prefix, path template and args. Ensures that
+    all args are url encoded.
+
+    Example:
+
+        _create_path(PREFIX, "/event/%s/", event_id)
+
+    Args:
+        prefix (str)
+        path (str): String template for the path
+        args: ([str]): Args to insert into path. Each arg will be url encoded
+
+    Returns:
+        str
+    """
+    return prefix + path % tuple(urllib.quote(arg) for arg in args)
-- 
cgit 1.4.1


From 11d2609da70af797405241cdf7d9df19db5628f2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 11:24:40 +0100
Subject: Ensure slashes are escaped

---
 synapse/federation/transport/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 1976e5d2fb..6db8efa6dd 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -949,4 +949,4 @@ def _create_path(prefix, path, *args):
     Returns:
         str
     """
-    return prefix + path % tuple(urllib.quote(arg) for arg in args)
+    return prefix + path % tuple(urllib.quote(arg, "") for arg in args)
-- 
cgit 1.4.1


From d49cbf712fcb152c7d3fb5b3b9bace277a3935d0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 12:03:33 +0100
Subject: Log event ID on exception

---
 synapse/federation/transaction_queue.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index d8e5c08a3c..8499d2429e 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -198,7 +198,10 @@ class TransactionQueue(object):
                             ],
                         )
                     except Exception:
-                        logger.exception("Failed to calculate hosts in room")
+                        logger.exception(
+                            "Failed to calculate hosts in room for event: %s",
+                            event.event_id,
+                        )
                         return
 
                     destinations = set(destinations)
-- 
cgit 1.4.1


From 1246d23710741be866692d47b15c432f48483a52 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 12:04:32 +0100
Subject: Preserve log contexts correctly

---
 synapse/federation/transaction_queue.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 8499d2429e..4dce984c1a 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -226,7 +226,10 @@ class TransactionQueue(object):
                     events_by_room.setdefault(event.room_id, []).append(event)
 
                 yield logcontext.make_deferred_yieldable(defer.gatherResults(
-                    [handle_room_events(evs) for evs in events_by_room.itervalues()],
+                    [
+                        logcontext.preserve_fn(handle_room_events)(evs)
+                        for evs in events_by_room.itervalues()
+                    ],
                     consumeErrors=True
                 ))
 
-- 
cgit 1.4.1


From f8e8ec013bad0afb010d8cfda3da63e36d37fea5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 14:00:24 +0100
Subject: Note why we're limiting concurrent event sends

---
 synapse/handlers/message.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 1814b205da..54cd691f91 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -557,6 +557,11 @@ class EventCreationHandler(object):
         See self.create_event and self.send_nonmember_event.
         """
 
+        # We limit the number of concurrent event sends in a room so that we
+        # don't fork the DAG too much. If we don't limit then we can end up in
+        # a situation where event persistence can't keep up, causing
+        # extremities to pile up, which in turn leads to state resolution
+        # taking longer.
         with (yield self.limiter.queue(event_dict["room_id"])):
             event, context = yield self.create_event(
                 requester,
-- 
cgit 1.4.1


From a060dfa1324b5ec12e0af96ec090a3b997582cb2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 10 Apr 2018 14:25:11 +0100
Subject: Use run_in_background instead

---
 synapse/federation/transaction_queue.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 4dce984c1a..5b0b798e57 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -227,7 +227,7 @@ class TransactionQueue(object):
 
                 yield logcontext.make_deferred_yieldable(defer.gatherResults(
                     [
-                        logcontext.preserve_fn(handle_room_events)(evs)
+                        logcontext.run_in_background(handle_room_events, evs)
                         for evs in events_by_room.itervalues()
                     ],
                     consumeErrors=True
-- 
cgit 1.4.1


From 7f8eebc8ee801467b0a04aa0fb07b1ba58866f22 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 7 Apr 2018 01:39:45 +0200
Subject: Open config file in non-bytes mode

Nothing written into it is encoded, so it makes little sense, but it
does break in python3 the way it was before.

The variable names were adjusted to be less misleading.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/config/_base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 32b439d20a..b748ed2b0a 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -281,15 +281,15 @@ class Config(object):
                     )
                 if not cls.path_exists(config_dir_path):
                     os.makedirs(config_dir_path)
-                with open(config_path, "wb") as config_file:
-                    config_bytes, config = obj.generate_config(
+                with open(config_path, "w") as config_file:
+                    config_str, config = obj.generate_config(
                         config_dir_path=config_dir_path,
                         server_name=server_name,
                         report_stats=(config_args.report_stats == "yes"),
                         is_generating_file=True
                     )
                     obj.invoke_all("generate_files", config)
-                    config_file.write(config_bytes)
+                    config_file.write(config_str)
                 print((
                     "A config file has been generated in %r for server name"
                     " %r with corresponding SSL keys and self-signed"
-- 
cgit 1.4.1


From a3f9ddbeded33c61a715c86ba5029954ba22c22d Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 10 Apr 2018 17:22:52 +0200
Subject: Open certificate files as bytes

That's what pyOpenSSL expects on python3

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/config/tls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 29eb012ddb..b66154bc7c 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -133,7 +133,7 @@ class TlsConfig(Config):
         tls_dh_params_path = config["tls_dh_params_path"]
 
         if not self.path_exists(tls_private_key_path):
-            with open(tls_private_key_path, "w") as private_key_file:
+            with open(tls_private_key_path, "wb") as private_key_file:
                 tls_private_key = crypto.PKey()
                 tls_private_key.generate_key(crypto.TYPE_RSA, 2048)
                 private_key_pem = crypto.dump_privatekey(
@@ -148,7 +148,7 @@ class TlsConfig(Config):
                 )
 
         if not self.path_exists(tls_certificate_path):
-            with open(tls_certificate_path, "w") as certificate_file:
+            with open(tls_certificate_path, "wb") as certificate_file:
                 cert = crypto.X509()
                 subject = cert.get_subject()
                 subject.CN = config["server_name"]
-- 
cgit 1.4.1


From d54cfbb7a826ccdff8b911b214bb7476a04a65a6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 10 Apr 2018 17:38:16 +0100
Subject: fix typo

---
 synapse/storage/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index eacd49d6a5..8cdfd50f90 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -266,9 +266,9 @@ class DataStore(RoomMemberStore, RoomStore,
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
-         * Users who have created their accounts more than 30 days
+         * Users who have created their accounts more than 30 days ago
          * Where last seen at most 30 days ago
-         * Where account creation and last_seen are > 30 days
+         * Where account creation and last_seen are > 30 days apart
 
          Returns counts globaly for a given user as well as breaking
          by platform
-- 
cgit 1.4.1


From b3384232a031cc209fb5f0e085bc073a220448be Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 10 Apr 2018 23:14:47 +0100
Subject: Add metrics for ResponseCache

---
 synapse/appservice/api.py               |  3 ++-
 synapse/federation/federation_server.py |  2 +-
 synapse/handlers/room_list.py           |  5 +++--
 synapse/handlers/sync.py                |  2 +-
 synapse/replication/http/send_event.py  |  2 +-
 synapse/util/caches/response_cache.py   | 14 +++++++++++++-
 6 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 40c433d7ae..11e9c37c63 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -73,7 +73,8 @@ class ApplicationServiceApi(SimpleHttpClient):
         super(ApplicationServiceApi, self).__init__(hs)
         self.clock = hs.get_clock()
 
-        self.protocol_meta_cache = ResponseCache(hs, timeout_ms=HOUR_IN_MS)
+        self.protocol_meta_cache = ResponseCache(hs, "as_protocol_meta",
+                                                 timeout_ms=HOUR_IN_MS)
 
     @defer.inlineCallbacks
     def query_user(self, service, user_id):
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index bea7fd0b71..e4ce037acf 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -65,7 +65,7 @@ class FederationServer(FederationBase):
 
         # We cache responses to state queries, as they take a while and often
         # come in waves.
-        self._state_resp_cache = ResponseCache(hs, timeout_ms=30000)
+        self._state_resp_cache = ResponseCache(hs, "state_resp", timeout_ms=30000)
 
     @defer.inlineCallbacks
     @log_function
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 5d81f59b44..8028d793c2 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -44,8 +44,9 @@ EMTPY_THIRD_PARTY_ID = ThirdPartyInstanceID(None, None)
 class RoomListHandler(BaseHandler):
     def __init__(self, hs):
         super(RoomListHandler, self).__init__(hs)
-        self.response_cache = ResponseCache(hs)
-        self.remote_response_cache = ResponseCache(hs, timeout_ms=30 * 1000)
+        self.response_cache = ResponseCache(hs, "room_list")
+        self.remote_response_cache = ResponseCache(hs, "remote_room_list",
+                                                   timeout_ms=30 * 1000)
 
     def get_local_public_room_list(self, limit=None, since_token=None,
                                    search_filter=None,
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 0f713ce038..06d17ab20c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -169,7 +169,7 @@ class SyncHandler(object):
         self.presence_handler = hs.get_presence_handler()
         self.event_sources = hs.get_event_sources()
         self.clock = hs.get_clock()
-        self.response_cache = ResponseCache(hs)
+        self.response_cache = ResponseCache(hs, "sync")
         self.state = hs.get_state_handler()
 
     def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index bbe2f967b7..c6a6551d24 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -115,7 +115,7 @@ class ReplicationSendEventRestServlet(RestServlet):
         self.clock = hs.get_clock()
 
         # The responses are tiny, so we may as well cache them for a while
-        self.response_cache = ResponseCache(hs, timeout_ms=30 * 60 * 1000)
+        self.response_cache = ResponseCache(hs, "send_event", timeout_ms=30 * 60 * 1000)
 
     def on_PUT(self, request, event_id):
         result = self.response_cache.get(event_id)
diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 00af539880..7f68289723 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from synapse.util.async import ObservableDeferred
+from synapse.util.caches import metrics as cache_metrics
 
 
 class ResponseCache(object):
@@ -24,17 +25,28 @@ class ResponseCache(object):
     used rather than trying to compute a new response.
     """
 
-    def __init__(self, hs, timeout_ms=0):
+    def __init__(self, hs, name, timeout_ms=0):
         self.pending_result_cache = {}  # Requests that haven't finished yet.
 
         self.clock = hs.get_clock()
         self.timeout_sec = timeout_ms / 1000.
 
+        self._metrics = cache_metrics.register_cache(
+            "response_cache",
+            size_callback=lambda: self.size(),
+            cache_name=name,
+        )
+
+    def size(self):
+        return len(self.pending_result_cache)
+
     def get(self, key):
         result = self.pending_result_cache.get(key)
         if result is not None:
+            self._metrics.inc_hits()
             return result.observe()
         else:
+            self._metrics.inc_misses()
             return None
 
     def set(self, key, deferred):
-- 
cgit 1.4.1


From 121591568bdc6f9abde0c09cd80e4a3aa74ee109 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Apr 2018 09:56:00 +0100
Subject: Send events to ASes concurrently

---
 synapse/handlers/appservice.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 3dd3fa2a27..69eacff949 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -18,7 +18,9 @@ from twisted.internet import defer
 import synapse
 from synapse.api.constants import EventTypes
 from synapse.util.metrics import Measure
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import (
+    make_deferred_yieldable, preserve_fn, run_in_background,
+)
 
 import logging
 
@@ -84,11 +86,16 @@ class ApplicationServicesHandler(object):
                     if not events:
                         break
 
+                    events_by_room = {}
                     for event in events:
+                        events_by_room.setdefault(event.room_id, []).append(event)
+
+                    @defer.inlineCallbacks
+                    def handle_event(event):
                         # Gather interested services
                         services = yield self._get_services_for_event(event)
                         if len(services) == 0:
-                            continue  # no services need notifying
+                            return  # no services need notifying
 
                         # Do we know this user exists? If not, poke the user
                         # query API for all services which match that user regex.
@@ -108,6 +115,16 @@ class ApplicationServicesHandler(object):
                                 service, event
                             )
 
+                    @defer.inlineCallbacks
+                    def handle_room_events(events):
+                        for event in events:
+                            yield handle_event(event)
+
+                    yield make_deferred_yieldable(defer.gatherResults([
+                        run_in_background(handle_room_events, evs)
+                        for evs in events_by_room.itervalues()
+                    ], consumeErrors=True))
+
                     events_processed_counter.inc_by(len(events))
 
                     yield self.store.set_appservice_last_pos(upper_bound)
-- 
cgit 1.4.1


From 781cd8c54f15378fe950a07a6e949fa65a499b12 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 11 Apr 2018 10:54:43 +0100
Subject: bump version/changelog

---
 CHANGES.rst         | 8 ++++++++
 synapse/__init__.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 684c5c7883..a838c092fe 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,5 +1,13 @@
+Changes in synapse v0.27.3 (2018-04-11)
+======================================
+
+Bug fixes:
+
+* URL quote path segments over federation (#3082)
+
 Changes in synapse v0.27.3-rc2 (2018-04-09)
 ==========================================
+
 v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates
 the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 7f6090baf8..1fa2de5268 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.3-rc2"
+__version__ = "0.27.3"
-- 
cgit 1.4.1


From ab825aa328e089b28cf858332bb95f47e1490377 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Apr 2018 11:07:33 +0100
Subject: Add GaugeMetric

---
 synapse/metrics/__init__.py |  9 ++++++++-
 synapse/metrics/metric.py   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 2ed82b04a5..2666f982a6 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -23,7 +23,7 @@ from twisted.internet import reactor
 
 from .metric import (
     CounterMetric, CallbackMetric, DistributionMetric, CacheMetric,
-    MemoryUsageMetric,
+    MemoryUsageMetric, GaugeMetric,
 )
 from .process_collector import register_process_collector
 
@@ -65,6 +65,13 @@ class Metrics(object):
         """
         return self._register(CounterMetric, *args, **kwargs)
 
+    def register_gauge(self, *args, **kwargs):
+        """
+        Returns:
+            GaugeMetric
+        """
+        return self._register(GaugeMetric, *args, **kwargs)
+
     def register_callback(self, *args, **kwargs):
         """
         Returns:
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index ff5aa8c0e1..9ebf2d6fdb 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -145,6 +145,36 @@ class CounterMetric(BaseMetric):
         )
 
 
+class GaugeMetric(BaseMetric):
+    """A metric that can go up or down
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(GaugeMetric, self).__init__(*args, **kwargs)
+
+        # dict[list[str]]: value for each set of label values. the keys are the
+        # label values, in the same order as the labels in self.labels.
+        #
+        # (if the metric is a scalar, the (single) key is the empty list).
+        self.guages = {}
+
+    def set(self, v, *values):
+        if len(values) != self.dimension():
+            raise ValueError(
+                "Expected as many values to inc() as labels (%d)" % (self.dimension())
+            )
+
+        # TODO: should assert that the tag values are all strings
+
+        self.guages[values] = v
+
+    def render(self):
+        return flatten(
+            self._render_for_labels(k, self.guages[k])
+            for k in sorted(self.guages.keys())
+        )
+
+
 class CallbackMetric(BaseMetric):
     """A metric that returns the numeric value returned by a callback whenever
     it is rendered. Typically this is used to implement gauges that yield the
-- 
cgit 1.4.1


From 92e34615c5c9ed5df2b0072a2686d3e42239e420 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Apr 2018 11:07:51 +0100
Subject: Track where event stream processing have gotten up to

---
 synapse/federation/transaction_queue.py |  4 ++++
 synapse/handlers/appservice.py          |  4 ++++
 synapse/metrics/__init__.py             | 13 +++++++++++++
 synapse/storage/events.py               |  3 +++
 4 files changed, 24 insertions(+)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 5b0b798e57..d68f0d8950 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -239,6 +239,10 @@ class TransactionQueue(object):
                     "events", next_token
                 )
 
+                synapse.metrics.event_processing_positions.set(
+                    next_token, "federation_sender",
+                )
+
         finally:
             self._is_processing = False
 
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 3dd3fa2a27..2a7e31e711 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -111,6 +111,10 @@ class ApplicationServicesHandler(object):
                     events_processed_counter.inc_by(len(events))
 
                     yield self.store.set_appservice_last_pos(upper_bound)
+
+                    synapse.metrics.event_processing_positions.set(
+                        upper_bound, "appservice_sender",
+                    )
             finally:
                 self.is_processing = False
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 2666f982a6..a6c0d7e1bf 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -151,6 +151,19 @@ reactor_metrics = get_metrics_for("python.twisted.reactor")
 tick_time = reactor_metrics.register_distribution("tick_time")
 pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
 
+synapse_metrics = get_metrics_for("synapse")
+
+# Used to track where various components have processed in the event stream,
+# e.g. federation sending, appservice sending, etc.
+event_processing_positions = synapse_metrics.register_gauge(
+    "event_processing_positions", labels=["name"],
+)
+
+# Used to track the current max events stream position
+event_persisted_position = synapse_metrics.register_gauge(
+    "event_persisted_position",
+)
+
 
 def runUntilCurrentTimer(func):
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ece5e6c41f..da44b52fd6 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -444,6 +444,9 @@ class EventsStore(EventsWorkerStore):
                     new_forward_extremeties=new_forward_extremeties,
                 )
                 persist_event_counter.inc_by(len(chunk))
+                synapse.metrics.event_persisted_position.set(
+                    chunk[-1][0].internal_metadata.stream_ordering,
+                )
                 for event, context in chunk:
                     if context.app_service:
                         origin_type = "local"
-- 
cgit 1.4.1


From 4dae4a97ed0e0b2cc9b5493172670ec7353ded2e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Apr 2018 11:52:19 +0100
Subject: Track last processed event received_ts

---
 synapse/federation/transaction_queue.py | 11 +++++++++++
 synapse/handlers/appservice.py          | 10 ++++++++++
 synapse/metrics/__init__.py             | 13 +++++++++++++
 synapse/storage/events_worker.py        | 18 ++++++++++++++++++
 4 files changed, 52 insertions(+)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index d68f0d8950..1c466cbded 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -243,6 +243,17 @@ class TransactionQueue(object):
                     next_token, "federation_sender",
                 )
 
+                if events:
+                    now = self.clock.time_msec()
+                    ts = yield self.store.get_received_ts(events[-1].event_id)
+
+                    synapse.metrics.event_processing_lag.set(
+                        now - ts, "federation_sender",
+                    )
+                    synapse.metrics.event_processing_last_ts.set(
+                        ts, "federation_sender",
+                    )
+
         finally:
             self._is_processing = False
 
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 2a7e31e711..fcc1246bee 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -115,6 +115,16 @@ class ApplicationServicesHandler(object):
                     synapse.metrics.event_processing_positions.set(
                         upper_bound, "appservice_sender",
                     )
+
+                    now = self.clock.time_msec()
+                    ts = yield self.store.get_received_ts(events[-1].event_id)
+
+                    synapse.metrics.event_processing_lag.set(
+                        now - ts, "appservice_sender",
+                    )
+                    synapse.metrics.event_processing_last_ts.set(
+                        ts, "appservice_sender",
+                    )
             finally:
                 self.is_processing = False
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index a6c0d7e1bf..e3b831db67 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -164,6 +164,19 @@ event_persisted_position = synapse_metrics.register_gauge(
     "event_persisted_position",
 )
 
+# Used to track the received_ts of the last event processed by various
+# components
+event_processing_last_ts = synapse_metrics.register_gauge(
+    "event_processing_last_ts", labels=["name"],
+)
+
+# Used to track the lag processing events. This is the time difference
+# between the last processed event's received_ts and the time it was
+# finished being processed.
+event_processing_lag = synapse_metrics.register_gauge(
+    "event_processing_lag", labels=["name"],
+)
+
 
 def runUntilCurrentTimer(func):
 
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 2e23dd78ba..769eb51489 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -51,6 +51,24 @@ _EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
 
 
 class EventsWorkerStore(SQLBaseStore):
+    def get_received_ts(self, event_id):
+        """Get received_ts (when it was persisted) for the event
+
+        Args:
+            event_id (str)
+
+        Returns:
+            Deferred[int|None]: Timstamp in milliseconds, or None for events
+            that were persisted before received_ts was implemented.
+        """
+        return self._simple_select_one_onecol(
+            table="events",
+            keyvalues={
+                "event_id": event_id,
+            },
+            retcol="received_ts",
+            desc="get_received_ts",
+        )
 
     @defer.inlineCallbacks
     def get_event(self, event_id, check_redacted=True,
-- 
cgit 1.4.1


From e939f3bca6d73bc4ec6450aeb41f5e121a29a662 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 11 Apr 2018 14:37:11 +0100
Subject: Fix tests

---
 tests/handlers/test_appservice.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py
index a667fb6f0e..b753455943 100644
--- a/tests/handlers/test_appservice.py
+++ b/tests/handlers/test_appservice.py
@@ -31,6 +31,7 @@ class AppServiceHandlerTestCase(unittest.TestCase):
         self.mock_scheduler = Mock()
         hs = Mock()
         hs.get_datastore = Mock(return_value=self.mock_store)
+        self.mock_store.get_received_ts.return_value = 0
         hs.get_application_service_api = Mock(return_value=self.mock_as_api)
         hs.get_application_service_scheduler = Mock(return_value=self.mock_scheduler)
         hs.get_clock.return_value = MockClock()
-- 
cgit 1.4.1


From f67e906e189cb528cab18cb5190be352dcc8914b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 12 Apr 2018 11:18:19 +0100
Subject: Set all metrics at the same time

---
 synapse/federation/transaction_queue.py | 12 ++++++------
 synapse/handlers/appservice.py          |  8 ++++----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 1c466cbded..963d938edd 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -233,16 +233,10 @@ class TransactionQueue(object):
                     consumeErrors=True
                 ))
 
-                events_processed_counter.inc_by(len(events))
-
                 yield self.store.update_federation_out_pos(
                     "events", next_token
                 )
 
-                synapse.metrics.event_processing_positions.set(
-                    next_token, "federation_sender",
-                )
-
                 if events:
                     now = self.clock.time_msec()
                     ts = yield self.store.get_received_ts(events[-1].event_id)
@@ -254,6 +248,12 @@ class TransactionQueue(object):
                         ts, "federation_sender",
                     )
 
+                events_processed_counter.inc_by(len(events))
+
+                synapse.metrics.event_processing_positions.set(
+                    next_token, "federation_sender",
+                )
+
         finally:
             self._is_processing = False
 
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index fcc1246bee..ce0814bc25 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -108,16 +108,16 @@ class ApplicationServicesHandler(object):
                                 service, event
                             )
 
-                    events_processed_counter.inc_by(len(events))
-
                     yield self.store.set_appservice_last_pos(upper_bound)
 
+                    now = self.clock.time_msec()
+                    ts = yield self.store.get_received_ts(events[-1].event_id)
+
                     synapse.metrics.event_processing_positions.set(
                         upper_bound, "appservice_sender",
                     )
 
-                    now = self.clock.time_msec()
-                    ts = yield self.store.get_received_ts(events[-1].event_id)
+                    events_processed_counter.inc_by(len(events))
 
                     synapse.metrics.event_processing_lag.set(
                         now - ts, "appservice_sender",
-- 
cgit 1.4.1


From d7bf3a68f002680ed0cd8a84c75b011371a50d6e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 12 Apr 2018 11:19:04 +0100
Subject: s/list/tuple

---
 synapse/metrics/metric.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 9ebf2d6fdb..89bd47c3f7 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -115,7 +115,7 @@ class CounterMetric(BaseMetric):
         # dict[list[str]]: value for each set of label values. the keys are the
         # label values, in the same order as the labels in self.labels.
         #
-        # (if the metric is a scalar, the (single) key is the empty list).
+        # (if the metric is a scalar, the (single) key is the empty tuple).
         self.counts = {}
 
         # Scalar metrics are never empty
@@ -155,7 +155,7 @@ class GaugeMetric(BaseMetric):
         # dict[list[str]]: value for each set of label values. the keys are the
         # label values, in the same order as the labels in self.labels.
         #
-        # (if the metric is a scalar, the (single) key is the empty list).
+        # (if the metric is a scalar, the (single) key is the empty tuple).
         self.guages = {}
 
     def set(self, v, *values):
-- 
cgit 1.4.1


From 23a7f9d7f4cfbf0333a1def526a59fb4b0f01067 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 12 Apr 2018 11:20:51 +0100
Subject: Doc we raise on unknown event

---
 synapse/storage/events_worker.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 769eb51489..88360b48b8 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -52,13 +52,14 @@ _EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
 
 class EventsWorkerStore(SQLBaseStore):
     def get_received_ts(self, event_id):
-        """Get received_ts (when it was persisted) for the event
+        """Get received_ts (when it was persisted) for the event. Raises an
+        exception for unknown events.
 
         Args:
             event_id (str)
 
         Returns:
-            Deferred[int|None]: Timstamp in milliseconds, or None for events
+            Deferred[int|None]: Timestamp in milliseconds, or None for events
             that were persisted before received_ts was implemented.
         """
         return self._simple_select_one_onecol(
-- 
cgit 1.4.1


From 415aeefd89ca48d3a3e9a7a981999ae071f6060b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 12 Apr 2018 12:07:09 +0100
Subject: Format docstring

---
 synapse/storage/events_worker.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 88360b48b8..a937b9bceb 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -52,8 +52,9 @@ _EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
 
 class EventsWorkerStore(SQLBaseStore):
     def get_received_ts(self, event_id):
-        """Get received_ts (when it was persisted) for the event. Raises an
-        exception for unknown events.
+        """Get received_ts (when it was persisted) for the event.
+
+        Raises an exception for unknown events.
 
         Args:
             event_id (str)
-- 
cgit 1.4.1


From b78395b7fe449d59a5c46c81a869f9f191cd934f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 12 Apr 2018 12:08:59 +0100
Subject: Refactor ResponseCache usage

Adds a `.wrap` method to ResponseCache which wraps up the boilerplate of a
(get, set) pair, and then use it throughout the codebase.

This will be largely non-functional, but does include the following functional
changes:

* federation_server.on_context_state_request: drops use of _server_linearizer
  which looked redundant and could cause incorrect cache misses by yielding
  between the get and the set.
* RoomListHandler.get_remote_public_room_list(): fixes logcontext leaks
* the wrap function includes some logging. I'm hoping this won't be too noisy
  on production.
---
 synapse/appservice/api.py               |  8 +----
 synapse/federation/federation_server.py | 16 +++------
 synapse/handlers/room_list.py           | 38 ++++++++-------------
 synapse/handlers/sync.py                | 16 ++++-----
 synapse/replication/http/send_event.py  | 18 ++++------
 synapse/util/caches/response_cache.py   | 58 +++++++++++++++++++++++++++++++--
 6 files changed, 87 insertions(+), 67 deletions(-)

diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 11e9c37c63..00efff1464 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -18,7 +18,6 @@ from synapse.api.constants import ThirdPartyEntityKind
 from synapse.api.errors import CodeMessageException
 from synapse.http.client import SimpleHttpClient
 from synapse.events.utils import serialize_event
-from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.types import ThirdPartyInstanceID
 
@@ -194,12 +193,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                 defer.returnValue(None)
 
         key = (service.id, protocol)
-        result = self.protocol_meta_cache.get(key)
-        if not result:
-            result = self.protocol_meta_cache.set(
-                key, preserve_fn(_get)()
-            )
-        return make_deferred_yieldable(result)
+        return self.protocol_meta_cache.wrap(key, _get)
 
     @defer.inlineCallbacks
     def push_bulk(self, service, events, txn_id=None):
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index e4ce037acf..d1611f39a9 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -30,7 +30,6 @@ import synapse.metrics
 from synapse.types import get_domain_from_id
 from synapse.util import async
 from synapse.util.caches.response_cache import ResponseCache
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 from synapse.util.logutils import log_function
 
 # when processing incoming transactions, we try to handle multiple rooms in
@@ -212,16 +211,11 @@ class FederationServer(FederationBase):
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        result = self._state_resp_cache.get((room_id, event_id))
-        if not result:
-            with (yield self._server_linearizer.queue((origin, room_id))):
-                d = self._state_resp_cache.set(
-                    (room_id, event_id),
-                    preserve_fn(self._on_context_state_request_compute)(room_id, event_id)
-                )
-                resp = yield make_deferred_yieldable(d)
-        else:
-            resp = yield make_deferred_yieldable(result)
+        resp = yield self._state_resp_cache.wrap(
+            (room_id, event_id),
+            self._on_context_state_request_compute,
+            room_id, event_id,
+        )
 
         defer.returnValue((200, resp))
 
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 8028d793c2..add3f9b009 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -20,7 +20,6 @@ from ._base import BaseHandler
 from synapse.api.constants import (
     EventTypes, JoinRules,
 )
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 from synapse.util.async import concurrently_execute
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 from synapse.util.caches.response_cache import ResponseCache
@@ -78,18 +77,11 @@ class RoomListHandler(BaseHandler):
             )
 
         key = (limit, since_token, network_tuple)
-        result = self.response_cache.get(key)
-        if not result:
-            logger.info("No cached result, calculating one.")
-            result = self.response_cache.set(
-                key,
-                preserve_fn(self._get_public_room_list)(
-                    limit, since_token, network_tuple=network_tuple
-                )
-            )
-        else:
-            logger.info("Using cached deferred result.")
-        return make_deferred_yieldable(result)
+        return self.response_cache.wrap(
+            key,
+            self._get_public_room_list,
+            limit, since_token, network_tuple=network_tuple,
+        )
 
     @defer.inlineCallbacks
     def _get_public_room_list(self, limit=None, since_token=None,
@@ -423,18 +415,14 @@ class RoomListHandler(BaseHandler):
             server_name, limit, since_token, include_all_networks,
             third_party_instance_id,
         )
-        result = self.remote_response_cache.get(key)
-        if not result:
-            result = self.remote_response_cache.set(
-                key,
-                repl_layer.get_public_rooms(
-                    server_name, limit=limit, since_token=since_token,
-                    search_filter=search_filter,
-                    include_all_networks=include_all_networks,
-                    third_party_instance_id=third_party_instance_id,
-                )
-            )
-        return result
+        return self.remote_response_cache.wrap(
+            key,
+            repl_layer.get_public_rooms,
+            server_name, limit=limit, since_token=since_token,
+            search_filter=search_filter,
+            include_all_networks=include_all_networks,
+            third_party_instance_id=third_party_instance_id,
+        )
 
 
 class RoomListNextBatch(namedtuple("RoomListNextBatch", (
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 06d17ab20c..c6946831ab 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -15,7 +15,7 @@
 
 from synapse.api.constants import Membership, EventTypes
 from synapse.util.async import concurrently_execute
-from synapse.util.logcontext import LoggingContext, make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import LoggingContext
 from synapse.util.metrics import Measure, measure_func
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.push.clientformat import format_push_rules_for_user
@@ -180,15 +180,11 @@ class SyncHandler(object):
         Returns:
             A Deferred SyncResult.
         """
-        result = self.response_cache.get(sync_config.request_key)
-        if not result:
-            result = self.response_cache.set(
-                sync_config.request_key,
-                preserve_fn(self._wait_for_sync_for_user)(
-                    sync_config, since_token, timeout, full_state
-                )
-            )
-        return make_deferred_yieldable(result)
+        return self.response_cache.wrap(
+            sync_config.request_key,
+            self._wait_for_sync_for_user,
+            sync_config, since_token, timeout, full_state,
+        )
 
     @defer.inlineCallbacks
     def _wait_for_sync_for_user(self, sync_config, since_token, timeout,
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index c6a6551d24..a9baa2c1c3 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -23,7 +23,6 @@ from synapse.events.snapshot import EventContext
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.util.async import sleep
 from synapse.util.caches.response_cache import ResponseCache
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 from synapse.util.metrics import Measure
 from synapse.types import Requester, UserID
 
@@ -118,17 +117,12 @@ class ReplicationSendEventRestServlet(RestServlet):
         self.response_cache = ResponseCache(hs, "send_event", timeout_ms=30 * 60 * 1000)
 
     def on_PUT(self, request, event_id):
-        result = self.response_cache.get(event_id)
-        if not result:
-            result = self.response_cache.set(
-                event_id,
-                self._handle_request(request)
-            )
-        else:
-            logger.warn("Returning cached response")
-        return make_deferred_yieldable(result)
-
-    @preserve_fn
+        return self.response_cache.wrap(
+            event_id,
+            self._handle_request,
+            request
+        )
+
     @defer.inlineCallbacks
     def _handle_request(self, request):
         with Measure(self.clock, "repl_send_event_parse"):
diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 066fa423fd..0c2c347953 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -12,9 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches import metrics as cache_metrics
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
+
+logger = logging.getLogger(__name__)
 
 
 class ResponseCache(object):
@@ -31,6 +35,7 @@ class ResponseCache(object):
         self.clock = hs.get_clock()
         self.timeout_sec = timeout_ms / 1000.
 
+        self._name = name
         self._metrics = cache_metrics.register_cache(
             "response_cache",
             size_callback=lambda: self.size(),
@@ -47,7 +52,7 @@ class ResponseCache(object):
         so you'll probably want to make_deferred_yieldable it.
 
         Args:
-            key (str):
+            key (hashable):
 
         Returns:
             twisted.internet.defer.Deferred|None: None if there is no entry
@@ -76,7 +81,7 @@ class ResponseCache(object):
         to do it everywhere ResponseCache is used.)
 
         Args:
-            key (str):
+            key (hashable):
             deferred (twisted.internet.defer.Deferred):
 
         Returns:
@@ -97,3 +102,52 @@ class ResponseCache(object):
 
         result.addBoth(remove)
         return result.observe()
+
+    def wrap(self, key, callback, *args, **kwargs):
+        """Wrap together a *get* and *set* call, taking care of logcontexts
+
+        First looks up the key in the cache, and if it is present makes it
+        follow the synapse logcontext rules and returns it.
+
+        Otherwise, makes a call to *callback(*args, **kwargs)*, which should
+        follow the synapse logcontext rules, and adds the result to the cache.
+
+        Example usage:
+
+            @defer.inlineCallbacks
+            def handle_request(request):
+                # etc
+                defer.returnValue(result)
+
+            result = yield response_cache.wrap(
+                key,
+                handle_request,
+                request,
+            )
+
+        Args:
+            key (hashable): key to get/set in the cache
+
+            callback (callable): function to call if the key is not found in
+                the cache
+
+            *args: positional parameters to pass to the callback, if it is used
+
+            **kwargs: named paramters to pass to the callback, if it is used
+
+        Returns:
+            twisted.internet.defer.Deferred: yieldable result
+        """
+        result = self.get(key)
+        if not result:
+            logger.info("[%s]: no cached result for [%s], calculating new one",
+                        self._name, key)
+            d = run_in_background(callback, *args, **kwargs)
+            result = self.set(key, d)
+        elif result.called:
+            logger.info("[%s]: using completed cached result for [%s]",
+                        self._name, key)
+        else:
+            logger.info("[%s]: using incomplete cached result for [%s]",
+                        self._name, key)
+        return make_deferred_yieldable(result)
-- 
cgit 1.4.1


From 119596ab8fc42d76950ca1e6db632d4e557d29af Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 12 Apr 2018 17:28:24 +0100
Subject: Update canonicaljson dependency

1.1.0 and 1.1.1 were broken, so we're updating this to help people make sure
they don't end up on a broken version.

Also, 1.1.0 is speedier...
---
 synapse/python_dependencies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 40eedb63cb..ab2545db91 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -22,7 +22,7 @@ REQUIREMENTS = {
     "jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
     "frozendict>=0.4": ["frozendict"],
     "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"],
-    "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"],
+    "canonicaljson>=1.1.2": ["canonicaljson>=1.1.2"],
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-- 
cgit 1.4.1


From 60f6014bb7912cf5629ae7d4ab2452ed67e5304a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 13 Apr 2018 07:32:29 +0100
Subject: ResponseCache: fix handling of completed results

Turns out that ObservableDeferred.observe doesn't return a deferred if the
result is already completed. Fix handling and improve documentation.
---
 synapse/util/caches/response_cache.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 0c2c347953..7f79333e96 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -14,6 +14,8 @@
 # limitations under the License.
 import logging
 
+from twisted.internet import defer
+
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches import metrics as cache_metrics
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
@@ -48,15 +50,21 @@ class ResponseCache(object):
     def get(self, key):
         """Look up the given key.
 
-        Returns a deferred which doesn't follow the synapse logcontext rules,
-        so you'll probably want to make_deferred_yieldable it.
+        Can return either a new Deferred (which also doesn't follow the synapse
+        logcontext rules), or, if the request has completed, the actual
+        result. You will probably want to make_deferred_yieldable the result.
+
+        If there is no entry for the key, returns None. It is worth noting that
+        this means there is no way to distinguish a completed result of None
+        from an absent cache entry.
 
         Args:
             key (hashable):
 
         Returns:
-            twisted.internet.defer.Deferred|None: None if there is no entry
-            for this key; otherwise a deferred result.
+            twisted.internet.defer.Deferred|None|E: None if there is no entry
+            for this key; otherwise either a deferred result or the result
+            itself.
         """
         result = self.pending_result_cache.get(key)
         if result is not None:
@@ -73,19 +81,17 @@ class ResponseCache(object):
         you should wrap normal synapse deferreds with
         logcontext.run_in_background).
 
-        Returns a new Deferred which also doesn't follow the synapse logcontext
-        rules, so you will want to make_deferred_yieldable it
-
-        (TODO: before using this more widely, it might make sense to refactor
-        it and get() so that they do the necessary wrapping rather than having
-        to do it everywhere ResponseCache is used.)
+        Can return either a new Deferred (which also doesn't follow the synapse
+        logcontext rules), or, if *deferred* was already complete, the actual
+        result. You will probably want to make_deferred_yieldable the result.
 
         Args:
             key (hashable):
-            deferred (twisted.internet.defer.Deferred):
+            deferred (twisted.internet.defer.Deferred[T):
 
         Returns:
-            twisted.internet.defer.Deferred
+            twisted.internet.defer.Deferred[T]|T: a new deferred, or the actual
+                result.
         """
         result = ObservableDeferred(deferred, consumeErrors=True)
         self.pending_result_cache[key] = result
@@ -144,7 +150,7 @@ class ResponseCache(object):
                         self._name, key)
             d = run_in_background(callback, *args, **kwargs)
             result = self.set(key, d)
-        elif result.called:
+        elif not isinstance(result, defer.Deferred) or result.called:
             logger.info("[%s]: using completed cached result for [%s]",
                         self._name, key)
         else:
-- 
cgit 1.4.1


From fac3f9e678452020e0e6b2c182913d4be965ab9b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 13 Apr 2018 10:21:38 +0100
Subject: Bump canonicaljson to 1.1.3

1.1.2 was a bit broken too :/
---
 synapse/python_dependencies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index ab2545db91..5cabf7dabe 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -22,7 +22,7 @@ REQUIREMENTS = {
     "jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
     "frozendict>=0.4": ["frozendict"],
     "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"],
-    "canonicaljson>=1.1.2": ["canonicaljson>=1.1.2"],
+    "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"],
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-- 
cgit 1.4.1


From d3347ad48553bd678fca7e3259d0824225cc6af2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 13 Apr 2018 11:16:43 +0100
Subject: Revert "Use sortedcontainers instead of blist"

This reverts commit 9fbe70a7dc3afabfdac176ba1f4be32dd44602aa.

It turns out that sortedcontainers.SortedDict is not an exact match for
blist.sorteddict; in particular, `popitem()` removes things from the opposite
end of the dict.

This is trivial to fix, but I want to add some unit tests, and potentially some
more thought about it, before we do so.
---
 synapse/federation/send_queue.py           | 14 +++++++-------
 synapse/python_dependencies.py             |  2 +-
 synapse/util/caches/stream_change_cache.py |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 945832283f..93e5acebc1 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -35,7 +35,7 @@ from synapse.storage.presence import UserPresenceState
 from synapse.util.metrics import Measure
 import synapse.metrics
 
-from sortedcontainers import SortedDict
+from blist import sorteddict
 from collections import namedtuple
 
 import logging
@@ -56,19 +56,19 @@ class FederationRemoteSendQueue(object):
         self.is_mine_id = hs.is_mine_id
 
         self.presence_map = {}  # Pending presence map user_id -> UserPresenceState
-        self.presence_changed = SortedDict()  # Stream position -> user_id
+        self.presence_changed = sorteddict()  # Stream position -> user_id
 
         self.keyed_edu = {}  # (destination, key) -> EDU
-        self.keyed_edu_changed = SortedDict()  # stream position -> (destination, key)
+        self.keyed_edu_changed = sorteddict()  # stream position -> (destination, key)
 
-        self.edus = SortedDict()  # stream position -> Edu
+        self.edus = sorteddict()  # stream position -> Edu
 
-        self.failures = SortedDict()  # stream position -> (destination, Failure)
+        self.failures = sorteddict()  # stream position -> (destination, Failure)
 
-        self.device_messages = SortedDict()  # stream position -> destination
+        self.device_messages = sorteddict()  # stream position -> destination
 
         self.pos = 1
-        self.pos_time = SortedDict()
+        self.pos_time = sorteddict()
 
         # EVERYTHING IS SAD. In particular, python only makes new scopes when
         # we make a new function, so we need to make a new function so the inner
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index f9596bddaf..40eedb63cb 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -34,8 +34,8 @@ REQUIREMENTS = {
     "bcrypt": ["bcrypt>=3.1.0"],
     "pillow": ["PIL"],
     "pydenticon": ["pydenticon"],
+    "blist": ["blist"],
     "pysaml2>=3.0.0": ["saml2>=3.0.0"],
-    "sortedcontainers": ["sortedcontainers"],
     "pymacaroons-pynacl": ["pymacaroons"],
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 2ff46090a6..941d873ab8 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -16,7 +16,7 @@
 from synapse.util.caches import register_cache, CACHE_SIZE_FACTOR
 
 
-from sortedcontainers import SortedDict
+from blist import sorteddict
 import logging
 
 
@@ -35,7 +35,7 @@ class StreamChangeCache(object):
     def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
         self._max_size = int(max_size * CACHE_SIZE_FACTOR)
         self._entity_to_key = {}
-        self._cache = SortedDict()
+        self._cache = sorteddict()
         self._earliest_known_stream_pos = current_stream_pos
         self.name = name
         self.metrics = register_cache(self.name, self._cache)
-- 
cgit 1.4.1


From 51d628d28dda540d71ac147f3a11b6a559f7bf8d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 13 Apr 2018 12:08:19 +0100
Subject: Bump version and Change log

---
 CHANGES.rst         | 8 ++++++++
 synapse/__init__.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index a838c092fe..19b7af606b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,11 @@
+Changes in synapse v0.27.4 (2018-04-13)
+======================================
+
+Changes:
+
+* Update canonicaljson dependency (#3095)
+
+
 Changes in synapse v0.27.3 (2018-04-11)
 ======================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 1fa2de5268..0c1c16b9a4 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.3"
+__version__ = "0.27.4"
-- 
cgit 1.4.1


From f8d46cad3c3b4318a6b63c55fe63f07d1ae91695 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 13 Apr 2018 15:41:52 +0100
Subject: correctly auth inbound federation_domain_whitelist reqs

---
 synapse/federation/transport/server.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 4c94d5a36c..d2a57d08d7 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -94,12 +94,6 @@ class Authenticator(object):
             "signatures": {},
         }
 
-        if (
-            self.federation_domain_whitelist is not None and
-            self.server_name not in self.federation_domain_whitelist
-        ):
-            raise FederationDeniedError(self.server_name)
-
         if content is not None:
             json_request["content"] = content
 
@@ -138,6 +132,12 @@ class Authenticator(object):
                 json_request["origin"] = origin
                 json_request["signatures"].setdefault(origin, {})[key] = sig
 
+        if (
+            self.federation_domain_whitelist is not None and
+            origin not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(self.server_name)
+
         if not json_request["signatures"]:
             raise NoAuthenticationError(
                 401, "Missing Authorization headers", Codes.UNAUTHORIZED,
-- 
cgit 1.4.1


From 25b0ba30b1ffab9cb799bd8fc331581b7ff6f7aa Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 13 Apr 2018 15:46:37 +0100
Subject: revert last to PR properly

---
 synapse/federation/transport/server.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index d2a57d08d7..4c94d5a36c 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -94,6 +94,12 @@ class Authenticator(object):
             "signatures": {},
         }
 
+        if (
+            self.federation_domain_whitelist is not None and
+            self.server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(self.server_name)
+
         if content is not None:
             json_request["content"] = content
 
@@ -132,12 +138,6 @@ class Authenticator(object):
                 json_request["origin"] = origin
                 json_request["signatures"].setdefault(origin, {})[key] = sig
 
-        if (
-            self.federation_domain_whitelist is not None and
-            origin not in self.federation_domain_whitelist
-        ):
-            raise FederationDeniedError(self.server_name)
-
         if not json_request["signatures"]:
             raise NoAuthenticationError(
                 401, "Missing Authorization headers", Codes.UNAUTHORIZED,
-- 
cgit 1.4.1


From 78a9698650b7a96e2a7814c3732c3ff8aa5a2f0f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 13 Apr 2018 15:47:43 +0100
Subject: fix federation_domain_whitelist

we were checking the wrong server_name on inbound requests
---
 synapse/federation/transport/server.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 4c94d5a36c..ff0656df3e 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -94,12 +94,6 @@ class Authenticator(object):
             "signatures": {},
         }
 
-        if (
-            self.federation_domain_whitelist is not None and
-            self.server_name not in self.federation_domain_whitelist
-        ):
-            raise FederationDeniedError(self.server_name)
-
         if content is not None:
             json_request["content"] = content
 
@@ -138,6 +132,12 @@ class Authenticator(object):
                 json_request["origin"] = origin
                 json_request["signatures"].setdefault(origin, {})[key] = sig
 
+        if (
+            self.federation_domain_whitelist is not None and
+            origin not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(origin)
+
         if not json_request["signatures"]:
             raise NoAuthenticationError(
                 401, "Missing Authorization headers", Codes.UNAUTHORIZED,
-- 
cgit 1.4.1


From c4bdbc2bd2a1c90467de4d81eb3bf5ee7f6f8fb9 Mon Sep 17 00:00:00 2001
From: Silke <silke@slxh.eu>
Date: Sat, 14 Apr 2018 14:36:11 +0200
Subject: Clarify that SRV may not point to a CNAME

Signed-off-by: Silke Hofstra <silke@slxh.eu>
---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index 8812cc1b4f..28fbe45de6 100644
--- a/README.rst
+++ b/README.rst
@@ -614,6 +614,9 @@ should have the format ``_matrix._tcp.<yourdomain.com> <ttl> IN SRV 10 0 <port>
     $ dig -t srv _matrix._tcp.example.com
     _matrix._tcp.example.com. 3600    IN      SRV     10 0 8448 synapse.example.com.
 
+Note that the server hostname cannot be an alias (CNAME record): it has to point
+directly to the server hosting the synapse instance.
+
 You can then configure your homeserver to use ``<yourdomain.com>`` as the domain in
 its user-ids, by setting ``server_name``::
 
-- 
cgit 1.4.1


From bfc2ade9b3784469a5f7b80119fbd1679b6268c8 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 17:24:25 +0200
Subject: Make event properties raise AttributeError instead

They raised KeyError before. I'm changing this because the code uses
hasattr() to check for the presence of a key. This worked accidentally
before, because hasattr() silences all exceptions in python 2. However,
in python3, this isn't the case anymore.

I had a look around to see if anything depended on this raising a
KeyError and I couldn't find anything. Of course, I could have simply
missed it.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/events/__init__.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index e673e96cc0..d4d1b92f7a 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -48,13 +48,22 @@ class _EventInternalMetadata(object):
 
 def _event_dict_property(key):
     def getter(self):
-        return self._event_dict[key]
+        try:
+            return self._event_dict[key]
+        except KeyError:
+            raise AttributeError(key)
 
     def setter(self, v):
-        self._event_dict[key] = v
+        try:
+            self._event_dict[key] = v
+        except KeyError:
+            raise AttributeError(key)
 
     def delete(self):
-        del self._event_dict[key]
+        try:
+            del self._event_dict[key]
+        except KeyError:
+            raise AttributeError(key)
 
     return property(
         getter,
-- 
cgit 1.4.1


From 1515560f5ce8054a539dd0fed9e88232883f079f Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 17:20:37 +0200
Subject: Use str(e) instead of e.message

Doing this I learned e.message was pretty shortlived, added in 2.6,
they realized it was a bad idea and deprecated it in 2.7

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/crypto/keyring.py        |  8 ++++----
 tests/storage/test_appservice.py | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 35f810b07b..fce83d445f 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -352,7 +352,7 @@ class Keyring(object):
                 logger.exception(
                     "Unable to get key from %r: %s %s",
                     perspective_name,
-                    type(e).__name__, str(e.message),
+                    type(e).__name__, str(e),
                 )
                 defer.returnValue({})
 
@@ -384,7 +384,7 @@ class Keyring(object):
                 logger.info(
                     "Unable to get key %r for %r directly: %s %s",
                     key_ids, server_name,
-                    type(e).__name__, str(e.message),
+                    type(e).__name__, str(e),
                 )
 
             if not keys:
@@ -734,7 +734,7 @@ def _handle_key_deferred(verify_request):
     except IOError as e:
         logger.warn(
             "Got IOError when downloading keys for %s: %s %s",
-            server_name, type(e).__name__, str(e.message),
+            server_name, type(e).__name__, str(e),
         )
         raise SynapseError(
             502,
@@ -744,7 +744,7 @@ def _handle_key_deferred(verify_request):
     except Exception as e:
         logger.exception(
             "Got Exception when downloading keys for %s: %s %s",
-            server_name, type(e).__name__, str(e.message),
+            server_name, type(e).__name__, str(e),
         )
         raise SynapseError(
             401,
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index c2e39a7288..00825498b1 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -480,9 +480,9 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
             ApplicationServiceStore(None, hs)
 
         e = cm.exception
-        self.assertIn(f1, e.message)
-        self.assertIn(f2, e.message)
-        self.assertIn("id", e.message)
+        self.assertIn(f1, str(e))
+        self.assertIn(f2, str(e))
+        self.assertIn("id", str(e))
 
     @defer.inlineCallbacks
     def test_duplicate_as_tokens(self):
@@ -504,6 +504,6 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
             ApplicationServiceStore(None, hs)
 
         e = cm.exception
-        self.assertIn(f1, e.message)
-        self.assertIn(f2, e.message)
-        self.assertIn("as_token", e.message)
+        self.assertIn(f1, str(e))
+        self.assertIn(f2, str(e))
+        self.assertIn("as_token", str(e))
-- 
cgit 1.4.1


From cb9cdfecd097c5a0c85840be9d37d590f0f4af2d Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 17:18:01 +0200
Subject: Add some more variables to the unittest config

These worked accidentally before (python2 doesn't complain if you
compare incompatible types) but under py3 this blows up spectacularly

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 tests/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/utils.py b/tests/utils.py
index f15317d27b..0cd9f7eeee 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -59,6 +59,10 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.email_enable_notifs = False
         config.block_non_admin_invites = False
         config.federation_domain_whitelist = None
+        config.federation_rc_reject_limit = 10
+        config.federation_rc_sleep_limit = 10
+        config.federation_rc_concurrent = 10
+        config.filter_timeline_limit = 5000
         config.user_directory_search_all_users = False
 
         # disable user directory updates, because they get done in the
-- 
cgit 1.4.1


From 36c59ce66908a770b580c95bee0dd3aaf9906f0e Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 16:51:07 +0200
Subject: Use six.itervalues in some places

There's more where that came from

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/api/errors.py                   |  3 ++-
 synapse/app/synchrotron.py              |  4 +++-
 synapse/federation/federation_server.py |  8 +++++---
 synapse/federation/send_queue.py        | 14 ++++++++------
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index bee59e80dd..a9ff5576f3 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -18,6 +18,7 @@
 import logging
 
 import simplejson as json
+from six import iteritems
 
 logger = logging.getLogger(__name__)
 
@@ -297,7 +298,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
         A dict representing the error response JSON.
     """
     err = {"error": msg, "errcode": code}
-    for key, value in kwargs.iteritems():
+    for key, value in iteritems(kwargs):
         err[key] = value
     return err
 
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 508b66613d..2fddcd935a 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -58,6 +58,8 @@ from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
 from twisted.web.resource import NoResource
 
+from six import iteritems
+
 logger = logging.getLogger("synapse.app.synchrotron")
 
 
@@ -211,7 +213,7 @@ class SynchrotronPresence(object):
 
     def get_currently_syncing_users(self):
         return [
-            user_id for user_id, count in self.user_to_num_current_syncs.iteritems()
+            user_id for user_id, count in iteritems(self.user_to_num_current_syncs)
             if count > 0
         ]
 
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index e4ce037acf..2dadf7deab 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -33,6 +33,8 @@ from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 from synapse.util.logutils import log_function
 
+from six import iteritems
+
 # when processing incoming transactions, we try to handle multiple rooms in
 # parallel, up to this limit.
 TRANSACTION_CONCURRENCY_LIMIT = 10
@@ -425,9 +427,9 @@ class FederationServer(FederationBase):
             "Claimed one-time-keys: %s",
             ",".join((
                 "%s for %s:%s" % (key_id, user_id, device_id)
-                for user_id, user_keys in json_result.iteritems()
-                for device_id, device_keys in user_keys.iteritems()
-                for key_id, _ in device_keys.iteritems()
+                for user_id, user_keys in iteritems(json_result)
+                for device_id, device_keys in iteritems(user_keys)
+                for key_id, _ in iteritems(device_keys)
             )),
         )
 
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 93e5acebc1..0f0c687b37 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -40,6 +40,8 @@ from collections import namedtuple
 
 import logging
 
+from six import itervalues, iteritems
+
 logger = logging.getLogger(__name__)
 
 
@@ -122,7 +124,7 @@ class FederationRemoteSendQueue(object):
 
             user_ids = set(
                 user_id
-                for uids in self.presence_changed.itervalues()
+                for uids in itervalues(self.presence_changed)
                 for user_id in uids
             )
 
@@ -276,7 +278,7 @@ class FederationRemoteSendQueue(object):
         # stream position.
         keyed_edus = {self.keyed_edu_changed[k]: k for k in keys[i:j]}
 
-        for ((destination, edu_key), pos) in keyed_edus.iteritems():
+        for ((destination, edu_key), pos) in iteritems(keyed_edus):
             rows.append((pos, KeyedEduRow(
                 key=edu_key,
                 edu=self.keyed_edu[(destination, edu_key)],
@@ -309,7 +311,7 @@ class FederationRemoteSendQueue(object):
         j = keys.bisect_right(to_token) + 1
         device_messages = {self.device_messages[k]: k for k in keys[i:j]}
 
-        for (destination, pos) in device_messages.iteritems():
+        for (destination, pos) in iteritems(device_messages):
             rows.append((pos, DeviceRow(
                 destination=destination,
             )))
@@ -528,19 +530,19 @@ def process_rows_for_federation(transaction_queue, rows):
     if buff.presence:
         transaction_queue.send_presence(buff.presence)
 
-    for destination, edu_map in buff.keyed_edus.iteritems():
+    for destination, edu_map in iteritems(buff.keyed_edus):
         for key, edu in edu_map.items():
             transaction_queue.send_edu(
                 edu.destination, edu.edu_type, edu.content, key=key,
             )
 
-    for destination, edu_list in buff.edus.iteritems():
+    for destination, edu_list in iteritems(buff.edus):
         for edu in edu_list:
             transaction_queue.send_edu(
                 edu.destination, edu.edu_type, edu.content, key=None,
             )
 
-    for destination, failure_list in buff.failures.iteritems():
+    for destination, failure_list in iteritems(buff.failures):
         for failure in failure_list:
             transaction_queue.send_failure(destination, failure)
 
-- 
cgit 1.4.1


From f63ff73c7fc9e27fa42ac73bf520796ff37bfcc2 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 16:39:30 +0200
Subject: add __bool__ alias to __nonzero__ methods

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/handlers/sync.py   | 7 +++++++
 synapse/notifier.py        | 1 +
 synapse/util/logcontext.py | 1 +
 3 files changed, 9 insertions(+)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 06d17ab20c..fe790b4c06 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -52,6 +52,7 @@ class TimelineBatch(collections.namedtuple("TimelineBatch", [
         to tell if room needs to be part of the sync result.
         """
         return bool(self.events)
+    __bool__ = __nonzero__  # python3
 
 
 class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
@@ -76,6 +77,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
             # nb the notification count does not, er, count: if there's nothing
             # else in the result, we don't need to send it.
         )
+    __bool__ = __nonzero__  # python3
 
 
 class ArchivedSyncResult(collections.namedtuple("ArchivedSyncResult", [
@@ -95,6 +97,7 @@ class ArchivedSyncResult(collections.namedtuple("ArchivedSyncResult", [
             or self.state
             or self.account_data
         )
+    __bool__ = __nonzero__  # python3
 
 
 class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [
@@ -106,6 +109,7 @@ class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [
     def __nonzero__(self):
         """Invited rooms should always be reported to the client"""
         return True
+    __bool__ = __nonzero__  # python3
 
 
 class GroupsSyncResult(collections.namedtuple("GroupsSyncResult", [
@@ -117,6 +121,7 @@ class GroupsSyncResult(collections.namedtuple("GroupsSyncResult", [
 
     def __nonzero__(self):
         return bool(self.join or self.invite or self.leave)
+    __bool__ = __nonzero__  # python3
 
 
 class DeviceLists(collections.namedtuple("DeviceLists", [
@@ -127,6 +132,7 @@ class DeviceLists(collections.namedtuple("DeviceLists", [
 
     def __nonzero__(self):
         return bool(self.changed or self.left)
+    __bool__ = __nonzero__  # python3
 
 
 class SyncResult(collections.namedtuple("SyncResult", [
@@ -159,6 +165,7 @@ class SyncResult(collections.namedtuple("SyncResult", [
             self.device_lists or
             self.groups
         )
+    __bool__ = __nonzero__  # python3
 
 
 class SyncHandler(object):
diff --git a/synapse/notifier.py b/synapse/notifier.py
index ef042681bc..0e40a4aad6 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -144,6 +144,7 @@ class _NotifierUserStream(object):
 class EventStreamResult(namedtuple("EventStreamResult", ("events", "tokens"))):
     def __nonzero__(self):
         return bool(self.events)
+    __bool__ = __nonzero__  # python3
 
 
 class Notifier(object):
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index d660ec785b..d59adc236e 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -92,6 +92,7 @@ class LoggingContext(object):
 
         def __nonzero__(self):
             return False
+        __bool__ = __nonzero__  # python3
 
     sentinel = Sentinel()
 
-- 
cgit 1.4.1


From 2a3c33ff03aa88317c30da43cd3773c2789f0fcf Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 17:15:16 +0200
Subject: Use six.moves.urlparse

The imports were shuffled around a bunch in py3

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/config/appservice.py              |  4 ++--
 synapse/http/matrixfederationclient.py    |  3 +--
 synapse/rest/client/v1/login.py           |  2 +-
 synapse/rest/client/v1/room.py            |  9 +++++----
 synapse/rest/media/v1/_base.py            |  2 +-
 synapse/rest/media/v1/media_repository.py |  2 +-
 tests/rest/client/v1/test_rooms.py        | 14 +++++++-------
 tests/utils.py                            |  5 ++---
 8 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 9a2359b6fd..277305e184 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -17,11 +17,11 @@ from ._base import Config, ConfigError
 from synapse.appservice import ApplicationService
 from synapse.types import UserID
 
-import urllib
 import yaml
 import logging
 
 from six import string_types
+from six.moves.urllib import parse as urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -105,7 +105,7 @@ def _load_appservice(hostname, as_info, config_filename):
         )
 
     localpart = as_info["sender_localpart"]
-    if urllib.quote(localpart) != localpart:
+    if urlparse.quote(localpart) != localpart:
         raise ValueError(
             "sender_localpart needs characters which are not URL encoded."
         )
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 60a29081e8..c2e5610f50 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -38,8 +38,7 @@ import logging
 import random
 import sys
 import urllib
-import urlparse
-
+from six.moves.urllib import parse as urlparse
 
 logger = logging.getLogger(__name__)
 outbound_logger = logging.getLogger("synapse.http.outbound")
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 45844aa2d2..34df5be4e9 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -25,7 +25,7 @@ from .base import ClientV1RestServlet, client_path_patterns
 
 import simplejson as json
 import urllib
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 import logging
 from saml2 import BINDING_HTTP_POST
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 2ad0e5943b..fcf9c9ab44 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -28,8 +28,9 @@ from synapse.http.servlet import (
     parse_json_object_from_request, parse_string, parse_integer
 )
 
+from six.moves.urllib import parse as urlparse
+
 import logging
-import urllib
 import simplejson as json
 
 logger = logging.getLogger(__name__)
@@ -433,7 +434,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet):
         as_client_event = "raw" not in request.args
         filter_bytes = request.args.get("filter", None)
         if filter_bytes:
-            filter_json = urllib.unquote(filter_bytes[-1]).decode("UTF-8")
+            filter_json = urlparse.unquote(filter_bytes[-1]).decode("UTF-8")
             event_filter = Filter(json.loads(filter_json))
         else:
             event_filter = None
@@ -718,8 +719,8 @@ class RoomTypingRestServlet(ClientV1RestServlet):
     def on_PUT(self, request, room_id, user_id):
         requester = yield self.auth.get_user_by_req(request)
 
-        room_id = urllib.unquote(room_id)
-        target_user = UserID.from_string(urllib.unquote(user_id))
+        room_id = urlparse.unquote(room_id)
+        target_user = UserID.from_string(urlparse.unquote(user_id))
 
         content = parse_json_object_from_request(request)
 
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index e7ac01da01..d9c4af9389 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -28,7 +28,7 @@ import os
 
 import logging
 import urllib
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index bb79599379..9800ce7581 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -47,7 +47,7 @@ import shutil
 
 import cgi
 import logging
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 logger = logging.getLogger(__name__)
 
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 7e8966a1a8..d763400eaf 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -24,7 +24,7 @@ from synapse.api.constants import Membership
 from synapse.types import UserID
 
 import json
-import urllib
+from six.moves.urllib import parse as urlparse
 
 from ....utils import MockHttpResource, setup_test_homeserver
 from .utils import RestTestCase
@@ -766,7 +766,7 @@ class RoomMemberStateTestCase(RestTestCase):
     @defer.inlineCallbacks
     def test_rooms_members_self(self):
         path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(self.room_id), self.user_id
+            urlparse.quote(self.room_id), self.user_id
         )
 
         # valid join message (NOOP since we made the room)
@@ -786,7 +786,7 @@ class RoomMemberStateTestCase(RestTestCase):
     def test_rooms_members_other(self):
         self.other_id = "@zzsid1:red"
         path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(self.room_id), self.other_id
+            urlparse.quote(self.room_id), self.other_id
         )
 
         # valid invite message
@@ -802,7 +802,7 @@ class RoomMemberStateTestCase(RestTestCase):
     def test_rooms_members_other_custom_keys(self):
         self.other_id = "@zzsid1:red"
         path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(self.room_id), self.other_id
+            urlparse.quote(self.room_id), self.other_id
         )
 
         # valid invite message with custom key
@@ -859,7 +859,7 @@ class RoomMessagesTestCase(RestTestCase):
     @defer.inlineCallbacks
     def test_invalid_puts(self):
         path = "/rooms/%s/send/m.room.message/mid1" % (
-            urllib.quote(self.room_id))
+            urlparse.quote(self.room_id))
         # missing keys or invalid json
         (code, response) = yield self.mock_resource.trigger(
             "PUT", path, '{}'
@@ -894,7 +894,7 @@ class RoomMessagesTestCase(RestTestCase):
     @defer.inlineCallbacks
     def test_rooms_messages_sent(self):
         path = "/rooms/%s/send/m.room.message/mid1" % (
-            urllib.quote(self.room_id))
+            urlparse.quote(self.room_id))
 
         content = '{"body":"test","msgtype":{"type":"a"}}'
         (code, response) = yield self.mock_resource.trigger("PUT", path, content)
@@ -911,7 +911,7 @@ class RoomMessagesTestCase(RestTestCase):
 
         # m.text message type
         path = "/rooms/%s/send/m.room.message/mid2" % (
-            urllib.quote(self.room_id))
+            urlparse.quote(self.room_id))
         content = '{"body":"test2","msgtype":"m.text"}'
         (code, response) = yield self.mock_resource.trigger("PUT", path, content)
         self.assertEquals(200, code, msg=str(response))
diff --git a/tests/utils.py b/tests/utils.py
index f15317d27b..9e815d8643 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -15,8 +15,7 @@
 
 import hashlib
 from inspect import getcallargs
-import urllib
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 from mock import Mock, patch
 from twisted.internet import defer, reactor
@@ -234,7 +233,7 @@ class MockHttpResource(HttpServer):
             if matcher:
                 try:
                     args = [
-                        urllib.unquote(u).decode("UTF-8")
+                        urlparse.unquote(u).decode("UTF-8")
                         for u in matcher.groups()
                     ]
 
-- 
cgit 1.4.1


From 512633ef44c0edb025aa5302e35ed9829a6d8502 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 15 Apr 2018 22:45:06 +0100
Subject: fix spurious changelog dup

---
 CHANGES.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 19b7af606b..c1ba8c75ce 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -61,7 +61,6 @@ Bug fixes:
 
 * Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte!
 * Fix replication after switch to simplejson (PR #3015)
-* Fix replication after switch to simplejson (PR #3015)
 * 404 correctly on missing paths via NoResource (PR #3022)
 * Fix error when claiming e2e keys from offline servers (PR #3034)
 * fix tests/storage/test_user_directory.py (PR #3042)
-- 
cgit 1.4.1


From a1a3c9660ffc1e99f656c1d39ebe57b8857b207d Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 21:46:23 +0200
Subject: Make tests py3 compatible

This is a mixed commit that fixes various small issues

 * print parentheses
 * 01 is invalid syntax (it was octal in py2)
 * [x for i in 1, 2] is invalid syntax
 * six moves

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 tests/config/test_load.py        | 2 +-
 tests/crypto/test_keyring.py     | 2 +-
 tests/test_distributor.py        | 2 +-
 tests/util/test_file_consumer.py | 2 +-
 tests/util/test_linearizer.py    | 3 ++-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/config/test_load.py b/tests/config/test_load.py
index 161a87d7e3..772afd2cf9 100644
--- a/tests/config/test_load.py
+++ b/tests/config/test_load.py
@@ -24,7 +24,7 @@ class ConfigLoadingTestCase(unittest.TestCase):
 
     def setUp(self):
         self.dir = tempfile.mkdtemp()
-        print self.dir
+        print(self.dir)
         self.file = os.path.join(self.dir, "homeserver.yaml")
 
     def tearDown(self):
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index d4ec02ffc2..149e443022 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -183,7 +183,7 @@ class KeyringTestCase(unittest.TestCase):
                 res_deferreds_2 = kr.verify_json_objects_for_server(
                     [("server10", json1)],
                 )
-                yield async.sleep(01)
+                yield async.sleep(1)
                 self.http_client.post_json.assert_not_called()
                 res_deferreds_2[0].addBoth(self.check_context, None)
 
diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index acebcf4a86..010aeaee7e 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -62,7 +62,7 @@ class DistributorTestCase(unittest.TestCase):
     def test_signal_catch(self):
         self.dist.declare("alarm")
 
-        observers = [Mock() for i in 1, 2]
+        observers = [Mock() for i in (1, 2)]
         for o in observers:
             self.dist.observe("alarm", o)
 
diff --git a/tests/util/test_file_consumer.py b/tests/util/test_file_consumer.py
index 76e2234255..d6e1082779 100644
--- a/tests/util/test_file_consumer.py
+++ b/tests/util/test_file_consumer.py
@@ -20,7 +20,7 @@ from mock import NonCallableMock
 from synapse.util.file_consumer import BackgroundFileConsumer
 
 from tests import unittest
-from StringIO import StringIO
+from six import StringIO
 
 import threading
 
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index 793a88e462..4865eb4bc6 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -18,6 +18,7 @@ from tests import unittest
 from twisted.internet import defer
 
 from synapse.util.async import Linearizer
+from six.moves import range
 
 
 class LinearizerTestCase(unittest.TestCase):
@@ -58,7 +59,7 @@ class LinearizerTestCase(unittest.TestCase):
                     logcontext.LoggingContext.current_context(), lc)
 
         func(0, sleep=True)
-        for i in xrange(1, 100):
+        for i in range(1, 100):
             func(i)
 
         return func(1000)
-- 
cgit 1.4.1


From 878995e660e46c5dedb56cde8bd7a3d46838cdd7 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 21:37:58 +0200
Subject: Replace Queue with six.moves.queue

and a six.range change which I missed the last time

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/storage/event_federation.py | 6 ++++--
 synapse/util/file_consumer.py       | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 00ee82d300..e828328243 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -24,7 +24,9 @@ from synapse.util.caches.descriptors import cached
 from unpaddedbase64 import encode_base64
 
 import logging
-from Queue import PriorityQueue, Empty
+from six.moves.queue import PriorityQueue, Empty
+
+from six.moves import range
 
 
 logger = logging.getLogger(__name__)
@@ -78,7 +80,7 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
             front_list = list(front)
             chunks = [
                 front_list[x:x + 100]
-                for x in xrange(0, len(front), 100)
+                for x in range(0, len(front), 100)
             ]
             for chunk in chunks:
                 txn.execute(
diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py
index 90a2608d6f..3c8a165331 100644
--- a/synapse/util/file_consumer.py
+++ b/synapse/util/file_consumer.py
@@ -17,7 +17,7 @@ from twisted.internet import threads, reactor
 
 from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 
-import Queue
+from six.moves import queue
 
 
 class BackgroundFileConsumer(object):
@@ -49,7 +49,7 @@ class BackgroundFileConsumer(object):
 
         # Queue of slices of bytes to be written. When producer calls
         # unregister a final None is sent.
-        self._bytes_queue = Queue.Queue()
+        self._bytes_queue = queue.Queue()
 
         # Deferred that is resolved when finished writing
         self._finished_deferred = None
-- 
cgit 1.4.1


From 639480e14a06723adf6817ddd2b2ff9e4f4cdf2a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 16 Apr 2018 18:41:37 +0100
Subject: Avoid creating events with huge numbers of prev_events

In most cases, we limit the number of prev_events for a given event to 10
events. This fixes a particular code path which created events with huge
numbers of prev_events.
---
 synapse/handlers/message.py            | 78 +++++++++++++++++++---------------
 synapse/handlers/room_member.py        | 13 ++++--
 synapse/storage/event_federation.py    | 57 ++++++++++++++++++-------
 tests/storage/test_event_federation.py | 68 +++++++++++++++++++++++++++++
 4 files changed, 162 insertions(+), 54 deletions(-)
 create mode 100644 tests/storage/test_event_federation.py

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 54cd691f91..21628a8540 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -37,7 +37,6 @@ from ._base import BaseHandler
 from canonicaljson import encode_canonical_json
 
 import logging
-import random
 import simplejson
 
 logger = logging.getLogger(__name__)
@@ -433,7 +432,7 @@ class EventCreationHandler(object):
 
     @defer.inlineCallbacks
     def create_event(self, requester, event_dict, token_id=None, txn_id=None,
-                     prev_event_ids=None):
+                     prev_events_and_hashes=None):
         """
         Given a dict from a client, create a new event.
 
@@ -447,7 +446,13 @@ class EventCreationHandler(object):
             event_dict (dict): An entire event
             token_id (str)
             txn_id (str)
-            prev_event_ids (list): The prev event ids to use when creating the event
+
+            prev_events_and_hashes (list[(str, dict[str, str], int)]|None):
+                the forward extremities to use as the prev_events for the
+                new event. For each event, a tuple of (event_id, hashes, depth)
+                where *hashes* is a map from algorithm to hash.
+
+                If None, they will be requested from the database.
 
         Returns:
             Tuple of created event (FrozenEvent), Context
@@ -485,7 +490,7 @@ class EventCreationHandler(object):
         event, context = yield self.create_new_client_event(
             builder=builder,
             requester=requester,
-            prev_event_ids=prev_event_ids,
+            prev_events_and_hashes=prev_events_and_hashes,
         )
 
         defer.returnValue((event, context))
@@ -588,39 +593,44 @@ class EventCreationHandler(object):
 
     @measure_func("create_new_client_event")
     @defer.inlineCallbacks
-    def create_new_client_event(self, builder, requester=None, prev_event_ids=None):
-        if prev_event_ids:
-            prev_events = yield self.store.add_event_hashes(prev_event_ids)
-            prev_max_depth = yield self.store.get_max_depth_of_events(prev_event_ids)
-            depth = prev_max_depth + 1
-        else:
-            latest_ret = yield self.store.get_latest_event_ids_and_hashes_in_room(
-                builder.room_id,
+    def create_new_client_event(self, builder, requester=None,
+                                prev_events_and_hashes=None):
+        """Create a new event for a local client
+
+        Args:
+            builder (EventBuilder):
+
+            requester (synapse.types.Requester|None):
+
+            prev_events_and_hashes (list[(str, dict[str, str], int)]|None):
+                the forward extremities to use as the prev_events for the
+                new event. For each event, a tuple of (event_id, hashes, depth)
+                where *hashes* is a map from algorithm to hash.
+
+                If None, they will be requested from the database.
+
+        Returns:
+            Deferred[(synapse.events.EventBase, synapse.events.snapshot.EventContext)]
+        """
+
+        if prev_events_and_hashes is not None:
+            assert len(prev_events_and_hashes) <= 10, \
+                "Attempting to create an event with %i prev_events" % (
+                    len(prev_events_and_hashes),
             )
+        else:
+            prev_events_and_hashes = \
+                yield self.store.get_prev_events_for_room(builder.room_id)
 
-            # We want to limit the max number of prev events we point to in our
-            # new event
-            if len(latest_ret) > 10:
-                # Sort by reverse depth, so we point to the most recent.
-                latest_ret.sort(key=lambda a: -a[2])
-                new_latest_ret = latest_ret[:5]
-
-                # We also randomly point to some of the older events, to make
-                # sure that we don't completely ignore the older events.
-                if latest_ret[5:]:
-                    sample_size = min(5, len(latest_ret[5:]))
-                    new_latest_ret.extend(random.sample(latest_ret[5:], sample_size))
-                latest_ret = new_latest_ret
-
-            if latest_ret:
-                depth = max([d for _, _, d in latest_ret]) + 1
-            else:
-                depth = 1
+        if prev_events_and_hashes:
+            depth = max([d for _, _, d in prev_events_and_hashes]) + 1
+        else:
+            depth = 1
 
-            prev_events = [
-                (event_id, prev_hashes)
-                for event_id, prev_hashes, _ in latest_ret
-            ]
+        prev_events = [
+            (event_id, prev_hashes)
+            for event_id, prev_hashes, _ in prev_events_and_hashes
+        ]
 
         builder.prev_events = prev_events
         builder.depth = depth
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index c45142d38d..714583f1d5 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -149,7 +149,7 @@ class RoomMemberHandler(object):
     @defer.inlineCallbacks
     def _local_membership_update(
         self, requester, target, room_id, membership,
-        prev_event_ids,
+        prev_events_and_hashes,
         txn_id=None,
         ratelimit=True,
         content=None,
@@ -175,7 +175,7 @@ class RoomMemberHandler(object):
             },
             token_id=requester.access_token_id,
             txn_id=txn_id,
-            prev_event_ids=prev_event_ids,
+            prev_events_and_hashes=prev_events_and_hashes,
         )
 
         # Check if this event matches the previous membership event for the user.
@@ -314,7 +314,12 @@ class RoomMemberHandler(object):
                     403, "Invites have been disabled on this server",
                 )
 
-        latest_event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
+        prev_events_and_hashes = yield self.store.get_prev_events_for_room(
+            room_id,
+        )
+        latest_event_ids = (
+            event_id for (event_id, _, _) in prev_events_and_hashes
+        )
         current_state_ids = yield self.state_handler.get_current_state_ids(
             room_id, latest_event_ids=latest_event_ids,
         )
@@ -403,7 +408,7 @@ class RoomMemberHandler(object):
             membership=effective_membership_state,
             txn_id=txn_id,
             ratelimit=ratelimit,
-            prev_event_ids=latest_event_ids,
+            prev_events_and_hashes=prev_events_and_hashes,
             content=content,
         )
         defer.returnValue(res)
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 00ee82d300..a183fc6b50 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import random
 
 from twisted.internet import defer
 
@@ -133,7 +134,47 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
             retcol="event_id",
         )
 
+    @defer.inlineCallbacks
+    def get_prev_events_for_room(self, room_id):
+        """
+        Gets a subset of the current forward extremities in the given room.
+
+        Limits the result to 10 extremities, so that we can avoid creating
+        events which refer to hundreds of prev_events.
+
+        Args:
+            room_id (str): room_id
+
+        Returns:
+            Deferred[list[(str, dict[str, str], int)]]
+                for each event, a tuple of (event_id, hashes, depth)
+                where *hashes* is a map from algorithm to hash.
+        """
+        res = yield self.get_latest_event_ids_and_hashes_in_room(room_id)
+        if len(res) > 10:
+            # Sort by reverse depth, so we point to the most recent.
+            res.sort(key=lambda a: -a[2])
+
+            # we use half of the limit for the actual most recent events, and
+            # the other half to randomly point to some of the older events, to
+            # make sure that we don't completely ignore the older events.
+            res = res[0:5] + random.sample(res[5:], 5)
+
+        defer.returnValue(res)
+
     def get_latest_event_ids_and_hashes_in_room(self, room_id):
+        """
+        Gets the current forward extremities in the given room
+
+        Args:
+            room_id (str): room_id
+
+        Returns:
+            Deferred[list[(str, dict[str, str], int)]]
+                for each event, a tuple of (event_id, hashes, depth)
+                where *hashes* is a map from algorithm to hash.
+        """
+
         return self.runInteraction(
             "get_latest_event_ids_and_hashes_in_room",
             self._get_latest_event_ids_and_hashes_in_room,
@@ -182,22 +223,6 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
             room_id,
         )
 
-    @defer.inlineCallbacks
-    def get_max_depth_of_events(self, event_ids):
-        sql = (
-            "SELECT MAX(depth) FROM events WHERE event_id IN (%s)"
-        ) % (",".join(["?"] * len(event_ids)),)
-
-        rows = yield self._execute(
-            "get_max_depth_of_events", None,
-            sql, *event_ids
-        )
-
-        if rows:
-            defer.returnValue(rows[0][0])
-        else:
-            defer.returnValue(1)
-
     def _get_min_depth_interaction(self, txn, room_id):
         min_depth = self._simple_select_one_onecol_txn(
             txn,
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
new file mode 100644
index 0000000000..30683e7888
--- /dev/null
+++ b/tests/storage/test_event_federation.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import tests.unittest
+import tests.utils
+
+
+class EventFederationWorkerStoreTestCase(tests.unittest.TestCase):
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def test_get_prev_events_for_room(self):
+        room_id = '@ROOM:local'
+
+        # add a bunch of events and hashes to act as forward extremities
+        def insert_event(txn, i):
+            event_id = '$event_%i:local' % i
+
+            txn.execute((
+                "INSERT INTO events ("
+                "   room_id, event_id, type, depth, topological_ordering,"
+                "   content, processed, outlier) "
+                "VALUES (?, ?, 'm.test', ?, ?, 'test', ?, ?)"
+            ), (room_id, event_id, i, i, True, False))
+
+            txn.execute((
+                'INSERT INTO event_forward_extremities (room_id, event_id) '
+                'VALUES (?, ?)'
+            ), (room_id, event_id))
+
+            txn.execute((
+                'INSERT INTO event_reference_hashes '
+                '(event_id, algorithm, hash) '
+                "VALUES (?, 'sha256', ?)"
+            ), (event_id, 'ffff'))
+
+        for i in range(0, 11):
+            yield self.store.runInteraction("insert", insert_event, i)
+
+        # this should get the last five and five others
+        r = yield self.store.get_prev_events_for_room(room_id)
+        self.assertEqual(10, len(r))
+        for i in range(0, 5):
+            el = r[i]
+            depth = el[2]
+            self.assertEqual(10 - i, depth)
+
+        for i in range(5, 5):
+            el = r[i]
+            depth = el[2]
+            self.assertLessEqual(5, depth)
-- 
cgit 1.4.1


From 9b7794262fa1a061f6668a6693a79b9bf8d3567a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 17 Apr 2018 22:11:19 +0100
Subject: Reject events which have too many auth_events or prev_events

... this should protect us from being dossed by people making silly events
(deliberately or otherwise)
---
 synapse/handlers/federation.py | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 080aca3d71..b7b0816449 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -15,8 +15,14 @@
 # limitations under the License.
 
 """Contains handlers for federation events."""
+
+import httplib
+import itertools
+import logging
+
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
+from twisted.internet import defer
 from unpaddedbase64 import decode_base64
 
 from ._base import BaseHandler
@@ -43,10 +49,6 @@ from synapse.util.retryutils import NotRetryingDestination
 
 from synapse.util.distributor import user_joined_room
 
-from twisted.internet import defer
-
-import itertools
-import logging
 
 logger = logging.getLogger(__name__)
 
@@ -115,6 +117,28 @@ class FederationHandler(BaseHandler):
             logger.debug("Already seen pdu %s", pdu.event_id)
             return
 
+        # do some initial sanity-checking of the event. In particular, make
+        # sure it doesn't have hundreds of prev_events or auth_events, which
+        # could cause a huge state resolution or cascade of event fetches
+        if len(pdu.prev_events) > 20:
+            logger.warn("Rejecting event %s which has %i prev_events",
+                        pdu.event_id, len(pdu.prev_events))
+            raise FederationError(
+                "ERROR",
+                httplib.BAD_REQUEST,
+                "Too many prev_events",
+                affected=pdu.event_id,
+            )
+        if len(pdu.auth_events) > 10:
+            logger.warn("Rejecting event %s which has %i auth_events",
+                        pdu.event_id, len(pdu.auth_events))
+            raise FederationError(
+                "ERROR",
+                httplib.BAD_REQUEST,
+                "Too many auth_events",
+                affected=pdu.event_id,
+            )
+
         # If we are currently in the process of joining this room, then we
         # queue up events for later processing.
         if pdu.room_id in self.room_queues:
-- 
cgit 1.4.1


From e585228860fe72ca8f2b95d49394a39f6329bf39 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 17 Apr 2018 23:41:12 +0100
Subject: Check events on backfill too

---
 synapse/handlers/federation.py | 57 +++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index b7b0816449..310db3fb46 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -119,23 +119,14 @@ class FederationHandler(BaseHandler):
 
         # do some initial sanity-checking of the event. In particular, make
         # sure it doesn't have hundreds of prev_events or auth_events, which
-        # could cause a huge state resolution or cascade of event fetches
-        if len(pdu.prev_events) > 20:
-            logger.warn("Rejecting event %s which has %i prev_events",
-                        pdu.event_id, len(pdu.prev_events))
+        # could cause a huge state resolution or cascade of event fetches.
+        try:
+            self._sanity_check_event(pdu)
+        except SynapseError as err:
             raise FederationError(
                 "ERROR",
-                httplib.BAD_REQUEST,
-                "Too many prev_events",
-                affected=pdu.event_id,
-            )
-        if len(pdu.auth_events) > 10:
-            logger.warn("Rejecting event %s which has %i auth_events",
-                        pdu.event_id, len(pdu.auth_events))
-            raise FederationError(
-                "ERROR",
-                httplib.BAD_REQUEST,
-                "Too many auth_events",
+                err.code,
+                err.msg,
                 affected=pdu.event_id,
             )
 
@@ -565,6 +556,9 @@ class FederationHandler(BaseHandler):
             extremities=extremities,
         )
 
+        for ev in events:
+            self._sanity_check_event(ev)
+
         # Don't bother processing events we already have.
         seen_events = yield self.store.have_events_in_timeline(
             set(e.event_id for e in events)
@@ -867,6 +861,39 @@ class FederationHandler(BaseHandler):
 
         defer.returnValue(False)
 
+    def _sanity_check_event(self, ev):
+        """
+        Do some early sanity checks of a received event
+
+        In particular, checks it doesn't have an excessive number of
+        prev_events or auth_events, which could cause a huge state resolution
+        or cascade of event fetches.
+
+        Args:
+            ev (synapse.events.EventBase): event to be checked
+
+        Returns: None
+
+        Raises:
+            SynapseError if the event does not pass muster
+        """
+        if len(ev.prev_events) > 20:
+            logger.warn("Rejecting event %s which has %i prev_events",
+                        ev.event_id, len(ev.prev_events))
+            raise SynapseError(
+                httplib.BAD_REQUEST,
+                "Too many prev_events",
+            )
+
+        if len(ev.auth_events) > 10:
+            logger.warn("Rejecting event %s which has %i auth_events",
+                        ev.event_id, len(ev.auth_events))
+            raise SynapseError(
+                "ERROR",
+                httplib.BAD_REQUEST,
+                "Too many auth_events",
+            )
+
     @defer.inlineCallbacks
     def send_invite(self, target_host, event):
         """ Sends the invite to the remote server for signing.
-- 
cgit 1.4.1


From 1f4b498b7313b9bff5bfc3eeeed25c7659276e82 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 17 Apr 2018 23:41:36 +0100
Subject: Add some comments

---
 synapse/federation/federation_server.py | 25 +++++++++++++++++++++++--
 synapse/handlers/federation.py          | 15 ++++++++++++---
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index e4ce037acf..682721cec7 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -494,13 +495,33 @@ class FederationServer(FederationBase):
     def _handle_received_pdu(self, origin, pdu):
         """ Process a PDU received in a federation /send/ transaction.
 
+        If the event is invalid, then this method throws a FederationError.
+        (The error will then be logged and sent back to the sender (which
+        probably won't do anything with it), and other events in the
+        transaction will be processed as normal).
+
+        It is likely that we'll then receive other events which refer to
+        this rejected_event in their prev_events, etc.  When that happens,
+        we'll attempt to fetch the rejected event again, which will presumably
+        fail, so those second-generation events will also get rejected.
+
+        Eventually, we get to the point where there are more than 10 events
+        between any new events and the original rejected event. Since we
+        only try to backfill 10 events deep on received pdu, we then accept the
+        new event, possibly introducing a discontinuity in the DAG, with new
+        forward extremities, so normal service is approximately returned,
+        until we try to backfill across the discontinuity.
+
         Args:
             origin (str): server which sent the pdu
             pdu (FrozenEvent): received pdu
 
         Returns (Deferred): completes with None
-        Raises: FederationError if the signatures / hash do not match
-    """
+
+        Raises: FederationError if the signatures / hash do not match, or
+            if the event was unacceptable for any other reason (eg, too large,
+            too many prev_events, couldn't find the prev_events)
+        """
         # check that it's actually being sent from a valid destination to
         # workaround bug #1753 in 0.18.5 and 0.18.6
         if origin != get_domain_from_id(pdu.event_id):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 310db3fb46..1296e22af6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -542,9 +542,16 @@ class FederationHandler(BaseHandler):
     def backfill(self, dest, room_id, limit, extremities):
         """ Trigger a backfill request to `dest` for the given `room_id`
 
-        This will attempt to get more events from the remote. This may return
-        be successfull and still return no events if the other side has no new
-        events to offer.
+        This will attempt to get more events from the remote. If the other side
+        has no new events to offer, this will return an empty list.
+
+        As the events are received, we check their signatures, and also do some
+        sanity-checking on them. If any of the backfilled events are invalid,
+        this method throws a SynapseError.
+
+        TODO: make this more useful to distinguish failures of the remote
+        server from invalid events (there is probably no point in trying to
+        re-fetch invalid events from every other HS in the room.)
         """
         if dest == self.server_name:
             raise SynapseError(400, "Can't backfill from self.")
@@ -556,6 +563,8 @@ class FederationHandler(BaseHandler):
             extremities=extremities,
         )
 
+        # do some sanity-checking of the received events, before we go and
+        # do state resolution across 1000 events.
         for ev in events:
             self._sanity_check_event(ev)
 
-- 
cgit 1.4.1


From b1dfbc3c4015865ab9798e5911b614e82a3dca2c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 17 Apr 2018 18:30:53 +0100
Subject: Refactor store.have_events

It turns out that most of the time we were calling have_events, we were only
using half of the result. Replace have_events with have_seen_events and
get_rejection_reasons, so that we can see what's going on a bit more clearly.
---
 synapse/federation/federation_client.py |  2 +-
 synapse/handlers/federation.py          | 31 ++++++++-------------
 synapse/storage/events.py               | 49 ++++++++++++++++++++++++++++-----
 3 files changed, 55 insertions(+), 27 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 38440da5b5..8e2c0c4cd2 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -394,7 +394,7 @@ class FederationClient(FederationBase):
             seen_events = yield self.store.get_events(event_ids, allow_rejected=True)
             signed_events = seen_events.values()
         else:
-            seen_events = yield self.store.have_events(event_ids)
+            seen_events = yield self.store.have_seen_events(event_ids)
             signed_events = []
 
         failed_to_fetch = set()
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 080aca3d71..ea6cb879fc 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -149,10 +149,6 @@ class FederationHandler(BaseHandler):
 
         auth_chain = []
 
-        have_seen = yield self.store.have_events(
-            [ev for ev, _ in pdu.prev_events]
-        )
-
         fetch_state = False
 
         # Get missing pdus if necessary.
@@ -168,7 +164,7 @@ class FederationHandler(BaseHandler):
             )
 
             prevs = {e_id for e_id, _ in pdu.prev_events}
-            seen = set(have_seen.keys())
+            seen = yield self.store.have_seen_events(prevs)
 
             if min_depth and pdu.depth < min_depth:
                 # This is so that we don't notify the user about this
@@ -196,8 +192,7 @@ class FederationHandler(BaseHandler):
 
                         # Update the set of things we've seen after trying to
                         # fetch the missing stuff
-                        have_seen = yield self.store.have_events(prevs)
-                        seen = set(have_seen.iterkeys())
+                        seen = yield self.store.have_seen_events(prevs)
 
                         if not prevs - seen:
                             logger.info(
@@ -248,8 +243,7 @@ class FederationHandler(BaseHandler):
             min_depth (int): Minimum depth of events to return.
         """
         # We recalculate seen, since it may have changed.
-        have_seen = yield self.store.have_events(prevs)
-        seen = set(have_seen.keys())
+        seen = yield self.store.have_seen_events(prevs)
 
         if not prevs - seen:
             return
@@ -361,9 +355,7 @@ class FederationHandler(BaseHandler):
             if auth_chain:
                 event_ids |= {e.event_id for e in auth_chain}
 
-            seen_ids = set(
-                (yield self.store.have_events(event_ids)).keys()
-            )
+            seen_ids = yield self.store.have_seen_events(event_ids)
 
             if state and auth_chain is not None:
                 # If we have any state or auth_chain given to us by the replication
@@ -633,7 +625,7 @@ class FederationHandler(BaseHandler):
 
                 failed_to_fetch = missing_auth - set(auth_events)
 
-        seen_events = yield self.store.have_events(
+        seen_events = yield self.store.have_seen_events(
             set(auth_events.keys()) | set(state_events.keys())
         )
 
@@ -1736,7 +1728,8 @@ class FederationHandler(BaseHandler):
             event_key = None
 
         if event_auth_events - current_state:
-            have_events = yield self.store.have_events(
+            # TODO: can we use store.have_seen_events here instead?
+            have_events = yield self.store.get_seen_events_with_rejections(
                 event_auth_events - current_state
             )
         else:
@@ -1759,12 +1752,12 @@ class FederationHandler(BaseHandler):
                     origin, event.room_id, event.event_id
                 )
 
-                seen_remotes = yield self.store.have_events(
+                seen_remotes = yield self.store.have_seen_events(
                     [e.event_id for e in remote_auth_chain]
                 )
 
                 for e in remote_auth_chain:
-                    if e.event_id in seen_remotes.keys():
+                    if e.event_id in seen_remotes:
                         continue
 
                     if e.event_id == event.event_id:
@@ -1791,7 +1784,7 @@ class FederationHandler(BaseHandler):
                     except AuthError:
                         pass
 
-                have_events = yield self.store.have_events(
+                have_events = yield self.store.get_seen_events_with_rejections(
                     [e_id for e_id, _ in event.auth_events]
                 )
                 seen_events = set(have_events.keys())
@@ -1876,13 +1869,13 @@ class FederationHandler(BaseHandler):
                         local_auth_chain,
                     )
 
-                    seen_remotes = yield self.store.have_events(
+                    seen_remotes = yield self.store.have_seen_events(
                         [e.event_id for e in result["auth_chain"]]
                     )
 
                     # 3. Process any remote auth chain events we haven't seen.
                     for ev in result["auth_chain"]:
-                        if ev.event_id in seen_remotes.keys():
+                        if ev.event_id in seen_remotes:
                             continue
 
                         if ev.event_id == event.event_id:
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index da44b52fd6..5fe4a0e56c 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -16,6 +16,7 @@
 
 from collections import OrderedDict, deque, namedtuple
 from functools import wraps
+import itertools
 import logging
 
 import simplejson as json
@@ -1320,13 +1321,49 @@ class EventsStore(EventsWorkerStore):
 
         defer.returnValue(set(r["event_id"] for r in rows))
 
-    def have_events(self, event_ids):
+    @defer.inlineCallbacks
+    def have_seen_events(self, event_ids):
         """Given a list of event ids, check if we have already processed them.
 
+        Args:
+            event_ids (iterable[str]):
+
         Returns:
-            dict: Has an entry for each event id we already have seen. Maps to
-            the rejected reason string if we rejected the event, else maps to
-            None.
+            Deferred[set[str]]: The events we have already seen.
+        """
+        results = set()
+
+        def have_seen_events_txn(txn, chunk):
+            sql = (
+                "SELECT event_id FROM events as e WHERE e.event_id IN (%s)"
+                % (",".join("?" * len(chunk)), )
+            )
+            txn.execute(sql, chunk)
+            for (event_id, ) in txn:
+                results.add(event_id)
+
+        # break the input up into chunks of 100
+        input_iterator = iter(event_ids)
+        for chunk in iter(lambda: list(itertools.islice(input_iterator, 100)),
+                          []):
+            yield self.runInteraction(
+                "have_seen_events",
+                have_seen_events_txn,
+                chunk,
+            )
+        defer.returnValue(results)
+
+    def get_seen_events_with_rejections(self, event_ids):
+        """Given a list of event ids, check if we rejected them.
+
+        Args:
+            event_ids (list[str])
+
+        Returns:
+            Deferred[dict[str, str|None):
+                Has an entry for each event id we already have seen. Maps to
+                the rejected reason string if we rejected the event, else maps
+                to None.
         """
         if not event_ids:
             return defer.succeed({})
@@ -1348,9 +1385,7 @@ class EventsStore(EventsWorkerStore):
 
             return res
 
-        return self.runInteraction(
-            "have_events", f,
-        )
+        return self.runInteraction("get_rejection_reasons", f)
 
     @defer.inlineCallbacks
     def count_daily_messages(self):
-- 
cgit 1.4.1


From 0c280d4d99e93d2fd9120a7ec93c0997d372adb4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Apr 2018 11:10:04 +0100
Subject: Reinstate linearizer for federation_server.on_context_state_request

---
 synapse/federation/federation_server.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index d1611f39a9..12843fe179 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -211,11 +211,17 @@ class FederationServer(FederationBase):
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        resp = yield self._state_resp_cache.wrap(
-            (room_id, event_id),
-            self._on_context_state_request_compute,
-            room_id, event_id,
-        )
+        # we grab the linearizer to protect ourselves from servers which hammer
+        # us. In theory we might already have the response to this query
+        # in the cache so we could return it without waiting for the linearizer
+        # - but that's non-trivial to get right, and anyway somewhat defeats
+        # the point of the linearizer.
+        with (yield self._server_linearizer.queue((origin, room_id))):
+            resp = yield self._state_resp_cache.wrap(
+                (room_id, event_id),
+                self._on_context_state_request_compute,
+                room_id, event_id,
+            )
 
         defer.returnValue((200, resp))
 
-- 
cgit 1.4.1


From 3de7d9fe99445b1229411e7a4706c46ea94ded00 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Apr 2018 11:41:03 +0100
Subject: accept stupid events over backfill

---
 synapse/handlers/federation.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 1296e22af6..2523ae4a09 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -563,10 +563,15 @@ class FederationHandler(BaseHandler):
             extremities=extremities,
         )
 
-        # do some sanity-checking of the received events, before we go and
-        # do state resolution across 1000 events.
-        for ev in events:
-            self._sanity_check_event(ev)
+        # ideally we'd sanity check the events here for excess prev_events etc,
+        # but it's hard to reject events at this point without completely
+        # breaking backfill in the same way that it is currently broken by
+        # events whose signature we cannot verify (#3121).
+        #
+        # So for now we accept the events anyway. #3124 tracks this.
+        #
+        # for ev in events:
+        #     self._sanity_check_event(ev)
 
         # Don't bother processing events we already have.
         seen_events = yield self.store.have_events_in_timeline(
-- 
cgit 1.4.1


From 67dabe143de0c28b027e9005fce0099a06f2e9e4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 20 Apr 2018 11:46:16 +0100
Subject: Document contrib directory

---
 contrib/README.rst              | 10 ++++++++++
 contrib/systemd/synapse.service |  4 ++++
 2 files changed, 14 insertions(+)
 create mode 100644 contrib/README.rst

diff --git a/contrib/README.rst b/contrib/README.rst
new file mode 100644
index 0000000000..c296c55628
--- /dev/null
+++ b/contrib/README.rst
@@ -0,0 +1,10 @@
+Community Contributions
+=======================
+
+Everything in this directory are projects submitted by the community that may be useful
+to others. As such, the project maintainers cannot guarantee support, stability
+or backwards compatibility of these projects. 
+
+Files in this directory should *not* be relied on directly, as they may not
+continue to work or exist in future. If you wish to use any of these files then
+they should be copied to avoid them breaking from underneath you.
diff --git a/contrib/systemd/synapse.service b/contrib/systemd/synapse.service
index 3f037055b9..b81ce3915d 100644
--- a/contrib/systemd/synapse.service
+++ b/contrib/systemd/synapse.service
@@ -2,6 +2,9 @@
 # (e.g. https://www.archlinux.org/packages/community/any/matrix-synapse/ for ArchLinux)
 # rather than in a user home directory or similar under virtualenv.
 
+# **NOTE:** This is an example service file that may change in the future. If you
+# wish to use this please copy rather than symlink it.
+
 [Unit]
 Description=Synapse Matrix homeserver
 
@@ -12,6 +15,7 @@ Group=synapse
 WorkingDirectory=/var/lib/synapse
 ExecStart=/usr/bin/python2.7 -m synapse.app.homeserver --config-path=/etc/synapse/homeserver.yaml
 ExecStop=/usr/bin/synctl stop /etc/synapse/homeserver.yaml
+# EnvironmentFile=-/etc/sysconfig/synapse  # Can be used to e.g. set SYNAPSE_CACHE_FACTOR 
 
 [Install]
 WantedBy=multi-user.target
-- 
cgit 1.4.1


From 1ea904b9f09ea6a9df7792f61029d0250602d46b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Apr 2018 00:53:18 +0100
Subject: Use deferred.addTimeout instead of time_bound_deferred

This doesn't feel like a wheel we need to reinvent.
---
 synapse/http/__init__.py               | 22 +++++++++++++
 synapse/http/client.py                 | 20 +++++-------
 synapse/http/matrixfederationclient.py | 35 ++++++++++-----------
 synapse/notifier.py                    | 23 +++++++-------
 synapse/util/__init__.py               | 56 ----------------------------------
 tests/util/test_clock.py               | 33 --------------------
 6 files changed, 59 insertions(+), 130 deletions(-)
 delete mode 100644 tests/util/test_clock.py

diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index bfebb0f644..20e568bc43 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,3 +13,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from twisted.internet.defer import CancelledError
+from twisted.python import failure
+
+from synapse.api.errors import SynapseError
+
+
+class RequestTimedOutError(SynapseError):
+    """Exception representing timeout of an outbound request"""
+    def __init__(self):
+        super(RequestTimedOutError, self).__init__(504, "Timed out")
+
+
+def cancelled_to_request_timed_out_error(value):
+    """Turns CancelledErrors into RequestTimedOutErrors.
+
+    For use with deferred.addTimeout()
+    """
+    if isinstance(value, failure.Failure):
+        value.trap(CancelledError)
+        raise RequestTimedOutError()
+    return value
diff --git a/synapse/http/client.py b/synapse/http/client.py
index f3e4973c2e..35c8d51e71 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,9 +19,9 @@ from OpenSSL.SSL import VERIFY_NONE
 from synapse.api.errors import (
     CodeMessageException, MatrixCodeMessageException, SynapseError, Codes,
 )
+from synapse.http import cancelled_to_request_timed_out_error
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
-from synapse.util import logcontext
 import synapse.metrics
 from synapse.http.endpoint import SpiderEndpoint
 
@@ -95,21 +96,16 @@ class SimpleHttpClient(object):
         # counters to it
         outgoing_requests_counter.inc(method)
 
-        def send_request():
+        logger.info("Sending request %s %s", method, uri)
+
+        try:
             request_deferred = self.agent.request(
                 method, uri, *args, **kwargs
             )
-
-            return self.clock.time_bound_deferred(
-                request_deferred,
-                time_out=60,
+            request_deferred.addTimeout(
+                60, reactor, cancelled_to_request_timed_out_error,
             )
-
-        logger.info("Sending request %s %s", method, uri)
-
-        try:
-            with logcontext.PreserveLoggingContext():
-                response = yield send_request()
+            response = yield make_deferred_yieldable(request_deferred)
 
             incoming_responses_counter.inc(method, response.code)
             logger.info(
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 60a29081e8..fe4b1636a6 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,17 +13,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import synapse.util.retryutils
 from twisted.internet import defer, reactor, protocol
 from twisted.internet.error import DNSLookupError
 from twisted.web.client import readBody, HTTPConnectionPool, Agent
 from twisted.web.http_headers import Headers
 from twisted.web._newclient import ResponseDone
 
+from synapse.http import cancelled_to_request_timed_out_error
 from synapse.http.endpoint import matrix_federation_endpoint
+import synapse.metrics
 from synapse.util.async import sleep
 from synapse.util import logcontext
-import synapse.metrics
+from synapse.util.logcontext import make_deferred_yieldable
+import synapse.util.retryutils
 
 from canonicaljson import encode_canonical_json
 
@@ -184,21 +187,19 @@ class MatrixFederationHttpClient(object):
                         producer = body_callback(method, http_url_bytes, headers_dict)
 
                     try:
-                        def send_request():
-                            request_deferred = self.agent.request(
-                                method,
-                                url_bytes,
-                                Headers(headers_dict),
-                                producer
-                            )
-
-                            return self.clock.time_bound_deferred(
-                                request_deferred,
-                                time_out=timeout / 1000. if timeout else 60,
-                            )
-
-                        with logcontext.PreserveLoggingContext():
-                            response = yield send_request()
+                        request_deferred = self.agent.request(
+                            method,
+                            url_bytes,
+                            Headers(headers_dict),
+                            producer
+                        )
+                        request_deferred.addTimeout(
+                            timeout / 1000. if timeout else 60,
+                            reactor, cancelled_to_request_timed_out_error,
+                        )
+                        response = yield make_deferred_yieldable(
+                            request_deferred,
+                        )
 
                         log_result = "%d %s" % (response.code, response.phrase,)
                         break
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 0e40a4aad6..1e4f78b993 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -13,12 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.defer import TimeoutError
+
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError
 from synapse.handlers.presence import format_user_presence_state
 
-from synapse.util import DeferredTimedOutError
 from synapse.util.logutils import log_function
 from synapse.util.async import ObservableDeferred
 from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
@@ -331,11 +332,11 @@ class Notifier(object):
                     # Now we wait for the _NotifierUserStream to be told there
                     # is a new token.
                     listener = user_stream.new_listener(prev_token)
+                    listener.deferred.addTimeout(
+                        (end_time - now) / 1000., reactor,
+                    )
                     with PreserveLoggingContext():
-                        yield self.clock.time_bound_deferred(
-                            listener.deferred,
-                            time_out=(end_time - now) / 1000.
-                        )
+                        yield listener.deferred
 
                     current_token = user_stream.current_token
 
@@ -346,7 +347,7 @@ class Notifier(object):
                     # Update the prev_token to the current_token since nothing
                     # has happened between the old prev_token and the current_token
                     prev_token = current_token
-                except DeferredTimedOutError:
+                except TimeoutError:
                     break
                 except defer.CancelledError:
                     break
@@ -551,13 +552,11 @@ class Notifier(object):
             if end_time <= now:
                 break
 
+            listener.deferred.addTimeout((end_time - now) / 1000., reactor)
             try:
                 with PreserveLoggingContext():
-                    yield self.clock.time_bound_deferred(
-                        listener.deferred,
-                        time_out=(end_time - now) / 1000.
-                    )
-            except DeferredTimedOutError:
+                    yield listener.deferred
+            except TimeoutError:
                 break
             except defer.CancelledError:
                 break
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 756d8ffa32..814a7bf71b 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.api.errors import SynapseError
 from synapse.util.logcontext import PreserveLoggingContext
 
 from twisted.internet import defer, reactor, task
@@ -24,11 +23,6 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-class DeferredTimedOutError(SynapseError):
-    def __init__(self):
-        super(DeferredTimedOutError, self).__init__(504, "Timed out")
-
-
 def unwrapFirstError(failure):
     # defer.gatherResults and DeferredLists wrap failures.
     failure.trap(defer.FirstError)
@@ -85,53 +79,3 @@ class Clock(object):
         except Exception:
             if not ignore_errs:
                 raise
-
-    def time_bound_deferred(self, given_deferred, time_out):
-        if given_deferred.called:
-            return given_deferred
-
-        ret_deferred = defer.Deferred()
-
-        def timed_out_fn():
-            e = DeferredTimedOutError()
-
-            try:
-                ret_deferred.errback(e)
-            except Exception:
-                pass
-
-            try:
-                given_deferred.cancel()
-            except Exception:
-                pass
-
-        timer = None
-
-        def cancel(res):
-            try:
-                self.cancel_call_later(timer)
-            except Exception:
-                pass
-            return res
-
-        ret_deferred.addBoth(cancel)
-
-        def success(res):
-            try:
-                ret_deferred.callback(res)
-            except Exception:
-                pass
-
-            return res
-
-        def err(res):
-            try:
-                ret_deferred.errback(res)
-            except Exception:
-                pass
-
-        given_deferred.addCallbacks(callback=success, errback=err)
-
-        timer = self.call_later(time_out, timed_out_fn)
-
-        return ret_deferred
diff --git a/tests/util/test_clock.py b/tests/util/test_clock.py
deleted file mode 100644
index 9672603579..0000000000
--- a/tests/util/test_clock.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2017 Vector Creations Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from synapse import util
-from twisted.internet import defer
-from tests import unittest
-
-
-class ClockTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def test_time_bound_deferred(self):
-        # just a deferred which never resolves
-        slow_deferred = defer.Deferred()
-
-        clock = util.Clock()
-        time_bound = clock.time_bound_deferred(slow_deferred, 0.001)
-
-        try:
-            yield time_bound
-            self.fail("Expected timedout error, but got nothing")
-        except util.DeferredTimedOutError:
-            pass
-- 
cgit 1.4.1


From 9b9c38373cf9cdc57e106c8cb85c46e6dfd5c329 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Apr 2018 12:00:06 +0100
Subject: Remove spurious param

---
 synapse/handlers/federation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2523ae4a09..b662d480b6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -903,7 +903,6 @@ class FederationHandler(BaseHandler):
             logger.warn("Rejecting event %s which has %i auth_events",
                         ev.event_id, len(ev.auth_events))
             raise SynapseError(
-                "ERROR",
                 httplib.BAD_REQUEST,
                 "Too many auth_events",
             )
-- 
cgit 1.4.1


From 08b29d45749d1944965e8d203155a4ea45a15423 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 24 Apr 2018 13:56:12 +0100
Subject: Bump version and update changelog

---
 CHANGES.rst         | 42 ++++++++++++++++++++++++++++++++++++++++--
 synapse/__init__.py |  2 +-
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index c1ba8c75ce..c62ab225c6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,41 @@
+Changes in synapse v0.28.0-rc1 (2018-04-24)
+===========================================
+
+Minor performance improvement to federation sending and bug fixes.
+
+(Note: This release does not include state resolutions discussed in matrix live)
+
+Features:
+* Add metrics for event processing lag (PR #3090)
+* Add metrics for ResponseCache (PR #3092)
+
+Changes:
+* Synapse on PyPy (PR #2760) Thanks to @Valodim!
+* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
+* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
+* Document the behaviour of ResponseCache (PR #3059)
+* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107
+#3109, #3110) Thanks to @NotAFile!
+* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
+* use python3-compatible prints (PR #3074) Thanks to @NotAFile!
+* Send federation events concurrently (PR #3078)
+* Limit concurrent event sends for a room (PR #3079)
+* Improve R30 stat definition (PR #3086)
+* Send events to ASes concurrently (PR #3088)
+* Refactor ResponseCache usage (PR #3093)
+* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh!
+* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile!
+* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile!
+* Refactor store.have_events (PR #3117)
+
+Bug Fixes:
+* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug!
+* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080)
+* fix federation_domain_whitelist (PR #3099)
+* Avoid creating events with huge numbers of prev_events (PR #3113)
+* Reject events which have lots of prev_events (PR #3118)
+
+
 Changes in synapse v0.27.4 (2018-04-13)
 ======================================
 
@@ -22,10 +60,10 @@ the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
 Changes in synapse v0.27.3-rc1 (2018-04-09)
 =======================================
 
-Notable changes include API support for joinability of groups. Also new metrics 
+Notable changes include API support for joinability of groups. Also new metrics
 and phone home stats. Phone home stats include better visibility of system usage
 so we can tweak synpase to work better for all users rather than our own experience
-with matrix.org. Also, recording 'r30' stat which is the measure we use to track 
+with matrix.org. Also, recording 'r30' stat which is the measure we use to track
 overal growth of the Matrix ecosystem. It is defined as:-
 
 Counts the number of native 30 day retained users, defined as:-
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 0c1c16b9a4..2b2c440eb8 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.4"
+__version__ = "0.28.0-rc1"
-- 
cgit 1.4.1


From 8e8b06715f25a620a46d01f7d7f00f229c57d593 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 24 Apr 2018 13:58:45 +0100
Subject: Revert "Bump version and update changelog"

This reverts commit 08b29d45749d1944965e8d203155a4ea45a15423.
---
 CHANGES.rst         | 42 ++----------------------------------------
 synapse/__init__.py |  2 +-
 2 files changed, 3 insertions(+), 41 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index c62ab225c6..c1ba8c75ce 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,41 +1,3 @@
-Changes in synapse v0.28.0-rc1 (2018-04-24)
-===========================================
-
-Minor performance improvement to federation sending and bug fixes.
-
-(Note: This release does not include state resolutions discussed in matrix live)
-
-Features:
-* Add metrics for event processing lag (PR #3090)
-* Add metrics for ResponseCache (PR #3092)
-
-Changes:
-* Synapse on PyPy (PR #2760) Thanks to @Valodim!
-* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
-* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
-* Document the behaviour of ResponseCache (PR #3059)
-* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107
-#3109, #3110) Thanks to @NotAFile!
-* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
-* use python3-compatible prints (PR #3074) Thanks to @NotAFile!
-* Send federation events concurrently (PR #3078)
-* Limit concurrent event sends for a room (PR #3079)
-* Improve R30 stat definition (PR #3086)
-* Send events to ASes concurrently (PR #3088)
-* Refactor ResponseCache usage (PR #3093)
-* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh!
-* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile!
-* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile!
-* Refactor store.have_events (PR #3117)
-
-Bug Fixes:
-* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug!
-* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080)
-* fix federation_domain_whitelist (PR #3099)
-* Avoid creating events with huge numbers of prev_events (PR #3113)
-* Reject events which have lots of prev_events (PR #3118)
-
-
 Changes in synapse v0.27.4 (2018-04-13)
 ======================================
 
@@ -60,10 +22,10 @@ the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
 Changes in synapse v0.27.3-rc1 (2018-04-09)
 =======================================
 
-Notable changes include API support for joinability of groups. Also new metrics
+Notable changes include API support for joinability of groups. Also new metrics 
 and phone home stats. Phone home stats include better visibility of system usage
 so we can tweak synpase to work better for all users rather than our own experience
-with matrix.org. Also, recording 'r30' stat which is the measure we use to track
+with matrix.org. Also, recording 'r30' stat which is the measure we use to track 
 overal growth of the Matrix ecosystem. It is defined as:-
 
 Counts the number of native 30 day retained users, defined as:-
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 2b2c440eb8..0c1c16b9a4 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.28.0-rc1"
+__version__ = "0.27.4"
-- 
cgit 1.4.1


From 2c3e995f38a411647ca08ae2959add577041ca97 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 24 Apr 2018 15:33:22 +0100
Subject: Bump version and update changelog

---
 CHANGES.rst         | 45 +++++++++++++++++++++++++++++++++++++++++++--
 synapse/__init__.py |  2 +-
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index c1ba8c75ce..cc2f4676ff 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,44 @@
+Changes in synapse v0.28.0-rc1 (2018-04-24)
+===========================================
+
+Minor performance improvement to federation sending and bug fixes.
+
+(Note: This release does not include state resolutions discussed in matrix live)
+
+Features:
+
+* Add metrics for event processing lag (PR #3090)
+* Add metrics for ResponseCache (PR #3092)
+
+Changes:
+
+* Synapse on PyPy (PR #2760) Thanks to @Valodim!
+* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
+* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
+* Document the behaviour of ResponseCache (PR #3059)
+* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107
+#3109, #3110) Thanks to @NotAFile!
+* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
+* use python3-compatible prints (PR #3074) Thanks to @NotAFile!
+* Send federation events concurrently (PR #3078)
+* Limit concurrent event sends for a room (PR #3079)
+* Improve R30 stat definition (PR #3086)
+* Send events to ASes concurrently (PR #3088)
+* Refactor ResponseCache usage (PR #3093)
+* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh!
+* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile!
+* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile!
+* Refactor store.have_events (PR #3117)
+
+Bug Fixes:
+
+* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug!
+* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080)
+* fix federation_domain_whitelist (PR #3099)
+* Avoid creating events with huge numbers of prev_events (PR #3113)
+* Reject events which have lots of prev_events (PR #3118)
+
+
 Changes in synapse v0.27.4 (2018-04-13)
 ======================================
 
@@ -22,10 +63,10 @@ the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
 Changes in synapse v0.27.3-rc1 (2018-04-09)
 =======================================
 
-Notable changes include API support for joinability of groups. Also new metrics 
+Notable changes include API support for joinability of groups. Also new metrics
 and phone home stats. Phone home stats include better visibility of system usage
 so we can tweak synpase to work better for all users rather than our own experience
-with matrix.org. Also, recording 'r30' stat which is the measure we use to track 
+with matrix.org. Also, recording 'r30' stat which is the measure we use to track
 overal growth of the Matrix ecosystem. It is defined as:-
 
 Counts the number of native 30 day retained users, defined as:-
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 0c1c16b9a4..2b2c440eb8 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.27.4"
+__version__ = "0.28.0-rc1"
-- 
cgit 1.4.1


From 13a2beabca14d31d6808022d4ca45dbe4f95d2d9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 24 Apr 2018 15:43:30 +0100
Subject: Update CHANGES.rst

fix formatting on line break
---
 CHANGES.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index cc2f4676ff..b7da58edbb 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -16,8 +16,7 @@ Changes:
 * move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
 * Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
 * Document the behaviour of ResponseCache (PR #3059)
-* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107
-#3109, #3110) Thanks to @NotAFile!
+* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107, #3109, #3110) Thanks to @NotAFile!
 * update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
 * use python3-compatible prints (PR #3074) Thanks to @NotAFile!
 * Send federation events concurrently (PR #3078)
-- 
cgit 1.4.1


From 6ab3b9c743753a9673192ef6ce41cfd4db443ca9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 24 Apr 2018 16:39:20 +0100
Subject: Update CHANGES.rst

Rephrase v0.28.0-rc1 summary
---
 CHANGES.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index cc2f4676ff..f4b63a512b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -3,7 +3,8 @@ Changes in synapse v0.28.0-rc1 (2018-04-24)
 
 Minor performance improvement to federation sending and bug fixes.
 
-(Note: This release does not include state resolutions discussed in matrix live)
+(Note: This release does not include the delta state resolution implementation discussed in matrix live)
+
 
 Features:
 
-- 
cgit 1.4.1


From 48c01ae8511dee744879ac7207872770a0e24705 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 25 Apr 2018 11:02:28 +0100
Subject: ignore atom editor python ide files

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index c8901eb206..d438c6aeea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,3 +48,4 @@ env/
 *.config
 
 .vscode/
+.ropeproject/
-- 
cgit 1.4.1


From e3a373f0023e99bdd2ff9bd37ded3571904b07cb Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 25 Apr 2018 14:58:43 +0100
Subject: remove duplicates from groups tables

and rename inconsistently named indexes.
Based on https://github.com/matrix-org/synapse/pull/3128 - thanks @vurpo\!
---
 .../schema/delta/48/group_unique_indexes.sql       | 34 ++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.sql

diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.sql b/synapse/storage/schema/delta/48/group_unique_indexes.sql
new file mode 100644
index 0000000000..9ea7a8f2e6
--- /dev/null
+++ b/synapse/storage/schema/delta/48/group_unique_indexes.sql
@@ -0,0 +1,34 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- remove duplicates from group_users table
+DELETE FROM group_users WHERE ctid NOT IN (
+   SELECT min(ctid) FROM group_users GROUP BY group_id, user_id
+);
+DROP INDEX groups_users_g_idx;
+CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
+
+-- remove duplicates from group_invites table
+DELETE FROM group_invites WHERE ctid NOT IN (
+   SELECT min(ctid) FROM group_invites GROUP BY group_id, user_id
+);
+DROP INDEX groups_invites_g_idx;
+CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
+
+-- rename other indexes to actually match their table names...
+ALTER INDEX groups_users_u_idx RENAME TO group_users_u_idx;
+ALTER INDEX groups_invites_u_idx RENAME TO group_invites_u_idx;
+ALTER INDEX groups_rooms_g_idx RENAME TO group_rooms_g_idx;
+ALTER INDEX groups_rooms_r_idx RENAME TO group_rooms_r_idx;
-- 
cgit 1.4.1


From ba3166743c50cb85c9ab2d35eb31968f892260bc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Apr 2018 15:11:18 +0100
Subject: Fix quarantine media admin API

---
 synapse/storage/room.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 740c036975..c0c01bcfbe 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -595,7 +595,7 @@ class RoomStore(RoomWorkerStore, SearchStore):
         while next_token:
             sql = """
                 SELECT stream_ordering, json FROM events
-                JOIN event_json USING (event_id)
+                JOIN event_json USING (room_id, event_id)
                 WHERE room_id = ?
                     AND stream_ordering < ?
                     AND contains_url = ? AND outlier = ?
-- 
cgit 1.4.1


From 22881b3d692f44696189a1e0eeb6e83999dae10c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Apr 2018 15:32:04 +0100
Subject: Also fix reindexing of search

---
 synapse/storage/search.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 426cbe6e1a..6ba3e59889 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -77,7 +77,7 @@ class SearchStore(BackgroundUpdateStore):
             sql = (
                 "SELECT stream_ordering, event_id, room_id, type, json, "
                 " origin_server_ts FROM events"
-                " JOIN event_json USING (event_id)"
+                " JOIN event_json USING (room_id, event_id)"
                 " WHERE ? <= stream_ordering AND stream_ordering < ?"
                 " AND (%s)"
                 " ORDER BY stream_ordering DESC"
-- 
cgit 1.4.1


From 617bf40924bc8d627734eb2fb25a9f3980ede5d5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 25 Apr 2018 17:37:29 +0100
Subject: Generate user daily stats

---
 synapse/app/homeserver.py                          |  6 +++
 synapse/storage/__init__.py                        | 60 ++++++++++++++++++++--
 synapse/storage/client_ips.py                      |  7 +++
 synapse/storage/prepare_database.py                |  2 +-
 .../schema/delta/49/add_user_daily_visits.sql      | 25 +++++++++
 .../delta/49/add_user_ips_last_seen_only_index.sql | 17 ++++++
 6 files changed, 112 insertions(+), 5 deletions(-)
 create mode 100644 synapse/storage/schema/delta/49/add_user_daily_visits.sql
 create mode 100644 synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a0e465d644..808567365a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -473,6 +473,9 @@ def run(hs):
                 " changes across releases."
             )
 
+    def generate_user_daily_visit_stats():
+        hs.get_datastore().generate_user_daily_visits()
+
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
         clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
@@ -485,6 +488,9 @@ def run(hs):
         # be quite busy the first few minutes
         clock.call_later(5 * 60, phone_stats_home)
 
+    clock.looping_call(generate_user_daily_visit_stats, 60 * 1000)
+    clock.call_later(5 * 60, generate_user_daily_visit_stats)
+
     if hs.config.daemonize and hs.config.print_pidfile:
         print (hs.config.pid_file)
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 8cdfd50f90..6f2f947433 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
+import time
+import logging
+
 from synapse.storage.devices import DeviceStore
 from .appservice import (
     ApplicationServiceStore, ApplicationServiceTransactionStore
@@ -55,10 +59,6 @@ from .engines import PostgresEngine
 from synapse.api.constants import PresenceState
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
-
-import logging
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -347,6 +347,58 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return self.runInteraction("count_r30_users", _count_r30_users)
 
+
+    def generate_user_daily_visits(self):
+        """
+        Generates daily visit data for use in cohort/ retention analysis
+        """
+        def _generate_user_daily_visits(txn):
+            logger.info("Calling _generate_user_daily_visits")
+            # determine timestamp of previous days
+            yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
+            yesterday_start = datetime.datetime(yesterday.year,
+                                                yesterday.month,
+                                                yesterday.day, 0, 0, 0, 0)
+            yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000
+
+            # Check that this job has not already been completed
+            sql = """
+                SELECT timestamp
+                FROM user_daily_visits
+                ORDER by timestamp desc limit 1
+            """
+            txn.execute(sql)
+            row = txn.fetchone()
+
+            # Bail if the most recent time is yesterday
+            if row and row[0] == yesterday_start_time:
+                logger.info("Bailing from _generate_user_daily_visits, already completed")
+                return
+            logger.info("inserting into user_daily_visits")
+            # Not specificying an upper bound means that if the update is run at
+            # 10 mins past midnight and the user is active during a 30 min session
+            # that the user is still included in the previous days stats
+            # This does mean that if the update is run hours late, then it is possible
+            # to overstate the cohort, but this seems a reasonable trade off
+            # The alternative is to insert on every request - but prefer to avoid
+            # for performance reasons
+            sql = """
+                    SELECT user_id, user_agent, device_id
+                    FROM user_ips
+                    WHERE last_seen > ?
+            """
+            txn.execute(sql, (yesterday_start_time,))
+
+            sql = """
+                    INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp)
+                    VALUES (?, ?, ?, ?)
+            """
+
+            for row in txn:
+                txn.execute(sql, (row + (yesterday_start_time,)))
+
+        return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits)
+
     def get_users(self):
         """Function to reterive a list of users in users table.
 
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 7b44dae0fc..ba46907737 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -55,6 +55,13 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
             columns=["user_id", "last_seen"],
         )
 
+        self.register_background_index_update(
+            "user_ips_last_seen_only_index",
+            index_name="user_ips_last_seen_only",
+            table="user_ips",
+            columns=["last_seen"],
+        )
+
         # (user_id, access_token, ip) -> (user_agent, device_id, last_seen)
         self._batch_row_update = {}
 
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 04411a665f..c08e9cd65a 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 48
+SCHEMA_VERSION = 49
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql
new file mode 100644
index 0000000000..11356a97fc
--- /dev/null
+++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql
@@ -0,0 +1,25 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL,
+                                 device_id TEXT,
+                                 user_agent TEXT NOT NULL,
+                                 timestamp BIGINT NOT NULL );
+
+/* What indexes should I include?
+ * Reads are offline so should optimise for writes
+ * Need to check if already an entry so user,day
+ */
diff --git a/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql
new file mode 100644
index 0000000000..3a4ed59b5b
--- /dev/null
+++ b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql
@@ -0,0 +1,17 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT into background_updates (update_name, progress_json)
+    VALUES ('user_ips_last_seen_only_index', '{}');
-- 
cgit 1.4.1


From fb6015d0a6828d57c18d601ea36b1a8a4cebc0e2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 25 Apr 2018 17:56:11 +0100
Subject: pep8

---
 synapse/storage/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6f2f947433..c22d38800c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -347,7 +347,6 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return self.runInteraction("count_r30_users", _count_r30_users)
 
-
     def generate_user_daily_visits(self):
         """
         Generates daily visit data for use in cohort/ retention analysis
@@ -390,14 +389,16 @@ class DataStore(RoomMemberStore, RoomStore,
             txn.execute(sql, (yesterday_start_time,))
 
             sql = """
-                    INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp)
+                    INSERT INTO user_daily_visits (user_id. user_agent,
+                                                   device_id, timestamp)
                     VALUES (?, ?, ?, ?)
             """
 
             for row in txn:
                 txn.execute(sql, (row + (yesterday_start_time,)))
 
-        return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits)
+        return self.runInteraction("generate_user_daily_visits",
+                                   _generate_user_daily_visits)
 
     def get_users(self):
         """Function to reterive a list of users in users table.
-- 
cgit 1.4.1


From 7ec8e798b4bc67b7365be3fc743e236b9f3c76ae Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 26 Apr 2018 11:31:22 +0100
Subject: Fix media admin APIs

---
 synapse/storage/room.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index c0c01bcfbe..ea6a189185 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -530,7 +530,7 @@ class RoomStore(RoomWorkerStore, SearchStore):
 
             # Convert the IDs to MXC URIs
             for media_id in local_mxcs:
-                local_media_mxcs.append("mxc://%s/%s" % (self.hostname, media_id))
+                local_media_mxcs.append("mxc://%s/%s" % (self.hs.hostname, media_id))
             for hostname, media_id in remote_mxcs:
                 remote_media_mxcs.append("mxc://%s/%s" % (hostname, media_id))
 
@@ -619,7 +619,7 @@ class RoomStore(RoomWorkerStore, SearchStore):
                     if matches:
                         hostname = matches.group(1)
                         media_id = matches.group(2)
-                        if hostname == self.hostname:
+                        if hostname == self.hs.hostname:
                             local_media_mxcs.append(media_id)
                         else:
                             remote_media_mxcs.append((hostname, media_id))
-- 
cgit 1.4.1


From d78ada31662e0aeeb5d13ddd16aabf432574fffd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Apr 2018 12:34:40 +0100
Subject: Miscellaneous fixes to python_dependencies

* add some doc about wtf this thing does
* pin Twisted to < 18.4
* add explicit dep on six (fixes #3089)
---
 synapse/python_dependencies.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 5cabf7dabe..711cbb6c50 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -1,5 +1,6 @@
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +19,18 @@ from distutils.version import LooseVersion
 
 logger = logging.getLogger(__name__)
 
+# this dict maps from python package name to a list of modules we expect it to
+# provide.
+#
+# the key is a "requirement specifier", as used as a parameter to `pip
+# install`[1], or an `install_requires` argument to `setuptools.setup` [2].
+#
+# the value is a sequence of strings; each entry should be the name of the
+# python module, optionally followed by a version assertion which can be either
+# ">=<ver>" or "==<ver>".
+#
+# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
+# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies
 REQUIREMENTS = {
     "jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
     "frozendict>=0.4": ["frozendict"],
@@ -26,7 +39,11 @@ REQUIREMENTS = {
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-    "Twisted>=16.0.0": ["twisted>=16.0.0"],
+
+    # we break under Twisted 18.4
+    # (https://github.com/matrix-org/synapse/issues/3135)
+    "Twisted>=16.0.0,<18.4": ["twisted>=16.0.0"],
+
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],
     "pyasn1": ["pyasn1"],
@@ -39,6 +56,7 @@ REQUIREMENTS = {
     "pymacaroons-pynacl": ["pymacaroons"],
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
+    "six": ["six"],
 }
 CONDITIONAL_REQUIREMENTS = {
     "web_client": {
-- 
cgit 1.4.1


From dbf76fd4b9aa555ef9ab11455a9ed77014d991bd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Apr 2018 13:05:30 +0100
Subject: jenkins build: make sure we have a recent setuptools

---
 jenkins/prepare_synapse.sh | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh
index ffcb1cfab9..219828fa7b 100755
--- a/jenkins/prepare_synapse.sh
+++ b/jenkins/prepare_synapse.sh
@@ -1,5 +1,7 @@
 #! /bin/bash
 
+set -eux
+
 cd "`dirname $0`/.."
 
 TOX_DIR=$WORKSPACE/.tox
@@ -14,7 +16,16 @@ fi
 tox -e py27 --notest -v
 
 TOX_BIN=$TOX_DIR/py27/bin
-$TOX_BIN/pip install setuptools
+
+# cryptography 2.2 requires setuptools >= 18.5.
+#
+# older versions of virtualenv (?) give us a virtualenv with the same version
+# of setuptools as is installed on the system python (and tox runs virtualenv
+# under python3, so we get the version of setuptools that is installed on that).
+#
+# anyway, make sure that we have a recent enough setuptools.
+$TOX_BIN/pip install 'setuptools>=18.5'
+
 { python synapse/python_dependencies.py
   echo lxml psycopg2
 } | xargs $TOX_BIN/pip install
-- 
cgit 1.4.1


From 28dd536e80167677ce45bd8eeee95c6ff6eff66f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 26 Apr 2018 15:51:39 +0100
Subject: update changelog and bump version to 0.28.0

---
 CHANGES.rst         | 9 +++++++++
 synapse/__init__.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index b7da58edbb..74f454cb5b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,12 @@
+Changes in synapse v0.28.0-rc1 (2018-04-26)
+===========================================
+
+Bug Fixes:
+
+* Fix quarantine media admin API and search reindex (PR #3130)
+* Fix media admin APIs (PR #3134)
+
+
 Changes in synapse v0.28.0-rc1 (2018-04-24)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 2b2c440eb8..4924f44d4e 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.28.0-rc1"
+__version__ = "0.28.0"
-- 
cgit 1.4.1


From 31c8be956f9addc801808f9b22108326287f7143 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 01:56:58 +0100
Subject: also upgrade pip when installing

---
 jenkins/prepare_synapse.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh
index 219828fa7b..a30179f2aa 100755
--- a/jenkins/prepare_synapse.sh
+++ b/jenkins/prepare_synapse.sh
@@ -26,6 +26,10 @@ TOX_BIN=$TOX_DIR/py27/bin
 # anyway, make sure that we have a recent enough setuptools.
 $TOX_BIN/pip install 'setuptools>=18.5'
 
+# we also need a semi-recent version of pip, because old ones fail to install
+# the "enum34" dependency of cryptography.
+$TOX_BIN/pip install 'pip>=10'
+
 { python synapse/python_dependencies.py
   echo lxml psycopg2
 } | xargs $TOX_BIN/pip install
-- 
cgit 1.4.1


From 9255a6cb17716c022ebae1dbe9c142b78ca86ea7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 11:07:40 +0100
Subject: Improve exception handling for background processes

There were a bunch of places where we fire off a process to happen in the
background, but don't have any exception handling on it - instead relying on
the unhandled error being logged when the relevent deferred gets
garbage-collected.

This is unsatisfactory for a number of reasons:
 - logging on garbage collection is best-effort and may happen some time after
   the error, if at all
 - it can be hard to figure out where the error actually happened.
 - it is logged as a scary CRITICAL error which (a) I always forget to grep for
   and (b) it's not really CRITICAL if a background process we don't care about
   fails.

So this is an attempt to add exception handling to everything we fire off into
the background.
---
 synapse/app/appservice.py                 | 15 +++--
 synapse/app/federation_sender.py          | 27 +++++----
 synapse/app/pusher.py                     | 31 +++++-----
 synapse/app/synchrotron.py                | 95 ++++++++++++++++---------------
 synapse/app/user_dir.py                   | 13 ++++-
 synapse/appservice/scheduler.py           | 25 ++++----
 synapse/crypto/keyring.py                 | 93 +++++++++++++++---------------
 synapse/federation/transaction_queue.py   |  2 +
 synapse/federation/transport/server.py    | 13 ++++-
 synapse/groups/attestations.py            | 44 +++++++-------
 synapse/handlers/message.py               | 22 +++++--
 synapse/handlers/presence.py              | 19 +++++--
 synapse/handlers/receipts.py              | 61 ++++++++++----------
 synapse/handlers/typing.py                | 43 +++++++-------
 synapse/notifier.py                       | 13 +++--
 synapse/push/emailpusher.py               | 11 ++--
 synapse/push/httppusher.py                |  5 +-
 synapse/rest/media/v1/storage_provider.py |  9 ++-
 synapse/storage/event_push_actions.py     | 24 +++++---
 synapse/util/logcontext.py                |  7 ++-
 20 files changed, 335 insertions(+), 237 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index f2540023a7..58f2c9d68c 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -32,10 +32,10 @@ from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, preserve_fn
+from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
-from twisted.internet import reactor
+from twisted.internet import reactor, defer
 from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.appservice")
@@ -112,9 +112,14 @@ class ASReplicationHandler(ReplicationClientHandler):
 
         if stream_name == "events":
             max_stream_id = self.store.get_room_max_stream_ordering()
-            preserve_fn(
-                self.appservice_handler.notify_interested_services
-            )(max_stream_id)
+            run_in_background(self._notify_app_services, max_stream_id)
+
+    @defer.inlineCallbacks
+    def _notify_app_services(self, room_stream_id):
+        try:
+            yield self.appservice_handler.notify_interested_services(room_stream_id)
+        except Exception:
+            logger.exception("Error notifying application services of event")
 
 
 def start(config_options):
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 0cc3331519..4f2a9ca21a 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -237,19 +237,22 @@ class FederationSenderHandler(object):
 
     @defer.inlineCallbacks
     def update_token(self, token):
-        self.federation_position = token
-
-        # We linearize here to ensure we don't have races updating the token
-        with (yield self._fed_position_linearizer.queue(None)):
-            if self._last_ack < self.federation_position:
-                yield self.store.update_federation_out_pos(
-                    "federation", self.federation_position
-                )
+        try:
+            self.federation_position = token
+
+            # We linearize here to ensure we don't have races updating the token
+            with (yield self._fed_position_linearizer.queue(None)):
+                if self._last_ack < self.federation_position:
+                    yield self.store.update_federation_out_pos(
+                        "federation", self.federation_position
+                    )
 
-                # We ACK this token over replication so that the master can drop
-                # its in memory queues
-                self.replication_client.send_federation_ack(self.federation_position)
-                self._last_ack = self.federation_position
+                    # We ACK this token over replication so that the master can drop
+                    # its in memory queues
+                    self.replication_client.send_federation_ack(self.federation_position)
+                    self._last_ack = self.federation_position
+        except Exception:
+            logger.exception("Error updating federation stream position")
 
 
 if __name__ == '__main__':
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index d5c3a85195..739d113ad5 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -144,20 +144,23 @@ class PusherReplicationHandler(ReplicationClientHandler):
 
     @defer.inlineCallbacks
     def poke_pushers(self, stream_name, token, rows):
-        if stream_name == "pushers":
-            for row in rows:
-                if row.deleted:
-                    yield self.stop_pusher(row.user_id, row.app_id, row.pushkey)
-                else:
-                    yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
-        elif stream_name == "events":
-            yield self.pusher_pool.on_new_notifications(
-                token, token,
-            )
-        elif stream_name == "receipts":
-            yield self.pusher_pool.on_new_receipts(
-                token, token, set(row.room_id for row in rows)
-            )
+        try:
+            if stream_name == "pushers":
+                for row in rows:
+                    if row.deleted:
+                        yield self.stop_pusher(row.user_id, row.app_id, row.pushkey)
+                    else:
+                        yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
+            elif stream_name == "events":
+                yield self.pusher_pool.on_new_notifications(
+                    token, token,
+                )
+            elif stream_name == "receipts":
+                yield self.pusher_pool.on_new_receipts(
+                    token, token, set(row.room_id for row in rows)
+                )
+        except Exception:
+            logger.exception("Error poking pushers")
 
     def stop_pusher(self, user_id, app_id, pushkey):
         key = "%s:%s" % (app_id, pushkey)
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 2fddcd935a..777da564d7 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -340,55 +340,58 @@ class SyncReplicationHandler(ReplicationClientHandler):
 
     @defer.inlineCallbacks
     def process_and_notify(self, stream_name, token, rows):
-        if stream_name == "events":
-            # We shouldn't get multiple rows per token for events stream, so
-            # we don't need to optimise this for multiple rows.
-            for row in rows:
-                event = yield self.store.get_event(row.event_id)
-                extra_users = ()
-                if event.type == EventTypes.Member:
-                    extra_users = (event.state_key,)
-                max_token = self.store.get_room_max_stream_ordering()
-                self.notifier.on_new_room_event(
-                    event, token, max_token, extra_users
+        try:
+            if stream_name == "events":
+                # We shouldn't get multiple rows per token for events stream, so
+                # we don't need to optimise this for multiple rows.
+                for row in rows:
+                    event = yield self.store.get_event(row.event_id)
+                    extra_users = ()
+                    if event.type == EventTypes.Member:
+                        extra_users = (event.state_key,)
+                    max_token = self.store.get_room_max_stream_ordering()
+                    self.notifier.on_new_room_event(
+                        event, token, max_token, extra_users
+                    )
+            elif stream_name == "push_rules":
+                self.notifier.on_new_event(
+                    "push_rules_key", token, users=[row.user_id for row in rows],
                 )
-        elif stream_name == "push_rules":
-            self.notifier.on_new_event(
-                "push_rules_key", token, users=[row.user_id for row in rows],
-            )
-        elif stream_name in ("account_data", "tag_account_data",):
-            self.notifier.on_new_event(
-                "account_data_key", token, users=[row.user_id for row in rows],
-            )
-        elif stream_name == "receipts":
-            self.notifier.on_new_event(
-                "receipt_key", token, rooms=[row.room_id for row in rows],
-            )
-        elif stream_name == "typing":
-            self.typing_handler.process_replication_rows(token, rows)
-            self.notifier.on_new_event(
-                "typing_key", token, rooms=[row.room_id for row in rows],
-            )
-        elif stream_name == "to_device":
-            entities = [row.entity for row in rows if row.entity.startswith("@")]
-            if entities:
+            elif stream_name in ("account_data", "tag_account_data",):
                 self.notifier.on_new_event(
-                    "to_device_key", token, users=entities,
+                    "account_data_key", token, users=[row.user_id for row in rows],
                 )
-        elif stream_name == "device_lists":
-            all_room_ids = set()
-            for row in rows:
-                room_ids = yield self.store.get_rooms_for_user(row.user_id)
-                all_room_ids.update(room_ids)
-            self.notifier.on_new_event(
-                "device_list_key", token, rooms=all_room_ids,
-            )
-        elif stream_name == "presence":
-            yield self.presence_handler.process_replication_rows(token, rows)
-        elif stream_name == "receipts":
-            self.notifier.on_new_event(
-                "groups_key", token, users=[row.user_id for row in rows],
-            )
+            elif stream_name == "receipts":
+                self.notifier.on_new_event(
+                    "receipt_key", token, rooms=[row.room_id for row in rows],
+                )
+            elif stream_name == "typing":
+                self.typing_handler.process_replication_rows(token, rows)
+                self.notifier.on_new_event(
+                    "typing_key", token, rooms=[row.room_id for row in rows],
+                )
+            elif stream_name == "to_device":
+                entities = [row.entity for row in rows if row.entity.startswith("@")]
+                if entities:
+                    self.notifier.on_new_event(
+                        "to_device_key", token, users=entities,
+                    )
+            elif stream_name == "device_lists":
+                all_room_ids = set()
+                for row in rows:
+                    room_ids = yield self.store.get_rooms_for_user(row.user_id)
+                    all_room_ids.update(room_ids)
+                self.notifier.on_new_event(
+                    "device_list_key", token, rooms=all_room_ids,
+                )
+            elif stream_name == "presence":
+                yield self.presence_handler.process_replication_rows(token, rows)
+            elif stream_name == "receipts":
+                self.notifier.on_new_event(
+                    "groups_key", token, users=[row.user_id for row in rows],
+                )
+        except Exception:
+            logger.exception("Error processing replication")
 
 
 def start(config_options):
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 5f845e80d1..5ba7e9b416 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -39,10 +39,10 @@ from synapse.storage.engines import create_engine
 from synapse.storage.user_directory import UserDirectoryStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, preserve_fn
+from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
-from twisted.internet import reactor
+from twisted.internet import reactor, defer
 from twisted.web.resource import NoResource
 
 logger = logging.getLogger("synapse.app.user_dir")
@@ -164,7 +164,14 @@ class UserDirectoryReplicationHandler(ReplicationClientHandler):
             stream_name, token, rows
         )
         if stream_name == "current_state_deltas":
-            preserve_fn(self.user_directory.notify_new_event)()
+            run_in_background(self._notify_directory)
+
+    @defer.inlineCallbacks
+    def _notify_directory(self):
+        try:
+            yield self.user_directory.notify_new_event()
+        except Exception:
+            logger.exception("Error notifiying user directory of state update")
 
 
 def start(config_options):
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 6da315473d..dfc8d1b42e 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -176,17 +176,20 @@ class _TransactionController(object):
 
     @defer.inlineCallbacks
     def _start_recoverer(self, service):
-        yield self.store.set_appservice_state(
-            service,
-            ApplicationServiceState.DOWN
-        )
-        logger.info(
-            "Application service falling behind. Starting recoverer. AS ID %s",
-            service.id
-        )
-        recoverer = self.recoverer_fn(service, self.on_recovered)
-        self.add_recoverers([recoverer])
-        recoverer.recover()
+        try:
+            yield self.store.set_appservice_state(
+                service,
+                ApplicationServiceState.DOWN
+            )
+            logger.info(
+                "Application service falling behind. Starting recoverer. AS ID %s",
+                service.id
+            )
+            recoverer = self.recoverer_fn(service, self.on_recovered)
+            self.add_recoverers([recoverer])
+            recoverer.recover()
+        except Exception:
+            logger.exception("Error starting AS recoverer")
 
     @defer.inlineCallbacks
     def _is_service_up(self, service):
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index fce83d445f..32cbddbc53 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -146,53 +146,56 @@ class Keyring(object):
             verify_requests (List[VerifyKeyRequest]):
         """
 
-        # create a deferred for each server we're going to look up the keys
-        # for; we'll resolve them once we have completed our lookups.
-        # These will be passed into wait_for_previous_lookups to block
-        # any other lookups until we have finished.
-        # The deferreds are called with no logcontext.
-        server_to_deferred = {
-            rq.server_name: defer.Deferred()
-            for rq in verify_requests
-        }
-
-        # We want to wait for any previous lookups to complete before
-        # proceeding.
-        yield self.wait_for_previous_lookups(
-            [rq.server_name for rq in verify_requests],
-            server_to_deferred,
-        )
-
-        # Actually start fetching keys.
-        self._get_server_verify_keys(verify_requests)
-
-        # When we've finished fetching all the keys for a given server_name,
-        # resolve the deferred passed to `wait_for_previous_lookups` so that
-        # any lookups waiting will proceed.
-        #
-        # map from server name to a set of request ids
-        server_to_request_ids = {}
-
-        for verify_request in verify_requests:
-            server_name = verify_request.server_name
-            request_id = id(verify_request)
-            server_to_request_ids.setdefault(server_name, set()).add(request_id)
-
-        def remove_deferreds(res, verify_request):
-            server_name = verify_request.server_name
-            request_id = id(verify_request)
-            server_to_request_ids[server_name].discard(request_id)
-            if not server_to_request_ids[server_name]:
-                d = server_to_deferred.pop(server_name, None)
-                if d:
-                    d.callback(None)
-            return res
-
-        for verify_request in verify_requests:
-            verify_request.deferred.addBoth(
-                remove_deferreds, verify_request,
+        try:
+            # create a deferred for each server we're going to look up the keys
+            # for; we'll resolve them once we have completed our lookups.
+            # These will be passed into wait_for_previous_lookups to block
+            # any other lookups until we have finished.
+            # The deferreds are called with no logcontext.
+            server_to_deferred = {
+                rq.server_name: defer.Deferred()
+                for rq in verify_requests
+            }
+
+            # We want to wait for any previous lookups to complete before
+            # proceeding.
+            yield self.wait_for_previous_lookups(
+                [rq.server_name for rq in verify_requests],
+                server_to_deferred,
             )
 
+            # Actually start fetching keys.
+            self._get_server_verify_keys(verify_requests)
+
+            # When we've finished fetching all the keys for a given server_name,
+            # resolve the deferred passed to `wait_for_previous_lookups` so that
+            # any lookups waiting will proceed.
+            #
+            # map from server name to a set of request ids
+            server_to_request_ids = {}
+
+            for verify_request in verify_requests:
+                server_name = verify_request.server_name
+                request_id = id(verify_request)
+                server_to_request_ids.setdefault(server_name, set()).add(request_id)
+
+            def remove_deferreds(res, verify_request):
+                server_name = verify_request.server_name
+                request_id = id(verify_request)
+                server_to_request_ids[server_name].discard(request_id)
+                if not server_to_request_ids[server_name]:
+                    d = server_to_deferred.pop(server_name, None)
+                    if d:
+                        d.callback(None)
+                return res
+
+            for verify_request in verify_requests:
+                verify_request.deferred.addBoth(
+                    remove_deferreds, verify_request,
+                )
+        except Exception:
+            logger.exception("Error starting key lookups")
+
     @defer.inlineCallbacks
     def wait_for_previous_lookups(self, server_names, server_to_deferred):
         """Waits for any previous key lookups for the given servers to finish.
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 963d938edd..ded2b1871a 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -323,6 +323,8 @@ class TransactionQueue(object):
                     break
 
                 yield self._process_presence_inner(states_map.values())
+        except Exception:
+            logger.exception("Error sending presence states to servers")
         finally:
             self._processing_pending_presence = False
 
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index ff0656df3e..19d09f5422 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -25,7 +25,7 @@ from synapse.http.servlet import (
 )
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.versionstring import get_version_string
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 from synapse.types import ThirdPartyInstanceID, get_domain_from_id
 
 import functools
@@ -152,11 +152,18 @@ class Authenticator(object):
         # alive
         retry_timings = yield self.store.get_destination_retry_timings(origin)
         if retry_timings and retry_timings["retry_last_ts"]:
-            logger.info("Marking origin %r as up", origin)
-            preserve_fn(self.store.set_destination_retry_timings)(origin, 0, 0)
+            run_in_background(self._reset_retry_timings, origin)
 
         defer.returnValue(origin)
 
+    @defer.inlineCallbacks
+    def _reset_retry_timings(self, origin):
+        try:
+            logger.info("Marking origin %r as up", origin)
+            yield self.store.set_destination_retry_timings(origin, 0, 0)
+        except Exception:
+            logger.exception("Error resetting retry timings on %s", origin)
+
 
 class BaseFederationServlet(object):
     REQUIRE_AUTH = True
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 1fb709e6c3..7187df2508 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -165,28 +165,32 @@ class GroupAttestionRenewer(object):
 
         @defer.inlineCallbacks
         def _renew_attestation(group_id, user_id):
-            if not self.is_mine_id(group_id):
-                destination = get_domain_from_id(group_id)
-            elif not self.is_mine_id(user_id):
-                destination = get_domain_from_id(user_id)
-            else:
-                logger.warn(
-                    "Incorrectly trying to do attestations for user: %r in %r",
-                    user_id, group_id,
+            try:
+                if not self.is_mine_id(group_id):
+                    destination = get_domain_from_id(group_id)
+                elif not self.is_mine_id(user_id):
+                    destination = get_domain_from_id(user_id)
+                else:
+                    logger.warn(
+                        "Incorrectly trying to do attestations for user: %r in %r",
+                        user_id, group_id,
+                    )
+                    yield self.store.remove_attestation_renewal(group_id, user_id)
+                    return
+
+                attestation = self.attestations.create_attestation(group_id, user_id)
+
+                yield self.transport_client.renew_group_attestation(
+                    destination, group_id, user_id,
+                    content={"attestation": attestation},
                 )
-                yield self.store.remove_attestation_renewal(group_id, user_id)
-                return
-
-            attestation = self.attestations.create_attestation(group_id, user_id)
 
-            yield self.transport_client.renew_group_attestation(
-                destination, group_id, user_id,
-                content={"attestation": attestation},
-            )
-
-            yield self.store.update_attestation_renewal(
-                group_id, user_id, attestation
-            )
+                yield self.store.update_attestation_renewal(
+                    group_id, user_id, attestation
+                )
+            except Exception:
+                logger.exception("Error renewing attestation of %r in %r",
+                                 user_id, group_id)
 
         for row in rows:
             group_id = row["group_id"]
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 21628a8540..d168ff5b86 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -857,15 +857,25 @@ class EventCreationHandler(object):
         @defer.inlineCallbacks
         def _notify():
             yield run_on_reactor()
-            self.notifier.on_new_room_event(
-                event, event_stream_id, max_stream_id,
-                extra_users=extra_users
-            )
+            try:
+                self.notifier.on_new_room_event(
+                    event, event_stream_id, max_stream_id,
+                    extra_users=extra_users
+                )
+            except Exception:
+                logger.exception("Error notifying about new room event")
 
         preserve_fn(_notify)()
 
         if event.type == EventTypes.Message:
-            presence = self.hs.get_presence_handler()
             # We don't want to block sending messages on any presence code. This
             # matters as sometimes presence code can take a while.
-            preserve_fn(presence.bump_presence_active_time)(requester.user)
+            run_in_background(self._bump_active_time, requester.user)
+
+    @defer.inlineCallbacks
+    def _bump_active_time(self, user):
+        try:
+            presence = self.hs.get_presence_handler()
+            yield presence.bump_presence_active_time(user)
+        except Exception:
+            logger.exception("Error bumping presence active time")
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index a5e501897c..585f3e4da2 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -31,7 +31,7 @@ from synapse.storage.presence import UserPresenceState
 
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 from synapse.util.async import Linearizer
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
@@ -254,6 +254,14 @@ class PresenceHandler(object):
 
         logger.info("Finished _persist_unpersisted_changes")
 
+    @defer.inlineCallbacks
+    def _update_states_and_catch_exception(self, new_states):
+        try:
+            res = yield self._update_states(new_states)
+            defer.returnValue(res)
+        except Exception:
+            logger.exception("Error updating presence")
+
     @defer.inlineCallbacks
     def _update_states(self, new_states):
         """Updates presence of users. Sets the appropriate timeouts. Pokes
@@ -364,7 +372,7 @@ class PresenceHandler(object):
                     now=now,
                 )
 
-            preserve_fn(self._update_states)(changes)
+            run_in_background(self._update_states_and_catch_exception, changes)
         except Exception:
             logger.exception("Exception in _handle_timeouts loop")
 
@@ -422,20 +430,23 @@ class PresenceHandler(object):
 
         @defer.inlineCallbacks
         def _end():
-            if affect_presence:
+            try:
                 self.user_to_num_current_syncs[user_id] -= 1
 
                 prev_state = yield self.current_state_for_user(user_id)
                 yield self._update_states([prev_state.copy_and_replace(
                     last_user_sync_ts=self.clock.time_msec(),
                 )])
+            except Exception:
+                logger.exception("Error updating presence after sync")
 
         @contextmanager
         def _user_syncing():
             try:
                 yield
             finally:
-                preserve_fn(_end)()
+                if affect_presence:
+                    run_in_background(_end)
 
         defer.returnValue(_user_syncing())
 
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index 3f215c2b4e..2e0672161c 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -135,37 +135,40 @@ class ReceiptsHandler(BaseHandler):
         """Given a list of receipts, works out which remote servers should be
         poked and pokes them.
         """
-        # TODO: Some of this stuff should be coallesced.
-        for receipt in receipts:
-            room_id = receipt["room_id"]
-            receipt_type = receipt["receipt_type"]
-            user_id = receipt["user_id"]
-            event_ids = receipt["event_ids"]
-            data = receipt["data"]
-
-            users = yield self.state.get_current_user_in_room(room_id)
-            remotedomains = set(get_domain_from_id(u) for u in users)
-            remotedomains = remotedomains.copy()
-            remotedomains.discard(self.server_name)
-
-            logger.debug("Sending receipt to: %r", remotedomains)
-
-            for domain in remotedomains:
-                self.federation.send_edu(
-                    destination=domain,
-                    edu_type="m.receipt",
-                    content={
-                        room_id: {
-                            receipt_type: {
-                                user_id: {
-                                    "event_ids": event_ids,
-                                    "data": data,
+        try:
+            # TODO: Some of this stuff should be coallesced.
+            for receipt in receipts:
+                room_id = receipt["room_id"]
+                receipt_type = receipt["receipt_type"]
+                user_id = receipt["user_id"]
+                event_ids = receipt["event_ids"]
+                data = receipt["data"]
+
+                users = yield self.state.get_current_user_in_room(room_id)
+                remotedomains = set(get_domain_from_id(u) for u in users)
+                remotedomains = remotedomains.copy()
+                remotedomains.discard(self.server_name)
+
+                logger.debug("Sending receipt to: %r", remotedomains)
+
+                for domain in remotedomains:
+                    self.federation.send_edu(
+                        destination=domain,
+                        edu_type="m.receipt",
+                        content={
+                            room_id: {
+                                receipt_type: {
+                                    user_id: {
+                                        "event_ids": event_ids,
+                                        "data": data,
+                                    }
                                 }
-                            }
+                            },
                         },
-                    },
-                    key=(room_id, receipt_type, user_id),
-                )
+                        key=(room_id, receipt_type, user_id),
+                    )
+        except Exception:
+            logger.exception("Error pushing receipts to remote servers")
 
     @defer.inlineCallbacks
     def get_receipts_for_room(self, room_id, to_key):
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 77c0cf146f..823e2e27e1 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -205,28 +205,31 @@ class TypingHandler(object):
 
     @defer.inlineCallbacks
     def _push_remote(self, member, typing):
-        users = yield self.state.get_current_user_in_room(member.room_id)
-        self._member_last_federation_poke[member] = self.clock.time_msec()
+        try:
+            users = yield self.state.get_current_user_in_room(member.room_id)
+            self._member_last_federation_poke[member] = self.clock.time_msec()
 
-        now = self.clock.time_msec()
-        self.wheel_timer.insert(
-            now=now,
-            obj=member,
-            then=now + FEDERATION_PING_INTERVAL,
-        )
+            now = self.clock.time_msec()
+            self.wheel_timer.insert(
+                now=now,
+                obj=member,
+                then=now + FEDERATION_PING_INTERVAL,
+            )
 
-        for domain in set(get_domain_from_id(u) for u in users):
-            if domain != self.server_name:
-                self.federation.send_edu(
-                    destination=domain,
-                    edu_type="m.typing",
-                    content={
-                        "room_id": member.room_id,
-                        "user_id": member.user_id,
-                        "typing": typing,
-                    },
-                    key=member,
-                )
+            for domain in set(get_domain_from_id(u) for u in users):
+                if domain != self.server_name:
+                    self.federation.send_edu(
+                        destination=domain,
+                        edu_type="m.typing",
+                        content={
+                            "room_id": member.room_id,
+                            "user_id": member.user_id,
+                            "typing": typing,
+                        },
+                        key=member,
+                    )
+        except Exception:
+            logger.exception("Error pushing typing notif to remotes")
 
     @defer.inlineCallbacks
     def _recv_edu(self, origin, content):
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 0e40a4aad6..939723a404 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -21,7 +21,7 @@ from synapse.handlers.presence import format_user_presence_state
 from synapse.util import DeferredTimedOutError
 from synapse.util.logutils import log_function
 from synapse.util.async import ObservableDeferred
-from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
+from synapse.util.logcontext import PreserveLoggingContext, run_in_background
 from synapse.util.metrics import Measure
 from synapse.types import StreamToken
 from synapse.visibility import filter_events_for_client
@@ -251,9 +251,7 @@ class Notifier(object):
     def _on_new_room_event(self, event, room_stream_id, extra_users=[]):
         """Notify any user streams that are interested in this room event"""
         # poke any interested application service.
-        preserve_fn(self.appservice_handler.notify_interested_services)(
-            room_stream_id
-        )
+        run_in_background(self._notify_app_services, room_stream_id)
 
         if self.federation_sender:
             self.federation_sender.notify_new_events(room_stream_id)
@@ -267,6 +265,13 @@ class Notifier(object):
             rooms=[event.room_id],
         )
 
+    @defer.inlineCallbacks
+    def _notify_app_services(self, room_stream_id):
+        try:
+            yield self.appservice_handler.notify_interested_services(room_stream_id)
+        except Exception:
+            logger.exception("Error notifying application services of event")
+
     def on_new_event(self, stream_key, new_token, users=[], rooms=[]):
         """ Used to inform listeners that something has happend event wise.
 
diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
index 58df98a793..ba7286cb72 100644
--- a/synapse/push/emailpusher.py
+++ b/synapse/push/emailpusher.py
@@ -77,10 +77,13 @@ class EmailPusher(object):
     @defer.inlineCallbacks
     def on_started(self):
         if self.mailer is not None:
-            self.throttle_params = yield self.store.get_throttle_params_by_room(
-                self.pusher_id
-            )
-            yield self._process()
+            try:
+                self.throttle_params = yield self.store.get_throttle_params_by_room(
+                    self.pusher_id
+                )
+                yield self._process()
+            except Exception:
+                logger.exception("Error starting email pusher")
 
     def on_stop(self):
         if self.timed_call:
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 2cbac571b8..1420d378ef 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -94,7 +94,10 @@ class HttpPusher(object):
 
     @defer.inlineCallbacks
     def on_started(self):
-        yield self._process()
+        try:
+            yield self._process()
+        except Exception:
+            logger.exception("Error starting http pusher")
 
     @defer.inlineCallbacks
     def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py
index c188192f2b..0252afd9d3 100644
--- a/synapse/rest/media/v1/storage_provider.py
+++ b/synapse/rest/media/v1/storage_provider.py
@@ -18,7 +18,7 @@ from twisted.internet import defer, threads
 from .media_storage import FileResponder
 
 from synapse.config._base import Config
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 
 import logging
 import os
@@ -87,7 +87,12 @@ class StorageProviderWrapper(StorageProvider):
             return self.backend.store_file(path, file_info)
         else:
             # TODO: Handle errors.
-            preserve_fn(self.backend.store_file)(path, file_info)
+            def store():
+                try:
+                    return self.backend.store_file(path, file_info)
+                except Exception:
+                    logger.exception("Error storing file")
+            run_in_background(store)
             return defer.succeed(None)
 
     def fetch(self, path, file_info):
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index e78f8d0114..c22762eb5c 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -448,6 +448,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
             "add_push_actions_to_staging", _add_push_actions_to_staging_txn
         )
 
+    @defer.inlineCallbacks
     def remove_push_actions_from_staging(self, event_id):
         """Called if we failed to persist the event to ensure that stale push
         actions don't build up in the DB
@@ -456,13 +457,22 @@ class EventPushActionsWorkerStore(SQLBaseStore):
             event_id (str)
         """
 
-        return self._simple_delete(
-            table="event_push_actions_staging",
-            keyvalues={
-                "event_id": event_id,
-            },
-            desc="remove_push_actions_from_staging",
-        )
+        try:
+            res = yield self._simple_delete(
+                table="event_push_actions_staging",
+                keyvalues={
+                    "event_id": event_id,
+                },
+                desc="remove_push_actions_from_staging",
+            )
+            defer.returnValue(res)
+        except Exception:
+            # this method is called from an exception handler, so propagating
+            # another exception here really isn't helpful - there's nothing
+            # the caller can do about it. Just log the exception and move on.
+            logger.exception(
+                "Error removing push actions after event persistence failure",
+            )
 
     @defer.inlineCallbacks
     def _find_stream_orderings_for_times(self):
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index d59adc236e..d6587e4409 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -305,7 +305,12 @@ def run_in_background(f, *args, **kwargs):
     deferred returned by the funtion completes.
 
     Useful for wrapping functions that return a deferred which you don't yield
-    on.
+    on (for instance because you want to pass it to deferred.gatherResults()).
+
+    Note that if you completely discard the result, you should make sure that
+    `f` doesn't raise any deferred exceptions, otherwise a scary-looking
+    CRITICAL error about an unhandled error will be logged without much
+    indication about where it came from.
     """
     current = LoggingContext.current_context()
     res = f(*args, **kwargs)
-- 
cgit 1.4.1


From 605defb9e4c274d61d9da86546e6fd6c78f3f6cf Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 11:16:28 +0100
Subject: Add missing consumeErrors

In general we want defer.gatherResults to consumeErrors, rather than having
exceptions hanging around and getting logged as CRITICAL unhandled errors.
---
 synapse/handlers/e2e_keys.py | 4 ++--
 synapse/push/pusherpool.py   | 8 ++++++--
 synapse/storage/stream.py    | 2 +-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 325c0c4a9f..7eb03ad32e 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -141,7 +141,7 @@ class E2eKeysHandler(object):
         yield make_deferred_yieldable(defer.gatherResults([
             preserve_fn(do_remote_query)(destination)
             for destination in remote_queries_not_in_cache
-        ]))
+        ], consumeErrors=True))
 
         defer.returnValue({
             "device_keys": results, "failures": failures,
@@ -244,7 +244,7 @@ class E2eKeysHandler(object):
         yield make_deferred_yieldable(defer.gatherResults([
             preserve_fn(claim_client_keys)(destination)
             for destination in remote_queries
-        ]))
+        ], consumeErrors=True))
 
         logger.info(
             "Claimed one-time-keys: %s",
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 134e89b371..2f467d1f9c 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -142,7 +142,9 @@ class PusherPool:
                             )
                         )
 
-            yield make_deferred_yieldable(defer.gatherResults(deferreds))
+            yield make_deferred_yieldable(
+                defer.gatherResults(deferreds, consumeErrors=True),
+            )
         except Exception:
             logger.exception("Exception in pusher on_new_notifications")
 
@@ -167,7 +169,9 @@ class PusherPool:
                             preserve_fn(p.on_new_receipts)(min_stream_id, max_stream_id)
                         )
 
-            yield make_deferred_yieldable(defer.gatherResults(deferreds))
+            yield make_deferred_yieldable(
+                defer.gatherResults(deferreds, consumeErrors=True),
+            )
         except Exception:
             logger.exception("Exception in pusher on_new_receipts")
 
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 2956c3b3e0..3b8b539993 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -202,7 +202,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                     room_id, from_key, to_key, limit, order=order,
                 )
                 for room_id in rm_ids
-            ]))
+            ], consumeErrors=True))
             results.update(dict(zip(rm_ids, res)))
 
         defer.returnValue(results)
-- 
cgit 1.4.1


From 6e10eed28e5da3700b6f545ad122dad1407cfb2c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 16 Feb 2018 11:16:29 +0000
Subject: Refactor event storage to not require state

This is in preparation for using contexts that may or may not have the
current_state_ids set. This will allow us to avoid unnecessarily pulling
out state for an event on the master process when using workers.

We also add a check to see if the state groups of the old extremities
are the same as the new ones.
---
 synapse/storage/events.py | 104 +++++++++++++++++++++++++++++-----------------
 1 file changed, 67 insertions(+), 37 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5fe4a0e56c..a9269707df 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -22,7 +22,6 @@ import logging
 import simplejson as json
 from twisted.internet import defer
 
-
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.util.async import ObservableDeferred
 from synapse.util.frozenutils import frozendict_json_encoder
@@ -425,7 +424,9 @@ class EventsStore(EventsWorkerStore):
                             )
                             current_state = yield self._get_new_state_after_events(
                                 room_id,
-                                ev_ctx_rm, new_latest_event_ids,
+                                ev_ctx_rm,
+                                latest_event_ids,
+                                new_latest_event_ids,
                             )
                             if current_state is not None:
                                 current_state_for_room[room_id] = current_state
@@ -513,7 +514,8 @@ class EventsStore(EventsWorkerStore):
         defer.returnValue(new_latest_event_ids)
 
     @defer.inlineCallbacks
-    def _get_new_state_after_events(self, room_id, events_context, new_latest_event_ids):
+    def _get_new_state_after_events(self, room_id, events_context, old_latest_event_ids,
+                                    new_latest_event_ids):
         """Calculate the current state dict after adding some new events to
         a room
 
@@ -524,6 +526,9 @@ class EventsStore(EventsWorkerStore):
             events_context (list[(EventBase, EventContext)]):
                 events and contexts which are being added to the room
 
+            old_latest_event_ids (iterable[str]):
+                the old forward extremities for the room.
+
             new_latest_event_ids (iterable[str]):
                 the new forward extremities for the room.
 
@@ -534,64 +539,89 @@ class EventsStore(EventsWorkerStore):
         """
 
         if not new_latest_event_ids:
-            defer.returnValue({})
+            return
 
         # map from state_group to ((type, key) -> event_id) state map
-        state_groups = {}
-        missing_event_ids = []
-        was_updated = False
+        state_groups_map = {}
+        for ev, ctx in events_context:
+            if ctx.state_group is None:
+                # I don't think this can happen, but let's double-check
+                raise Exception(
+                    "Context for new extremity event %s has no state "
+                    "group" % (ev.event_id, ),
+                )
+
+            if ctx.state_group in state_groups_map:
+                continue
+
+            state_groups_map[ctx.state_group] = ctx.current_state_ids
+
+        # We need to map the event_ids to their state groups. First, lets
+        # check if the event is one we're persisting and then we can pull the
+        # state group from its context.
+        # Otherwise we need to pull the state group from the database.
+
+        # Set of events we need to fetch groups for. (We know none of the old
+        # extremities are going to be in events_context).
+        missing_event_ids = set(old_latest_event_ids)
+
+        event_id_to_state_group = {}
         for event_id in new_latest_event_ids:
-            # First search in the list of new events we're adding,
-            # and then use the current state from that
+            # First search in the list of new events we're adding.
             for ev, ctx in events_context:
                 if event_id == ev.event_id:
-                    if ctx.current_state_ids is None:
-                        raise Exception("Unknown current state")
-
-                    if ctx.state_group is None:
-                        # I don't think this can happen, but let's double-check
-                        raise Exception(
-                            "Context for new extremity event %s has no state "
-                            "group" % (event_id, ),
-                        )
-
-                    # If we've already seen the state group don't bother adding
-                    # it to the state sets again
-                    if ctx.state_group not in state_groups:
-                        state_groups[ctx.state_group] = ctx.current_state_ids
-                        if ctx.delta_ids or hasattr(ev, "state_key"):
-                            was_updated = True
+                    event_id_to_state_group[event_id] = ctx.state_group
                     break
             else:
                 # If we couldn't find it, then we'll need to pull
                 # the state from the database
-                was_updated = True
-                missing_event_ids.append(event_id)
-
-        if not was_updated:
-            return
+                missing_event_ids.add(event_id)
 
         if missing_event_ids:
-            # Now pull out the state for any missing events from DB
+            # Now pull out the state groups for any missing events from DB
             event_to_groups = yield self._get_state_group_for_events(
                 missing_event_ids,
             )
+            event_id_to_state_group.update(event_to_groups)
+
+        # State groups of old_latest_event_ids
+        old_state_groups = set(
+            event_id_to_state_group[evid] for evid in old_latest_event_ids
+        )
+
+        # State groups of new_latest_event_ids
+        new_state_groups = set(
+            event_id_to_state_group[evid] for evid in new_latest_event_ids
+        )
 
-            groups = set(event_to_groups.itervalues()) - set(state_groups.iterkeys())
+        # If they old and new groups are the same then we don't need to do
+        # anything.
+        if old_state_groups == new_state_groups:
+            return
 
-            if groups:
-                group_to_state = yield self._get_state_for_groups(groups)
-                state_groups.update(group_to_state)
+        # Now that we have calculated new_state_groups we need to get
+        # their state IDs so we can resolve to a single state set.
+        missing_state = new_state_groups - set(state_groups_map)
+        if missing_state:
+            group_to_state = yield self._get_state_for_groups(missing_state)
+            state_groups_map.update(group_to_state)
 
-        if len(state_groups) == 1:
+        if len(new_state_groups) == 1:
             # If there is only one state group, then we know what the current
             # state is.
-            defer.returnValue(state_groups.values()[0])
+            defer.returnValue(state_groups_map[new_state_groups.pop()])
+
+        # Ok, we need to defer to the state handler to resolve our state sets.
 
         def get_events(ev_ids):
             return self.get_events(
                 ev_ids, get_prev_content=False, check_redacted=False,
             )
+
+        state_groups = {
+            sg: state_groups_map[sg] for sg in new_state_groups
+        }
+
         events_map = {ev.event_id: ev for ev, _ in events_context}
         logger.debug("calling resolve_state_groups from preserve_events")
         res = yield self._state_resolution_handler.resolve_state_groups(
-- 
cgit 1.4.1


From 6493b22b42685a6cc06c1113196d305e4d52deed Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 11:40:06 +0100
Subject: reraise exceptions more carefully

We need to be careful (under python 2, at least) that when we reraise an
exception after doing some error handling, we actually reraise the original
exception rather than anything that might have been raised (and handled) during
the error handling.
---
 synapse/handlers/federation.py | 16 ++++++++++------
 synapse/handlers/message.py    | 21 ++++++++++++++-------
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ae7e0d6da2..260df025f9 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -19,9 +19,11 @@
 import httplib
 import itertools
 import logging
+import sys
 
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
+import six
 from twisted.internet import defer
 from unpaddedbase64 import decode_base64
 
@@ -1513,12 +1515,14 @@ class FederationHandler(BaseHandler):
                 backfilled=backfilled,
             )
         except:  # noqa: E722, as we reraise the exception this is fine.
-            # Ensure that we actually remove the entries in the push actions
-            # staging area
-            logcontext.preserve_fn(
-                self.store.remove_push_actions_from_staging
-            )(event.event_id)
-            raise
+            tp, value, tb = sys.exc_info()
+
+            logcontext.run_in_background(
+                self.store.remove_push_actions_from_staging,
+                event.event_id,
+            )
+
+            six.reraise(tp, value, tb)
 
         if not backfilled:
             # this intentionally does not yield: we don't care about the result
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 21628a8540..8e2e44bdcd 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -13,6 +13,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
+import simplejson
+import sys
+
+from canonicaljson import encode_canonical_json
+import six
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 
@@ -34,11 +40,6 @@ from synapse.replication.http.send_event import send_event_to_master
 
 from ._base import BaseHandler
 
-from canonicaljson import encode_canonical_json
-
-import logging
-import simplejson
-
 logger = logging.getLogger(__name__)
 
 
@@ -729,8 +730,14 @@ class EventCreationHandler(object):
         except:  # noqa: E722, as we reraise the exception this is fine.
             # Ensure that we actually remove the entries in the push actions
             # staging area, if we calculated them.
-            preserve_fn(self.store.remove_push_actions_from_staging)(event.event_id)
-            raise
+            tp, value, tb = sys.exc_info()
+
+            run_in_background(
+                self.store.remove_push_actions_from_staging,
+                event.event_id,
+            )
+
+            six.reraise(tp, value, tb)
 
     @defer.inlineCallbacks
     def persist_and_notify_client_event(
-- 
cgit 1.4.1


From 53849ea9d396874ae132ae3cd0a9e8d3ecb7e6d2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 27 Apr 2018 12:11:39 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 0c8307e439..40d13c6484 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.28.0-rc1 (2018-04-26)
+Changes in synapse v0.28.0 (2018-04-26)
 ===========================================
 
 Bug Fixes:
-- 
cgit 1.4.1


From 05ba7e3a44a689c67b67c617a9a23ce44da5a15e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 27 Apr 2018 12:13:12 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 0c8307e439..40d13c6484 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.28.0-rc1 (2018-04-26)
+Changes in synapse v0.28.0 (2018-04-26)
 ===========================================
 
 Bug Fixes:
-- 
cgit 1.4.1


From 13843f771ea64cfd4fb7e2f03854b9506f63fcfd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 12:17:13 +0100
Subject: Trap exceptions thrown within run_in_background

Turn any exceptions that get thrown synchronously within run_in_background into
Failures instead.
---
 synapse/util/logcontext.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index d59adc236e..127581a0b4 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -308,7 +308,13 @@ def run_in_background(f, *args, **kwargs):
     on.
     """
     current = LoggingContext.current_context()
-    res = f(*args, **kwargs)
+    try:
+        res = f(*args, **kwargs)
+    except:   # noqa: E722
+        # the assumption here is that the caller doesn't want to be disturbed
+        # by synchronous exceptions, so let's turn them into Failures.
+        return defer.fail()
+
     if isinstance(res, defer.Deferred) and not res.called:
         # The function will have reset the context before returning, so
         # we need to restore it now.
-- 
cgit 1.4.1


From 9d2c1b8429996cb9766ac636034485e2a1d685cc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 12:52:30 +0100
Subject: Backport deferred.addTimeout

Twisted 16.0 doesn't have addTimeout, so let's backport it.
---
 synapse/http/__init__.py               |  2 +-
 synapse/http/client.py                 |  6 ++-
 synapse/http/matrixfederationclient.py |  7 ++--
 synapse/notifier.py                    | 22 +++++++----
 synapse/util/async.py                  | 67 ++++++++++++++++++++++++++++++++++
 5 files changed, 90 insertions(+), 14 deletions(-)

diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index 20e568bc43..0d47ccdb59 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -28,7 +28,7 @@ class RequestTimedOutError(SynapseError):
 def cancelled_to_request_timed_out_error(value):
     """Turns CancelledErrors into RequestTimedOutErrors.
 
-    For use with deferred.addTimeout()
+    For use with async.add_timeout_to_deferred
     """
     if isinstance(value, failure.Failure):
         value.trap(CancelledError)
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 35c8d51e71..62309c3365 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -20,6 +20,7 @@ from synapse.api.errors import (
     CodeMessageException, MatrixCodeMessageException, SynapseError, Codes,
 )
 from synapse.http import cancelled_to_request_timed_out_error
+from synapse.util.async import add_timeout_to_deferred
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
 import synapse.metrics
@@ -102,8 +103,9 @@ class SimpleHttpClient(object):
             request_deferred = self.agent.request(
                 method, uri, *args, **kwargs
             )
-            request_deferred.addTimeout(
-                60, reactor, cancelled_to_request_timed_out_error,
+            add_timeout_to_deferred(
+                request_deferred,
+                60, cancelled_to_request_timed_out_error,
             )
             response = yield make_deferred_yieldable(request_deferred)
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index fe4b1636a6..30036fe81c 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone
 from synapse.http import cancelled_to_request_timed_out_error
 from synapse.http.endpoint import matrix_federation_endpoint
 import synapse.metrics
-from synapse.util.async import sleep
+from synapse.util.async import sleep, add_timeout_to_deferred
 from synapse.util import logcontext
 from synapse.util.logcontext import make_deferred_yieldable
 import synapse.util.retryutils
@@ -193,9 +193,10 @@ class MatrixFederationHttpClient(object):
                             Headers(headers_dict),
                             producer
                         )
-                        request_deferred.addTimeout(
+                        add_timeout_to_deferred(
+                            request_deferred,
                             timeout / 1000. if timeout else 60,
-                            reactor, cancelled_to_request_timed_out_error,
+                            cancelled_to_request_timed_out_error,
                         )
                         response = yield make_deferred_yieldable(
                             request_deferred,
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1e4f78b993..a1c06e8fca 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -13,15 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer, reactor
-from twisted.internet.defer import TimeoutError
+from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError
 from synapse.handlers.presence import format_user_presence_state
 
 from synapse.util.logutils import log_function
-from synapse.util.async import ObservableDeferred
+from synapse.util.async import (
+    ObservableDeferred, add_timeout_to_deferred,
+    DeferredTimeoutError,
+)
 from synapse.util.logcontext import PreserveLoggingContext, preserve_fn
 from synapse.util.metrics import Measure
 from synapse.types import StreamToken
@@ -332,8 +334,9 @@ class Notifier(object):
                     # Now we wait for the _NotifierUserStream to be told there
                     # is a new token.
                     listener = user_stream.new_listener(prev_token)
-                    listener.deferred.addTimeout(
-                        (end_time - now) / 1000., reactor,
+                    add_timeout_to_deferred(
+                        listener.deferred,
+                        (end_time - now) / 1000.,
                     )
                     with PreserveLoggingContext():
                         yield listener.deferred
@@ -347,7 +350,7 @@ class Notifier(object):
                     # Update the prev_token to the current_token since nothing
                     # has happened between the old prev_token and the current_token
                     prev_token = current_token
-                except TimeoutError:
+                except DeferredTimeoutError:
                     break
                 except defer.CancelledError:
                     break
@@ -552,11 +555,14 @@ class Notifier(object):
             if end_time <= now:
                 break
 
-            listener.deferred.addTimeout((end_time - now) / 1000., reactor)
+            add_timeout_to_deferred(
+                listener.deferred.addTimeout,
+                (end_time - now) / 1000.,
+            )
             try:
                 with PreserveLoggingContext():
                     yield listener.deferred
-            except TimeoutError:
+            except DeferredTimeoutError:
                 break
             except defer.CancelledError:
                 break
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 0729bb2863..1df5c5600c 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -15,6 +15,8 @@
 
 
 from twisted.internet import defer, reactor
+from twisted.internet.defer import CancelledError
+from twisted.python import failure
 
 from .logcontext import (
     PreserveLoggingContext, make_deferred_yieldable, preserve_fn
@@ -392,3 +394,68 @@ class ReadWriteLock(object):
                     self.key_to_current_writer.pop(key)
 
         defer.returnValue(_ctx_manager())
+
+
+class DeferredTimeoutError(Exception):
+    """
+    This error is raised by default when a L{Deferred} times out.
+    """
+
+
+def add_timeout_to_deferred(deferred, timeout, on_timeout_cancel=None):
+    """
+    Add a timeout to a deferred by scheduling it to be cancelled after
+    timeout seconds.
+
+    This is essentially a backport of deferred.addTimeout, which was introduced
+    in twisted 16.5.
+
+    If the deferred gets timed out, it errbacks with a DeferredTimeoutError,
+    unless a cancelable function was passed to its initialization or unless
+    a different on_timeout_cancel callable is provided.
+
+    Args:
+        deferred (defer.Deferred): deferred to be timed out
+        timeout (Number): seconds to time out after
+
+        on_timeout_cancel (callable): A callable which is called immediately
+            after the deferred times out, and not if this deferred is
+            otherwise cancelled before the timeout.
+
+            It takes an arbitrary value, which is the value of the deferred at
+            that exact point in time (probably a CancelledError Failure), and
+            the timeout.
+
+            The default callable (if none is provided) will translate a
+            CancelledError Failure into a DeferredTimeoutError.
+    """
+    timed_out = [False]
+
+    def time_it_out():
+        timed_out[0] = True
+        deferred.cancel()
+
+    delayed_call = reactor.callLater(timeout, time_it_out)
+
+    def convert_cancelled(value):
+        if timed_out[0]:
+            to_call = on_timeout_cancel or _cancelled_to_timed_out_error
+            return to_call(value, timeout)
+        return value
+
+    deferred.addBoth(convert_cancelled)
+
+    def cancel_timeout(result):
+        # stop the pending call to cancel the deferred if it's been fired
+        if delayed_call.active():
+            delayed_call.cancel()
+        return result
+
+    deferred.addBoth(cancel_timeout)
+
+
+def _cancelled_to_timed_out_error(value, timeout):
+    if isinstance(value, failure.Failure):
+        value.trap(CancelledError)
+        raise DeferredTimeoutError(timeout, "Deferred")
+    return value
-- 
cgit 1.4.1


From 2a13af23bc0561ab48e0a90528231c40ee209724 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 11:29:27 +0100
Subject: Use run_in_background in preference to preserve_fn

While I was going through uses of preserve_fn for other PRs, I converted places
which only use the wrapped function once to use run_in_background, to avoid
creating the function object.
---
 synapse/app/federation_sender.py              |  4 ++--
 synapse/app/pusher.py                         |  4 ++--
 synapse/app/synchrotron.py                    |  5 ++---
 synapse/appservice/scheduler.py               | 12 ++++++------
 synapse/crypto/keyring.py                     | 28 ++++++++++++++++-----------
 synapse/federation/federation_client.py       |  5 +++--
 synapse/groups/attestations.py                |  4 ++--
 synapse/handlers/appservice.py                |  5 ++++-
 synapse/handlers/e2e_keys.py                  |  6 +++---
 synapse/handlers/federation.py                | 16 +++++++++------
 synapse/handlers/initial_sync.py              | 12 +++++++-----
 synapse/handlers/message.py                   |  5 +++--
 synapse/handlers/typing.py                    |  7 ++++---
 synapse/push/pusherpool.py                    | 20 +++++++++++--------
 synapse/rest/media/v1/preview_url_resource.py |  5 +++--
 synapse/storage/events_worker.py              |  5 +++--
 synapse/storage/stream.py                     |  5 +++--
 synapse/util/async.py                         |  4 ++--
 synapse/util/file_consumer.py                 |  6 ++++--
 synapse/util/logcontext.py                    |  2 +-
 synapse/util/ratelimitutils.py                |  4 ++--
 synapse/util/retryutils.py                    |  4 ++--
 22 files changed, 97 insertions(+), 71 deletions(-)

diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 0cc3331519..c6daa0d43f 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -38,7 +38,7 @@ from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.async import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, preserve_fn
+from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
@@ -229,7 +229,7 @@ class FederationSenderHandler(object):
         # presence, typing, etc.
         if stream_name == "federation":
             send_queue.process_rows_for_federation(self.federation_sender, rows)
-            preserve_fn(self.update_token)(token)
+            run_in_background(self.update_token, token)
 
         # We also need to poke the federation sender when new events happen
         elif stream_name == "events":
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index d5c3a85195..8bd5c0c2b7 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -33,7 +33,7 @@ from synapse.server import HomeServer
 from synapse.storage import DataStore
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, preserve_fn
+from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 from twisted.internet import defer, reactor
@@ -140,7 +140,7 @@ class PusherReplicationHandler(ReplicationClientHandler):
 
     def on_rdata(self, stream_name, token, rows):
         super(PusherReplicationHandler, self).on_rdata(stream_name, token, rows)
-        preserve_fn(self.poke_pushers)(stream_name, token, rows)
+        run_in_background(self.poke_pushers, stream_name, token, rows)
 
     @defer.inlineCallbacks
     def poke_pushers(self, stream_name, token, rows):
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 2fddcd935a..0c4ccc58bc 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -51,7 +51,7 @@ from synapse.storage.engines import create_engine
 from synapse.storage.presence import UserPresenceState
 from synapse.storage.roommember import RoomMemberStore
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.logcontext import LoggingContext, preserve_fn
+from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
 from synapse.util.stringutils import random_string
 from synapse.util.versionstring import get_version_string
@@ -327,8 +327,7 @@ class SyncReplicationHandler(ReplicationClientHandler):
 
     def on_rdata(self, stream_name, token, rows):
         super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows)
-
-        preserve_fn(self.process_and_notify)(stream_name, token, rows)
+        run_in_background(self.process_and_notify, stream_name, token, rows)
 
     def get_streams_to_replicate(self):
         args = super(SyncReplicationHandler, self).get_streams_to_replicate()
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 6da315473d..ba1631b5c8 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -51,7 +51,7 @@ components.
 from twisted.internet import defer
 
 from synapse.appservice import ApplicationServiceState
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import Measure
 
 import logging
@@ -106,7 +106,7 @@ class _ServiceQueuer(object):
     def enqueue(self, service, event):
         # if this service isn't being sent something
         self.queued_events.setdefault(service.id, []).append(event)
-        preserve_fn(self._send_request)(service)
+        run_in_background(self._send_request, service)
 
     @defer.inlineCallbacks
     def _send_request(self, service):
@@ -152,10 +152,10 @@ class _TransactionController(object):
                 if sent:
                     yield txn.complete(self.store)
                 else:
-                    preserve_fn(self._start_recoverer)(service)
-        except Exception as e:
-            logger.exception(e)
-            preserve_fn(self._start_recoverer)(service)
+                    run_in_background(self._start_recoverer, service)
+        except Exception:
+            logger.exception("Error creating appservice transaction")
+            run_in_background(self._start_recoverer, service)
 
     @defer.inlineCallbacks
     def on_recovered(self, recoverer):
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index fce83d445f..38944a7326 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -19,7 +19,8 @@ from synapse.api.errors import SynapseError, Codes
 from synapse.util import unwrapFirstError, logcontext
 from synapse.util.logcontext import (
     PreserveLoggingContext,
-    preserve_fn
+    preserve_fn,
+    run_in_background,
 )
 from synapse.util.metrics import Measure
 
@@ -127,7 +128,7 @@ class Keyring(object):
 
             verify_requests.append(verify_request)
 
-        preserve_fn(self._start_key_lookups)(verify_requests)
+        run_in_background(self._start_key_lookups, verify_requests)
 
         # Pass those keys to handle_key_deferred so that the json object
         # signatures can be verified
@@ -313,7 +314,7 @@ class Keyring(object):
                     if not verify_request.deferred.called:
                         verify_request.deferred.errback(err)
 
-        preserve_fn(do_iterations)().addErrback(on_err)
+        run_in_background(do_iterations).addErrback(on_err)
 
     @defer.inlineCallbacks
     def get_keys_from_store(self, server_name_and_key_ids):
@@ -329,8 +330,9 @@ class Keyring(object):
         """
         res = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self.store.get_server_verify_keys)(
-                    server_name, key_ids
+                run_in_background(
+                    self.store.get_server_verify_keys,
+                    server_name, key_ids,
                 ).addCallback(lambda ks, server: (server, ks), server_name)
                 for server_name, key_ids in server_name_and_key_ids
             ],
@@ -358,7 +360,7 @@ class Keyring(object):
 
         results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(get_key)(p_name, p_keys)
+                run_in_background(get_key, p_name, p_keys)
                 for p_name, p_keys in self.perspective_servers.items()
             ],
             consumeErrors=True,
@@ -398,7 +400,7 @@ class Keyring(object):
 
         results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(get_key)(server_name, key_ids)
+                run_in_background(get_key, server_name, key_ids)
                 for server_name, key_ids in server_name_and_key_ids
             ],
             consumeErrors=True,
@@ -481,7 +483,8 @@ class Keyring(object):
 
         yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self.store_keys)(
+                run_in_background(
+                    self.store_keys,
                     server_name=server_name,
                     from_server=perspective_name,
                     verify_keys=response_keys,
@@ -539,7 +542,8 @@ class Keyring(object):
 
         yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self.store_keys)(
+                run_in_background(
+                    self.store_keys,
                     server_name=key_server_name,
                     from_server=server_name,
                     verify_keys=verify_keys,
@@ -615,7 +619,8 @@ class Keyring(object):
 
         yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self.store.store_server_keys_json)(
+                run_in_background(
+                    self.store.store_server_keys_json,
                     server_name=server_name,
                     key_id=key_id,
                     from_server=server_name,
@@ -716,7 +721,8 @@ class Keyring(object):
         # TODO(markjh): Store whether the keys have expired.
         return logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self.store.store_server_verify_key)(
+                run_in_background(
+                    self.store.store_server_verify_key,
                     server_name, server_name, key.time_added, key
                 )
                 for key_id, key in verify_keys.items()
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 8e2c0c4cd2..8adc60863e 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -33,7 +33,7 @@ from synapse.federation.federation_base import (
 import synapse.metrics
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.logutils import log_function
 from synapse.util.retryutils import NotRetryingDestination
 
@@ -417,7 +417,8 @@ class FederationClient(FederationBase):
             batch = set(missing_events[i:i + batch_size])
 
             deferreds = [
-                preserve_fn(self.get_pdu)(
+                run_in_background(
+                    self.get_pdu,
                     destinations=random_server_list(),
                     event_id=e_id,
                 )
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 1fb709e6c3..5f53f17954 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -42,7 +42,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
 from synapse.types import get_domain_from_id
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 
 from signedjson.sign import sign_json
 
@@ -192,4 +192,4 @@ class GroupAttestionRenewer(object):
             group_id = row["group_id"]
             user_id = row["user_id"]
 
-            preserve_fn(_renew_attestation)(group_id, user_id)
+            run_in_background(_renew_attestation, group_id, user_id)
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 0245197c02..6cc2388306 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -198,7 +198,10 @@ class ApplicationServicesHandler(object):
         services = yield self._get_services_for_3pn(protocol)
 
         results = yield make_deferred_yieldable(defer.DeferredList([
-            preserve_fn(self.appservice_api.query_3pe)(service, kind, protocol, fields)
+            run_in_background(
+                self.appservice_api.query_3pe,
+                service, kind, protocol, fields,
+            )
             for service in services
         ], consumeErrors=True))
 
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 325c0c4a9f..fc958404a1 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -24,7 +24,7 @@ from synapse.api.errors import (
     SynapseError, CodeMessageException, FederationDeniedError,
 )
 from synapse.types import get_domain_from_id, UserID
-from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
@@ -139,7 +139,7 @@ class E2eKeysHandler(object):
                 failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(defer.gatherResults([
-            preserve_fn(do_remote_query)(destination)
+            run_in_background(do_remote_query, destination)
             for destination in remote_queries_not_in_cache
         ]))
 
@@ -242,7 +242,7 @@ class E2eKeysHandler(object):
                 failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(defer.gatherResults([
-            preserve_fn(claim_client_keys)(destination)
+            run_in_background(claim_client_keys, destination)
             for destination in remote_queries
         ]))
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ae7e0d6da2..c66ca0f381 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -637,7 +637,8 @@ class FederationHandler(BaseHandler):
 
                 results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
                     [
-                        logcontext.preserve_fn(self.replication_layer.get_pdu)(
+                        logcontext.run_in_background(
+                            self.replication_layer.get_pdu,
                             [dest],
                             event_id,
                             outlier=True,
@@ -1023,7 +1024,7 @@ class FederationHandler(BaseHandler):
             # lots of requests for missing prev_events which we do actually
             # have. Hence we fire off the deferred, but don't wait for it.
 
-            logcontext.preserve_fn(self._handle_queued_pdus)(room_queue)
+            logcontext.run_in_background(self._handle_queued_pdus, room_queue)
 
         defer.returnValue(True)
 
@@ -1523,8 +1524,9 @@ class FederationHandler(BaseHandler):
         if not backfilled:
             # this intentionally does not yield: we don't care about the result
             # and don't need to wait for it.
-            logcontext.preserve_fn(self.pusher_pool.on_new_notifications)(
-                event_stream_id, max_stream_id
+            logcontext.run_in_background(
+                self.pusher_pool.on_new_notifications,
+                event_stream_id, max_stream_id,
             )
 
         defer.returnValue((context, event_stream_id, max_stream_id))
@@ -1538,7 +1540,8 @@ class FederationHandler(BaseHandler):
         """
         contexts = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
-                logcontext.preserve_fn(self._prep_event)(
+                logcontext.run_in_background(
+                    self._prep_event,
                     origin,
                     ev_info["event"],
                     state=ev_info.get("state"),
@@ -1867,7 +1870,8 @@ class FederationHandler(BaseHandler):
 
             different_events = yield logcontext.make_deferred_yieldable(
                 defer.gatherResults([
-                    logcontext.preserve_fn(self.store.get_event)(
+                    logcontext.run_in_background(
+                        self.store.get_event,
                         d,
                         allow_none=True,
                         allow_rejected=False,
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index c5267b4b84..cd33a86599 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -27,7 +27,7 @@ from synapse.types import (
 from synapse.util import unwrapFirstError
 from synapse.util.async import concurrently_execute
 from synapse.util.caches.snapshot_cache import SnapshotCache
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.visibility import filter_events_for_client
 
 from ._base import BaseHandler
@@ -166,7 +166,8 @@ class InitialSyncHandler(BaseHandler):
                 (messages, token), current_state = yield make_deferred_yieldable(
                     defer.gatherResults(
                         [
-                            preserve_fn(self.store.get_recent_events_for_room)(
+                            run_in_background(
+                                self.store.get_recent_events_for_room,
                                 event.room_id,
                                 limit=limit,
                                 end_token=room_end_token,
@@ -391,9 +392,10 @@ class InitialSyncHandler(BaseHandler):
 
         presence, receipts, (messages, token) = yield defer.gatherResults(
             [
-                preserve_fn(get_presence)(),
-                preserve_fn(get_receipts)(),
-                preserve_fn(self.store.get_recent_events_for_room)(
+                run_in_background(get_presence),
+                run_in_background(get_receipts),
+                run_in_background(
+                    self.store.get_recent_events_for_room,
                     room_id,
                     limit=limit,
                     end_token=now_token.room_key,
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 21628a8540..244b98dd8d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -850,7 +850,8 @@ class EventCreationHandler(object):
 
         # this intentionally does not yield: we don't care about the result
         # and don't need to wait for it.
-        preserve_fn(self.pusher_pool.on_new_notifications)(
+        run_in_background(
+            self.pusher_pool.on_new_notifications,
             event_stream_id, max_stream_id
         )
 
@@ -862,7 +863,7 @@ class EventCreationHandler(object):
                 extra_users=extra_users
             )
 
-        preserve_fn(_notify)()
+        run_in_background(_notify)
 
         if event.type == EventTypes.Message:
             presence = self.hs.get_presence_handler()
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 77c0cf146f..19cde70adf 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, AuthError
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
 from synapse.types import UserID, get_domain_from_id
@@ -97,7 +97,8 @@ class TypingHandler(object):
             if self.hs.is_mine_id(member.user_id):
                 last_fed_poke = self._member_last_federation_poke.get(member, None)
                 if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now:
-                    preserve_fn(self._push_remote)(
+                    run_in_background(
+                        self._push_remote,
                         member=member,
                         typing=True
                     )
@@ -196,7 +197,7 @@ class TypingHandler(object):
     def _push_update(self, member, typing):
         if self.hs.is_mine_id(member.user_id):
             # Only send updates for changes to our own users.
-            preserve_fn(self._push_remote)(member, typing)
+            run_in_background(self._push_remote, member, typing)
 
         self._push_update_local(
             member=member,
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 134e89b371..7bb5733090 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -14,13 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from twisted.internet import defer
 
-from .pusher import PusherFactory
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.push.pusher import PusherFactory
 from synapse.util.async import run_on_reactor
-
-import logging
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
 
@@ -137,8 +137,9 @@ class PusherPool:
                 if u in self.pushers:
                     for p in self.pushers[u].values():
                         deferreds.append(
-                            preserve_fn(p.on_new_notifications)(
-                                min_stream_id, max_stream_id
+                            run_in_background(
+                                p.on_new_notifications,
+                                min_stream_id, max_stream_id,
                             )
                         )
 
@@ -164,7 +165,10 @@ class PusherPool:
                 if u in self.pushers:
                     for p in self.pushers[u].values():
                         deferreds.append(
-                            preserve_fn(p.on_new_receipts)(min_stream_id, max_stream_id)
+                            run_in_background(
+                                p.on_new_receipts,
+                                min_stream_id, max_stream_id,
+                            )
                         )
 
             yield make_deferred_yieldable(defer.gatherResults(deferreds))
@@ -207,7 +211,7 @@ class PusherPool:
                 if appid_pushkey in byuser:
                     byuser[appid_pushkey].on_stop()
                 byuser[appid_pushkey] = p
-                preserve_fn(p.on_started)()
+                run_in_background(p.on_started)
 
         logger.info("Started pushers")
 
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 0fc21540c6..9290d7946f 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -35,7 +35,7 @@ from ._base import FileInfo
 from synapse.api.errors import (
     SynapseError, Codes,
 )
-from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.stringutils import random_string
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.http.client import SpiderHttpClient
@@ -144,7 +144,8 @@ class PreviewUrlResource(Resource):
         observable = self._cache.get(url)
 
         if not observable:
-            download = preserve_fn(self._do_preview)(
+            download = run_in_background(
+                self._do_preview,
                 url, requester.user, ts,
             )
             observable = ObservableDeferred(
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index a937b9bceb..ba834854e1 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -20,7 +20,7 @@ from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 
 from synapse.util.logcontext import (
-    preserve_fn, PreserveLoggingContext, make_deferred_yieldable
+    PreserveLoggingContext, make_deferred_yieldable, run_in_background,
 )
 from synapse.util.metrics import Measure
 from synapse.api.errors import SynapseError
@@ -319,7 +319,8 @@ class EventsWorkerStore(SQLBaseStore):
 
         res = yield make_deferred_yieldable(defer.gatherResults(
             [
-                preserve_fn(self._get_event_from_row)(
+                run_in_background(
+                    self._get_event_from_row,
                     row["internal_metadata"], row["json"], row["redacts"],
                     rejected_reason=row["rejects"],
                 )
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 2956c3b3e0..5b245a936c 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -41,7 +41,7 @@ from synapse.storage.events import EventsWorkerStore
 from synapse.util.caches.descriptors import cached
 from synapse.types import RoomStreamToken
 from synapse.util.caches.stream_change_cache import StreamChangeCache
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
 import abc
@@ -198,7 +198,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         room_ids = list(room_ids)
         for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)):
             res = yield make_deferred_yieldable(defer.gatherResults([
-                preserve_fn(self.get_room_events_stream_for_room)(
+                run_in_background(
+                    self.get_room_events_stream_for_room,
                     room_id, from_key, to_key, limit, order=order,
                 )
                 for room_id in rm_ids
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 0729bb2863..bd07067328 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -17,7 +17,7 @@
 from twisted.internet import defer, reactor
 
 from .logcontext import (
-    PreserveLoggingContext, make_deferred_yieldable, preserve_fn
+    PreserveLoggingContext, make_deferred_yieldable, run_in_background
 )
 from synapse.util import logcontext, unwrapFirstError
 
@@ -161,7 +161,7 @@ def concurrently_execute(func, args, limit):
             pass
 
     return logcontext.make_deferred_yieldable(defer.gatherResults([
-        preserve_fn(_concurrently_execute_inner)()
+        run_in_background(_concurrently_execute_inner)
         for _ in xrange(limit)
     ], consumeErrors=True)).addErrback(unwrapFirstError)
 
diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py
index 3c8a165331..3380970e4e 100644
--- a/synapse/util/file_consumer.py
+++ b/synapse/util/file_consumer.py
@@ -15,7 +15,7 @@
 
 from twisted.internet import threads, reactor
 
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 from six.moves import queue
 
@@ -70,7 +70,9 @@ class BackgroundFileConsumer(object):
 
         self._producer = producer
         self.streaming = streaming
-        self._finished_deferred = preserve_fn(threads.deferToThread)(self._writer)
+        self._finished_deferred = run_in_background(
+            threads.deferToThread, self._writer
+        )
         if not streaming:
             self._producer.resumeProducing()
 
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index d59adc236e..c2edf87e58 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -341,7 +341,7 @@ def make_deferred_yieldable(deferred):
     returning a deferred. Then, when the deferred completes, restores the
     current logcontext before running callbacks/errbacks.
 
-    (This is more-or-less the opposite operation to preserve_fn.)
+    (This is more-or-less the opposite operation to run_in_background.)
     """
     if isinstance(deferred, defer.Deferred) and not deferred.called:
         prev_context = LoggingContext.set_current_context(LoggingContext.sentinel)
diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py
index 1101881a2d..18424f6c36 100644
--- a/synapse/util/ratelimitutils.py
+++ b/synapse/util/ratelimitutils.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from synapse.api.errors import LimitExceededError
 
 from synapse.util.async import sleep
-from synapse.util.logcontext import preserve_fn
+from synapse.util.logcontext import run_in_background
 
 import collections
 import contextlib
@@ -150,7 +150,7 @@ class _PerHostRatelimiter(object):
                 "Ratelimit [%s]: sleeping req",
                 id(request_id),
             )
-            ret_defer = preserve_fn(sleep)(self.sleep_msec / 1000.0)
+            ret_defer = run_in_background(sleep, self.sleep_msec / 1000.0)
 
             self.sleeping_requests.add(request_id)
 
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 47b0bb5eb3..4e93f69d3a 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -203,8 +203,8 @@ class RetryDestinationLimiter(object):
                 )
             except Exception:
                 logger.exception(
-                    "Failed to store set_destination_retry_timings",
+                    "Failed to store destination_retry_timings",
                 )
 
         # we deliberately do this in the background.
-        synapse.util.logcontext.preserve_fn(store_retry_timings)()
+        synapse.util.logcontext.run_in_background(store_retry_timings)
-- 
cgit 1.4.1


From 453adf00b67fc9156050678640caa38a71181ebf Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 14:32:08 +0100
Subject: pep8; remove spurious import

---
 synapse/handlers/message.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 0f26b91862..23502eda70 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -31,7 +31,7 @@ from synapse.types import (
     UserID, RoomAlias, RoomStreamToken,
 )
 from synapse.util.async import run_on_reactor, ReadWriteLock, Limiter
-from synapse.util.logcontext import preserve_fn, run_in_background
+from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import measure_func
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.stringutils import random_string
-- 
cgit 1.4.1


From 4f5694e2cecc8530d09fd00e8bb6f165764a6042 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Fri, 27 Apr 2018 16:22:26 +0200
Subject: Add py3 tests to tox with folders that work

It's just a few tests, but it will at least prevent a few files from
regressing. Also, it makes it easiert to check your code against py36
while writing it.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 tox.ini | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index f408defc8f..c6cee92568 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = packaging, py27, pep8
+envlist = packaging, py27, py36 pep8
 
 [testenv]
 deps =
@@ -46,6 +46,14 @@ commands =
 # )
 usedevelop=true
 
+[testenv:py36]
+usedevelop=true
+commands =
+    /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
+    coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
+        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics} {env:TOXSUFFIX:}
+    {env:DUMP_COVERAGE_COMMAND:coverage report -m}
+
 [testenv:packaging]
 deps =
     check-manifest
-- 
cgit 1.4.1


From a376d8f7615e9715d6c5c3fa2d95f9d48ff87a61 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 12:29:02 +0200
Subject: open log_config in text mode too

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/config/logger.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 3f70039acd..6a7228dc2f 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -117,7 +117,7 @@ class LoggingConfig(Config):
         log_config = config.get("log_config")
         if log_config and not os.path.exists(log_config):
             log_file = self.abspath("homeserver.log")
-            with open(log_config, "wb") as log_config_file:
+            with open(log_config, "w") as log_config_file:
                 log_config_file.write(
                     DEFAULT_LOG_CONFIG.substitute(log_file=log_file)
                 )
-- 
cgit 1.4.1


From cdb4647a80dfccf4f8865a04c28bfaffa378ef10 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 13:04:40 +0200
Subject: Don't yield in list comprehensions

I've tried to grep for more of this with no success.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/handlers/appservice.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 0245197c02..b7776693de 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -259,11 +259,15 @@ class ApplicationServicesHandler(object):
             event based on the service regex.
         """
         services = self.store.get_app_services()
-        interested_list = [
-            s for s in services if (
-                yield s.is_interested(event, self.store)
-            )
-        ]
+
+        # we can't use a list comprehension here. Since python 3, list
+        # comprehensions use a generator internally. This means you can't yield
+        # inside of a list comprehension anymore.
+        interested_list = []
+        for s in services:
+            if (yield s.is_interested(event, self.store)):
+                interested_list.append(s)
+
         defer.returnValue(interested_list)
 
     def _get_services_for_user(self, user_id):
-- 
cgit 1.4.1


From 57b58e2174f120fb13fbe2f6d57e8647b69921ec Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 12:17:56 +0200
Subject: make imports local

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/push/httppusher.py          | 4 ++--
 synapse/push/pusher.py              | 2 +-
 synapse/replication/tcp/protocol.py | 4 ++--
 synapse/replication/tcp/resource.py | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 1420d378ef..b077e1a446 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -18,8 +18,8 @@ import logging
 from twisted.internet import defer, reactor
 from twisted.internet.error import AlreadyCalled, AlreadyCancelled
 
-import push_rule_evaluator
-import push_tools
+from . import push_rule_evaluator
+from . import push_tools
 import synapse
 from synapse.push import PusherConfigException
 from synapse.util.logcontext import LoggingContext
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index 71576330a9..5aa6667e91 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from httppusher import HttpPusher
+from .httppusher import HttpPusher
 
 import logging
 logger = logging.getLogger(__name__)
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 0a9a290af4..d7d38464b2 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -53,12 +53,12 @@ from twisted.internet import defer
 from twisted.protocols.basic import LineOnlyReceiver
 from twisted.python.failure import Failure
 
-from commands import (
+from .commands import (
     COMMAND_MAP, VALID_CLIENT_COMMANDS, VALID_SERVER_COMMANDS,
     ErrorCommand, ServerCommand, RdataCommand, PositionCommand, PingCommand,
     NameCommand, ReplicateCommand, UserSyncCommand, SyncCommand,
 )
-from streams import STREAMS_MAP
+from .streams import STREAMS_MAP
 
 from synapse.util.stringutils import random_string
 from synapse.metrics.metric import CounterMetric
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index 786c3fe864..a41af4fd6c 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -18,8 +18,8 @@
 from twisted.internet import defer, reactor
 from twisted.internet.protocol import Factory
 
-from streams import STREAMS_MAP, FederationStream
-from protocol import ServerReplicationStreamProtocol
+from .streams import STREAMS_MAP, FederationStream
+from .protocol import ServerReplicationStreamProtocol
 
 from synapse.util.metrics import Measure, measure_func
 
-- 
cgit 1.4.1


From 94f4d7f49ed322dac38e8946bba590ac41c0f40f Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 11:46:48 +0200
Subject: move httplib import to six

---
 synapse/handlers/federation.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 260df025f9..24ff6782c5 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -16,7 +16,6 @@
 
 """Contains handlers for federation events."""
 
-import httplib
 import itertools
 import logging
 import sys
@@ -24,6 +23,7 @@ import sys
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 import six
+from six.moves import http_client
 from twisted.internet import defer
 from unpaddedbase64 import decode_base64
 
@@ -889,7 +889,7 @@ class FederationHandler(BaseHandler):
             logger.warn("Rejecting event %s which has %i prev_events",
                         ev.event_id, len(ev.prev_events))
             raise SynapseError(
-                httplib.BAD_REQUEST,
+                http_client.BAD_REQUEST,
                 "Too many prev_events",
             )
 
@@ -897,7 +897,7 @@ class FederationHandler(BaseHandler):
             logger.warn("Rejecting event %s which has %i auth_events",
                         ev.event_id, len(ev.auth_events))
             raise SynapseError(
-                httplib.BAD_REQUEST,
+                http_client.BAD_REQUEST,
                 "Too many auth_events",
             )
 
-- 
cgit 1.4.1


From 4f2f5171b752169e265b5d93c9b4e788171b5326 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 11:39:37 +0200
Subject: replace stringIO imports

---
 synapse/http/client.py       | 4 ++--
 synapse/util/logformatter.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/http/client.py b/synapse/http/client.py
index 62309c3365..70a19d9b74 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -40,7 +40,7 @@ from twisted.web.http import PotentialDataLoss
 from twisted.web.http_headers import Headers
 from twisted.web._newclient import ResponseDone
 
-from StringIO import StringIO
+from six import StringIO
 
 import simplejson as json
 import logging
@@ -507,7 +507,7 @@ class SpiderHttpClient(SimpleHttpClient):
                     reactor,
                     SpiderEndpointFactory(hs)
                 )
-            ), [('gzip', GzipDecoder)]
+            ), [(b'gzip', GzipDecoder)]
         )
         # We could look like Chrome:
         # self.user_agent = ("Mozilla/5.0 (%s) (KHTML, like Gecko)
diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py
index cdbc4bffd7..59ab3c6968 100644
--- a/synapse/util/logformatter.py
+++ b/synapse/util/logformatter.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 
-import StringIO
+from six import StringIO
 import logging
 import traceback
 
-- 
cgit 1.4.1


From d82b6ea9e68150aece1fc46cb0821b31cf728910 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 13:57:00 +0200
Subject: Move more xrange to six

plus a bonus next()

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/federation/federation_client.py     | 4 +++-
 synapse/handlers/room_list.py               | 4 +++-
 synapse/storage/registration.py             | 4 +++-
 synapse/storage/schema/delta/30/as_users.py | 4 +++-
 synapse/storage/stream.py                   | 4 +++-
 synapse/storage/tags.py                     | 4 +++-
 synapse/util/async.py                       | 6 ++++--
 synapse/util/stringutils.py                 | 5 +++--
 synapse/util/wheel_timer.py                 | 4 +++-
 9 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 8e2c0c4cd2..a2067fc390 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -19,6 +19,8 @@ import itertools
 import logging
 import random
 
+from six.moves import range
+
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
@@ -413,7 +415,7 @@ class FederationClient(FederationBase):
 
         batch_size = 20
         missing_events = list(missing_events)
-        for i in xrange(0, len(missing_events), batch_size):
+        for i in range(0, len(missing_events), batch_size):
             batch = set(missing_events[i:i + batch_size])
 
             deferreds = [
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index add3f9b009..5757bb7f8a 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -15,6 +15,8 @@
 
 from twisted.internet import defer
 
+from six.moves import range
+
 from ._base import BaseHandler
 
 from synapse.api.constants import (
@@ -200,7 +202,7 @@ class RoomListHandler(BaseHandler):
             step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
 
         chunk = []
-        for i in xrange(0, len(rooms_to_scan), step):
+        for i in range(0, len(rooms_to_scan), step):
             batch = rooms_to_scan[i:i + step]
             logger.info("Processing %i rooms for result", len(batch))
             yield concurrently_execute(
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 6b557ca0cf..a50717db2d 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -22,6 +22,8 @@ from synapse.storage import background_updates
 from synapse.storage._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
+from six.moves import range
+
 
 class RegistrationWorkerStore(SQLBaseStore):
     @cached()
@@ -469,7 +471,7 @@ class RegistrationStore(RegistrationWorkerStore,
                 match = regex.search(user_id)
                 if match:
                     found.add(int(match.group(1)))
-            for i in xrange(len(found) + 1):
+            for i in range(len(found) + 1):
                 if i not in found:
                     return i
 
diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py
index c53e53c94f..85bd1a2006 100644
--- a/synapse/storage/schema/delta/30/as_users.py
+++ b/synapse/storage/schema/delta/30/as_users.py
@@ -14,6 +14,8 @@
 import logging
 from synapse.config.appservice import load_appservices
 
+from six.moves import range
+
 
 logger = logging.getLogger(__name__)
 
@@ -58,7 +60,7 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs):
 
     for as_id, user_ids in owned.items():
         n = 100
-        user_chunks = (user_ids[i:i + 100] for i in xrange(0, len(user_ids), n))
+        user_chunks = (user_ids[i:i + 100] for i in range(0, len(user_ids), n))
         for chunk in user_chunks:
             cur.execute(
                 database_engine.convert_param_style(
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 3b8b539993..52c90d8e04 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -47,6 +47,8 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 import abc
 import logging
 
+from six.moves import range
+
 
 logger = logging.getLogger(__name__)
 
@@ -196,7 +198,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         results = {}
         room_ids = list(room_ids)
-        for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)):
+        for rm_ids in (room_ids[i:i + 20] for i in range(0, len(room_ids), 20)):
             res = yield make_deferred_yieldable(defer.gatherResults([
                 preserve_fn(self.get_room_events_stream_for_room)(
                     room_id, from_key, to_key, limit, order=order,
diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py
index 13bff9f055..6671d3cfca 100644
--- a/synapse/storage/tags.py
+++ b/synapse/storage/tags.py
@@ -22,6 +22,8 @@ from twisted.internet import defer
 import simplejson as json
 import logging
 
+from six.moves import range
+
 logger = logging.getLogger(__name__)
 
 
@@ -98,7 +100,7 @@ class TagsWorkerStore(AccountDataWorkerStore):
 
         batch_size = 50
         results = []
-        for i in xrange(0, len(tag_ids), batch_size):
+        for i in range(0, len(tag_ids), batch_size):
             tags = yield self.runInteraction(
                 "get_all_updated_tag_content",
                 get_tag_content,
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 1df5c5600c..cb53c31123 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -27,6 +27,8 @@ from contextlib import contextmanager
 
 import logging
 
+from six.moves import range
+
 logger = logging.getLogger(__name__)
 
 
@@ -158,13 +160,13 @@ def concurrently_execute(func, args, limit):
     def _concurrently_execute_inner():
         try:
             while True:
-                yield func(it.next())
+                yield func(next(it))
         except StopIteration:
             pass
 
     return logcontext.make_deferred_yieldable(defer.gatherResults([
         preserve_fn(_concurrently_execute_inner)()
-        for _ in xrange(limit)
+        for _ in range(limit)
     ], consumeErrors=True)).addErrback(unwrapFirstError)
 
 
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 95a6168e16..b98b9dc6e4 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -15,6 +15,7 @@
 
 import random
 import string
+from six.moves import range
 
 _string_with_symbols = (
     string.digits + string.ascii_letters + ".,;:^&*-_+=#~@"
@@ -22,12 +23,12 @@ _string_with_symbols = (
 
 
 def random_string(length):
-    return ''.join(random.choice(string.ascii_letters) for _ in xrange(length))
+    return ''.join(random.choice(string.ascii_letters) for _ in range(length))
 
 
 def random_string_with_symbols(length):
     return ''.join(
-        random.choice(_string_with_symbols) for _ in xrange(length)
+        random.choice(_string_with_symbols) for _ in range(length)
     )
 
 
diff --git a/synapse/util/wheel_timer.py b/synapse/util/wheel_timer.py
index b70f9a6b0a..7a9e45aca9 100644
--- a/synapse/util/wheel_timer.py
+++ b/synapse/util/wheel_timer.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from six.moves import range
+
 
 class _Entry(object):
     __slots__ = ["end_key", "queue"]
@@ -68,7 +70,7 @@ class WheelTimer(object):
         # Add empty entries between the end of the current list and when we want
         # to insert. This ensures there are no gaps.
         self.entries.extend(
-            _Entry(key) for key in xrange(last_key, then_key + 1)
+            _Entry(key) for key in range(last_key, then_key + 1)
         )
 
         self.entries[-1].queue.append(obj)
-- 
cgit 1.4.1


From 42c89c8215533e3ef48cf8c9bf33595d4826b905 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 28 Apr 2018 22:27:30 +0100
Subject: make it work with sqlite

---
 .../schema/delta/48/group_unique_indexes.py        | 54 ++++++++++++++++++++++
 .../schema/delta/48/group_unique_indexes.sql       | 34 --------------
 2 files changed, 54 insertions(+), 34 deletions(-)
 create mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.py
 delete mode 100644 synapse/storage/schema/delta/48/group_unique_indexes.sql

diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py
new file mode 100644
index 0000000000..c1f5881b3e
--- /dev/null
+++ b/synapse/storage/schema/delta/48/group_unique_indexes.py
@@ -0,0 +1,54 @@
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.storage.engines import PostgresEngine
+
+FIX_INDEXES = """
+-- rebuild indexes as uniques
+DROP INDEX groups_invites_g_idx;
+CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
+DROP INDEX groups_users_g_idx;
+CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
+
+-- rename other indexes to actually match their table names..
+DROP INDEX groups_users_u_idx;
+CREATE INDEX group_users_u_idx ON group_users(user_id);
+DROP INDEX groups_invites_u_idx;
+CREATE INDEX group_invites_u_idx ON group_invites(user_id);
+DROP INDEX groups_rooms_g_idx;
+CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id);
+DROP INDEX groups_rooms_r_idx;
+CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
+"""
+
+def run_create(cur, database_engine, *args, **kwargs):
+    rowid = "ctid" if isinstance(database_engine, PostgresEngine) then "rowid"
+
+    # remove duplicates from group_users & group_invites tables
+    cur.execute("""
+        DELETE FROM group_users WHERE %s NOT IN (
+           SELECT min(%s) FROM group_users GROUP BY group_id, user_id
+        );
+    """ % (rowid, rowid));
+    cur.execute("""
+        DELETE FROM group_invites WHERE %s NOT IN (
+           SELECT min(%s) FROM group_invites GROUP BY group_id, user_id
+        );
+    """ % (rowid, rowid));
+
+    for statement in get_statements(FIX_INDEXES.splitlines()):
+        cur.execute(statement)
+
+def run_upgrade(*args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.sql b/synapse/storage/schema/delta/48/group_unique_indexes.sql
deleted file mode 100644
index 9ea7a8f2e6..0000000000
--- a/synapse/storage/schema/delta/48/group_unique_indexes.sql
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2018 New Vector Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
--- remove duplicates from group_users table
-DELETE FROM group_users WHERE ctid NOT IN (
-   SELECT min(ctid) FROM group_users GROUP BY group_id, user_id
-);
-DROP INDEX groups_users_g_idx;
-CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id);
-
--- remove duplicates from group_invites table
-DELETE FROM group_invites WHERE ctid NOT IN (
-   SELECT min(ctid) FROM group_invites GROUP BY group_id, user_id
-);
-DROP INDEX groups_invites_g_idx;
-CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id);
-
--- rename other indexes to actually match their table names...
-ALTER INDEX groups_users_u_idx RENAME TO group_users_u_idx;
-ALTER INDEX groups_invites_u_idx RENAME TO group_invites_u_idx;
-ALTER INDEX groups_rooms_g_idx RENAME TO group_rooms_g_idx;
-ALTER INDEX groups_rooms_r_idx RENAME TO group_rooms_r_idx;
-- 
cgit 1.4.1


From 006e18b6bbca0721e4de957e641ef64a2f0903cf Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 28 Apr 2018 22:32:24 +0100
Subject: pep8

---
 synapse/storage/schema/delta/48/group_unique_indexes.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py
index c1f5881b3e..b653a987b1 100644
--- a/synapse/storage/schema/delta/48/group_unique_indexes.py
+++ b/synapse/storage/schema/delta/48/group_unique_indexes.py
@@ -32,23 +32,25 @@ DROP INDEX groups_rooms_r_idx;
 CREATE INDEX group_rooms_r_idx ON group_rooms(room_id);
 """
 
+
 def run_create(cur, database_engine, *args, **kwargs):
-    rowid = "ctid" if isinstance(database_engine, PostgresEngine) then "rowid"
+    rowid = "ctid" if isinstance(database_engine, PostgresEngine) else "rowid"
 
     # remove duplicates from group_users & group_invites tables
     cur.execute("""
         DELETE FROM group_users WHERE %s NOT IN (
            SELECT min(%s) FROM group_users GROUP BY group_id, user_id
         );
-    """ % (rowid, rowid));
+    """ % (rowid, rowid))
     cur.execute("""
         DELETE FROM group_invites WHERE %s NOT IN (
            SELECT min(%s) FROM group_invites GROUP BY group_id, user_id
         );
-    """ % (rowid, rowid));
+    """ % (rowid, rowid))
 
     for statement in get_statements(FIX_INDEXES.splitlines()):
         cur.execute(statement)
 
+
 def run_upgrade(*args, **kwargs):
     pass
-- 
cgit 1.4.1


From adaf3ec87fe59c59ad533cab99b5f2b1d455ab08 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 28 Apr 2018 22:39:15 +0100
Subject: fix missing import

---
 synapse/storage/schema/delta/48/group_unique_indexes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/storage/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/delta/48/group_unique_indexes.py
index b653a987b1..2233af87d7 100644
--- a/synapse/storage/schema/delta/48/group_unique_indexes.py
+++ b/synapse/storage/schema/delta/48/group_unique_indexes.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from synapse.storage.engines import PostgresEngine
+from synapse.storage.prepare_database import get_statements
 
 FIX_INDEXES = """
 -- rebuild indexes as uniques
-- 
cgit 1.4.1


From e9143b659352e87fd9e26c8e5a771c78011bc945 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 23:56:59 +0200
Subject: more bytes strings

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/http/endpoint.py                 | 2 +-
 synapse/http/server.py                   | 2 +-
 synapse/rest/media/v1/upload_resource.py | 6 +++---
 synapse/util/httpresourcetree.py         | 7 +++++--
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index 00572c2897..db455e5909 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -286,7 +286,7 @@ def resolve_service(service_name, dns_client=client, cache=SERVER_CACHE, clock=t
         if (len(answers) == 1
                 and answers[0].type == dns.SRV
                 and answers[0].payload
-                and answers[0].payload.target == dns.Name('.')):
+                and answers[0].payload.target == dns.Name(b'.')):
             raise ConnectError("Service %s unavailable" % service_name)
 
         for answer in answers:
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 8d632290de..55b9ad5251 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -546,6 +546,6 @@ def _request_user_agent_is_curl(request):
         b"User-Agent", default=[]
     )
     for user_agent in user_agents:
-        if "curl" in user_agent:
+        if b"curl" in user_agent:
             return True
     return False
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index f6f498cdc5..a31e75cb46 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -81,15 +81,15 @@ class UploadResource(Resource):
         headers = request.requestHeaders
 
         if headers.hasHeader("Content-Type"):
-            media_type = headers.getRawHeaders("Content-Type")[0]
+            media_type = headers.getRawHeaders(b"Content-Type")[0]
         else:
             raise SynapseError(
                 msg="Upload request missing 'Content-Type'",
                 code=400,
             )
 
-        # if headers.hasHeader("Content-Disposition"):
-        #     disposition = headers.getRawHeaders("Content-Disposition")[0]
+        # if headers.hasHeader(b"Content-Disposition"):
+        #     disposition = headers.getRawHeaders(b"Content-Disposition")[0]
         # TODO(markjh): parse content-dispostion
 
         content_uri = yield self.media_repo.create_content(
diff --git a/synapse/util/httpresourcetree.py b/synapse/util/httpresourcetree.py
index d747849553..e9f0f292ee 100644
--- a/synapse/util/httpresourcetree.py
+++ b/synapse/util/httpresourcetree.py
@@ -40,9 +40,12 @@ def create_resource_tree(desired_tree, root_resource):
     # extra resources to existing nodes. See self._resource_id for the key.
     resource_mappings = {}
     for full_path, res in desired_tree.items():
+        # twisted requires all resources to be bytes
+        full_path = full_path.encode("utf-8")
+
         logger.info("Attaching %s to path %s", res, full_path)
         last_resource = root_resource
-        for path_seg in full_path.split('/')[1:-1]:
+        for path_seg in full_path.split(b'/')[1:-1]:
             if path_seg not in last_resource.listNames():
                 # resource doesn't exist, so make a "dummy resource"
                 child_resource = NoResource()
@@ -57,7 +60,7 @@ def create_resource_tree(desired_tree, root_resource):
 
         # ===========================
         # now attach the actual desired resource
-        last_path_seg = full_path.split('/')[-1]
+        last_path_seg = full_path.split(b'/')[-1]
 
         # if there is already a resource here, thieve its children and
         # replace it
-- 
cgit 1.4.1


From 122593265b4a10732b2e7fcfe2f5b2eba83d61d7 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 23:53:07 +0200
Subject: Construct HMAC as bytes on py3

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/rest/client/v1/register.py       | 16 +++++++++-------
 synapse/rest/client/v2_alpha/register.py | 10 ++++++----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index 8a82097178..9b3022e0b0 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -30,6 +30,8 @@ from hashlib import sha1
 import hmac
 import logging
 
+from six import string_types
+
 logger = logging.getLogger(__name__)
 
 
@@ -333,11 +335,11 @@ class RegisterRestServlet(ClientV1RestServlet):
     def _do_shared_secret(self, request, register_json, session):
         yield run_on_reactor()
 
-        if not isinstance(register_json.get("mac", None), basestring):
+        if not isinstance(register_json.get("mac", None), string_types):
             raise SynapseError(400, "Expected mac.")
-        if not isinstance(register_json.get("user", None), basestring):
+        if not isinstance(register_json.get("user", None), string_types):
             raise SynapseError(400, "Expected 'user' key.")
-        if not isinstance(register_json.get("password", None), basestring):
+        if not isinstance(register_json.get("password", None), string_types):
             raise SynapseError(400, "Expected 'password' key.")
 
         if not self.hs.config.registration_shared_secret:
@@ -358,14 +360,14 @@ class RegisterRestServlet(ClientV1RestServlet):
         got_mac = str(register_json["mac"])
 
         want_mac = hmac.new(
-            key=self.hs.config.registration_shared_secret,
+            key=self.hs.config.registration_shared_secret.encode(),
             digestmod=sha1,
         )
         want_mac.update(user)
-        want_mac.update("\x00")
+        want_mac.update(b"\x00")
         want_mac.update(password)
-        want_mac.update("\x00")
-        want_mac.update("admin" if admin else "notadmin")
+        want_mac.update(b"\x00")
+        want_mac.update(b"admin" if admin else b"notadmin")
         want_mac = want_mac.hexdigest()
 
         if compare_digest(want_mac, got_mac):
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index f317c919dc..5cab00aea9 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -35,6 +35,8 @@ from hashlib import sha1
 from synapse.util.async import run_on_reactor
 from synapse.util.ratelimitutils import FederationRateLimiter
 
+from six import string_types
+
 
 # We ought to be using hmac.compare_digest() but on older pythons it doesn't
 # exist. It's a _really minor_ security flaw to use plain string comparison
@@ -210,14 +212,14 @@ class RegisterRestServlet(RestServlet):
         # in sessions. Pull out the username/password provided to us.
         desired_password = None
         if 'password' in body:
-            if (not isinstance(body['password'], basestring) or
+            if (not isinstance(body['password'], string_types) or
                     len(body['password']) > 512):
                 raise SynapseError(400, "Invalid password")
             desired_password = body["password"]
 
         desired_username = None
         if 'username' in body:
-            if (not isinstance(body['username'], basestring) or
+            if (not isinstance(body['username'], string_types) or
                     len(body['username']) > 512):
                 raise SynapseError(400, "Invalid username")
             desired_username = body['username']
@@ -243,7 +245,7 @@ class RegisterRestServlet(RestServlet):
 
             access_token = get_access_token_from_request(request)
 
-            if isinstance(desired_username, basestring):
+            if isinstance(desired_username, string_types):
                 result = yield self._do_appservice_registration(
                     desired_username, access_token, body
                 )
@@ -464,7 +466,7 @@ class RegisterRestServlet(RestServlet):
         # includes the password and admin flag in the hashed text. Why are
         # these different?
         want_mac = hmac.new(
-            key=self.hs.config.registration_shared_secret,
+            key=self.hs.config.registration_shared_secret.encode(),
             msg=user,
             digestmod=sha1,
         ).hexdigest()
-- 
cgit 1.4.1


From 7767a9fc0e2adbe1828e0bd86e83446a9fb66bc7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 30 Apr 2018 00:37:32 +0100
Subject: Update tox.ini

add missing comma
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c6cee92568..ca8373662b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = packaging, py27, py36 pep8
+envlist = packaging, py27, py36, pep8
 
 [testenv]
 deps =
-- 
cgit 1.4.1


From af3cc5051124982182e7151e35fb72f857d81392 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Apr 2018 12:09:47 +0100
Subject: Remove redundant call to preserve_fn

submit_event_for_as doesn't return a deferred anyway, so this is pointless.
---
 synapse/handlers/appservice.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 6cc2388306..f6c7b89270 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -19,7 +19,7 @@ import synapse
 from synapse.api.constants import EventTypes
 from synapse.util.metrics import Measure
 from synapse.util.logcontext import (
-    make_deferred_yieldable, preserve_fn, run_in_background,
+    make_deferred_yieldable, run_in_background,
 )
 
 import logging
@@ -111,9 +111,7 @@ class ApplicationServicesHandler(object):
 
                         # Fork off pushes to these services
                         for service in services:
-                            preserve_fn(self.scheduler.submit_event_for_as)(
-                                service, event
-                            )
+                            self.scheduler.submit_event_for_as(service, event)
 
                     @defer.inlineCallbacks
                     def handle_room_events(events):
-- 
cgit 1.4.1


From d1d54d60889f94228adff55f64116aa66d07ac3a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 30 Apr 2018 00:58:31 +0100
Subject: add py36 to build matrix

---
 .travis.yml | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3ce93cb434..e6ba6f4752 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,22 @@
 sudo: false
 language: python
-python: 2.7
 
 # tell travis to cache ~/.cache/pip
 cache: pip
 
-env:
-  - TOX_ENV=packaging
-  - TOX_ENV=pep8
-  - TOX_ENV=py27
+matrix:
+  include:
+  - python: 2.7
+    env: TOX_ENV=packaging
+
+  - python: 2.7
+    env: TOX_ENV=pep8
+
+  - python: 2.7
+    env: TOX_ENV=py27
+    
+  - python: 3.6
+    env: TOX_ENV=py36
 
 install:
   - pip install tox
-- 
cgit 1.4.1


From 0c9db26260210bd2066048333b2644a2511b1801 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 09:49:10 +0200
Subject: add comment explaining attributeerror

---
 synapse/events/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index d4d1b92f7a..c3ff85c49a 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -47,6 +47,9 @@ class _EventInternalMetadata(object):
 
 
 def _event_dict_property(key):
+    # We want to be able to use hasattr with the event dict properties.
+    # However, (on python3) hasattr expects AttributeError to be raised. Hence,
+    # we need to transform the KeyError into an AttributeError
     def getter(self):
         try:
             return self._event_dict[key]
-- 
cgit 1.4.1


From 6e005d13821b86e4727f4f28f4c7b1e51c05d096 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 10:37:41 +0200
Subject: run config tests on py3

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index ca8373662b..89ca6d7a19 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,8 @@ usedevelop=true
 commands =
     /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
     coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
-        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics} {env:TOXSUFFIX:}
+        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics tests/config} \
+        {env:TOXSUFFIX:}
     {env:DUMP_COVERAGE_COMMAND:coverage report -m}
 
 [testenv:packaging]
-- 
cgit 1.4.1


From 576b71dd3d7139ae246b900fea4533d813f38a2a Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Mon, 30 Apr 2018 14:29:48 +0200
Subject: add guard for None on purge_history api

---
 synapse/rest/client/v1/admin.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 303419d281..efd5c9873d 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -168,11 +168,24 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
                 yield self.store.find_first_stream_ordering_after_ts(ts)
             )
 
-            (_, depth, _) = (
+            room_event_after_stream_ordering = (
                 yield self.store.get_room_event_after_stream_ordering(
                     room_id, stream_ordering,
                 )
             )
+            if room_event_after_stream_ordering:
+                (_, depth, _) = room_event_after_stream_ordering
+            else:
+                logger.warn(
+                    "[purge] purging events not possible: No event found "
+                    "(received_ts %i => stream_ordering %i)",
+                    ts, stream_ordering,
+                )
+                raise SynapseError(
+                    404,
+                    "there is no event to be purged",
+                    errcode=Codes.NOT_FOUND,
+                )
             logger.info(
                 "[purge] purging up to depth %i (received_ts %i => "
                 "stream_ordering %i)",
-- 
cgit 1.4.1


From 2ad3fc36e66fcc86cc3fde99760e4851402a8d56 Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Mon, 30 Apr 2018 16:21:11 +0100
Subject: Fixes #3135 - Replace _OpenSSLECCurve with crypto.get_elliptic_curve
 (#3157)

fixes #3135

Signed-off-by: Will Hunt will@half-shot.uk
---
 synapse/crypto/context_factory.py | 9 +++++----
 synapse/python_dependencies.py    | 7 +++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index cff3ca809a..0397f73ab4 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 from twisted.internet import ssl
-from OpenSSL import SSL
-from twisted.internet._sslverify import _OpenSSLECCurve, _defaultCurveName
+from OpenSSL import SSL, crypto
+from twisted.internet._sslverify import _defaultCurveName
 
 import logging
 
@@ -32,8 +32,9 @@ class ServerContextFactory(ssl.ContextFactory):
     @staticmethod
     def configure_context(context, config):
         try:
-            _ecCurve = _OpenSSLECCurve(_defaultCurveName)
-            _ecCurve.addECKeyToContext(context)
+            _ecCurve = crypto.get_elliptic_curve(_defaultCurveName)
+            context.set_tmp_ecdh(_ecCurve)
+
         except Exception:
             logger.exception("Failed to enable elliptic curve for TLS")
         context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 711cbb6c50..216db4d164 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -39,12 +39,11 @@ REQUIREMENTS = {
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
+    "Twisted>=16.0.0": ["twisted>=16.0.0"],
 
-    # we break under Twisted 18.4
-    # (https://github.com/matrix-org/synapse/issues/3135)
-    "Twisted>=16.0.0,<18.4": ["twisted>=16.0.0"],
+    # We use crypto.get_elliptic_curve which is only supported in >=0.15
+    "pyopenssl>=0.15": ["OpenSSL>=0.15"],
 
-    "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],
     "pyasn1": ["pyasn1"],
     "daemonize": ["daemonize"],
-- 
cgit 1.4.1


From 6495dbb326dd2b5d58e5de25107f7fe6d13b6ca4 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 21:58:30 +0200
Subject: Burminate v1auth

This closes #2602

v1auth was created to account for the differences in status code between
the v1 and v2_alpha revisions of the protocol (401 vs 403 for invalid
tokens). However since those protocols were merged, this makes the r0
version/endpoint internally inconsistent, and violates the
specification for the r0 endpoint.

This might break clients that rely on this inconsistency with the
specification. This is said to affect the legacy angular reference
client. However, I feel that restoring parity with the spec is more
important. Either way, it is critical to inform developers about this
change, in case they rely on the illegal behaviour.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/rest/client/v1/base.py       |  6 +++++-
 synapse/rest/client/v1/pusher.py     |  2 +-
 synapse/server.py                    | 10 ----------
 tests/rest/client/v1/test_events.py  |  9 +++++++--
 tests/rest/client/v1/test_profile.py |  2 +-
 tests/rest/client/v1/test_rooms.py   | 18 +++++++++---------
 tests/rest/client/v1/test_typing.py  |  2 +-
 7 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py
index c7aa0bbf59..197335d7aa 100644
--- a/synapse/rest/client/v1/base.py
+++ b/synapse/rest/client/v1/base.py
@@ -52,6 +52,10 @@ class ClientV1RestServlet(RestServlet):
     """A base Synapse REST Servlet for the client version 1 API.
     """
 
+    # This subclass was presumably created to allow the auth for the v1
+    # protocol version to be different, however this behaviour was removed.
+    # it may no longer be necessary
+
     def __init__(self, hs):
         """
         Args:
@@ -59,5 +63,5 @@ class ClientV1RestServlet(RestServlet):
         """
         self.hs = hs
         self.builder_factory = hs.get_event_builder_factory()
-        self.auth = hs.get_v1auth()
+        self.auth = hs.get_auth()
         self.txns = HttpTransactionCache(hs.get_clock())
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 1819a560cb..0206e664c1 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -150,7 +150,7 @@ class PushersRemoveRestServlet(RestServlet):
         super(RestServlet, self).__init__()
         self.hs = hs
         self.notifier = hs.get_notifier()
-        self.auth = hs.get_v1auth()
+        self.auth = hs.get_auth()
         self.pusher_pool = self.hs.get_pusherpool()
 
     @defer.inlineCallbacks
diff --git a/synapse/server.py b/synapse/server.py
index cd0c1a51be..ebdea6b0c4 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -105,7 +105,6 @@ class HomeServer(object):
         'federation_client',
         'federation_server',
         'handlers',
-        'v1auth',
         'auth',
         'state_handler',
         'state_resolution_handler',
@@ -225,15 +224,6 @@ class HomeServer(object):
     def build_simple_http_client(self):
         return SimpleHttpClient(self)
 
-    def build_v1auth(self):
-        orf = Auth(self)
-        # Matrix spec makes no reference to what HTTP status code is returned,
-        # but the V1 API uses 403 where it means 401, and the webclient
-        # relies on this behaviour, so V1 gets its own copy of the auth
-        # with backwards compat behaviour.
-        orf.TOKEN_NOT_FOUND_HTTP_STATUS = 403
-        return orf
-
     def build_state_handler(self):
         return StateHandler(self)
 
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index a8d09600bd..f5a7258e68 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -148,11 +148,16 @@ class EventStreamPermissionsTestCase(RestTestCase):
 
     @defer.inlineCallbacks
     def test_stream_basic_permissions(self):
-        # invalid token, expect 403
+        # invalid token, expect 401
+        # note: this is in violation of the original v1 spec, which expected
+        # 403. However, since the v1 spec no longer exists and the v1
+        # implementation is now part of the r0 implementation, the newer
+        # behaviour is used instead to be consistent with the r0 spec.
+        # see issue #2602
         (code, response) = yield self.mock_resource.trigger_get(
             "/events?access_token=%s" % ("invalid" + self.token, )
         )
-        self.assertEquals(403, code, msg=str(response))
+        self.assertEquals(401, code, msg=str(response))
 
         # valid token, expect content
         (code, response) = yield self.mock_resource.trigger_get(
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index deac7f100c..dc94b8bd19 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -52,7 +52,7 @@ class ProfileTestCase(unittest.TestCase):
         def _get_user_by_req(request=None, allow_guest=False):
             return synapse.types.create_requester(myid)
 
-        hs.get_v1auth().get_user_by_req = _get_user_by_req
+        hs.get_auth().get_user_by_req = _get_user_by_req
 
         profile.register_servlets(hs, self.mock_resource)
 
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index d763400eaf..61d737725b 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -60,7 +60,7 @@ class RoomPermissionsTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -70,7 +70,7 @@ class RoomPermissionsTestCase(RestTestCase):
 
         synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource)
 
-        self.auth = hs.get_v1auth()
+        self.auth = hs.get_auth()
 
         # create some rooms under the name rmcreator_id
         self.uncreated_rmid = "!aa:test"
@@ -425,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -507,7 +507,7 @@ class RoomsCreateTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -597,7 +597,7 @@ class RoomTopicTestCase(RestTestCase):
                 "is_guest": False,
             }
 
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -711,7 +711,7 @@ class RoomMemberStateTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -843,7 +843,7 @@ class RoomMessagesTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -945,7 +945,7 @@ class RoomInitialSyncTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
@@ -1017,7 +1017,7 @@ class RoomMessageListTestCase(RestTestCase):
                 "token_id": 1,
                 "is_guest": False,
             }
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 2ec4ecab5b..fe161ee5cb 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -68,7 +68,7 @@ class RoomTypingTestCase(RestTestCase):
                 "is_guest": False,
             }
 
-        hs.get_v1auth().get_user_by_access_token = get_user_by_access_token
+        hs.get_auth().get_user_by_access_token = get_user_by_access_token
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
-- 
cgit 1.4.1


From 5917562b6040448cd26b67fac52d61b41ba08f6d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 1 May 2018 12:12:22 +0100
Subject: 10 mins seems more reasonable that every minute

---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 808567365a..328ab8032c 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -488,7 +488,7 @@ def run(hs):
         # be quite busy the first few minutes
         clock.call_later(5 * 60, phone_stats_home)
 
-    clock.looping_call(generate_user_daily_visit_stats, 60 * 1000)
+    clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000)
     clock.call_later(5 * 60, generate_user_daily_visit_stats)
 
     if hs.config.daemonize and hs.config.print_pidfile:
-- 
cgit 1.4.1


From d0857702e8e22b8bd3dae7fc3cd3718da1e94f19 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 1 May 2018 12:12:57 +0100
Subject: add inidexes based on usage

---
 synapse/storage/schema/delta/49/add_user_daily_visits.sql | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql
index 11356a97fc..3dd478196f 100644
--- a/synapse/storage/schema/delta/49/add_user_daily_visits.sql
+++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql
@@ -16,10 +16,6 @@
 
 CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL,
                                  device_id TEXT,
-                                 user_agent TEXT NOT NULL,
                                  timestamp BIGINT NOT NULL );
-
-/* What indexes should I include?
- * Reads are offline so should optimise for writes
- * Need to check if already an entry so user,day
- */
+CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp);
+CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp);
-- 
cgit 1.4.1


From dd1a8324195f37d508d518db10ba37d9028afc86 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 1 May 2018 12:13:49 +0100
Subject: remove user agent from data model, will just join on user_ips

---
 synapse/storage/__init__.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index c22d38800c..b51cf70336 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import datetime
+from dateutil import tz
 import time
 import logging
 
@@ -354,10 +355,9 @@ class DataStore(RoomMemberStore, RoomStore,
         def _generate_user_daily_visits(txn):
             logger.info("Calling _generate_user_daily_visits")
             # determine timestamp of previous days
-            yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
-            yesterday_start = datetime.datetime(yesterday.year,
-                                                yesterday.month,
-                                                yesterday.day, 0, 0, 0, 0)
+            yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
+            yesterday_start = datetime.datetime(yesterday.year, yesterday.month,
+                                                yesterday.day, tzinfo=tz.tzutc())
             yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000
 
             # Check that this job has not already been completed
@@ -371,9 +371,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
             # Bail if the most recent time is yesterday
             if row and row[0] == yesterday_start_time:
-                logger.info("Bailing from _generate_user_daily_visits, already completed")
                 return
-            logger.info("inserting into user_daily_visits")
+
             # Not specificying an upper bound means that if the update is run at
             # 10 mins past midnight and the user is active during a 30 min session
             # that the user is still included in the previous days stats
@@ -382,20 +381,20 @@ class DataStore(RoomMemberStore, RoomStore,
             # The alternative is to insert on every request - but prefer to avoid
             # for performance reasons
             sql = """
-                    SELECT user_id, user_agent, device_id
+                    SELECT user_id, device_id
                     FROM user_ips
                     WHERE last_seen > ?
             """
             txn.execute(sql, (yesterday_start_time,))
+            user_visits = txn.fetchall()
 
             sql = """
-                    INSERT INTO user_daily_visits (user_id. user_agent,
-                                                   device_id, timestamp)
-                    VALUES (?, ?, ?, ?)
+                    INSERT INTO user_daily_visits (user_id, device_id, timestamp)
+                    VALUES (?, ?, ?)
             """
 
-            for row in txn:
-                txn.execute(sql, (row + (yesterday_start_time,)))
+            for visit in user_visits:
+                txn.execute(sql, (visit + (yesterday_start_time,)))
 
         return self.runInteraction("generate_user_daily_visits",
                                    _generate_user_daily_visits)
-- 
cgit 1.4.1


From 33f469ba19586bbafa0cf2c7d7c35463bdab87eb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 1 May 2018 16:19:39 +0100
Subject: Apply some limits to depth to counter abuse

* When creating a new event, cap its depth to 2^63 - 1
* When receiving events, reject any without a sensible depth

As per https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI
---
 synapse/api/constants.py              |  3 +++
 synapse/federation/federation_base.py | 21 ++++++++++++++++++---
 synapse/handlers/message.py           |  6 +++++-
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 489efb7f86..5baba43966 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -16,6 +16,9 @@
 
 """Contains constants from the specification."""
 
+# the "depth" field on events is limited to 2**63 - 1
+MAX_DEPTH = 2**63 - 1
+
 
 class Membership(object):
 
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 79eaa31031..4cc98a3fe8 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -14,7 +14,10 @@
 # limitations under the License.
 import logging
 
-from synapse.api.errors import SynapseError
+import six
+
+from synapse.api.constants import MAX_DEPTH
+from synapse.api.errors import SynapseError, Codes
 from synapse.crypto.event_signing import check_event_content_hash
 from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
@@ -190,11 +193,23 @@ def event_from_pdu_json(pdu_json, outlier=False):
         FrozenEvent
 
     Raises:
-        SynapseError: if the pdu is missing required fields
+        SynapseError: if the pdu is missing required fields or is otherwise
+            not a valid matrix event
     """
     # we could probably enforce a bunch of other fields here (room_id, sender,
     # origin, etc etc)
-    assert_params_in_request(pdu_json, ('event_id', 'type'))
+    assert_params_in_request(pdu_json, ('event_id', 'type', 'depth'))
+
+    depth = pdu_json['depth']
+    if not isinstance(depth, six.integer_types):
+        raise SynapseError(400, "Depth %r not an intger" % (depth, ),
+                           Codes.BAD_JSON)
+
+    if depth < 0:
+        raise SynapseError(400, "Depth too small", Codes.BAD_JSON)
+    elif depth > MAX_DEPTH:
+        raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
+
     event = FrozenEvent(
         pdu_json
     )
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 21628a8540..53beb2b9ab 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, MAX_DEPTH
 from synapse.api.errors import AuthError, Codes, SynapseError
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
@@ -624,6 +624,10 @@ class EventCreationHandler(object):
 
         if prev_events_and_hashes:
             depth = max([d for _, _, d in prev_events_and_hashes]) + 1
+            # we cap depth of generated events, to ensure that they are not
+            # rejected by other servers (and so that they can be persisted in
+            # the db)
+            depth = min(depth, MAX_DEPTH)
         else:
             depth = 1
 
-- 
cgit 1.4.1


From d858f3bd4e0513bea2fae43b23250b979b971407 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Apr 2018 12:34:40 +0100
Subject: Miscellaneous fixes to python_dependencies

* add some doc about wtf this thing does
* pin Twisted to < 18.4
* add explicit dep on six (fixes #3089)
---
 synapse/python_dependencies.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 5cabf7dabe..711cbb6c50 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -1,5 +1,6 @@
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +19,18 @@ from distutils.version import LooseVersion
 
 logger = logging.getLogger(__name__)
 
+# this dict maps from python package name to a list of modules we expect it to
+# provide.
+#
+# the key is a "requirement specifier", as used as a parameter to `pip
+# install`[1], or an `install_requires` argument to `setuptools.setup` [2].
+#
+# the value is a sequence of strings; each entry should be the name of the
+# python module, optionally followed by a version assertion which can be either
+# ">=<ver>" or "==<ver>".
+#
+# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
+# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies
 REQUIREMENTS = {
     "jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
     "frozendict>=0.4": ["frozendict"],
@@ -26,7 +39,11 @@ REQUIREMENTS = {
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-    "Twisted>=16.0.0": ["twisted>=16.0.0"],
+
+    # we break under Twisted 18.4
+    # (https://github.com/matrix-org/synapse/issues/3135)
+    "Twisted>=16.0.0,<18.4": ["twisted>=16.0.0"],
+
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],
     "pyasn1": ["pyasn1"],
@@ -39,6 +56,7 @@ REQUIREMENTS = {
     "pymacaroons-pynacl": ["pymacaroons"],
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
+    "six": ["six"],
 }
 CONDITIONAL_REQUIREMENTS = {
     "web_client": {
-- 
cgit 1.4.1


From 8570bb84ccba5c7e53161e445d13e3aaffbcab1b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 1 May 2018 18:22:53 +0100
Subject: Update __init__.py

bump version
---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 4924f44d4e..f31cb9a3cb 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.28.0"
+__version__ = "0.28.1"
-- 
cgit 1.4.1


From 8e6bd0e32456e66ec8df3af62ecacf291cf4632f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 1 May 2018 18:28:14 +0100
Subject: changelog for 0.28.1

---
 CHANGES.rst | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 74f454cb5b..8da5e0dbf6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,5 +1,26 @@
-Changes in synapse v0.28.0-rc1 (2018-04-26)
-===========================================
+Changes in synapse v0.28.1 (2018-05-01)
+=======================================
+
+SECURITY UPDATE
+
+* Clamp the allowed values of event depth received over federation to be
+  [0, 2**63 - 1].  This mitigates an attack where malicious events
+  injected with depth = 2**63 - 1 render rooms unusable.  Depth is used to
+  determine the cosmetic ordering of events within a room, and so the ordering
+  of events in such a room will default to using stream_ordering rather than depth
+  (topological_ordering).
+
+  This is a temporary solution to mitigate abuse in the wild, whilst a long solution
+  is being implemented to improve how the depth parameter is used.
+
+  Full details at
+  https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI/edit#
+
+* Pin Twisted to <18.4 until we stop using the private _OpenSSLECCurve API.
+
+
+Changes in synapse v0.28.0 (2018-04-26)
+=======================================
 
 Bug Fixes:
 
-- 
cgit 1.4.1


From 5c2214f4c73e67ea907aea740e8c168fd4735299 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 1 May 2018 19:03:35 +0100
Subject: fix markdown

---
 CHANGES.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 8da5e0dbf6..cc40855387 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -4,8 +4,8 @@ Changes in synapse v0.28.1 (2018-05-01)
 SECURITY UPDATE
 
 * Clamp the allowed values of event depth received over federation to be
-  [0, 2**63 - 1].  This mitigates an attack where malicious events
-  injected with depth = 2**63 - 1 render rooms unusable.  Depth is used to
+  [0, 2^63 - 1].  This mitigates an attack where malicious events
+  injected with depth = 2^63 - 1 render rooms unusable.  Depth is used to
   determine the cosmetic ordering of events within a room, and so the ordering
   of events in such a room will default to using stream_ordering rather than depth
   (topological_ordering).
@@ -14,7 +14,7 @@ SECURITY UPDATE
   is being implemented to improve how the depth parameter is used.
 
   Full details at
-  https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI/edit#
+  https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI
 
 * Pin Twisted to <18.4 until we stop using the private _OpenSSLECCurve API.
 
-- 
cgit 1.4.1


From da602419b277308d06ba5e2a147e6d128eaa2802 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 1 May 2018 19:19:23 +0100
Subject: missing word :|

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 0dd2efd9d2..317846d2a2 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,7 +10,7 @@ SECURITY UPDATE
   of events in such a room will default to using stream_ordering rather than depth
   (topological_ordering).
 
-  This is a temporary solution to mitigate abuse in the wild, whilst a long solution
+  This is a temporary solution to mitigate abuse in the wild, whilst a long term solution
   is being implemented to improve how the depth parameter is used.
 
   Full details at
-- 
cgit 1.4.1


From 9f21de6a015a210d9ce5ae71948af98a56046ea2 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 1 May 2018 19:19:46 +0100
Subject: missing word :|

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 0dd2efd9d2..317846d2a2 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -10,7 +10,7 @@ SECURITY UPDATE
   of events in such a room will default to using stream_ordering rather than depth
   (topological_ordering).
 
-  This is a temporary solution to mitigate abuse in the wild, whilst a long solution
+  This is a temporary solution to mitigate abuse in the wild, whilst a long term solution
   is being implemented to improve how the depth parameter is used.
 
   Full details at
-- 
cgit 1.4.1


From d4c14e143825c679a7909b009aec2e51f829857f Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Tue, 1 May 2018 20:47:58 +0200
Subject: Fix the documentation about 'POSTGRES_DB'

---
 contrib/docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 25c358c847..aed56646c2 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -135,7 +135,7 @@ Shared secrets, that will be initialized to random values if not set:
 
 Database specific values (will use SQLite if not set):
 
-* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`]
+* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
 * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
 * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
 * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-- 
cgit 1.4.1


From 4f2e898c29de5b51e8c5893afa64e5b43d25048f Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Tue, 1 May 2018 20:49:52 +0200
Subject: Make the logging level configurable

---
 contrib/docker/conf/log.config    | 6 +++---
 contrib/docker/docker-compose.yml | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config
index b5c907c4f9..1851995802 100644
--- a/contrib/docker/conf/log.config
+++ b/contrib/docker/conf/log.config
@@ -17,13 +17,13 @@ handlers:
 
 loggers:
     synapse:
-        level: INFO
+        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
 
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
-        level: INFO
+        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
 
 root:
-    level: INFO
+    level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
     handlers: [console]
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 46e72601d3..0b531949e0 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -15,6 +15,7 @@ services:
       - SYNAPSE_SERVER_NAME=my.matrix.host
       - SYNAPSE_REPORT_STATS=no
       - SYNAPSE_ENABLE_REGISTRATION=yes
+      - SYNAPSE_LOG_LEVEL=INFO
       - POSTGRES_PASSWORD=changeme
     volumes:
       # You may either store all the files in a local folder
-- 
cgit 1.4.1


From e482f8cd8504b36dc1ce2c1e51e0dee479d33249 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 09:12:26 +0100
Subject: Fix incorrect reference to StringIO

This was introduced in 4f2f5171
---
 synapse/util/logformatter.py    |  2 +-
 tests/util/test_logformatter.py | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 tests/util/test_logformatter.py

diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py
index 59ab3c6968..3e42868ea9 100644
--- a/synapse/util/logformatter.py
+++ b/synapse/util/logformatter.py
@@ -32,7 +32,7 @@ class LogFormatter(logging.Formatter):
         super(LogFormatter, self).__init__(*args, **kwargs)
 
     def formatException(self, ei):
-        sio = StringIO.StringIO()
+        sio = StringIO()
         (typ, val, tb) = ei
 
         # log the stack above the exception capture point if possible, but
diff --git a/tests/util/test_logformatter.py b/tests/util/test_logformatter.py
new file mode 100644
index 0000000000..1a1a8412f2
--- /dev/null
+++ b/tests/util/test_logformatter.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from synapse.util.logformatter import LogFormatter
+from tests import unittest
+
+
+class TestException(Exception):
+    pass
+
+
+class LogFormatterTestCase(unittest.TestCase):
+    def test_formatter(self):
+        formatter = LogFormatter()
+
+        try:
+            raise TestException("testytest")
+        except TestException:
+            ei = sys.exc_info()
+
+        output = formatter.formatException(ei)
+
+        # check the output looks vaguely sane
+        self.assertIn("testytest", output)
+        self.assertIn("Capture point", output)
-- 
cgit 1.4.1


From f22e7cda2c72d461acba664cb083e8c4e3c7572a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 11:46:23 +0100
Subject: Fix a class of logcontext leaks

So, it turns out that if you have a first `Deferred` `D1`, you can add a
callback which returns another `Deferred` `D2`, and `D2` must then complete
before any further callbacks on `D1` will execute (and later callbacks on `D1`
get the *result* of `D2` rather than `D2` itself).

So, `D1` might have `called=True` (as in, it has started running its
callbacks), but any new callbacks added to `D1` won't get run until `D2`
completes - so if you `yield D1` in an `inlineCallbacks` function, your `yield`
will 'block'.

In conclusion: some of our assumptions in `logcontext` were invalid. We need to
make sure that we don't optimise out the logcontext juggling when this
situation happens. Fortunately, it is easy to detect by checking `D1.paused`.
---
 synapse/util/logcontext.py    | 60 ++++++++++++++++++++++++--------------
 tests/util/test_logcontext.py | 67 ++++++++++++++++++++++++++++++++++++-------
 2 files changed, 94 insertions(+), 33 deletions(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 01ac71e53e..e086e12213 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -302,7 +302,7 @@ def preserve_fn(f):
 def run_in_background(f, *args, **kwargs):
     """Calls a function, ensuring that the current context is restored after
     return from the function, and that the sentinel context is set once the
-    deferred returned by the funtion completes.
+    deferred returned by the function completes.
 
     Useful for wrapping functions that return a deferred which you don't yield
     on (for instance because you want to pass it to deferred.gatherResults()).
@@ -320,24 +320,31 @@ def run_in_background(f, *args, **kwargs):
         # by synchronous exceptions, so let's turn them into Failures.
         return defer.fail()
 
-    if isinstance(res, defer.Deferred) and not res.called:
-        # The function will have reset the context before returning, so
-        # we need to restore it now.
-        LoggingContext.set_current_context(current)
-
-        # The original context will be restored when the deferred
-        # completes, but there is nothing waiting for it, so it will
-        # get leaked into the reactor or some other function which
-        # wasn't expecting it. We therefore need to reset the context
-        # here.
-        #
-        # (If this feels asymmetric, consider it this way: we are
-        # effectively forking a new thread of execution. We are
-        # probably currently within a ``with LoggingContext()`` block,
-        # which is supposed to have a single entry and exit point. But
-        # by spawning off another deferred, we are effectively
-        # adding a new exit point.)
-        res.addBoth(_set_context_cb, LoggingContext.sentinel)
+    if not isinstance(res, defer.Deferred):
+        return res
+
+    if res.called and not res.paused:
+        # The function should have maintained the logcontext, so we can
+        # optimise out the messing about
+        return res
+
+    # The function may have reset the context before returning, so
+    # we need to restore it now.
+    ctx = LoggingContext.set_current_context(current)
+
+    # The original context will be restored when the deferred
+    # completes, but there is nothing waiting for it, so it will
+    # get leaked into the reactor or some other function which
+    # wasn't expecting it. We therefore need to reset the context
+    # here.
+    #
+    # (If this feels asymmetric, consider it this way: we are
+    # effectively forking a new thread of execution. We are
+    # probably currently within a ``with LoggingContext()`` block,
+    # which is supposed to have a single entry and exit point. But
+    # by spawning off another deferred, we are effectively
+    # adding a new exit point.)
+    res.addBoth(_set_context_cb, ctx)
     return res
 
 
@@ -354,9 +361,18 @@ def make_deferred_yieldable(deferred):
 
     (This is more-or-less the opposite operation to run_in_background.)
     """
-    if isinstance(deferred, defer.Deferred) and not deferred.called:
-        prev_context = LoggingContext.set_current_context(LoggingContext.sentinel)
-        deferred.addBoth(_set_context_cb, prev_context)
+    if not isinstance(deferred, defer.Deferred):
+        return deferred
+
+    if deferred.called and not deferred.paused:
+        # it looks like this deferred is ready to run any callbacks we give it
+        # immediately. We may as well optimise out the logcontext faffery.
+        return deferred
+
+    # ok, we can't be sure that a yield won't block, so let's reset the
+    # logcontext, and add a callback to the deferred to restore it.
+    prev_context = LoggingContext.set_current_context(LoggingContext.sentinel)
+    deferred.addBoth(_set_context_cb, prev_context)
     return deferred
 
 
diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py
index 4850722bc5..7707fbb1f0 100644
--- a/tests/util/test_logcontext.py
+++ b/tests/util/test_logcontext.py
@@ -36,24 +36,28 @@ class LoggingContextTestCase(unittest.TestCase):
             yield sleep(0)
             self._check_test_key("one")
 
-    def _test_preserve_fn(self, function):
+    def _test_run_in_background(self, function):
         sentinel_context = LoggingContext.current_context()
 
         callback_completed = [False]
 
-        @defer.inlineCallbacks
-        def cb():
+        def test():
             context_one.request = "one"
-            yield function()
-            self._check_test_key("one")
+            d = function()
 
-            callback_completed[0] = True
+            def cb(res):
+                self._check_test_key("one")
+                callback_completed[0] = True
+                return res
+            d.addCallback(cb)
+
+            return d
 
         with LoggingContext() as context_one:
             context_one.request = "one"
 
             # fire off function, but don't wait on it.
-            logcontext.preserve_fn(cb)()
+            logcontext.run_in_background(test)
 
             self._check_test_key("one")
 
@@ -80,20 +84,31 @@ class LoggingContextTestCase(unittest.TestCase):
         # test is done once d2 finishes
         return d2
 
-    def test_preserve_fn_with_blocking_fn(self):
+    def test_run_in_background_with_blocking_fn(self):
         @defer.inlineCallbacks
         def blocking_function():
             yield sleep(0)
 
-        return self._test_preserve_fn(blocking_function)
+        return self._test_run_in_background(blocking_function)
 
-    def test_preserve_fn_with_non_blocking_fn(self):
+    def test_run_in_background_with_non_blocking_fn(self):
         @defer.inlineCallbacks
         def nonblocking_function():
             with logcontext.PreserveLoggingContext():
                 yield defer.succeed(None)
 
-        return self._test_preserve_fn(nonblocking_function)
+        return self._test_run_in_background(nonblocking_function)
+
+    @unittest.DEBUG
+    def test_run_in_background_with_chained_deferred(self):
+        # a function which returns a deferred which looks like it has been
+        # called, but is actually paused
+        def testfunc():
+            return logcontext.make_deferred_yieldable(
+                _chained_deferred_function()
+            )
+
+        return self._test_run_in_background(testfunc)
 
     @defer.inlineCallbacks
     def test_make_deferred_yieldable(self):
@@ -118,6 +133,22 @@ class LoggingContextTestCase(unittest.TestCase):
             # now it should be restored
             self._check_test_key("one")
 
+    @defer.inlineCallbacks
+    def test_make_deferred_yieldable_with_chained_deferreds(self):
+        sentinel_context = LoggingContext.current_context()
+
+        with LoggingContext() as context_one:
+            context_one.request = "one"
+
+            d1 = logcontext.make_deferred_yieldable(_chained_deferred_function())
+            # make sure that the context was reset by make_deferred_yieldable
+            self.assertIs(LoggingContext.current_context(), sentinel_context)
+
+            yield d1
+
+            # now it should be restored
+            self._check_test_key("one")
+
     @defer.inlineCallbacks
     def test_make_deferred_yieldable_on_non_deferred(self):
         """Check that make_deferred_yieldable does the right thing when its
@@ -132,3 +163,17 @@ class LoggingContextTestCase(unittest.TestCase):
             r = yield d1
             self.assertEqual(r, "bum")
             self._check_test_key("one")
+
+
+# a function which returns a deferred which has been "called", but
+# which had a function which returned another incomplete deferred on
+# its callback list, so won't yet call any other new callbacks.
+def _chained_deferred_function():
+    d = defer.succeed(None)
+
+    def cb(res):
+        d2 = defer.Deferred()
+        reactor.callLater(0, d2.callback, res)
+        return d2
+    d.addCallback(cb)
+    return d
-- 
cgit 1.4.1


From 46beeb9a307febb679fc25565aca439f8af044ed Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 15:46:22 +0100
Subject: Fix a couple of logcontext leaks in unit tests

... which were making other, innocent, tests, fail.

Plus remove a spurious unittest.DEBUG which was making the output noisy.
---
 tests/appservice/test_scheduler.py       | 11 +++++++++--
 tests/storage/test_event_push_actions.py |  1 -
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py
index e5a902f734..9181692771 100644
--- a/tests/appservice/test_scheduler.py
+++ b/tests/appservice/test_scheduler.py
@@ -17,6 +17,8 @@ from synapse.appservice.scheduler import (
     _ServiceQueuer, _TransactionController, _Recoverer
 )
 from twisted.internet import defer
+
+from synapse.util.logcontext import make_deferred_yieldable
 from ..utils import MockClock
 from mock import Mock
 from tests import unittest
@@ -204,7 +206,9 @@ class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase):
 
     def test_send_single_event_with_queue(self):
         d = defer.Deferred()
-        self.txn_ctrl.send = Mock(return_value=d)
+        self.txn_ctrl.send = Mock(
+            side_effect=lambda x, y: make_deferred_yieldable(d),
+        )
         service = Mock(id=4)
         event = Mock(event_id="first")
         event2 = Mock(event_id="second")
@@ -235,7 +239,10 @@ class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase):
         srv_2_event2 = Mock(event_id="srv2b")
 
         send_return_list = [srv_1_defer, srv_2_defer]
-        self.txn_ctrl.send = Mock(side_effect=lambda x, y: send_return_list.pop(0))
+
+        def do_send(x, y):
+            return make_deferred_yieldable(send_return_list.pop(0))
+        self.txn_ctrl.send = Mock(side_effect=do_send)
 
         # send events for different ASes and make sure they are sent
         self.queuer.enqueue(srv1, srv_1_event)
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 575374c6a6..9962ce8a5d 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -128,7 +128,6 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase):
         yield _rotate(10)
         yield _assert_counts(1, 1)
 
-    @tests.unittest.DEBUG
     @defer.inlineCallbacks
     def test_find_first_stream_ordering_after_ts(self):
         def add_event(so, ts):
-- 
cgit 1.4.1


From 11607006d9570f76a46b44c9c8de13f13550dbcf Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 15:48:47 +0100
Subject: Remove spurious unittest.DEBUG

---
 tests/util/test_logcontext.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py
index 7707fbb1f0..ad78d884e0 100644
--- a/tests/util/test_logcontext.py
+++ b/tests/util/test_logcontext.py
@@ -99,7 +99,6 @@ class LoggingContextTestCase(unittest.TestCase):
 
         return self._test_run_in_background(nonblocking_function)
 
-    @unittest.DEBUG
     def test_run_in_background_with_chained_deferred(self):
         # a function which returns a deferred which looks like it has been
         # called, but is actually paused
-- 
cgit 1.4.1


From be31adb036cab08c4c70cb1e0d0f6550f0a75ef1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 13:29:16 +0100
Subject: Fix logcontext leak in media repo

Make FileResponder.write_to_consumer uphold the logcontext contract
---
 synapse/rest/media/v1/_base.py         | 1 +
 synapse/rest/media/v1/media_storage.py | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index d9c4af9389..c0d2f06855 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -143,6 +143,7 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam
         respond_404(request)
         return
 
+    logger.debug("Responding to media request with responder %s")
     add_file_headers(request, media_type, file_size, upload_name)
     with responder:
         yield responder.write_to_consumer(request)
diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index 7f263db239..d23fe10b07 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -255,7 +255,9 @@ class FileResponder(Responder):
         self.open_file = open_file
 
     def write_to_consumer(self, consumer):
-        return FileSender().beginFileTransfer(self.open_file, consumer)
+        return make_deferred_yieldable(
+            FileSender().beginFileTransfer(self.open_file, consumer)
+        )
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.open_file.close()
-- 
cgit 1.4.1


From 32015e1109bc955697353d8f8088e3f6b538d12c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 2 May 2018 16:52:42 +0100
Subject: Escape label values in prometheus metrics

---
 synapse/metrics/metric.py    | 22 ++++++++++++++++++++--
 tests/metrics/test_metric.py | 21 ++++++++++++++++++++-
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 89bd47c3f7..1a09e417c9 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -16,6 +16,7 @@
 
 from itertools import chain
 import logging
+import re
 
 logger = logging.getLogger(__name__)
 
@@ -56,8 +57,7 @@ class BaseMetric(object):
         return not len(self.labels)
 
     def _render_labelvalue(self, value):
-        # TODO: escape backslashes, quotes and newlines
-        return '"%s"' % (value)
+        return '"%s"' % (_escape_label_value(value),)
 
     def _render_key(self, values):
         if self.is_scalar():
@@ -299,3 +299,21 @@ class MemoryUsageMetric(object):
             "process_psutil_rss:total %d" % sum_rss,
             "process_psutil_rss:count %d" % len_rss,
         ]
+
+
+def _escape_character(c):
+    """Replaces a single character with its escape sequence.
+    """
+    if c == "\\":
+        return "\\\\"
+    elif c == "\"":
+        return "\\\""
+    elif c == "\n":
+        return "\\n"
+    return c
+
+
+def _escape_label_value(value):
+    """Takes a label value and escapes quotes, newlines and backslashes
+    """
+    return re.sub(r"([\n\"\\])", lambda m: _escape_character(m.group(1)), value)
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
index 39bde6e3f8..069c0be762 100644
--- a/tests/metrics/test_metric.py
+++ b/tests/metrics/test_metric.py
@@ -16,7 +16,8 @@
 from tests import unittest
 
 from synapse.metrics.metric import (
-    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric
+    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric,
+    _escape_label_value,
 )
 
 
@@ -171,3 +172,21 @@ class CacheMetricTestCase(unittest.TestCase):
             'cache:size{name="cache_name"} 1',
             'cache:evicted_size{name="cache_name"} 2',
         ])
+
+
+class LabelValueEscapeTestCase(unittest.TestCase):
+    def test_simple(self):
+        string = "safjhsdlifhyskljfksdfh"
+        self.assertEqual(string, _escape_label_value(string))
+
+    def test_escape(self):
+        self.assertEqual(
+            "abc\\\"def\\nghi\\\\",
+            _escape_label_value("abc\"def\nghi\\"),
+        )
+
+    def test_sequence_of_escapes(self):
+        self.assertEqual(
+            "abc\\\"def\\nghi\\\\\\n",
+            _escape_label_value("abc\"def\nghi\\\n"),
+        )
-- 
cgit 1.4.1


From a41117c63b4e7ef2021fb59cb521c77d72f2bf70 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 2 May 2018 17:27:27 +0100
Subject: Make _escape_character take MatchObject

---
 synapse/metrics/metric.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 1a09e417c9..fbba94e633 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -301,9 +301,17 @@ class MemoryUsageMetric(object):
         ]
 
 
-def _escape_character(c):
+def _escape_character(m):
     """Replaces a single character with its escape sequence.
+
+    Args:
+        m (re.MatchObject): A match object whose first group is the single
+            character to replace
+
+    Returns:
+        str
     """
+    c = m.group(1)
     if c == "\\":
         return "\\\\"
     elif c == "\"":
@@ -316,4 +324,4 @@ def _escape_character(c):
 def _escape_label_value(value):
     """Takes a label value and escapes quotes, newlines and backslashes
     """
-    return re.sub(r"([\n\"\\])", lambda m: _escape_character(m.group(1)), value)
+    return re.sub(r"([\n\"\\])", _escape_character, value)
-- 
cgit 1.4.1


From 1dfd650348251508ac044b3db114092b1e857a74 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 2 May 2018 22:42:36 +0100
Subject: add missing param to cancelled_to_request_timed_out_error

This gets two arguments, not one.
---
 synapse/http/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index 0d47ccdb59..054372e179 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -25,7 +25,7 @@ class RequestTimedOutError(SynapseError):
         super(RequestTimedOutError, self).__init__(504, "Timed out")
 
 
-def cancelled_to_request_timed_out_error(value):
+def cancelled_to_request_timed_out_error(value, timeout):
     """Turns CancelledErrors into RequestTimedOutErrors.
 
     For use with async.add_timeout_to_deferred
-- 
cgit 1.4.1


From 31c7c29d4304f7577a1f5dc1740fe3a248351500 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 3 May 2018 10:38:08 +0100
Subject: Fix up grammar

---
 synapse/storage/events.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a9269707df..05cde96afc 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -556,9 +556,9 @@ class EventsStore(EventsWorkerStore):
 
             state_groups_map[ctx.state_group] = ctx.current_state_ids
 
-        # We need to map the event_ids to their state groups. First, lets
-        # check if the event is one we're persisting and then we can pull the
-        # state group from its context.
+        # We need to map the event_ids to their state groups. First, let's
+        # check if the event is one we're persisting, in which case we can
+        # pull the state group from its context.
         # Otherwise we need to pull the state group from the database.
 
         # Set of events we need to fetch groups for. (We know none of the old
-- 
cgit 1.4.1


From a0501ac57ea597efaa32e66c57c2f87036ec8b32 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 3 May 2018 10:51:39 +0100
Subject: Warn of potential client incompatibility from #3161

---
 CHANGES.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 317846d2a2..94b83027e2 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,18 @@
+Changes in synapse <unreleased>
+===============================
+
+Potentially breaking change:
+
+* Make Client-Server API return 403 for invalid token (PR #3161).
+
+  This changes the Client-server spec to return a 403 error code instead of 401
+  when the access token is unrecognised. This is the behaviour required by the
+  specification, but some clients may be relying on the old, incorrect
+  behaviour.
+
+  Thanks to @NotAFile for fixing this.
+
+
 Changes in synapse v0.28.1 (2018-05-01)
 =======================================
 
-- 
cgit 1.4.1


From d72faf2fad9cd4a2f7a82bc35d7e74c36fc49e4f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 3 May 2018 10:56:42 +0100
Subject: Fix changes warning

---
 CHANGES.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 94b83027e2..9d40b2ac1e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -3,9 +3,9 @@ Changes in synapse <unreleased>
 
 Potentially breaking change:
 
-* Make Client-Server API return 403 for invalid token (PR #3161).
+* Make Client-Server API return 401 for invalid token (PR #3161).
 
-  This changes the Client-server spec to return a 403 error code instead of 401
+  This changes the Client-server spec to return a 401 error code instead of 403
   when the access token is unrecognised. This is the behaviour required by the
   specification, but some clients may be relying on the old, incorrect
   behaviour.
-- 
cgit 1.4.1


From 2e7a94c36ba75a94114e4a9fb1aa6d4e3444e8b4 Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Thu, 3 May 2018 12:31:47 +0100
Subject: Don't abortConnection() if the transport connection has already
 closed.

---
 synapse/http/endpoint.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index d8923c9abb..564ae4c10d 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -113,10 +113,15 @@ class _WrappedConnection(object):
         if time.time() - self.last_request >= 2.5 * 60:
             self.abort()
             # Abort the underlying TLS connection. The abort() method calls
-            # loseConnection() on the underlying TLS connection which tries to
+            # loseConnection() on the TLS connection which tries to
             # shutdown the connection cleanly. We call abortConnection()
-            # since that will promptly close the underlying TCP connection.
-            self.transport.abortConnection()
+            # since that will promptly close the TLS connection.
+            #
+            # In Twisted >18.4; the TLS connection will be None if it has closed
+            # which will make abortConnection() throw. Check that the TLS connection
+            # is not None before trying to close it.
+            if self.transport.getHandle() is not None:
+                self.transport.abortConnection()
 
     def request(self, request):
         self.last_request = time.time()
-- 
cgit 1.4.1


From a7fe62f0cb7bdb40e5c9b7f09f694e9c0913610c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 3 May 2018 11:16:36 +0100
Subject: Fix logcontext leaks in rate limiter

---
 synapse/util/ratelimitutils.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py
index 18424f6c36..0ab63c3d7d 100644
--- a/synapse/util/ratelimitutils.py
+++ b/synapse/util/ratelimitutils.py
@@ -18,7 +18,10 @@ from twisted.internet import defer
 from synapse.api.errors import LimitExceededError
 
 from synapse.util.async import sleep
-from synapse.util.logcontext import run_in_background
+from synapse.util.logcontext import (
+    run_in_background, make_deferred_yieldable,
+    PreserveLoggingContext,
+)
 
 import collections
 import contextlib
@@ -176,6 +179,9 @@ class _PerHostRatelimiter(object):
             return r
 
         def on_err(r):
+            # XXX: why is this necessary? this is called before we start
+            # processing the request so why would the request be in
+            # current_processing?
             self.current_processing.discard(request_id)
             return r
 
@@ -187,7 +193,7 @@ class _PerHostRatelimiter(object):
 
         ret_defer.addCallbacks(on_start, on_err)
         ret_defer.addBoth(on_both)
-        return ret_defer
+        return make_deferred_yieldable(ret_defer)
 
     def _on_exit(self, request_id):
         logger.debug(
@@ -197,7 +203,12 @@ class _PerHostRatelimiter(object):
         self.current_processing.discard(request_id)
         try:
             request_id, deferred = self.ready_request_queue.popitem()
+
+            # XXX: why do we do the following? the on_start callback above will
+            # do it for us.
             self.current_processing.add(request_id)
-            deferred.callback(None)
+
+            with PreserveLoggingContext():
+                deferred.callback(None)
         except KeyError:
             pass
-- 
cgit 1.4.1


From 95b6912045b5d23ae823d6733beb7f315b010528 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 3 May 2018 15:51:04 +0100
Subject: Fix metrics that have integer value labels

---
 synapse/metrics/metric.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index fbba94e633..98d5ca7b6c 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -324,4 +324,4 @@ def _escape_character(m):
 def _escape_label_value(value):
     """Takes a label value and escapes quotes, newlines and backslashes
     """
-    return re.sub(r"([\n\"\\])", _escape_character, value)
+    return re.sub(r"([\n\"\\])", _escape_character, str(value))
-- 
cgit 1.4.1


From 6d8ec3462d33daa5d390fca206ec2e281c27c38f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 3 May 2018 16:25:05 +0100
Subject: Note that label values can be anything

---
 synapse/metrics/metric.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 98d5ca7b6c..f421e7a93f 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -71,7 +71,8 @@ class BaseMetric(object):
         """Render this metric for a single set of labels
 
         Args:
-            label_values (list[str]): values for each of the labels
+            label_values (list[object]): values for each of the labels,
+                (which get stringified).
             value: value of the metric at with these labels
 
         Returns:
-- 
cgit 1.4.1


From 5addeaa02c432241af282900b19172ba7158118c Mon Sep 17 00:00:00 2001
From: kaiyou <pierre@jaury.eu>
Date: Fri, 4 May 2018 21:23:01 +0200
Subject: Add Docker packaging in the author list

---
 AUTHORS.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/AUTHORS.rst b/AUTHORS.rst
index 3dcb1c2a89..e13ac5ad34 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -60,3 +60,6 @@ Niklas Riekenbrauck <nikriek at gmail dot.com>
 
 Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
+
+Pierre Jaury <pierre at jaury.eu>
+* Docker packaging
\ No newline at end of file
-- 
cgit 1.4.1


From 88868b283994b9a73634d7153fd7104257475349 Mon Sep 17 00:00:00 2001
From: Konstantinos Sideris <sideris.konstantin@gmail.com>
Date: Sat, 5 May 2018 12:55:02 +0300
Subject: notifications: Convert next_token to string according to the spec

Currently the parameter is serialized as an integer.

Signed-off-by: Konstantinos Sideris <sideris.konstantin@gmail.com>
---
 synapse/rest/client/v2_alpha/notifications.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/notifications.py b/synapse/rest/client/v2_alpha/notifications.py
index ec170109fe..66583d6778 100644
--- a/synapse/rest/client/v2_alpha/notifications.py
+++ b/synapse/rest/client/v2_alpha/notifications.py
@@ -88,7 +88,7 @@ class NotificationsServlet(RestServlet):
                     pa["topological_ordering"], pa["stream_ordering"]
                 )
             returned_push_actions.append(returned_pa)
-            next_token = pa["stream_ordering"]
+            next_token = str(pa["stream_ordering"])
 
         defer.returnValue((200, {
             "notifications": returned_push_actions,
-- 
cgit 1.4.1


From bf98fa0864fe62dbf1303acc778912d69fd4a738 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 8 May 2018 15:58:35 +0100
Subject: Part user from rooms on account deactivate

This implements this very crudely: this probably isn't viable
because parting a user from all their rooms could take a long time,
and if the HS gets restarted in that time the process will be
aborted.
---
 synapse/handlers/deactivate_account.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index b1d3814909..387620c9c6 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017 New Vector Ltd
+# Copyright 2017, 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 from twisted.internet import defer
 
 from ._base import BaseHandler
+from synapse.types import UserID, create_requester
 
 import logging
 
@@ -27,6 +28,7 @@ class DeactivateAccountHandler(BaseHandler):
         super(DeactivateAccountHandler, self).__init__(hs)
         self._auth_handler = hs.get_auth_handler()
         self._device_handler = hs.get_device_handler()
+        self._room_member_handler = hs.get_room_member_handler()
 
     @defer.inlineCallbacks
     def deactivate_account(self, user_id):
@@ -50,3 +52,15 @@ class DeactivateAccountHandler(BaseHandler):
 
         yield self.store.user_delete_threepids(user_id)
         yield self.store.user_set_password_hash(user_id, None)
+
+        user = UserID.from_string(user_id)
+
+        rooms_for_user = yield self.store.get_rooms_for_user(user_id)
+        for room_id in rooms_for_user:
+            yield self._room_member_handler.update_membership(
+                create_requester(user),
+                user,
+                room_id,
+                "leave",
+                ratelimit=False,
+            )
-- 
cgit 1.4.1


From 06c0d0ed081b56716135c4881e600861b2e8cad5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 8 May 2018 15:45:38 +0100
Subject: Split paginate_room_events storage function

---
 synapse/storage/stream.py | 100 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 72 insertions(+), 28 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index f0784ba137..b57a8a7ef6 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -738,17 +738,28 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
     def has_room_changed_since(self, room_id, stream_id):
         return self._events_stream_cache.has_entity_changed(room_id, stream_id)
 
+    def paginate_room_events_txn(self, txn, room_id, from_key, to_key=None,
+                                 direction='b', limit=-1, event_filter=None):
+        """Returns list of events before or after a given token.
 
-class StreamStore(StreamWorkerStore):
-    def get_room_max_stream_ordering(self):
-        return self._stream_id_gen.get_current_token()
-
-    def get_room_min_stream_ordering(self):
-        return self._backfill_id_gen.get_current_token()
+        Args:
+            txn
+            room_id (str)
+            from_key (str): The token used to stream from
+            to_key (str|None): A token which if given limits the results to
+                only those before
+            direction(char): Either 'b' or 'f' to indicate whether we are
+                paginating forwards or backwards from `from_key`.
+            limit (int): The maximum number of events to return. Zero or less
+                means no limit.
+            event_filter (Filter|None): If provided filters the events to
+                those that match the filter.
 
-    @defer.inlineCallbacks
-    def paginate_room_events(self, room_id, from_key, to_key=None,
-                             direction='b', limit=-1, event_filter=None):
+        Returns:
+            tuple[list[dict], str]: Returns the results as a list of dicts and
+            a token that points to the end of the result set. The dicts have
+            the keys "event_id", "toplogical_ordering" and "stream_orderign".
+        """
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
         # we have a bit of asymmetry when it comes to equalities.
@@ -795,29 +806,54 @@ class StreamStore(StreamWorkerStore):
             "limit": limit_str
         }
 
-        def f(txn):
-            txn.execute(sql, args)
+        txn.execute(sql, args)
 
-            rows = self.cursor_to_dict(txn)
+        rows = self.cursor_to_dict(txn)
 
-            if rows:
-                topo = rows[-1]["topological_ordering"]
-                toke = rows[-1]["stream_ordering"]
-                if direction == 'b':
-                    # Tokens are positions between events.
-                    # This token points *after* the last event in the chunk.
-                    # We need it to point to the event before it in the chunk
-                    # when we are going backwards so we subtract one from the
-                    # stream part.
-                    toke -= 1
-                next_token = str(RoomStreamToken(topo, toke))
-            else:
-                # TODO (erikj): We should work out what to do here instead.
-                next_token = to_key if to_key else from_key
+        if rows:
+            topo = rows[-1]["topological_ordering"]
+            toke = rows[-1]["stream_ordering"]
+            if direction == 'b':
+                # Tokens are positions between events.
+                # This token points *after* the last event in the chunk.
+                # We need it to point to the event before it in the chunk
+                # when we are going backwards so we subtract one from the
+                # stream part.
+                toke -= 1
+            next_token = str(RoomStreamToken(topo, toke))
+        else:
+            # TODO (erikj): We should work out what to do here instead.
+            next_token = to_key if to_key else from_key
+
+        return rows, next_token,
 
-            return rows, next_token,
+    @defer.inlineCallbacks
+    def paginate_room_events(self, room_id, from_key, to_key=None,
+                             direction='b', limit=-1, event_filter=None):
+        """Returns list of events before or after a given token.
 
-        rows, token = yield self.runInteraction("paginate_room_events", f)
+        Args:
+            room_id (str)
+            from_key (str): The token used to stream from
+            to_key (str|None): A token which if given limits the results to
+                only those before
+            direction(char): Either 'b' or 'f' to indicate whether we are
+                paginating forwards or backwards from `from_key`.
+            limit (int): The maximum number of events to return. Zero or less
+                means no limit.
+            event_filter (Filter|None): If provided filters the events to
+                those that match the filter.
+
+        Returns:
+            tuple[list[dict], str]: Returns the results as a list of dicts and
+            a token that points to the end of the result set. The dicts have
+            the keys "event_id", "toplogical_ordering" and "stream_orderign".
+        """
+
+        rows, token = yield self.runInteraction(
+            "paginate_room_events", self.paginate_room_events_txn,
+            room_id, from_key, to_key, direction, limit, event_filter,
+        )
 
         events = yield self._get_events(
             [r["event_id"] for r in rows],
@@ -827,3 +863,11 @@ class StreamStore(StreamWorkerStore):
         self._set_before_and_after(events, rows)
 
         defer.returnValue((events, token))
+
+
+class StreamStore(StreamWorkerStore):
+    def get_room_max_stream_ordering(self):
+        return self._stream_id_gen.get_current_token()
+
+    def get_room_min_stream_ordering(self):
+        return self._backfill_id_gen.get_current_token()
-- 
cgit 1.4.1


From 274b8c6025e15eede0137bcb6a73ac00bf370ee8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 8 May 2018 16:15:07 +0100
Subject: Only fetch required fields from database

---
 synapse/storage/stream.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index b57a8a7ef6..d3adb0bf37 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -796,7 +796,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             limit_str = ""
 
         sql = (
-            "SELECT * FROM events"
+            "SELECT event_id, topological_ordering, stream_ordering"
+            " FROM events"
             " WHERE outlier = ? AND room_id = ? AND %(bounds)s"
             " ORDER BY topological_ordering %(order)s,"
             " stream_ordering %(order)s %(limit)s"
-- 
cgit 1.4.1


From 3e6d306e94327983b1f6b66cec9faace642d3f16 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 8 May 2018 16:18:58 +0100
Subject: Parse tokens before calling DB function

---
 synapse/storage/stream.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index d3adb0bf37..ce98587608 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -738,16 +738,16 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
     def has_room_changed_since(self, room_id, stream_id):
         return self._events_stream_cache.has_entity_changed(room_id, stream_id)
 
-    def paginate_room_events_txn(self, txn, room_id, from_key, to_key=None,
+    def paginate_room_events_txn(self, txn, room_id, from_token, to_token=None,
                                  direction='b', limit=-1, event_filter=None):
         """Returns list of events before or after a given token.
 
         Args:
             txn
             room_id (str)
-            from_key (str): The token used to stream from
-            to_key (str|None): A token which if given limits the results to
-                only those before
+            from_token (RoomStreamToken): The token used to stream from
+            to_token (RoomStreamToken|None): A token which if given limits the
+                results to only those before
             direction(char): Either 'b' or 'f' to indicate whether we are
                 paginating forwards or backwards from `from_key`.
             limit (int): The maximum number of events to return. Zero or less
@@ -757,7 +757,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         Returns:
             tuple[list[dict], str]: Returns the results as a list of dicts and
-            a token that points to the end of the result set. The dicts have
+            a token that points to the end of the result set. The dicts haveq
             the keys "event_id", "toplogical_ordering" and "stream_orderign".
         """
         # Tokens really represent positions between elements, but we use
@@ -767,20 +767,20 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         if direction == 'b':
             order = "DESC"
             bounds = upper_bound(
-                RoomStreamToken.parse(from_key), self.database_engine
+                from_token, self.database_engine
             )
-            if to_key:
+            if to_token:
                 bounds = "%s AND %s" % (bounds, lower_bound(
-                    RoomStreamToken.parse(to_key), self.database_engine
+                    to_token, self.database_engine
                 ))
         else:
             order = "ASC"
             bounds = lower_bound(
-                RoomStreamToken.parse(from_key), self.database_engine
+                from_token, self.database_engine
             )
-            if to_key:
+            if to_token:
                 bounds = "%s AND %s" % (bounds, upper_bound(
-                    RoomStreamToken.parse(to_key), self.database_engine
+                    to_token, self.database_engine
                 ))
 
         filter_clause, filter_args = filter_to_clause(event_filter)
@@ -821,12 +821,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 # when we are going backwards so we subtract one from the
                 # stream part.
                 toke -= 1
-            next_token = str(RoomStreamToken(topo, toke))
+            next_token = RoomStreamToken(topo, toke)
         else:
             # TODO (erikj): We should work out what to do here instead.
-            next_token = to_key if to_key else from_key
+            next_token = to_token if to_token else from_token
 
-        return rows, next_token,
+        return rows, str(next_token),
 
     @defer.inlineCallbacks
     def paginate_room_events(self, room_id, from_key, to_key=None,
@@ -851,6 +851,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             the keys "event_id", "toplogical_ordering" and "stream_orderign".
         """
 
+        from_key = RoomStreamToken.parse(from_key)
+        if to_key:
+            to_key = RoomStreamToken.parse(to_key)
+
         rows, token = yield self.runInteraction(
             "paginate_room_events", self.paginate_room_events_txn,
             room_id, from_key, to_key, direction, limit, event_filter,
-- 
cgit 1.4.1


From 696f5324539b014a5a9613c3a534abb2b3725dac Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 8 May 2018 16:19:33 +0100
Subject: Reuse existing pagination code for context API

---
 synapse/storage/stream.py | 90 ++++++++---------------------------------------
 1 file changed, 15 insertions(+), 75 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index ce98587608..938a8809dc 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -42,7 +42,7 @@ from synapse.util.caches.descriptors import cached
 from synapse.types import RoomStreamToken
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.engines import PostgresEngine
 
 import abc
 import logging
@@ -595,88 +595,28 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             retcols=["stream_ordering", "topological_ordering"],
         )
 
-        token = RoomStreamToken(
-            results["topological_ordering"],
+        # Paginating backwards includes the event at the token, but paginating
+        # forward doesn't.
+        before_token = RoomStreamToken(
+            results["topological_ordering"] - 1,
             results["stream_ordering"],
         )
 
-        if isinstance(self.database_engine, Sqlite3Engine):
-            # SQLite3 doesn't optimise ``(x < a) OR (x = a AND y < b)``
-            # So we give pass it to SQLite3 as the UNION ALL of the two queries.
-
-            query_before = (
-                "SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND topological_ordering < ?"
-                " UNION ALL"
-                " SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND topological_ordering = ? AND stream_ordering < ?"
-                " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
-            )
-            before_args = (
-                room_id, token.topological,
-                room_id, token.topological, token.stream,
-                before_limit,
-            )
-
-            query_after = (
-                "SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND topological_ordering > ?"
-                " UNION ALL"
-                " SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND topological_ordering = ? AND stream_ordering > ?"
-                " ORDER BY topological_ordering ASC, stream_ordering ASC LIMIT ?"
-            )
-            after_args = (
-                room_id, token.topological,
-                room_id, token.topological, token.stream,
-                after_limit,
-            )
-        else:
-            query_before = (
-                "SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND %s"
-                " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
-            ) % (upper_bound(token, self.database_engine, inclusive=False),)
-
-            before_args = (room_id, before_limit)
-
-            query_after = (
-                "SELECT topological_ordering, stream_ordering, event_id FROM events"
-                " WHERE room_id = ? AND %s"
-                " ORDER BY topological_ordering ASC, stream_ordering ASC LIMIT ?"
-            ) % (lower_bound(token, self.database_engine, inclusive=False),)
-
-            after_args = (room_id, after_limit)
-
-        txn.execute(query_before, before_args)
+        after_token = RoomStreamToken(
+            results["topological_ordering"],
+            results["stream_ordering"],
+        )
 
-        rows = self.cursor_to_dict(txn)
+        rows, start_token = self.paginate_room_events_txn(
+            txn, room_id, before_token, direction='b', limit=before_limit,
+        )
         events_before = [r["event_id"] for r in rows]
 
-        if rows:
-            start_token = str(RoomStreamToken(
-                rows[0]["topological_ordering"],
-                rows[0]["stream_ordering"] - 1,
-            ))
-        else:
-            start_token = str(RoomStreamToken(
-                token.topological,
-                token.stream - 1,
-            ))
-
-        txn.execute(query_after, after_args)
-
-        rows = self.cursor_to_dict(txn)
+        rows, end_token = self.paginate_room_events_txn(
+            txn, room_id, after_token, direction='f', limit=after_limit,
+        )
         events_after = [r["event_id"] for r in rows]
 
-        if rows:
-            end_token = str(RoomStreamToken(
-                rows[-1]["topological_ordering"],
-                rows[-1]["stream_ordering"],
-            ))
-        else:
-            end_token = str(token)
-
         return {
             "before": {
                 "event_ids": events_before,
-- 
cgit 1.4.1


From 6c957e26f06f5806ae2e69838865ee2990a75258 Mon Sep 17 00:00:00 2001
From: rubo77 <github@r.z11.de>
Date: Wed, 9 May 2018 00:14:01 +0200
Subject: nuke-room-from-db.sh: added postgresql option and help

---
 scripts-dev/nuke-room-from-db.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh
index 1201d176c2..cd74d78d6a 100755
--- a/scripts-dev/nuke-room-from-db.sh
+++ b/scripts-dev/nuke-room-from-db.sh
@@ -6,9 +6,19 @@
 
 ## Do not run it lightly.
 
+set -e
+
+if [ "$1" == "-h" ] || [ "$1" == "" ]; then
+  echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
+  echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
+  echo "or"
+  echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
+  exit
+fi
+
 ROOMID="$1"
 
-sqlite3 homeserver.db <<EOF
+cat <<EOF
 DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
 DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
 DELETE FROM event_edges WHERE room_id = '$ROOMID';
-- 
cgit 1.4.1


From 8ff8ab3bcec7c0ac731b88bc0fbe6338d82ab962 Mon Sep 17 00:00:00 2001
From: rubo77 <github@r.z11.de>
Date: Wed, 9 May 2018 00:21:00 +0200
Subject: Dont nuke non-existing table event_search_content

---
 scripts-dev/nuke-room-from-db.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh
index 1201d176c2..53617f21f1 100755
--- a/scripts-dev/nuke-room-from-db.sh
+++ b/scripts-dev/nuke-room-from-db.sh
@@ -29,7 +29,6 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID';
 DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
 DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
 DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
-DELETE FROM event_search_content WHERE c1room_id = '$ROOMID';
 DELETE FROM guest_access WHERE room_id = '$ROOMID';
 DELETE FROM history_visibility WHERE room_id = '$ROOMID';
 DELETE FROM room_tags WHERE room_id = '$ROOMID';
-- 
cgit 1.4.1


From d11b8b6b659957b28c2bbf8bc6ca060284097377 Mon Sep 17 00:00:00 2001
From: rubo77 <github@r.z11.de>
Date: Wed, 9 May 2018 00:46:47 +0200
Subject: nuke-room-from-db.sh: nuke from table event_search too

---
 scripts-dev/nuke-room-from-db.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh
index 53617f21f1..2cfd71b0b8 100755
--- a/scripts-dev/nuke-room-from-db.sh
+++ b/scripts-dev/nuke-room-from-db.sh
@@ -29,6 +29,7 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID';
 DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
 DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
 DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
+DELETE FROM event_search WHERE room_id = '$ROOMID'; 
 DELETE FROM guest_access WHERE room_id = '$ROOMID';
 DELETE FROM history_visibility WHERE room_id = '$ROOMID';
 DELETE FROM room_tags WHERE room_id = '$ROOMID';
-- 
cgit 1.4.1


From 23ec51c94ce90fd70931fbe98cadb7dd51a9db4e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 09:55:19 +0100
Subject: Fix up comments and make function private

---
 synapse/storage/stream.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 938a8809dc..54be025401 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -607,12 +607,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             results["stream_ordering"],
         )
 
-        rows, start_token = self.paginate_room_events_txn(
+        rows, start_token = self._paginate_room_events_txn(
             txn, room_id, before_token, direction='b', limit=before_limit,
         )
         events_before = [r["event_id"] for r in rows]
 
-        rows, end_token = self.paginate_room_events_txn(
+        rows, end_token = self._paginate_room_events_txn(
             txn, room_id, after_token, direction='f', limit=after_limit,
         )
         events_after = [r["event_id"] for r in rows]
@@ -678,8 +678,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
     def has_room_changed_since(self, room_id, stream_id):
         return self._events_stream_cache.has_entity_changed(room_id, stream_id)
 
-    def paginate_room_events_txn(self, txn, room_id, from_token, to_token=None,
-                                 direction='b', limit=-1, event_filter=None):
+    def _paginate_room_events_txn(self, txn, room_id, from_token, to_token=None,
+                                  direction='b', limit=-1, event_filter=None):
         """Returns list of events before or after a given token.
 
         Args:
@@ -697,8 +697,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         Returns:
             tuple[list[dict], str]: Returns the results as a list of dicts and
-            a token that points to the end of the result set. The dicts haveq
-            the keys "event_id", "toplogical_ordering" and "stream_orderign".
+            a token that points to the end of the result set. The dicts have
+            the keys "event_id", "toplogical_ordering" and "stream_ordering".
         """
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
@@ -796,7 +796,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             to_key = RoomStreamToken.parse(to_key)
 
         rows, token = yield self.runInteraction(
-            "paginate_room_events", self.paginate_room_events_txn,
+            "paginate_room_events", self._paginate_room_events_txn,
             room_id, from_key, to_key, direction, limit, event_filter,
         )
 
-- 
cgit 1.4.1


From 27cf170558865d0afeea5d4c2524e8664c28323b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 10:53:29 +0100
Subject: Refactor recent events func to use pagination func

This also removes a cache that is unlikely to ever get hit.
---
 synapse/storage/stream.py | 75 +++++++++++++++++------------------------------
 1 file changed, 27 insertions(+), 48 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 54be025401..da43bb1321 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -38,7 +38,6 @@ from twisted.internet import defer
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.events import EventsWorkerStore
 
-from synapse.util.caches.descriptors import cached
 from synapse.types import RoomStreamToken
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
@@ -363,60 +362,40 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         defer.returnValue((events, token))
 
-    @cached(num_args=4)
+    @defer.inlineCallbacks
     def get_recent_event_ids_for_room(self, room_id, limit, end_token, from_token=None):
-        end_token = RoomStreamToken.parse_stream_token(end_token)
-
-        if from_token is None:
-            sql = (
-                "SELECT stream_ordering, topological_ordering, event_id"
-                " FROM events"
-                " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?"
-                " ORDER BY topological_ordering DESC, stream_ordering DESC"
-                " LIMIT ?"
-            )
-        else:
-            from_token = RoomStreamToken.parse_stream_token(from_token)
-            sql = (
-                "SELECT stream_ordering, topological_ordering, event_id"
-                " FROM events"
-                " WHERE room_id = ? AND stream_ordering > ?"
-                " AND stream_ordering <= ? AND outlier = ?"
-                " ORDER BY topological_ordering DESC, stream_ordering DESC"
-                " LIMIT ?"
-            )
-
-        def get_recent_events_for_room_txn(txn):
-            if from_token is None:
-                txn.execute(sql, (room_id, end_token.stream, False, limit,))
-            else:
-                txn.execute(sql, (
-                    room_id, from_token.stream, end_token.stream, False, limit
-                ))
+        """Get the most recent events in the room in topological ordering.
 
-            rows = self.cursor_to_dict(txn)
+        Args:
+            room_id (str)
+            limit (int)
+            end_token (str): The stream token representing now.
+            from_token(str|None): Token to not return events before, if given.
 
-            rows.reverse()  # As we selected with reverse ordering
+        Returns:
+            Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of
+            dicts (which include event_ids, etc), and a tuple for
+            `(start_token, end_token)` representing the range of rows
+            returned.
+            The returned events are in ascending order.
+        """
+        # Allow a zero limit here, and no-op.
+        if limit == 0:
+            defer.returnValue(([], (end_token, end_token)))
 
-            if rows:
-                # Tokens are positions between events.
-                # This token points *after* the last event in the chunk.
-                # We need it to point to the event before it in the chunk
-                # since we are going backwards so we subtract one from the
-                # stream part.
-                topo = rows[0]["topological_ordering"]
-                toke = rows[0]["stream_ordering"] - 1
-                start_token = str(RoomStreamToken(topo, toke))
+        end_token = RoomStreamToken.parse_stream_token(end_token)
+        if from_token is not None:
+            from_token = RoomStreamToken.parse(from_token)
 
-                token = (start_token, str(end_token))
-            else:
-                token = (str(end_token), str(end_token))
+        rows, token = yield self.runInteraction(
+            "get_recent_event_ids_for_room", self._paginate_room_events_txn,
+            room_id, from_token=end_token, to_token=from_token, limit=limit,
+        )
 
-            return rows, token
+        # We want to return the results in ascending order.
+        rows.reverse()
 
-        return self.runInteraction(
-            "get_recent_events_for_room", get_recent_events_for_room_txn
-        )
+        defer.returnValue((rows, (token, str(end_token))))
 
     def get_room_event_after_stream_ordering(self, room_id, stream_ordering):
         """Gets details of the first event in a room at or after a stream ordering
-- 
cgit 1.4.1


From 7dd13415dbf82edeaae0b94f835e20025964e1b4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 10:58:16 +0100
Subject: Remove unused from_token param

---
 synapse/storage/stream.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index da43bb1321..ecd39074b8 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -346,9 +346,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         defer.returnValue(ret)
 
     @defer.inlineCallbacks
-    def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None):
+    def get_recent_events_for_room(self, room_id, limit, end_token):
         rows, token = yield self.get_recent_event_ids_for_room(
-            room_id, limit, end_token, from_token
+            room_id, limit, end_token,
         )
 
         logger.debug("stream before")
@@ -363,14 +363,13 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         defer.returnValue((events, token))
 
     @defer.inlineCallbacks
-    def get_recent_event_ids_for_room(self, room_id, limit, end_token, from_token=None):
+    def get_recent_event_ids_for_room(self, room_id, limit, end_token):
         """Get the most recent events in the room in topological ordering.
 
         Args:
             room_id (str)
             limit (int)
             end_token (str): The stream token representing now.
-            from_token(str|None): Token to not return events before, if given.
 
         Returns:
             Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of
@@ -384,12 +383,10 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             defer.returnValue(([], (end_token, end_token)))
 
         end_token = RoomStreamToken.parse_stream_token(end_token)
-        if from_token is not None:
-            from_token = RoomStreamToken.parse(from_token)
 
         rows, token = yield self.runInteraction(
             "get_recent_event_ids_for_room", self._paginate_room_events_txn,
-            room_id, from_token=end_token, to_token=from_token, limit=limit,
+            room_id, from_token=end_token, limit=limit,
         )
 
         # We want to return the results in ascending order.
-- 
cgit 1.4.1


From 05e0a2462c76be6987c7ec3d9517d500583bac65 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 11:18:23 +0100
Subject: Refactor pagination DB API to return concrete type

This makes it easier to document what is being returned by the storage
functions and what some functions expect as arguments.
---
 synapse/storage/stream.py | 76 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 48 insertions(+), 28 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index ecd39074b8..772d2c6198 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -47,6 +47,7 @@ import abc
 import logging
 
 from six.moves import range
+from collections import namedtuple
 
 
 logger = logging.getLogger(__name__)
@@ -59,6 +60,12 @@ _STREAM_TOKEN = "stream"
 _TOPOLOGICAL_TOKEN = "topological"
 
 
+# Used as return values for pagination APIs
+_EventDictReturn = namedtuple("_EventDictReturn", (
+    "event_id", "topological_ordering", "stream_ordering",
+))
+
+
 def lower_bound(token, engine, inclusive=False):
     inclusive = "=" if inclusive else ""
     if token.topological is None:
@@ -256,9 +263,13 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                     " ORDER BY stream_ordering %s LIMIT ?"
                 ) % (order,)
                 txn.execute(sql, (room_id, from_id, to_id, limit))
+
+                rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
             else:
                 sql = (
-                    "SELECT event_id, stream_ordering FROM events WHERE"
+                    "SELECT event_id, topological_ordering, stream_ordering"
+                    " FROM events"
+                    " WHERE"
                     " room_id = ?"
                     " AND not outlier"
                     " AND stream_ordering <= ?"
@@ -266,14 +277,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 ) % (order, order,)
                 txn.execute(sql, (room_id, to_id, limit))
 
-            rows = self.cursor_to_dict(txn)
+                rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn]
 
             return rows
 
         rows = yield self.runInteraction("get_room_events_stream_for_room", f)
 
         ret = yield self._get_events(
-            [r["event_id"] for r in rows],
+            [r.event_id for r in rows],
             get_prev_content=True
         )
 
@@ -283,7 +294,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             ret.reverse()
 
         if rows:
-            key = "s%d" % min(r["stream_ordering"] for r in rows)
+            key = "s%d" % min(r.stream_ordering for r in rows)
         else:
             # Assume we didn't get anything because there was nothing to
             # get.
@@ -330,14 +341,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                     " ORDER BY stream_ordering ASC"
                 )
                 txn.execute(sql, (user_id, to_id,))
-            rows = self.cursor_to_dict(txn)
+
+            rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
 
             return rows
 
         rows = yield self.runInteraction("get_membership_changes_for_user", f)
 
         ret = yield self._get_events(
-            [r["event_id"] for r in rows],
+            [r.event_id for r in rows],
             get_prev_content=True
         )
 
@@ -353,14 +365,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         logger.debug("stream before")
         events = yield self._get_events(
-            [r["event_id"] for r in rows],
+            [r.event_id for r in rows],
             get_prev_content=True
         )
         logger.debug("stream after")
 
         self._set_before_and_after(events, rows)
 
-        defer.returnValue((events, token))
+        defer.returnValue((events, (token, end_token)))
 
     @defer.inlineCallbacks
     def get_recent_event_ids_for_room(self, room_id, limit, end_token):
@@ -372,15 +384,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             end_token (str): The stream token representing now.
 
         Returns:
-            Deferred[tuple[list[dict], tuple[str, str]]]: Returns a list of
-            dicts (which include event_ids, etc), and a tuple for
-            `(start_token, end_token)` representing the range of rows
-            returned.
-            The returned events are in ascending order.
+            Deferred[tuple[list[_EventDictReturn],  str]]: Returns a list of
+            _EventDictReturn and a token pointint to the start of the returned
+            events.
+            The events returned are in ascending order.
         """
         # Allow a zero limit here, and no-op.
         if limit == 0:
-            defer.returnValue(([], (end_token, end_token)))
+            defer.returnValue(([], end_token))
 
         end_token = RoomStreamToken.parse_stream_token(end_token)
 
@@ -392,7 +403,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         # We want to return the results in ascending order.
         rows.reverse()
 
-        defer.returnValue((rows, (token, str(end_token))))
+        defer.returnValue((rows, token))
 
     def get_room_event_after_stream_ordering(self, room_id, stream_ordering):
         """Gets details of the first event in a room at or after a stream ordering
@@ -496,10 +507,19 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
     @staticmethod
     def _set_before_and_after(events, rows, topo_order=True):
+        """Inserts ordering information to events' internal metadata from
+        the DB rows.
+
+        Args:
+            events (list[FrozenEvent])
+            rows (list[_EventDictReturn])
+            topo_order (bool): Whether the events were ordered topologically
+                or by stream ordering
+        """
         for event, row in zip(events, rows):
-            stream = row["stream_ordering"]
-            if topo_order:
-                topo = event.depth
+            stream = row.stream_ordering
+            if topo_order and row.topological_ordering:
+                topo = row.topological_ordering
             else:
                 topo = None
             internal = event.internal_metadata
@@ -586,12 +606,12 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         rows, start_token = self._paginate_room_events_txn(
             txn, room_id, before_token, direction='b', limit=before_limit,
         )
-        events_before = [r["event_id"] for r in rows]
+        events_before = [r.event_id for r in rows]
 
         rows, end_token = self._paginate_room_events_txn(
             txn, room_id, after_token, direction='f', limit=after_limit,
         )
-        events_after = [r["event_id"] for r in rows]
+        events_after = [r.event_id for r in rows]
 
         return {
             "before": {
@@ -672,9 +692,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 those that match the filter.
 
         Returns:
-            tuple[list[dict], str]: Returns the results as a list of dicts and
-            a token that points to the end of the result set. The dicts have
-            the keys "event_id", "toplogical_ordering" and "stream_ordering".
+            tuple[list[_EventDictReturn], str]: Returns the results as a list
+            of _EventDictReturn and a token that points to the end of the
+            result set.
         """
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
@@ -725,11 +745,11 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         txn.execute(sql, args)
 
-        rows = self.cursor_to_dict(txn)
+        rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn]
 
         if rows:
-            topo = rows[-1]["topological_ordering"]
-            toke = rows[-1]["stream_ordering"]
+            topo = rows[-1].topological_ordering
+            toke = rows[-1].stream_ordering
             if direction == 'b':
                 # Tokens are positions between events.
                 # This token points *after* the last event in the chunk.
@@ -764,7 +784,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             tuple[list[dict], str]: Returns the results as a list of dicts and
             a token that points to the end of the result set. The dicts have
-            the keys "event_id", "toplogical_ordering" and "stream_orderign".
+            the keys "event_id", "topological_ordering" and "stream_orderign".
         """
 
         from_key = RoomStreamToken.parse(from_key)
@@ -777,7 +797,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         )
 
         events = yield self._get_events(
-            [r["event_id"] for r in rows],
+            [r.event_id for r in rows],
             get_prev_content=True
         )
 
-- 
cgit 1.4.1


From c4af4c24ca988832018feaf0ac5a2f6dbb8bfe68 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 11:55:34 +0100
Subject: Refactor get_recent_events_for_room return type

There is no reason to return a tuple of tokens when the last token is
always the token passed as an argument. Changing it makes it consistent
with other storage APIs
---
 synapse/handlers/initial_sync.py | 10 +++++-----
 synapse/handlers/sync.py         |  2 +-
 synapse/storage/stream.py        | 16 +++++++++++++++-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index cd33a86599..5a9aa0c16d 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -181,8 +181,8 @@ class InitialSyncHandler(BaseHandler):
                     self.store, user_id, messages
                 )
 
-                start_token = now_token.copy_and_replace("room_key", token[0])
-                end_token = now_token.copy_and_replace("room_key", token[1])
+                start_token = now_token.copy_and_replace("room_key", token)
+                end_token = now_token.copy_and_replace("room_key", room_end_token)
                 time_now = self.clock.time_msec()
 
                 d["messages"] = {
@@ -325,8 +325,8 @@ class InitialSyncHandler(BaseHandler):
             self.store, user_id, messages, is_peeking=is_peeking
         )
 
-        start_token = StreamToken.START.copy_and_replace("room_key", token[0])
-        end_token = StreamToken.START.copy_and_replace("room_key", token[1])
+        start_token = StreamToken.START.copy_and_replace("room_key", token)
+        end_token = StreamToken.START.copy_and_replace("room_key", stream_token)
 
         time_now = self.clock.time_msec()
 
@@ -409,7 +409,7 @@ class InitialSyncHandler(BaseHandler):
         )
 
         start_token = now_token.copy_and_replace("room_key", token[0])
-        end_token = now_token.copy_and_replace("room_key", token[1])
+        end_token = now_token
 
         time_now = self.clock.time_msec()
 
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index b52e4c2aff..c25a76d215 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -429,7 +429,7 @@ class SyncHandler(object):
         Returns:
             A Deferred map from ((type, state_key)->Event)
         """
-        last_events, token = yield self.store.get_recent_events_for_room(
+        last_events, _ = yield self.store.get_recent_events_for_room(
             room_id, end_token=stream_position.room_key, limit=1,
         )
 
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 772d2c6198..b5baacd32c 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -359,6 +359,20 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
     @defer.inlineCallbacks
     def get_recent_events_for_room(self, room_id, limit, end_token):
+        """Get the most recent events in the room in topological ordering.
+
+        Args:
+            room_id (str)
+            limit (int)
+            end_token (str): The stream token representing now.
+
+        Returns:
+            Deferred[tuple[list[FrozenEvent],  str]]: Returns a list of
+            events and a token pointint to the start of the returned
+            events.
+            The events returned are in ascending order.
+        """
+
         rows, token = yield self.get_recent_event_ids_for_room(
             room_id, limit, end_token,
         )
@@ -372,7 +386,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         self._set_before_and_after(events, rows)
 
-        defer.returnValue((events, (token, end_token)))
+        defer.returnValue((events, token))
 
     @defer.inlineCallbacks
     def get_recent_event_ids_for_room(self, room_id, limit, end_token):
-- 
cgit 1.4.1


From e5ab9cd24b2f63c6ca00ae6354dbcbfcd9127dfb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 11:58:35 +0100
Subject: Don't unnecessarily require token to be stream token

This allows calling the `get_recent_event_ids_for_room` function in more
situations.
---
 synapse/storage/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index b5baacd32c..5e4327bb96 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -407,7 +407,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         if limit == 0:
             defer.returnValue(([], end_token))
 
-        end_token = RoomStreamToken.parse_stream_token(end_token)
+        end_token = RoomStreamToken.parse(end_token)
 
         rows, token = yield self.runInteraction(
             "get_recent_event_ids_for_room", self._paginate_room_events_txn,
-- 
cgit 1.4.1


From e2accd7f1d21e34181dd4543eca30ad1ea971b4c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 11:59:45 +0100
Subject: Refactor sync APIs to reuse pagination API

The sync API often returns events in a topological rather than stream
ordering, e.g. when the user joined the room or on initial sync. When
this happens we can reuse existing pagination storage functions.
---
 synapse/handlers/sync.py  | 19 ++++++++----
 synapse/storage/stream.py | 73 +++++++++++++++++++++++------------------------
 2 files changed, 48 insertions(+), 44 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index c25a76d215..b75daa340d 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -354,12 +354,19 @@ class SyncHandler(object):
                 since_key = since_token.room_key
 
             while limited and len(recents) < timeline_limit and max_repeat:
-                events, end_key = yield self.store.get_room_events_stream_for_room(
-                    room_id,
-                    limit=load_limit + 1,
-                    from_key=since_key,
-                    to_key=end_key,
-                )
+                if since_key:
+                    events, end_key = yield self.store.get_room_events_stream_for_room(
+                        room_id,
+                        limit=load_limit + 1,
+                        from_key=since_key,
+                        to_key=end_key,
+                    )
+                else:
+                    events, end_key = yield self.store.get_recent_events_for_room(
+                        room_id,
+                        limit=load_limit + 1,
+                        end_token=end_key,
+                    )
                 loaded_recents = sync_config.filter_collection.filter_room_timeline(
                     events
                 )
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 5e4327bb96..8bb4e85709 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -233,52 +233,49 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
     @defer.inlineCallbacks
     def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0,
                                         order='DESC'):
-        # Note: If from_key is None then we return in topological order. This
-        # is because in that case we're using this as a "get the last few messages
-        # in a room" function, rather than "get new messages since last sync"
-        if from_key is not None:
-            from_id = RoomStreamToken.parse_stream_token(from_key).stream
-        else:
-            from_id = None
-        to_id = RoomStreamToken.parse_stream_token(to_key).stream
 
+        """Get new room events in stream ordering since `from_key`.
+
+        Args:
+            room_id (str)
+            from_key (str): Token from which no events are returned before
+            to_key (str): Token from which no events are returned after. (This
+                is typically the current stream token)
+            limit (int): Maximum number of events to return
+            order (str): Either "DESC" or "ASC". Determines which events are
+                returned when the result is limited. If "DESC" then the most
+                recent `limit` events are returned, otherwise returns the
+                oldest `limit` events.
+
+        Returns:
+            Deferred[tuple[list[FrozenEvent], str]]: Returns the list of
+            events (in ascending order) and the token from the start of
+            the chunk of events returned.
+        """
         if from_key == to_key:
             defer.returnValue(([], from_key))
 
-        if from_id:
-            has_changed = yield self._events_stream_cache.has_entity_changed(
-                room_id, from_id
-            )
+        from_id = RoomStreamToken.parse_stream_token(from_key).stream
+        to_id = RoomStreamToken.parse_stream_token(to_key).stream
 
-            if not has_changed:
-                defer.returnValue(([], from_key))
+        has_changed = yield self._events_stream_cache.has_entity_changed(
+            room_id, from_id
+        )
 
-        def f(txn):
-            if from_id is not None:
-                sql = (
-                    "SELECT event_id, stream_ordering FROM events WHERE"
-                    " room_id = ?"
-                    " AND not outlier"
-                    " AND stream_ordering > ? AND stream_ordering <= ?"
-                    " ORDER BY stream_ordering %s LIMIT ?"
-                ) % (order,)
-                txn.execute(sql, (room_id, from_id, to_id, limit))
-
-                rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
-            else:
-                sql = (
-                    "SELECT event_id, topological_ordering, stream_ordering"
-                    " FROM events"
-                    " WHERE"
-                    " room_id = ?"
-                    " AND not outlier"
-                    " AND stream_ordering <= ?"
-                    " ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?"
-                ) % (order, order,)
-                txn.execute(sql, (room_id, to_id, limit))
+        if not has_changed:
+            defer.returnValue(([], from_key))
 
-                rows = [_EventDictReturn(row[0], row[1], row[2]) for row in txn]
+        def f(txn):
+            sql = (
+                "SELECT event_id, stream_ordering FROM events WHERE"
+                " room_id = ?"
+                " AND not outlier"
+                " AND stream_ordering > ? AND stream_ordering <= ?"
+                " ORDER BY stream_ordering %s LIMIT ?"
+            ) % (order,)
+            txn.execute(sql, (room_id, from_id, to_id, limit))
 
+            rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
             return rows
 
         rows = yield self.runInteraction("get_room_events_stream_for_room", f)
-- 
cgit 1.4.1


From c0e08dc45b8996e88ff3fd6c7584c26f5ec2e839 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 14:31:32 +0100
Subject: Remove unused code path from member change DB func

The function is never called without a from_key, so we can remove all
the handling for that scenario.
---
 synapse/storage/stream.py | 34 ++++++++++------------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 772d2c6198..25aef32551 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -304,10 +304,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
     @defer.inlineCallbacks
     def get_membership_changes_for_user(self, user_id, from_key, to_key):
-        if from_key is not None:
-            from_id = RoomStreamToken.parse_stream_token(from_key).stream
-        else:
-            from_id = None
+        from_id = RoomStreamToken.parse_stream_token(from_key).stream
         to_id = RoomStreamToken.parse_stream_token(to_key).stream
 
         if from_key == to_key:
@@ -321,26 +318,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 defer.returnValue([])
 
         def f(txn):
-            if from_id is not None:
-                sql = (
-                    "SELECT m.event_id, stream_ordering FROM events AS e,"
-                    " room_memberships AS m"
-                    " WHERE e.event_id = m.event_id"
-                    " AND m.user_id = ?"
-                    " AND e.stream_ordering > ? AND e.stream_ordering <= ?"
-                    " ORDER BY e.stream_ordering ASC"
-                )
-                txn.execute(sql, (user_id, from_id, to_id,))
-            else:
-                sql = (
-                    "SELECT m.event_id, stream_ordering FROM events AS e,"
-                    " room_memberships AS m"
-                    " WHERE e.event_id = m.event_id"
-                    " AND m.user_id = ?"
-                    " AND stream_ordering <= ?"
-                    " ORDER BY stream_ordering ASC"
-                )
-                txn.execute(sql, (user_id, to_id,))
+            sql = (
+                "SELECT m.event_id, stream_ordering FROM events AS e,"
+                " room_memberships AS m"
+                " WHERE e.event_id = m.event_id"
+                " AND m.user_id = ?"
+                " AND e.stream_ordering > ? AND e.stream_ordering <= ?"
+                " ORDER BY e.stream_ordering ASC"
+            )
+            txn.execute(sql, (user_id, from_id, to_id,))
 
             rows = [_EventDictReturn(row[0], None, row[1]) for row in txn]
 
-- 
cgit 1.4.1


From 7e8726b8fb297f255ed1fd78f8e3be1d7f21dcc7 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 9 May 2018 14:54:28 +0100
Subject: Part deactivated users in the background

One room at a time so we don't take out the whole server with leave
events, and restart at server restart.
---
 synapse/handlers/deactivate_account.py | 35 +++++++++++++++++++++++++++++++++-
 synapse/storage/registration.py        | 27 ++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 387620c9c6..5bb6c11e8f 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -12,10 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 
 from ._base import BaseHandler
 from synapse.types import UserID, create_requester
+from synapse.util.logcontext import run_in_background
 
 import logging
 
@@ -30,6 +31,10 @@ class DeactivateAccountHandler(BaseHandler):
         self._device_handler = hs.get_device_handler()
         self._room_member_handler = hs.get_room_member_handler()
 
+        self._user_parter_running = False
+
+        reactor.callWhenRunning(self.start_user_parting)
+
     @defer.inlineCallbacks
     def deactivate_account(self, user_id):
         """Deactivate a user's account
@@ -53,10 +58,38 @@ class DeactivateAccountHandler(BaseHandler):
         yield self.store.user_delete_threepids(user_id)
         yield self.store.user_set_password_hash(user_id, None)
 
+        yield self.store.add_user_pending_deactivation(user_id)
+
+        self.start_user_parting()
+
+    def start_user_parting(self):
+       if not self._user_parter_running:
+            run_in_background(self.user_parter_loop())
+
+    @defer.inlineCallbacks
+    def user_parter_loop(self):
+        self._user_parter_running = True
+        logger.info("Starting user parter")
+        try:
+            while True:
+                user_id = yield self.store.get_user_pending_deactivation()
+                if user_id is None:
+                    break
+                logger.info("User parter parting %r", user_id)
+                yield self.part_user(user_id)
+                yield self.store.del_user_pending_deactivation(user_id)
+                logger.info("User parter finished parting %r", user_id)
+            logger.info("User parter finished: stopping")
+        finally:
+            self._user_parter_running = False
+
+    @defer.inlineCallbacks
+    def part_user(self, user_id):
         user = UserID.from_string(user_id)
 
         rooms_for_user = yield self.store.get_rooms_for_user(user_id)
         for room_id in rooms_for_user:
+            logger.info("User parter parting %r from %r", user_id, room_id)
             yield self._room_member_handler.update_membership(
                 create_requester(user),
                 user,
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index a50717db2d..de068a55d2 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -526,3 +526,30 @@ class RegistrationStore(RegistrationWorkerStore,
         except self.database_engine.module.IntegrityError:
             ret = yield self.get_3pid_guest_access_token(medium, address)
             defer.returnValue(ret)
+
+    def add_user_pending_deactivation(self, user_id):
+        return self._simple_insert(
+            "users_pending_deactivation",
+            values={
+                "user_id": user_id,
+            },
+            desc="add_user_pending_deactivation",
+        )
+
+    def del_user_pending_deactivation(self, user_id):
+        return self._simple_delete_one(
+            "users_pending_deactivation",
+            keyvalues={
+                "user_id": user_id,
+            },
+            desc="del_user_pending_deactivation",
+        )
+
+    def get_user_pending_deactivation(self):
+        return self._simple_select_one_onecol(
+            "users_pending_deactivation",
+            keyvalues={},
+            retcol="user_id",
+            allow_none=True,
+            desc="get_users_pending_deactivation",
+        )
-- 
cgit 1.4.1


From 52281e4c548233087f03a014fdb21783a471b2a7 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 9 May 2018 15:06:16 +0100
Subject: Indent fail

---
 synapse/handlers/deactivate_account.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 5bb6c11e8f..f99935b4ee 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -63,7 +63,7 @@ class DeactivateAccountHandler(BaseHandler):
         self.start_user_parting()
 
     def start_user_parting(self):
-       if not self._user_parter_running:
+        if not self._user_parter_running:
             run_in_background(self.user_parter_loop())
 
     @defer.inlineCallbacks
-- 
cgit 1.4.1


From 46df23f581bc52313cb1c9366e967ebcddef07cf Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 9 May 2018 15:07:54 +0100
Subject: Add the schema file

---
 .../storage/schema/delta/48/deactivated_users.sql  | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 synapse/storage/schema/delta/48/deactivated_users.sql

diff --git a/synapse/storage/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/delta/48/deactivated_users.sql
new file mode 100644
index 0000000000..e9013a6969
--- /dev/null
+++ b/synapse/storage/schema/delta/48/deactivated_users.sql
@@ -0,0 +1,25 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Store any accounts that have been requested to be deactivated.
+ * We part the account from all the rooms its in when its
+ * deactivated. This can take some time and synapse may be restarted
+ * before it completes, so store the user IDs here until the process
+ * is complete.
+ */
+CREATE TABLE users_pending_deactivation (
+    user_id TEXT NOT NULL
+);
-- 
cgit 1.4.1


From 294e9a0c9ba0c4ad69605eb781d9ae56b8982a96 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 9 May 2018 15:10:37 +0100
Subject: Prefix internal functions

---
 synapse/handlers/deactivate_account.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index f99935b4ee..81e0a75e78 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -33,7 +33,7 @@ class DeactivateAccountHandler(BaseHandler):
 
         self._user_parter_running = False
 
-        reactor.callWhenRunning(self.start_user_parting)
+        reactor.callWhenRunning(self._start_user_parting)
 
     @defer.inlineCallbacks
     def deactivate_account(self, user_id):
@@ -60,14 +60,14 @@ class DeactivateAccountHandler(BaseHandler):
 
         yield self.store.add_user_pending_deactivation(user_id)
 
-        self.start_user_parting()
+        self._start_user_parting()
 
-    def start_user_parting(self):
+    def _start_user_parting(self):
         if not self._user_parter_running:
-            run_in_background(self.user_parter_loop())
+            run_in_background(self._user_parter_loop())
 
     @defer.inlineCallbacks
-    def user_parter_loop(self):
+    def _user_parter_loop(self):
         self._user_parter_running = True
         logger.info("Starting user parter")
         try:
@@ -76,7 +76,7 @@ class DeactivateAccountHandler(BaseHandler):
                 if user_id is None:
                     break
                 logger.info("User parter parting %r", user_id)
-                yield self.part_user(user_id)
+                yield self._part_user(user_id)
                 yield self.store.del_user_pending_deactivation(user_id)
                 logger.info("User parter finished parting %r", user_id)
             logger.info("User parter finished: stopping")
@@ -84,7 +84,7 @@ class DeactivateAccountHandler(BaseHandler):
             self._user_parter_running = False
 
     @defer.inlineCallbacks
-    def part_user(self, user_id):
+    def _part_user(self, user_id):
         user = UserID.from_string(user_id)
 
         rooms_for_user = yield self.store.get_rooms_for_user(user_id)
-- 
cgit 1.4.1


From 75552d2148f9f48add495cb7b18a378b0abf70b9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 15:15:38 +0100
Subject: Update comments

---
 synapse/storage/stream.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 772d2c6198..60d2dca154 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -385,7 +385,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         Returns:
             Deferred[tuple[list[_EventDictReturn],  str]]: Returns a list of
-            _EventDictReturn and a token pointint to the start of the returned
+            _EventDictReturn and a token pointing to the start of the returned
             events.
             The events returned are in ascending order.
         """
@@ -514,7 +514,8 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             events (list[FrozenEvent])
             rows (list[_EventDictReturn])
             topo_order (bool): Whether the events were ordered topologically
-                or by stream ordering
+                or by stream ordering. If true then all rows should have a non
+                null topological_ordering.
         """
         for event, row in zip(events, rows):
             stream = row.stream_ordering
@@ -692,9 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 those that match the filter.
 
         Returns:
-            tuple[list[_EventDictReturn], str]: Returns the results as a list
-            of _EventDictReturn and a token that points to the end of the
-            result set.
+            Deferred[tuple[list[_EventDictReturn], str]]: Returns the results
+            as a list of _EventDictReturn and a token that points to the end
+            of the result set.
         """
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
-- 
cgit 1.4.1


From 7ce98804ff2ffa8ecacf63ddad469dc0221b655e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 15:42:39 +0100
Subject: Fix up comment

---
 synapse/storage/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index f529642101..b6d171eb9b 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -368,7 +368,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         Returns:
             Deferred[tuple[list[FrozenEvent],  str]]: Returns a list of
-            events and a token pointint to the start of the returned
+            events and a token pointing to the start of the returned
             events.
             The events returned are in ascending order.
         """
-- 
cgit 1.4.1


From fcf55f225541bcffa4bcb32213e0fde74f7b5827 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 15:43:00 +0100
Subject: Fix returned token is no longer a tuple

---
 synapse/handlers/initial_sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 5a9aa0c16d..71af86fe21 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -408,7 +408,7 @@ class InitialSyncHandler(BaseHandler):
             self.store, user_id, messages, is_peeking=is_peeking,
         )
 
-        start_token = now_token.copy_and_replace("room_key", token[0])
+        start_token = now_token.copy_and_replace("room_key", token)
         end_token = now_token
 
         time_now = self.clock.time_msec()
-- 
cgit 1.4.1


From d26bec8a437e44a05febd02473a8dcd8ff2c9f58 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 9 May 2018 15:56:07 +0100
Subject: Add comment to sync as to why code path is split

---
 synapse/handlers/sync.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index b75daa340d..263e42dded 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -354,6 +354,11 @@ class SyncHandler(object):
                 since_key = since_token.room_key
 
             while limited and len(recents) < timeline_limit and max_repeat:
+                # If we have a since_key then we are trying to get any events
+                # that have happened since `since_key` up to `end_key`, so we
+                # can just use `get_room_events_stream_for_room`.
+                # Otherwise, we want to return the last N events in the room
+                # in toplogical ordering.
                 if since_key:
                     events, end_key = yield self.store.get_room_events_stream_for_room(
                         room_id,
-- 
cgit 1.4.1


From 18e144fe088bda9e28697062cda62dca1e03724b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 19:55:03 +0100
Subject: Move RequestsMetrics to its own file

This is useful in its own right, because server.py is full of stuff; but more
importantly, I want to do some refactoring that will cause a circular reference
as it is.
---
 synapse/http/request_metrics.py | 147 ++++++++++++++++++++++++++++++++++++++++
 synapse/http/server.py          | 128 ++--------------------------------
 2 files changed, 151 insertions(+), 124 deletions(-)
 create mode 100644 synapse/http/request_metrics.py

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
new file mode 100644
index 0000000000..4a843a36a7
--- /dev/null
+++ b/synapse/http/request_metrics.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import synapse.metrics
+from synapse.util.logcontext import LoggingContext
+
+logger = logging.getLogger(__name__)
+
+metrics = synapse.metrics.get_metrics_for("synapse.http.server")
+
+# total number of responses served, split by method/servlet/tag
+response_count = metrics.register_counter(
+    "response_count",
+    labels=["method", "servlet", "tag"],
+    alternative_names=(
+        # the following are all deprecated aliases for the same metric
+        metrics.name_prefix + x for x in (
+            "_requests",
+            "_response_time:count",
+            "_response_ru_utime:count",
+            "_response_ru_stime:count",
+            "_response_db_txn_count:count",
+            "_response_db_txn_duration:count",
+        )
+    )
+)
+
+requests_counter = metrics.register_counter(
+    "requests_received",
+    labels=["method", "servlet", ],
+)
+
+outgoing_responses_counter = metrics.register_counter(
+    "responses",
+    labels=["method", "code"],
+)
+
+response_timer = metrics.register_counter(
+    "response_time_seconds",
+    labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_time:total",
+    ),
+)
+
+response_ru_utime = metrics.register_counter(
+    "response_ru_utime_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_ru_utime:total",
+    ),
+)
+
+response_ru_stime = metrics.register_counter(
+    "response_ru_stime_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_ru_stime:total",
+    ),
+)
+
+response_db_txn_count = metrics.register_counter(
+    "response_db_txn_count", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_db_txn_count:total",
+    ),
+)
+
+# seconds spent waiting for db txns, excluding scheduling time, when processing
+# this request
+response_db_txn_duration = metrics.register_counter(
+    "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"],
+    alternative_names=(
+        metrics.name_prefix + "_response_db_txn_duration:total",
+    ),
+)
+
+# seconds spent waiting for a db connection, when processing this request
+response_db_sched_duration = metrics.register_counter(
+    "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"]
+)
+
+# size in bytes of the response written
+response_size = metrics.register_counter(
+    "response_size", labels=["method", "servlet", "tag"]
+)
+
+
+class RequestMetrics(object):
+    def start(self, clock, name):
+        self.start = clock.time_msec()
+        self.start_context = LoggingContext.current_context()
+        self.name = name
+
+    def stop(self, clock, request):
+        context = LoggingContext.current_context()
+
+        tag = ""
+        if context:
+            tag = context.tag
+
+            if context != self.start_context:
+                logger.warn(
+                    "Context have unexpectedly changed %r, %r",
+                    context, self.start_context
+                )
+                return
+
+        response_count.inc(request.method, self.name, tag)
+
+        response_timer.inc_by(
+            clock.time_msec() - self.start, request.method,
+            self.name, tag
+        )
+
+        ru_utime, ru_stime = context.get_resource_usage()
+
+        response_ru_utime.inc_by(
+            ru_utime, request.method, self.name, tag
+        )
+        response_ru_stime.inc_by(
+            ru_stime, request.method, self.name, tag
+        )
+        response_db_txn_count.inc_by(
+            context.db_txn_count, request.method, self.name, tag
+        )
+        response_db_txn_duration.inc_by(
+            context.db_txn_duration_ms / 1000., request.method, self.name, tag
+        )
+        response_db_sched_duration.inc_by(
+            context.db_sched_duration_ms / 1000., request.method, self.name, tag
+        )
+
+        response_size.inc_by(request.sentLength, request.method, self.name, tag)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 55b9ad5251..37b26b908e 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -18,6 +18,10 @@
 from synapse.api.errors import (
     cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
 )
+from synapse.http.request_metrics import (
+    RequestMetrics, requests_counter,
+    outgoing_responses_counter,
+)
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 from synapse.util.caches import intern_dict
 from synapse.util.metrics import Measure
@@ -41,82 +45,6 @@ import simplejson
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-# total number of responses served, split by method/servlet/tag
-response_count = metrics.register_counter(
-    "response_count",
-    labels=["method", "servlet", "tag"],
-    alternative_names=(
-        # the following are all deprecated aliases for the same metric
-        metrics.name_prefix + x for x in (
-            "_requests",
-            "_response_time:count",
-            "_response_ru_utime:count",
-            "_response_ru_stime:count",
-            "_response_db_txn_count:count",
-            "_response_db_txn_duration:count",
-        )
-    )
-)
-
-requests_counter = metrics.register_counter(
-    "requests_received",
-    labels=["method", "servlet", ],
-)
-
-outgoing_responses_counter = metrics.register_counter(
-    "responses",
-    labels=["method", "code"],
-)
-
-response_timer = metrics.register_counter(
-    "response_time_seconds",
-    labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_time:total",
-    ),
-)
-
-response_ru_utime = metrics.register_counter(
-    "response_ru_utime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_utime:total",
-    ),
-)
-
-response_ru_stime = metrics.register_counter(
-    "response_ru_stime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_stime:total",
-    ),
-)
-
-response_db_txn_count = metrics.register_counter(
-    "response_db_txn_count", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_count:total",
-    ),
-)
-
-# seconds spent waiting for db txns, excluding scheduling time, when processing
-# this request
-response_db_txn_duration = metrics.register_counter(
-    "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_duration:total",
-    ),
-)
-
-# seconds spent waiting for a db connection, when processing this request
-response_db_sched_duration = metrics.register_counter(
-    "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"]
-)
-
-# size in bytes of the response written
-response_size = metrics.register_counter(
-    "response_size", labels=["method", "servlet", "tag"]
-)
 
 _next_request_id = 0
 
@@ -386,54 +314,6 @@ def _unrecognised_request_handler(request):
     raise UnrecognizedRequestError()
 
 
-class RequestMetrics(object):
-    def start(self, clock, name):
-        self.start = clock.time_msec()
-        self.start_context = LoggingContext.current_context()
-        self.name = name
-
-    def stop(self, clock, request):
-        context = LoggingContext.current_context()
-
-        tag = ""
-        if context:
-            tag = context.tag
-
-            if context != self.start_context:
-                logger.warn(
-                    "Context have unexpectedly changed %r, %r",
-                    context, self.start_context
-                )
-                return
-
-        response_count.inc(request.method, self.name, tag)
-
-        response_timer.inc_by(
-            clock.time_msec() - self.start, request.method,
-            self.name, tag
-        )
-
-        ru_utime, ru_stime = context.get_resource_usage()
-
-        response_ru_utime.inc_by(
-            ru_utime, request.method, self.name, tag
-        )
-        response_ru_stime.inc_by(
-            ru_stime, request.method, self.name, tag
-        )
-        response_db_txn_count.inc_by(
-            context.db_txn_count, request.method, self.name, tag
-        )
-        response_db_txn_duration.inc_by(
-            context.db_txn_duration_ms / 1000., request.method, self.name, tag
-        )
-        response_db_sched_duration.inc_by(
-            context.db_sched_duration_ms / 1000., request.method, self.name, tag
-        )
-
-        response_size.inc_by(request.sentLength, request.method, self.name, tag)
-
-
 class RootRedirect(resource.Resource):
     """Redirects the root '/' path to another path."""
 
-- 
cgit 1.4.1


From 8460e48d0639a7f575fd3e57dcd5cf263a6d6a19 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 23:00:11 +0100
Subject: Move request_id management into SynapseRequest

---
 synapse/http/server.py | 31 ++++++++++++++++---------------
 synapse/http/site.py   |  9 +++++++++
 2 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 37b26b908e..369f7bbdd1 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -46,38 +46,40 @@ import simplejson
 logger = logging.getLogger(__name__)
 
 
-_next_request_id = 0
-
-
 def request_handler(include_metrics=False):
     """Decorator for ``wrap_request_handler``"""
     return lambda request_handler: wrap_request_handler(request_handler, include_metrics)
 
 
 def wrap_request_handler(request_handler, include_metrics=False):
-    """Wraps a method that acts as a request handler with the necessary logging
-    and exception handling.
+    """Wraps a request handler method with the necessary logging and exception
+     handling.
 
-    The method must have a signature of "handle_foo(self, request)". The
-    argument "self" must have "version_string" and "clock" attributes. The
-    argument "request" must be a twisted HTTP request.
+    The handler method must have a signature of "handle_foo(self, request)",
+    where "self" must have "version_string" and "clock" attributes (and
+    "request" must be a SynapseRequest).
 
-    The method must return a deferred. If the deferred succeeds we assume that
+    The handler must return a deferred. If the deferred succeeds we assume that
     a response has been sent. If the deferred fails with a SynapseError we use
     it to send a JSON response with the appropriate HTTP reponse code. If the
     deferred fails with any other type of error we send a 500 reponse.
 
-    We insert a unique request-id into the logging context for this request and
-    log the response and duration for this request.
+    As well as calling `request.processing` (which will log the response and
+    duration for this request), the wrapped request handler will insert the
+    request id into the logging context.
     """
 
     @defer.inlineCallbacks
     def wrapped_request_handler(self, request):
-        global _next_request_id
-        request_id = "%s-%s" % (request.method, _next_request_id)
-        _next_request_id += 1
+        """
+        Args:
+            self:
+            request (synapse.http.site.SynapseRequest):
+        """
 
+        request_id = request.get_request_id()
         with LoggingContext(request_id) as request_context:
+            request_context.request = request_id
             with Measure(self.clock, "wrapped_request_handler"):
                 request_metrics = RequestMetrics()
                 # we start the request metrics timer here with an initial stab
@@ -87,7 +89,6 @@ def wrap_request_handler(request_handler, include_metrics=False):
                 servlet_name = self.__class__.__name__
                 request_metrics.start(self.clock, name=servlet_name)
 
-                request_context.request = request_id
                 with request.processing():
                     try:
                         with PreserveLoggingContext(request_context):
diff --git a/synapse/http/site.py b/synapse/http/site.py
index c8b46e1af2..6af276e69a 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -22,6 +22,8 @@ import time
 
 ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
 
+_next_request_seq = 0
+
 
 class SynapseRequest(Request):
     def __init__(self, site, *args, **kw):
@@ -30,6 +32,10 @@ class SynapseRequest(Request):
         self.authenticated_entity = None
         self.start_time = 0
 
+        global _next_request_seq
+        self.request_seq = _next_request_seq
+        _next_request_seq += 1
+
     def __repr__(self):
         # We overwrite this so that we don't log ``access_token``
         return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % (
@@ -41,6 +47,9 @@ class SynapseRequest(Request):
             self.site.site_tag,
         )
 
+    def get_request_id(self):
+        return "%s-%i" % (self.method, self.request_seq)
+
     def get_redacted_uri(self):
         return ACCESS_TOKEN_RE.sub(
             br'\1<redacted>\3',
-- 
cgit 1.4.1


From 4d298506ddb65f642d899dffa236e4499f0c427b Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 10 May 2018 11:57:13 +0100
Subject: Oops, don't call function passed to run_in_background

---
 synapse/handlers/deactivate_account.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 81e0a75e78..78558dd309 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -64,7 +64,7 @@ class DeactivateAccountHandler(BaseHandler):
 
     def _start_user_parting(self):
         if not self._user_parter_running:
-            run_in_background(self._user_parter_loop())
+            run_in_background(self._user_parter_loop)
 
     @defer.inlineCallbacks
     def _user_parter_loop(self):
-- 
cgit 1.4.1


From 09b29f9c4a941ae5721b1f5e296156dfe92e3395 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 23:03:11 +0100
Subject: Make RequestMetrics take a raw time rather than a clock

... which is going to make it easier to move around.
---
 synapse/http/request_metrics.py | 8 ++++----
 synapse/http/server.py          | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 4a843a36a7..4e8a5f5306 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -100,12 +100,12 @@ response_size = metrics.register_counter(
 
 
 class RequestMetrics(object):
-    def start(self, clock, name):
-        self.start = clock.time_msec()
+    def start(self, time_msec, name):
+        self.start = time_msec
         self.start_context = LoggingContext.current_context()
         self.name = name
 
-    def stop(self, clock, request):
+    def stop(self, time_msec, request):
         context = LoggingContext.current_context()
 
         tag = ""
@@ -122,7 +122,7 @@ class RequestMetrics(object):
         response_count.inc(request.method, self.name, tag)
 
         response_timer.inc_by(
-            clock.time_msec() - self.start, request.method,
+            time_msec - self.start, request.method,
             self.name, tag
         )
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 369f7bbdd1..b16c9c17f6 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -87,7 +87,7 @@ def wrap_request_handler(request_handler, include_metrics=False):
                 # JsonResource (or a subclass), and JsonResource._async_render
                 # will update it once it picks a servlet.
                 servlet_name = self.__class__.__name__
-                request_metrics.start(self.clock, name=servlet_name)
+                request_metrics.start(self.clock.time_msec(), name=servlet_name)
 
                 with request.processing():
                     try:
@@ -138,7 +138,7 @@ def wrap_request_handler(request_handler, include_metrics=False):
                     finally:
                         try:
                             request_metrics.stop(
-                                self.clock, request
+                                self.clock.time_msec(), request
                             )
                         except Exception as e:
                             logger.warn("Failed to stop metrics: %r", e)
-- 
cgit 1.4.1


From c6f730282c3a25468df0e624293aefd60cef7840 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 23:05:14 +0100
Subject: Move RequestMetrics handling into SynapseRequest.processing()

It fits quite nicely here, and opens the path to getting rid of the
"include_metrics" mess.
---
 synapse/http/server.py | 19 +++++---------
 synapse/http/site.py   | 69 ++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 64 insertions(+), 24 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index b16c9c17f6..8e5d1d58f5 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -19,7 +19,7 @@ from synapse.api.errors import (
     cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
 )
 from synapse.http.request_metrics import (
-    RequestMetrics, requests_counter,
+    requests_counter,
     outgoing_responses_counter,
 )
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
@@ -81,19 +81,18 @@ def wrap_request_handler(request_handler, include_metrics=False):
         with LoggingContext(request_id) as request_context:
             request_context.request = request_id
             with Measure(self.clock, "wrapped_request_handler"):
-                request_metrics = RequestMetrics()
                 # we start the request metrics timer here with an initial stab
                 # at the servlet name. For most requests that name will be
                 # JsonResource (or a subclass), and JsonResource._async_render
                 # will update it once it picks a servlet.
                 servlet_name = self.__class__.__name__
-                request_metrics.start(self.clock.time_msec(), name=servlet_name)
-
-                with request.processing():
+                with request.processing(servlet_name):
                     try:
                         with PreserveLoggingContext(request_context):
                             if include_metrics:
-                                yield request_handler(self, request, request_metrics)
+                                yield request_handler(
+                                    self, request, request.request_metrics,
+                                )
                             else:
                                 requests_counter.inc(request.method, servlet_name)
                                 yield request_handler(self, request)
@@ -135,13 +134,7 @@ def wrap_request_handler(request_handler, include_metrics=False):
                             pretty_print=_request_user_agent_is_curl(request),
                             version_string=self.version_string,
                         )
-                    finally:
-                        try:
-                            request_metrics.stop(
-                                self.clock.time_msec(), request
-                            )
-                        except Exception as e:
-                            logger.warn("Failed to stop metrics: %r", e)
+
     return wrapped_request_handler
 
 
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 6af276e69a..bfd9832aa0 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -12,20 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.util.logcontext import LoggingContext
-from twisted.web.server import Site, Request
-
 import contextlib
 import logging
 import re
 import time
 
+from twisted.web.server import Site, Request
+
+from synapse.http.request_metrics import RequestMetrics
+from synapse.util.logcontext import LoggingContext
+
+logger = logging.getLogger(__name__)
+
 ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
 
 _next_request_seq = 0
 
 
 class SynapseRequest(Request):
+    """Class which encapsulates an HTTP request to synapse.
+
+    All of the requests processed in synapse are of this type.
+
+    It extends twisted's twisted.web.server.Request, and adds:
+     * Unique request ID
+     * Redaction of access_token query-params in __repr__
+     * Logging at start and end
+     * Metrics to record CPU, wallclock and DB time by endpoint.
+
+    It provides a method `processing` which should be called by the Resource
+    which is handling the request, and returns a context manager.
+
+    """
     def __init__(self, site, *args, **kw):
         Request.__init__(self, *args, **kw)
         self.site = site
@@ -59,7 +77,11 @@ class SynapseRequest(Request):
     def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]
 
-    def started_processing(self):
+    def _started_processing(self, servlet_name):
+        self.start_time = int(time.time() * 1000)
+        self.request_metrics = RequestMetrics()
+        self.request_metrics.start(self.start_time, name=servlet_name)
+
         self.site.access_logger.info(
             "%s - %s - Received request: %s %s",
             self.getClientIP(),
@@ -67,10 +89,8 @@ class SynapseRequest(Request):
             self.method,
             self.get_redacted_uri()
         )
-        self.start_time = int(time.time() * 1000)
-
-    def finished_processing(self):
 
+    def _finished_processing(self):
         try:
             context = LoggingContext.current_context()
             ru_utime, ru_stime = context.get_resource_usage()
@@ -81,6 +101,8 @@ class SynapseRequest(Request):
             ru_utime, ru_stime = (0, 0)
             db_txn_count, db_txn_duration_ms = (0, 0)
 
+        end_time = int(time.time() * 1000)
+
         self.site.access_logger.info(
             "%s - %s - {%s}"
             " Processed request: %dms (%dms, %dms) (%dms/%dms/%d)"
@@ -88,7 +110,7 @@ class SynapseRequest(Request):
             self.getClientIP(),
             self.site.site_tag,
             self.authenticated_entity,
-            int(time.time() * 1000) - self.start_time,
+            end_time - self.start_time,
             int(ru_utime * 1000),
             int(ru_stime * 1000),
             db_sched_duration_ms,
@@ -102,11 +124,36 @@ class SynapseRequest(Request):
             self.get_user_agent(),
         )
 
+        try:
+            self.request_metrics.stop(end_time, self)
+        except Exception as e:
+            logger.warn("Failed to stop metrics: %r", e)
+
     @contextlib.contextmanager
-    def processing(self):
-        self.started_processing()
+    def processing(self, servlet_name):
+        """Record the fact that we are processing this request.
+
+        Returns a context manager; the correct way to use this is:
+
+        @defer.inlineCallbacks
+        def handle_request(request):
+            with request.processing("FooServlet"):
+                yield really_handle_the_request()
+
+        This will log the request's arrival. Once the context manager is
+        closed, the completion of the request will be logged, and the various
+        metrics will be updated.
+
+        Args:
+            servlet_name (str): the name of the servlet which will be
+                processing this request. This is used in the metrics.
+
+                It is possible to update this afterwards by updating
+                self.request_metrics.servlet_name.
+        """
+        self._started_processing(servlet_name)
         yield
-        self.finished_processing()
+        self._finished_processing()
 
 
 class XForwardedForRequest(SynapseRequest):
-- 
cgit 1.4.1


From b8700dd7d0482813beb9c780b411de5108ae078a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 18:03:04 +0100
Subject: Bump requests_counter in wrapped_request_handler

less magic
---
 synapse/http/server.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 8e5d1d58f5..1d05b53873 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -90,12 +90,20 @@ def wrap_request_handler(request_handler, include_metrics=False):
                     try:
                         with PreserveLoggingContext(request_context):
                             if include_metrics:
-                                yield request_handler(
+                                d = request_handler(
                                     self, request, request.request_metrics,
                                 )
                             else:
-                                requests_counter.inc(request.method, servlet_name)
-                                yield request_handler(self, request)
+                                d = request_handler(self, request)
+
+                            # record the arrival of the request *after*
+                            # dispatching to the handler, so that the handler
+                            # can update the servlet name in the request
+                            # metrics
+                            requests_counter.inc(request.method,
+                                                 request.request_metrics.name)
+                            yield d
+
                     except CodeMessageException as e:
                         code = e.code
                         if isinstance(e, SynapseError):
@@ -220,7 +228,6 @@ class JsonResource(HttpServer, resource.Resource):
             servlet_classname = "%r" % callback
 
         request_metrics.name = servlet_classname
-        requests_counter.inc(request.method, servlet_classname)
 
         # Now trigger the callback. If it returns a response, we send it
         # here. If it throws an exception, that is handled by the wrapper
-- 
cgit 1.4.1


From 49e5a613f1ac53498b31150f78332fc64932e61b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 23:44:22 +0100
Subject: Move outgoing_responses_counter handling to RequestMetrics

it's much neater there.
---
 synapse/http/request_metrics.py | 2 ++
 synapse/http/server.py          | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 4e8a5f5306..8c850bf23f 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -119,6 +119,8 @@ class RequestMetrics(object):
                 )
                 return
 
+        outgoing_responses_counter.inc(request.method, str(request.code))
+
         response_count.inc(request.method, self.name, tag)
 
         response_timer.inc_by(
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 1d05b53873..200b2c4837 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -20,7 +20,6 @@ from synapse.api.errors import (
 )
 from synapse.http.request_metrics import (
     requests_counter,
-    outgoing_responses_counter,
 )
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 from synapse.util.caches import intern_dict
@@ -112,7 +111,6 @@ def wrap_request_handler(request_handler, include_metrics=False):
                             )
                         else:
                             logger.exception(e)
-                        outgoing_responses_counter.inc(request.method, str(code))
                         respond_with_json(
                             request, code, cs_exception(e), send_cors=True,
                             pretty_print=_request_user_agent_is_curl(request),
@@ -274,8 +272,6 @@ class JsonResource(HttpServer, resource.Resource):
 
     def _send_response(self, request, code, response_json_object,
                        response_code_message=None):
-        outgoing_responses_counter.inc(request.method, str(code))
-
         # TODO: Only enable CORS for the requests that need it.
         respond_with_json(
             request, code, response_json_object,
-- 
cgit 1.4.1


From 9589a1925e944360c68f7eb8a65f8ba94ec9bb84 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 May 2018 23:49:29 +0100
Subject: Remove include_metrics param

The metrics are now available via the request, so this is redundant and can go
away at last.
---
 synapse/http/server.py | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 200b2c4837..9598969de8 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -45,12 +45,12 @@ import simplejson
 logger = logging.getLogger(__name__)
 
 
-def request_handler(include_metrics=False):
+def request_handler():
     """Decorator for ``wrap_request_handler``"""
-    return lambda request_handler: wrap_request_handler(request_handler, include_metrics)
+    return wrap_request_handler
 
 
-def wrap_request_handler(request_handler, include_metrics=False):
+def wrap_request_handler(request_handler):
     """Wraps a request handler method with the necessary logging and exception
      handling.
 
@@ -88,12 +88,7 @@ def wrap_request_handler(request_handler, include_metrics=False):
                 with request.processing(servlet_name):
                     try:
                         with PreserveLoggingContext(request_context):
-                            if include_metrics:
-                                d = request_handler(
-                                    self, request, request.request_metrics,
-                                )
-                            else:
-                                d = request_handler(self, request)
+                            d = request_handler(self, request)
 
                             # record the arrival of the request *after*
                             # dispatching to the handler, so that the handler
@@ -206,13 +201,9 @@ class JsonResource(HttpServer, resource.Resource):
         self._async_render(request)
         return server.NOT_DONE_YET
 
-    # Disable metric reporting because _async_render does its own metrics.
-    # It does its own metric reporting because _async_render dispatches to
-    # a callback and it's the class name of that callback we want to report
-    # against rather than the JsonResource itself.
-    @request_handler(include_metrics=True)
+    @request_handler()
     @defer.inlineCallbacks
-    def _async_render(self, request, request_metrics):
+    def _async_render(self, request):
         """ This gets called from render() every time someone sends us a request.
             This checks if anyone has registered a callback for that method and
             path.
@@ -224,8 +215,7 @@ class JsonResource(HttpServer, resource.Resource):
             servlet_classname = servlet_instance.__class__.__name__
         else:
             servlet_classname = "%r" % callback
-
-        request_metrics.name = servlet_classname
+        request.request_metrics.name = servlet_classname
 
         # Now trigger the callback. If it returns a response, we send it
         # here. If it throws an exception, that is handled by the wrapper
-- 
cgit 1.4.1


From 09f570b9357d40764a0716f7e287fa4dde44610a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 10 May 2018 11:59:51 +0100
Subject: Factor wrap_request_handler_with_logging out of wrap_request_handler

... so that it can be used on non-JSON endpoints
---
 synapse/http/server.py | 120 +++++++++++++++++++++++++++----------------------
 1 file changed, 66 insertions(+), 54 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 9598969de8..fd58e65c4b 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -50,9 +50,10 @@ def request_handler():
     return wrap_request_handler
 
 
-def wrap_request_handler(request_handler):
-    """Wraps a request handler method with the necessary logging and exception
-     handling.
+def wrap_request_handler(h):
+    """Wraps a request handler method with exception handling.
+
+    Also adds logging as per wrap_request_handler_with_logging.
 
     The handler method must have a signature of "handle_foo(self, request)",
     where "self" must have "version_string" and "clock" attributes (and
@@ -62,12 +63,63 @@ def wrap_request_handler(request_handler):
     a response has been sent. If the deferred fails with a SynapseError we use
     it to send a JSON response with the appropriate HTTP reponse code. If the
     deferred fails with any other type of error we send a 500 reponse.
+    """
+
+    @defer.inlineCallbacks
+    def wrapped_request_handler(self, request):
+        try:
+            yield h(self, request)
+        except CodeMessageException as e:
+            code = e.code
+            if isinstance(e, SynapseError):
+                logger.info(
+                    "%s SynapseError: %s - %s", request, code, e.msg
+                )
+            else:
+                logger.exception(e)
+            respond_with_json(
+                request, code, cs_exception(e), send_cors=True,
+                pretty_print=_request_user_agent_is_curl(request),
+                version_string=self.version_string,
+            )
+
+        except Exception:
+            # failure.Failure() fishes the original Failure out
+            # of our stack, and thus gives us a sensible stack
+            # trace.
+            f = failure.Failure()
+            logger.error(
+                "Failed handle request via %r: %r: %s",
+                h,
+                request,
+                f.getTraceback().rstrip(),
+            )
+            respond_with_json(
+                request,
+                500,
+                {
+                    "error": "Internal server error",
+                    "errcode": Codes.UNKNOWN,
+                },
+                send_cors=True,
+                pretty_print=_request_user_agent_is_curl(request),
+                version_string=self.version_string,
+            )
+
+    return wrap_request_handler_with_logging(wrapped_request_handler)
+
+
+def wrap_request_handler_with_logging(h):
+    """Wraps a request handler to provide logging and metrics
+
+    The handler method must have a signature of "handle_foo(self, request)",
+    where "self" must have a "clock" attribute (and "request" must be a
+    SynapseRequest).
 
     As well as calling `request.processing` (which will log the response and
     duration for this request), the wrapped request handler will insert the
     request id into the logging context.
     """
-
     @defer.inlineCallbacks
     def wrapped_request_handler(self, request):
         """
@@ -86,56 +138,16 @@ def wrap_request_handler(request_handler):
                 # will update it once it picks a servlet.
                 servlet_name = self.__class__.__name__
                 with request.processing(servlet_name):
-                    try:
-                        with PreserveLoggingContext(request_context):
-                            d = request_handler(self, request)
-
-                            # record the arrival of the request *after*
-                            # dispatching to the handler, so that the handler
-                            # can update the servlet name in the request
-                            # metrics
-                            requests_counter.inc(request.method,
-                                                 request.request_metrics.name)
-                            yield d
-
-                    except CodeMessageException as e:
-                        code = e.code
-                        if isinstance(e, SynapseError):
-                            logger.info(
-                                "%s SynapseError: %s - %s", request, code, e.msg
-                            )
-                        else:
-                            logger.exception(e)
-                        respond_with_json(
-                            request, code, cs_exception(e), send_cors=True,
-                            pretty_print=_request_user_agent_is_curl(request),
-                            version_string=self.version_string,
-                        )
-                    except Exception:
-                        # failure.Failure() fishes the original Failure out
-                        # of our stack, and thus gives us a sensible stack
-                        # trace.
-                        f = failure.Failure()
-                        logger.error(
-                            "Failed handle request %s.%s on %r: %r: %s",
-                            request_handler.__module__,
-                            request_handler.__name__,
-                            self,
-                            request,
-                            f.getTraceback().rstrip(),
-                        )
-                        respond_with_json(
-                            request,
-                            500,
-                            {
-                                "error": "Internal server error",
-                                "errcode": Codes.UNKNOWN,
-                            },
-                            send_cors=True,
-                            pretty_print=_request_user_agent_is_curl(request),
-                            version_string=self.version_string,
-                        )
-
+                    with PreserveLoggingContext(request_context):
+                        d = h(self, request)
+
+                        # record the arrival of the request *after*
+                        # dispatching to the handler, so that the handler
+                        # can update the servlet name in the request
+                        # metrics
+                        requests_counter.inc(request.method,
+                                             request.request_metrics.name)
+                        yield d
     return wrapped_request_handler
 
 
-- 
cgit 1.4.1


From 645cb4bf06deee1c4c10ecc3d7df2c914168f19a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 10 May 2018 12:10:27 +0100
Subject: Remove redundant request_handler decorator

This is needless complexity; we might as well use the wrapper directly.

Also rename wrap_request_handler->wrap_json_request_handler.
---
 synapse/http/additional_resource.py           |  4 ++--
 synapse/http/server.py                        |  9 ++-------
 synapse/rest/key/v2/remote_key_resource.py    |  8 +++++---
 synapse/rest/media/v1/download_resource.py    | 16 +++++++++-------
 synapse/rest/media/v1/preview_url_resource.py |  5 +++--
 synapse/rest/media/v1/thumbnail_resource.py   | 23 +++++++++++++----------
 synapse/rest/media/v1/upload_resource.py      | 15 ++++++++-------
 7 files changed, 42 insertions(+), 38 deletions(-)

diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py
index 343e932cb1..d9e7f5dfb7 100644
--- a/synapse/http/additional_resource.py
+++ b/synapse/http/additional_resource.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.http.server import wrap_request_handler
+from synapse.http.server import wrap_json_request_handler
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET
 
@@ -50,6 +50,6 @@ class AdditionalResource(Resource):
         self._async_render(request)
         return NOT_DONE_YET
 
-    @wrap_request_handler
+    @wrap_json_request_handler
     def _async_render(self, request):
         return self._handler(request)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index fd58e65c4b..f29e36f490 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -45,12 +45,7 @@ import simplejson
 logger = logging.getLogger(__name__)
 
 
-def request_handler():
-    """Decorator for ``wrap_request_handler``"""
-    return wrap_request_handler
-
-
-def wrap_request_handler(h):
+def wrap_json_request_handler(h):
     """Wraps a request handler method with exception handling.
 
     Also adds logging as per wrap_request_handler_with_logging.
@@ -213,7 +208,7 @@ class JsonResource(HttpServer, resource.Resource):
         self._async_render(request)
         return server.NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def _async_render(self, request):
         """ This gets called from render() every time someone sends us a request.
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index 17e6079cba..17b3077926 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.http.server import request_handler, respond_with_json_bytes
+from synapse.http.server import (
+    respond_with_json_bytes, wrap_json_request_handler,
+)
 from synapse.http.servlet import parse_integer, parse_json_object_from_request
 from synapse.api.errors import SynapseError, Codes
 from synapse.crypto.keyring import KeyLookupError
@@ -99,7 +101,7 @@ class RemoteKey(Resource):
         self.async_render_GET(request)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def async_render_GET(self, request):
         if len(request.postpath) == 1:
@@ -124,7 +126,7 @@ class RemoteKey(Resource):
         self.async_render_POST(request)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def async_render_POST(self, request):
         content = parse_json_object_from_request(request)
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index fe7e17596f..3fc3f64d62 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -12,16 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import synapse.http.servlet
+import logging
 
-from ._base import parse_media_id, respond_404
+from twisted.internet import defer
 from twisted.web.resource import Resource
-from synapse.http.server import request_handler, set_cors_headers
-
 from twisted.web.server import NOT_DONE_YET
-from twisted.internet import defer
 
-import logging
+from synapse.http.server import (
+    set_cors_headers,
+    wrap_json_request_handler,
+)
+import synapse.http.servlet
+from ._base import parse_media_id, respond_404
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +45,7 @@ class DownloadResource(Resource):
         self._async_render_GET(request)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def _async_render_GET(self, request):
         set_cors_headers(request)
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 9290d7946f..6b089689b4 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -40,8 +40,9 @@ from synapse.util.stringutils import random_string
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.http.client import SpiderHttpClient
 from synapse.http.server import (
-    request_handler, respond_with_json_bytes,
+    respond_with_json_bytes,
     respond_with_json,
+    wrap_json_request_handler,
 )
 from synapse.util.async import ObservableDeferred
 from synapse.util.stringutils import is_ascii
@@ -90,7 +91,7 @@ class PreviewUrlResource(Resource):
         self._async_render_GET(request)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def _async_render_GET(self, request):
 
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index 58ada49711..6c12d79f56 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -14,18 +14,21 @@
 # limitations under the License.
 
 
-from ._base import (
-    parse_media_id, respond_404, respond_with_file, FileInfo,
-    respond_with_responder,
-)
-from twisted.web.resource import Resource
-from synapse.http.servlet import parse_string, parse_integer
-from synapse.http.server import request_handler, set_cors_headers
+import logging
 
-from twisted.web.server import NOT_DONE_YET
 from twisted.internet import defer
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
 
-import logging
+from synapse.http.server import (
+    set_cors_headers,
+    wrap_json_request_handler,
+)
+from synapse.http.servlet import parse_integer, parse_string
+from ._base import (
+    FileInfo, parse_media_id, respond_404, respond_with_file,
+    respond_with_responder,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +51,7 @@ class ThumbnailResource(Resource):
         self._async_render_GET(request)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def _async_render_GET(self, request):
         set_cors_headers(request)
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index a31e75cb46..7d01c57fd1 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -13,16 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.http.server import respond_with_json, request_handler
-
-from synapse.api.errors import SynapseError
+import logging
 
-from twisted.web.server import NOT_DONE_YET
 from twisted.internet import defer
-
 from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
 
-import logging
+from synapse.api.errors import SynapseError
+from synapse.http.server import (
+    respond_with_json,
+    wrap_json_request_handler,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ class UploadResource(Resource):
         respond_with_json(request, 200, {}, send_cors=True)
         return NOT_DONE_YET
 
-    @request_handler()
+    @wrap_json_request_handler
     @defer.inlineCallbacks
     def _async_render_POST(self, request):
         requester = yield self.auth.get_user_by_req(request)
-- 
cgit 1.4.1


From 217bc53c981c03ce880d1fa6afad2970be3f1d78 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 10 May 2018 12:20:40 +0100
Subject: Many docstrings

---
 synapse/handlers/deactivate_account.py | 24 ++++++++++++++++++++++++
 synapse/storage/registration.py        | 12 ++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 78558dd309..c00ec18627 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -31,8 +31,11 @@ class DeactivateAccountHandler(BaseHandler):
         self._device_handler = hs.get_device_handler()
         self._room_member_handler = hs.get_room_member_handler()
 
+        # Flag that indicates whether the process to part users from rooms is running
         self._user_parter_running = False
 
+        # Start the user parter loop so it can resume parting users from rooms where
+        # it left off (if it has work left to do).
         reactor.callWhenRunning(self._start_user_parting)
 
     @defer.inlineCallbacks
@@ -58,16 +61,32 @@ class DeactivateAccountHandler(BaseHandler):
         yield self.store.user_delete_threepids(user_id)
         yield self.store.user_set_password_hash(user_id, None)
 
+        # Add the user to a table of users penpding deactivation (ie.
+        # removal from all the rooms they're a member of)
         yield self.store.add_user_pending_deactivation(user_id)
 
+        # Now start the process that goes through that list and
+        # parts users from rooms (if it isn't already running)
         self._start_user_parting()
 
     def _start_user_parting(self):
+        """
+        Start the process that goes through the table of users
+        pending deactivation, if it isn't already running.
+
+        Returns:
+            None
+        """
         if not self._user_parter_running:
             run_in_background(self._user_parter_loop)
 
     @defer.inlineCallbacks
     def _user_parter_loop(self):
+        """Loop that parts deactivated users from rooms
+
+        Returns:
+            None
+        """
         self._user_parter_running = True
         logger.info("Starting user parter")
         try:
@@ -85,6 +104,11 @@ class DeactivateAccountHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _part_user(self, user_id):
+        """Causes the given user_id to leave all the rooms they're joined to
+
+        Returns:
+            None
+        """
         user = UserID.from_string(user_id)
 
         rooms_for_user = yield self.store.get_rooms_for_user(user_id)
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index de068a55d2..c05ce4612f 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -528,6 +528,10 @@ class RegistrationStore(RegistrationWorkerStore,
             defer.returnValue(ret)
 
     def add_user_pending_deactivation(self, user_id):
+        """
+        Adds a user to the table of users who need to be parted from all the rooms they're
+        in
+        """
         return self._simple_insert(
             "users_pending_deactivation",
             values={
@@ -537,6 +541,10 @@ class RegistrationStore(RegistrationWorkerStore,
         )
 
     def del_user_pending_deactivation(self, user_id):
+        """
+        Removes the given user to the table of users who need to be parted from all the
+        rooms they're in, effectively marking that user as fully deactivated.
+        """
         return self._simple_delete_one(
             "users_pending_deactivation",
             keyvalues={
@@ -546,6 +554,10 @@ class RegistrationStore(RegistrationWorkerStore,
         )
 
     def get_user_pending_deactivation(self):
+        """
+        Gets one user from the table of users waiting to be parted from all the rooms
+        they're in.
+        """
         return self._simple_select_one_onecol(
             "users_pending_deactivation",
             keyvalues={},
-- 
cgit 1.4.1


From 6b49628e3bf18f6cc1a1347fef8e4180e854d245 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 10 May 2018 12:23:53 +0100
Subject: Catch failure to part user from room

---
 synapse/handlers/deactivate_account.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index c00ec18627..4eb18775e8 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -114,10 +114,16 @@ class DeactivateAccountHandler(BaseHandler):
         rooms_for_user = yield self.store.get_rooms_for_user(user_id)
         for room_id in rooms_for_user:
             logger.info("User parter parting %r from %r", user_id, room_id)
-            yield self._room_member_handler.update_membership(
-                create_requester(user),
-                user,
-                room_id,
-                "leave",
-                ratelimit=False,
-            )
+            try:
+                yield self._room_member_handler.update_membership(
+                    create_requester(user),
+                    user,
+                    room_id,
+                    "leave",
+                    ratelimit=False,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to part user %r from room %r: ignoring and continuing",
+                    user_id, room_id,
+                )
-- 
cgit 1.4.1


From 318711e1399da009910c3a9e5fa297c28a2d0a97 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 10 May 2018 18:46:59 +0100
Subject: Set Server header in SynapseRequest

(instead of everywhere that writes a response. Or rather, the subset of places
which write responses where we haven't forgotten it).

This also means that we don't have to have the mysterious version_string
attribute in anything with a request handler.

Unfortunately it does mean that we have to pass the version string wherever we
instantiate a SynapseSite, which has been c&ped 150 times, but that is code
that ought to be cleaned up anyway really.
---
 synapse/app/appservice.py                     |  1 +
 synapse/app/client_reader.py                  |  1 +
 synapse/app/event_creator.py                  |  1 +
 synapse/app/federation_reader.py              |  1 +
 synapse/app/federation_sender.py              |  1 +
 synapse/app/frontend_proxy.py                 |  1 +
 synapse/app/homeserver.py                     |  2 ++
 synapse/app/media_repository.py               |  1 +
 synapse/app/pusher.py                         |  1 +
 synapse/app/synchrotron.py                    |  1 +
 synapse/app/user_dir.py                       |  1 +
 synapse/http/additional_resource.py           |  3 +--
 synapse/http/server.py                        | 14 ++++----------
 synapse/http/site.py                          | 11 ++++++++++-
 synapse/rest/client/v1/pusher.py              |  1 -
 synapse/rest/client/v2_alpha/auth.py          |  2 --
 synapse/rest/key/v1/server_key_resource.py    |  2 --
 synapse/rest/key/v2/local_key_resource.py     |  2 --
 synapse/rest/key/v2/remote_key_resource.py    |  2 --
 synapse/rest/media/v1/download_resource.py    |  3 +--
 synapse/rest/media/v1/preview_url_resource.py |  1 -
 synapse/rest/media/v1/thumbnail_resource.py   |  1 -
 synapse/rest/media/v1/upload_resource.py      |  1 -
 23 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 58f2c9d68c..b1efacc9f8 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -74,6 +74,7 @@ class AppserviceServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 267d34c881..38b98382c6 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -98,6 +98,7 @@ class ClientReaderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index b915d12d53..bd7f3d5679 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -114,6 +114,7 @@ class EventCreatorServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index c1dc66dd17..6e10b27b9e 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -87,6 +87,7 @@ class FederationReaderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index a08af83a4c..6f24e32d6d 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -101,6 +101,7 @@ class FederationSenderServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index b349e3e3ce..0f700ee786 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -152,6 +152,7 @@ class FrontendProxyServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a0e465d644..75f40fd5a4 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -140,6 +140,7 @@ class SynapseHomeServer(HomeServer):
                     site_tag,
                     listener_config,
                     root_resource,
+                    self.version_string,
                 ),
                 self.tls_server_context_factory,
             )
@@ -153,6 +154,7 @@ class SynapseHomeServer(HomeServer):
                     site_tag,
                     listener_config,
                     root_resource,
+                    self.version_string,
                 )
             )
         logger.info("Synapse now listening on port %d", port)
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index fc8282bbc1..9c93195f0a 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -94,6 +94,7 @@ class MediaRepositoryServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 26930d1b3b..3912eae48c 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -104,6 +104,7 @@ class PusherServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 7152b1deb4..c6294a7a0c 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -281,6 +281,7 @@ class SynchrotronServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 5ba7e9b416..53eb3474da 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -126,6 +126,7 @@ class UserDirectoryServer(HomeServer):
                 site_tag,
                 listener_config,
                 root_resource,
+                self.version_string,
             )
         )
 
diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py
index d9e7f5dfb7..a797396ade 100644
--- a/synapse/http/additional_resource.py
+++ b/synapse/http/additional_resource.py
@@ -42,8 +42,7 @@ class AdditionalResource(Resource):
         Resource.__init__(self)
         self._handler = handler
 
-        # these are required by the request_handler wrapper
-        self.version_string = hs.version_string
+        # required by the request_handler wrapper
         self.clock = hs.get_clock()
 
     def render(self, request):
diff --git a/synapse/http/server.py b/synapse/http/server.py
index f29e36f490..b6e2ae14a2 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -51,8 +51,8 @@ def wrap_json_request_handler(h):
     Also adds logging as per wrap_request_handler_with_logging.
 
     The handler method must have a signature of "handle_foo(self, request)",
-    where "self" must have "version_string" and "clock" attributes (and
-    "request" must be a SynapseRequest).
+    where "self" must have a "clock" attribute (and "request" must be a
+    SynapseRequest).
 
     The handler must return a deferred. If the deferred succeeds we assume that
     a response has been sent. If the deferred fails with a SynapseError we use
@@ -75,7 +75,6 @@ def wrap_json_request_handler(h):
             respond_with_json(
                 request, code, cs_exception(e), send_cors=True,
                 pretty_print=_request_user_agent_is_curl(request),
-                version_string=self.version_string,
             )
 
         except Exception:
@@ -98,7 +97,6 @@ def wrap_json_request_handler(h):
                 },
                 send_cors=True,
                 pretty_print=_request_user_agent_is_curl(request),
-                version_string=self.version_string,
             )
 
     return wrap_request_handler_with_logging(wrapped_request_handler)
@@ -192,7 +190,6 @@ class JsonResource(HttpServer, resource.Resource):
         self.canonical_json = canonical_json
         self.clock = hs.get_clock()
         self.path_regexs = {}
-        self.version_string = hs.version_string
         self.hs = hs
 
     def register_paths(self, method, path_patterns, callback):
@@ -275,7 +272,6 @@ class JsonResource(HttpServer, resource.Resource):
             send_cors=True,
             response_code_message=response_code_message,
             pretty_print=_request_user_agent_is_curl(request),
-            version_string=self.version_string,
             canonical_json=self.canonical_json,
         )
 
@@ -326,7 +322,7 @@ class RootRedirect(resource.Resource):
 
 def respond_with_json(request, code, json_object, send_cors=False,
                       response_code_message=None, pretty_print=False,
-                      version_string="", canonical_json=True):
+                      canonical_json=True):
     # could alternatively use request.notifyFinish() and flip a flag when
     # the Deferred fires, but since the flag is RIGHT THERE it seems like
     # a waste.
@@ -348,12 +344,11 @@ def respond_with_json(request, code, json_object, send_cors=False,
         request, code, json_bytes,
         send_cors=send_cors,
         response_code_message=response_code_message,
-        version_string=version_string
     )
 
 
 def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
-                            version_string="", response_code_message=None):
+                            response_code_message=None):
     """Sends encoded JSON in response to the given request.
 
     Args:
@@ -367,7 +362,6 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
 
     request.setResponseCode(code, message=response_code_message)
     request.setHeader(b"Content-Type", b"application/json")
-    request.setHeader(b"Server", version_string)
     request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
     request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate")
 
diff --git a/synapse/http/site.py b/synapse/http/site.py
index bfd9832aa0..202a990508 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -77,6 +77,11 @@ class SynapseRequest(Request):
     def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]
 
+    def render(self, resrc):
+        # override the Server header which is set by twisted
+        self.setHeader("Server", self.site.server_version_string)
+        return Request.render(self, resrc)
+
     def _started_processing(self, servlet_name):
         self.start_time = int(time.time() * 1000)
         self.request_metrics = RequestMetrics()
@@ -151,6 +156,8 @@ class SynapseRequest(Request):
                 It is possible to update this afterwards by updating
                 self.request_metrics.servlet_name.
         """
+        # TODO: we should probably just move this into render() and finish(),
+        # to save having to call a separate method.
         self._started_processing(servlet_name)
         yield
         self._finished_processing()
@@ -191,7 +198,8 @@ class SynapseSite(Site):
     Subclass of a twisted http Site that does access logging with python's
     standard logging
     """
-    def __init__(self, logger_name, site_tag, config, resource, *args, **kwargs):
+    def __init__(self, logger_name, site_tag, config, resource,
+                 server_version_string, *args, **kwargs):
         Site.__init__(self, resource, *args, **kwargs)
 
         self.site_tag = site_tag
@@ -199,6 +207,7 @@ class SynapseSite(Site):
         proxied = config.get("x_forwarded", False)
         self.requestFactory = SynapseRequestFactory(self, proxied)
         self.access_logger = logging.getLogger(logger_name)
+        self.server_version_string = server_version_string
 
     def log(self, request):
         pass
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 0206e664c1..40e523cc5f 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -176,7 +176,6 @@ class PushersRemoveRestServlet(RestServlet):
 
         request.setResponseCode(200)
         request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
-        request.setHeader(b"Server", self.hs.version_string)
         request.setHeader(b"Content-Length", b"%d" % (
             len(PushersRemoveRestServlet.SUCCESS_HTML),
         ))
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
index 8e5577148f..d6f3a19648 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -129,7 +129,6 @@ class AuthRestServlet(RestServlet):
             html_bytes = html.encode("utf8")
             request.setResponseCode(200)
             request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
-            request.setHeader(b"Server", self.hs.version_string)
             request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
 
             request.write(html_bytes)
@@ -175,7 +174,6 @@ class AuthRestServlet(RestServlet):
             html_bytes = html.encode("utf8")
             request.setResponseCode(200)
             request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
-            request.setHeader(b"Server", self.hs.version_string)
             request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
 
             request.write(html_bytes)
diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py
index bd4fea5774..1498d188c1 100644
--- a/synapse/rest/key/v1/server_key_resource.py
+++ b/synapse/rest/key/v1/server_key_resource.py
@@ -49,7 +49,6 @@ class LocalKey(Resource):
     """
 
     def __init__(self, hs):
-        self.version_string = hs.version_string
         self.response_body = encode_canonical_json(
             self.response_json_object(hs.config)
         )
@@ -84,7 +83,6 @@ class LocalKey(Resource):
     def render_GET(self, request):
         return respond_with_json_bytes(
             request, 200, self.response_body,
-            version_string=self.version_string
         )
 
     def getChild(self, name, request):
diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index be68d9a096..04775b3c45 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -63,7 +63,6 @@ class LocalKey(Resource):
     isLeaf = True
 
     def __init__(self, hs):
-        self.version_string = hs.version_string
         self.config = hs.config
         self.clock = hs.clock
         self.update_response_body(self.clock.time_msec())
@@ -115,5 +114,4 @@ class LocalKey(Resource):
             self.update_response_body(time_now)
         return respond_with_json_bytes(
             request, 200, self.response_body,
-            version_string=self.version_string
         )
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index 17b3077926..21b4c1175e 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -93,7 +93,6 @@ class RemoteKey(Resource):
     def __init__(self, hs):
         self.keyring = hs.get_keyring()
         self.store = hs.get_datastore()
-        self.version_string = hs.version_string
         self.clock = hs.get_clock()
         self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
@@ -242,5 +241,4 @@ class RemoteKey(Resource):
 
             respond_with_json_bytes(
                 request, 200, result_io.getvalue(),
-                version_string=self.version_string
             )
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index 3fc3f64d62..8cf8820c31 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -37,9 +37,8 @@ class DownloadResource(Resource):
         self.media_repo = media_repo
         self.server_name = hs.hostname
 
-        # Both of these are expected by @request_handler()
+        # this is expected by @wrap_json_request_handler
         self.clock = hs.get_clock()
-        self.version_string = hs.version_string
 
     def render_GET(self, request):
         self._async_render_GET(request)
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 6b089689b4..2839207abc 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -58,7 +58,6 @@ class PreviewUrlResource(Resource):
 
         self.auth = hs.get_auth()
         self.clock = hs.get_clock()
-        self.version_string = hs.version_string
         self.filepaths = media_repo.filepaths
         self.max_spider_size = hs.config.max_spider_size
         self.server_name = hs.hostname
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index 6c12d79f56..aae6e464e8 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -44,7 +44,6 @@ class ThumbnailResource(Resource):
         self.media_storage = media_storage
         self.dynamic_thumbnails = hs.config.dynamic_thumbnails
         self.server_name = hs.hostname
-        self.version_string = hs.version_string
         self.clock = hs.get_clock()
 
     def render_GET(self, request):
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 7d01c57fd1..7567476fce 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -41,7 +41,6 @@ class UploadResource(Resource):
         self.server_name = hs.hostname
         self.auth = hs.get_auth()
         self.max_upload_size = hs.config.max_upload_size
-        self.version_string = hs.version_string
         self.clock = hs.get_clock()
 
     def render_POST(self, request):
-- 
cgit 1.4.1


From db18d854cd8e197cb3d77441b2a7ebd6f81beb81 Mon Sep 17 00:00:00 2001
From: Damir Manapov <tebuchet@gmail.com>
Date: Thu, 10 May 2018 22:13:31 +0300
Subject: transaction_id, destination twice

---
 synapse/federation/units.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index 3f645acc43..01c5b8fe17 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -74,8 +74,6 @@ class Transaction(JsonEncodedObject):
         "previous_ids",
         "pdus",
         "edus",
-        "transaction_id",
-        "destination",
         "pdu_failures",
     ]
 
-- 
cgit 1.4.1


From 6406b70aebd61ecddf61e45196d7cb13428b509e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 11 May 2018 15:30:11 +0100
Subject: Use stream rather depth ordering for push actions

This simplifies things as it is, but will also allow us to change the
way we traverse topologically without having to update the way push
actions work.
---
 synapse/storage/event_push_actions.py    | 53 ++++++++++----------------------
 synapse/storage/receipts.py              |  1 -
 tests/storage/test_event_push_actions.py |  4 +--
 3 files changed, 18 insertions(+), 40 deletions(-)

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index c22762eb5c..f084a5f54b 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -18,8 +18,6 @@ from synapse.storage._base import SQLBaseStore, LoggingTransaction
 from twisted.internet import defer
 from synapse.util.async import sleep
 from synapse.util.caches.descriptors import cachedInlineCallbacks
-from synapse.types import RoomStreamToken
-from .stream import lower_bound
 
 import logging
 import simplejson as json
@@ -99,7 +97,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
     def _get_unread_counts_by_receipt_txn(self, txn, room_id, user_id,
                                           last_read_event_id):
         sql = (
-            "SELECT stream_ordering, topological_ordering"
+            "SELECT stream_ordering"
             " FROM events"
             " WHERE room_id = ? AND event_id = ?"
         )
@@ -111,17 +109,12 @@ class EventPushActionsWorkerStore(SQLBaseStore):
             return {"notify_count": 0, "highlight_count": 0}
 
         stream_ordering = results[0][0]
-        topological_ordering = results[0][1]
 
         return self._get_unread_counts_by_pos_txn(
-            txn, room_id, user_id, topological_ordering, stream_ordering
+            txn, room_id, user_id, stream_ordering
         )
 
-    def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, topological_ordering,
-                                      stream_ordering):
-        token = RoomStreamToken(
-            topological_ordering, stream_ordering
-        )
+    def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, stream_ordering):
 
         # First get number of notifications.
         # We don't need to put a notif=1 clause as all rows always have
@@ -132,10 +125,10 @@ class EventPushActionsWorkerStore(SQLBaseStore):
             " WHERE"
             " user_id = ?"
             " AND room_id = ?"
-            " AND %s"
-        ) % (lower_bound(token, self.database_engine, inclusive=False),)
+            " AND stream_ordering > ?"
+        )
 
-        txn.execute(sql, (user_id, room_id))
+        txn.execute(sql, (user_id, room_id, stream_ordering))
         row = txn.fetchone()
         notify_count = row[0] if row else 0
 
@@ -155,10 +148,10 @@ class EventPushActionsWorkerStore(SQLBaseStore):
             " highlight = 1"
             " AND user_id = ?"
             " AND room_id = ?"
-            " AND %s"
-        ) % (lower_bound(token, self.database_engine, inclusive=False),)
+            " AND stream_ordering > ?"
+        )
 
-        txn.execute(sql, (user_id, room_id))
+        txn.execute(sql, (user_id, room_id, stream_ordering))
         row = txn.fetchone()
         highlight_count = row[0] if row else 0
 
@@ -209,7 +202,6 @@ class EventPushActionsWorkerStore(SQLBaseStore):
                 "   ep.highlight "
                 " FROM ("
                 "   SELECT room_id,"
-                "       MAX(topological_ordering) as topological_ordering,"
                 "       MAX(stream_ordering) as stream_ordering"
                 "   FROM events"
                 "   INNER JOIN receipts_linearized USING (room_id, event_id)"
@@ -219,13 +211,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
                 " event_push_actions AS ep"
                 " WHERE"
                 "   ep.room_id = rl.room_id"
-                "   AND ("
-                "       ep.topological_ordering > rl.topological_ordering"
-                "       OR ("
-                "           ep.topological_ordering = rl.topological_ordering"
-                "           AND ep.stream_ordering > rl.stream_ordering"
-                "       )"
-                "   )"
+                "   AND ep.stream_ordering > rl.stream_ordering"
                 "   AND ep.user_id = ?"
                 "   AND ep.stream_ordering > ?"
                 "   AND ep.stream_ordering <= ?"
@@ -318,7 +304,6 @@ class EventPushActionsWorkerStore(SQLBaseStore):
                 "  ep.highlight, e.received_ts"
                 " FROM ("
                 "   SELECT room_id,"
-                "       MAX(topological_ordering) as topological_ordering,"
                 "       MAX(stream_ordering) as stream_ordering"
                 "   FROM events"
                 "   INNER JOIN receipts_linearized USING (room_id, event_id)"
@@ -329,13 +314,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
                 " INNER JOIN events AS e USING (room_id, event_id)"
                 " WHERE"
                 "   ep.room_id = rl.room_id"
-                "   AND ("
-                "       ep.topological_ordering > rl.topological_ordering"
-                "       OR ("
-                "           ep.topological_ordering = rl.topological_ordering"
-                "           AND ep.stream_ordering > rl.stream_ordering"
-                "       )"
-                "   )"
+                "   AND ep.stream_ordering > rl.stream_ordering"
                 "   AND ep.user_id = ?"
                 "   AND ep.stream_ordering > ?"
                 "   AND ep.stream_ordering <= ?"
@@ -762,10 +741,10 @@ class EventPushActionsStore(EventPushActionsWorkerStore):
         )
 
     def _remove_old_push_actions_before_txn(self, txn, room_id, user_id,
-                                            topological_ordering, stream_ordering):
+                                            stream_ordering):
         """
         Purges old push actions for a user and room before a given
-        topological_ordering.
+        stream_ordering.
 
         We however keep a months worth of highlighted notifications, so that
         users can still get a list of recent highlights.
@@ -774,7 +753,7 @@ class EventPushActionsStore(EventPushActionsWorkerStore):
             txn: The transcation
             room_id: Room ID to delete from
             user_id: user ID to delete for
-            topological_ordering: The lowest topological ordering which will
+            stream_ordering: The lowest stream ordering which will
                                   not be deleted.
         """
         txn.call_after(
@@ -793,9 +772,9 @@ class EventPushActionsStore(EventPushActionsWorkerStore):
         txn.execute(
             "DELETE FROM event_push_actions "
             " WHERE user_id = ? AND room_id = ? AND "
-            " topological_ordering <= ?"
+            " stream_ordering <= ?"
             " AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)",
-            (user_id, room_id, topological_ordering, self.stream_ordering_month_ago)
+            (user_id, room_id, stream_ordering, self.stream_ordering_month_ago)
         )
 
         txn.execute("""
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 63997ed449..2f95e7e82a 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -407,7 +407,6 @@ class ReceiptsStore(ReceiptsWorkerStore):
                 txn,
                 room_id=room_id,
                 user_id=user_id,
-                topological_ordering=topological_ordering,
                 stream_ordering=stream_ordering,
             )
 
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 9962ce8a5d..3cbf9a78b1 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -55,7 +55,7 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase):
         def _assert_counts(noitf_count, highlight_count):
             counts = yield self.store.runInteraction(
                 "", self.store._get_unread_counts_by_pos_txn,
-                room_id, user_id, 0, 0
+                room_id, user_id, 0
             )
             self.assertEquals(
                 counts,
@@ -86,7 +86,7 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase):
         def _mark_read(stream, depth):
             return self.store.runInteraction(
                 "", self.store._remove_old_push_actions_before_txn,
-                room_id, user_id, depth, stream
+                room_id, user_id, stream
             )
 
         yield _assert_counts(0, 0)
-- 
cgit 1.4.1


From f077e97914c9b5c82c94786130d98af52516cde0 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 14 May 2018 13:50:58 +0100
Subject: instead of inserting user daily visit data at the end of the day,
 instead insert incrementally through the day

---
 synapse/app/homeserver.py   | 19 +++++++++++++---
 synapse/storage/__init__.py | 54 ++++++++++++---------------------------------
 2 files changed, 30 insertions(+), 43 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f785a7a22b..bfc79a5e81 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -17,6 +17,7 @@ import gc
 import logging
 import os
 import sys
+import datetime
 
 import synapse
 import synapse.config.logger
@@ -475,9 +476,24 @@ def run(hs):
                 " changes across releases."
             )
 
+    # def recurring_user_daily_visit_stats():
+
     def generate_user_daily_visit_stats():
         hs.get_datastore().generate_user_daily_visits()
 
+    # Since user daily stats are bucketed at midnight UTC,
+    # and user_ips.last_seen can be updated at any time, it is important to call
+    # generate_user_daily_visit_stats immediately prior to the day end. Assuming
+    # an hourly cadence, the simplist way is to allign all calls to the hour
+    # end
+    end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \
+        + datetime.timedelta(hours=1) \
+        - datetime.timedelta(seconds=10)  # Ensure method fires before day transistion
+
+    time_to_next_hour = end_of_hour - datetime.datetime.now()
+    clock.call_later(time_to_next_hour.seconds,
+                     clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000))
+
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
         clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
@@ -490,9 +506,6 @@ def run(hs):
         # be quite busy the first few minutes
         clock.call_later(5 * 60, phone_stats_home)
 
-    clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000)
-    clock.call_later(5 * 60, generate_user_daily_visit_stats)
-
     if hs.config.daemonize and hs.config.print_pidfile:
         print (hs.config.pid_file)
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b51cf70336..6949876c13 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -353,48 +353,22 @@ class DataStore(RoomMemberStore, RoomStore,
         Generates daily visit data for use in cohort/ retention analysis
         """
         def _generate_user_daily_visits(txn):
-            logger.info("Calling _generate_user_daily_visits")
-            # determine timestamp of previous days
-            yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
-            yesterday_start = datetime.datetime(yesterday.year, yesterday.month,
-                                                yesterday.day, tzinfo=tz.tzutc())
-            yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000
-
-            # Check that this job has not already been completed
-            sql = """
-                SELECT timestamp
-                FROM user_daily_visits
-                ORDER by timestamp desc limit 1
-            """
-            txn.execute(sql)
-            row = txn.fetchone()
-
-            # Bail if the most recent time is yesterday
-            if row and row[0] == yesterday_start_time:
-                return
-
-            # Not specificying an upper bound means that if the update is run at
-            # 10 mins past midnight and the user is active during a 30 min session
-            # that the user is still included in the previous days stats
-            # This does mean that if the update is run hours late, then it is possible
-            # to overstate the cohort, but this seems a reasonable trade off
-            # The alternative is to insert on every request - but prefer to avoid
-            # for performance reasons
-            sql = """
-                    SELECT user_id, device_id
-                    FROM user_ips
-                    WHERE last_seen > ?
-            """
-            txn.execute(sql, (yesterday_start_time,))
-            user_visits = txn.fetchall()
 
+            # determine timestamp of the day start
+            now = datetime.datetime.utcnow()
+            today_start = datetime.datetime(now.year, now.month,
+                                            now.day, tzinfo=tz.tzutc())
+            today_start_time = int(time.mktime(today_start.timetuple())) * 1000
+            logger.info(today_start_time)
             sql = """
-                    INSERT INTO user_daily_visits (user_id, device_id, timestamp)
-                    VALUES (?, ?, ?)
-            """
-
-            for visit in user_visits:
-                txn.execute(sql, (visit + (yesterday_start_time,)))
+                INSERT INTO user_daily_visits (user_id, device_id, timestamp)
+                SELECT user_id, device_id, ?
+                FROM user_ips AS u
+                LEFT JOIN user_daily_visits USING (user_id, device_id)
+                WHERE last_seen > ? AND timestamp IS NULL
+                GROUP BY user_id, device_id;
+                """
+            txn.execute(sql, (today_start_time, today_start_time))
 
         return self.runInteraction("generate_user_daily_visits",
                                    _generate_user_daily_visits)
-- 
cgit 1.4.1


From aea80a0118270807feb6e679d3d65562e9c9e512 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 14 May 2018 15:50:57 +0100
Subject: v0.29.0-rc1: bump version and change log

---
 CHANGES.rst         | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 synapse/__init__.py |  2 +-
 2 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 9d40b2ac1e..a5c3848bce 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,5 +1,9 @@
-Changes in synapse <unreleased>
-===============================
+Changes in synapse v0.29.0-rc1 (2018-05-14)
+===========================================
+
+Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a
+closed spec bug in the Client Server API. Additionally further prep for Python 3
+migration.
 
 Potentially breaking change:
 
@@ -12,6 +16,64 @@ Potentially breaking change:
 
   Thanks to @NotAFile for fixing this.
 
+Features:
+
+* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou!
+
+Changes:
+
+* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77!
+* Make 'unexpected logging context' into warnings (PR #3007)
+* Open certificate files as bytes (PR #3084) Thanks to @NotAFile!
+* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile!
+* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile!
+* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile!
+* Use deferred.addTimeout instead of time_bound_deferred (PR #3127)
+* remove duplicates from groups tables (PR #3129)
+* Miscellaneous fixes to python_dependencies (PR #3136)
+* Improve exception handling for background processes (PR #3138)
+* Add missing consumeErrors to improve exception handling (PR #3139)
+* Use run_in_background in preference to preserve_fn (PR #3140)
+* Refactor event storage to prepare for changes in state calculations (PR #3141)
+* reraise exceptions more carefully (PR #3142)
+* Remove redundant call to preserve_fn (PR #3143)
+* Trap exceptions thrown within run_in_background (PR #3144)
+* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile!
+* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile!
+* Move more xrange to six (PR #3151) Thanks to @NotAFile!
+* make imports local (PR #3152) Thanks to @NotAFile!
+* move httplib import to six (PR #3153) Thanks to @NotAFile!
+* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile!
+* more bytes strings (PR #3155) Thanks to @NotAFile!
+* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile!
+* run config tests on py3 (PR #3159) Thanks to @NotAFile!
+* add guard for None on purge_history api (PR #3160) Thanks to @krombel!
+* Refactor /context to reuse pagination storage functions (PR #3193)
+* Refactor recent events func to use pagination func (PR #3195)
+* Refactor pagination DB API to return concrete type (PR #3196)
+* Refactor get_recent_events_for_room return type (PR #3198)
+* Refactor sync APIs to reuse pagination API (PR #3199)
+* Remove unused code path from member change DB func (PR #3200)
+* Part user from rooms on account deactivate (PR #3201)
+* Refactor request handling wrappers (PR #3203)
+* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov!
+
+Bug Fixes:
+
+* synapse fails to start under Twisted >= 18.4 (PR #3135)
+* Fix incorrect reference to StringIO (PR #3168)
+* Fix a class of logcontext leaks (PR #3170)
+* Fix a couple of logcontext leaks in unit tests (PR #3172)
+* Fix logcontext leak in media repo (PR #3174)
+* Escape label values in prometheus metrics (PR #3175)
+* fix http request timeout code (PR #3178)
+* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot!
+* Fix logcontext leaks in rate limiter (PR #3183)
+* Fix metrics that have integer value labels (PR #3186)
+* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx!
+* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77!
+* Set Server header in SynapseRequest (PR #3208)
+
 
 Changes in synapse v0.28.1 (2018-05-01)
 =======================================
diff --git a/synapse/__init__.py b/synapse/__init__.py
index f31cb9a3cb..69c081eb37 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.28.1"
+__version__ = "0.29.0-rc1"
-- 
cgit 1.4.1


From e71fb118f4d57f0c77647aa6fcabe36c08e31b20 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 14 May 2018 17:39:22 +0100
Subject: rearrange and collect related PRs

---
 CHANGES.rst | 53 +++++++++++++++++++++++++++--------------------------
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index a5c3848bce..aed8525f2d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -20,60 +20,61 @@ Features:
 
 * Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou!
 
-Changes:
+Changes - General:
 
 * nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77!
+* Part user from rooms on account deactivate (PR #3201)
 * Make 'unexpected logging context' into warnings (PR #3007)
-* Open certificate files as bytes (PR #3084) Thanks to @NotAFile!
-* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile!
-* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile!
-* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile!
-* Use deferred.addTimeout instead of time_bound_deferred (PR #3127)
+* Set Server header in SynapseRequest (PR #3208)
+* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178)
 * remove duplicates from groups tables (PR #3129)
-* Miscellaneous fixes to python_dependencies (PR #3136)
 * Improve exception handling for background processes (PR #3138)
 * Add missing consumeErrors to improve exception handling (PR #3139)
 * Use run_in_background in preference to preserve_fn (PR #3140)
-* Refactor event storage to prepare for changes in state calculations (PR #3141)
 * reraise exceptions more carefully (PR #3142)
 * Remove redundant call to preserve_fn (PR #3143)
 * Trap exceptions thrown within run_in_background (PR #3144)
-* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile!
-* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile!
-* Move more xrange to six (PR #3151) Thanks to @NotAFile!
-* make imports local (PR #3152) Thanks to @NotAFile!
-* move httplib import to six (PR #3153) Thanks to @NotAFile!
-* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile!
-* more bytes strings (PR #3155) Thanks to @NotAFile!
-* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile!
-* run config tests on py3 (PR #3159) Thanks to @NotAFile!
-* add guard for None on purge_history api (PR #3160) Thanks to @krombel!
+
+Changes - Refactors:
+
 * Refactor /context to reuse pagination storage functions (PR #3193)
 * Refactor recent events func to use pagination func (PR #3195)
 * Refactor pagination DB API to return concrete type (PR #3196)
 * Refactor get_recent_events_for_room return type (PR #3198)
 * Refactor sync APIs to reuse pagination API (PR #3199)
 * Remove unused code path from member change DB func (PR #3200)
-* Part user from rooms on account deactivate (PR #3201)
 * Refactor request handling wrappers (PR #3203)
 * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov!
+* Refactor event storage to prepare for changes in state calculations (PR #3141)
+
+Changes - Python 3 migration:
+
+* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile!
+* run config tests on py3 (PR #3159) Thanks to @NotAFile!
+* Open certificate files as bytes (PR #3084) Thanks to @NotAFile!
+* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile!
+* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile!
+* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile!
+* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile!
+* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile!
+* Move more xrange to six (PR #3151) Thanks to @NotAFile!
+* make imports local (PR #3152) Thanks to @NotAFile!
+* move httplib import to six (PR #3153) Thanks to @NotAFile!
+* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile!
+* more bytes strings (PR #3155) Thanks to @NotAFile!
 
 Bug Fixes:
 
-* synapse fails to start under Twisted >= 18.4 (PR #3135)
-* Fix incorrect reference to StringIO (PR #3168)
+* synapse fails to start under Twisted >= 18.4 (PR #3157)
 * Fix a class of logcontext leaks (PR #3170)
 * Fix a couple of logcontext leaks in unit tests (PR #3172)
 * Fix logcontext leak in media repo (PR #3174)
-* Escape label values in prometheus metrics (PR #3175)
-* fix http request timeout code (PR #3178)
+* Escape label values in prometheus metrics (PR #3175, #3186)
 * Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot!
 * Fix logcontext leaks in rate limiter (PR #3183)
-* Fix metrics that have integer value labels (PR #3186)
 * notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx!
 * nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77!
-* Set Server header in SynapseRequest (PR #3208)
-
+* add guard for None on purge_history api (PR #3160) Thanks to @krombel!
 
 Changes in synapse v0.28.1 (2018-05-01)
 =======================================
-- 
cgit 1.4.1


From 589ecc5b5800a86406a0d1464fd6974c029a1163 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 14 May 2018 17:49:59 +0100
Subject: further musical chairs

---
 CHANGES.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index aed8525f2d..78d015db9a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -26,11 +26,9 @@ Changes - General:
 * Part user from rooms on account deactivate (PR #3201)
 * Make 'unexpected logging context' into warnings (PR #3007)
 * Set Server header in SynapseRequest (PR #3208)
-* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178)
 * remove duplicates from groups tables (PR #3129)
 * Improve exception handling for background processes (PR #3138)
 * Add missing consumeErrors to improve exception handling (PR #3139)
-* Use run_in_background in preference to preserve_fn (PR #3140)
 * reraise exceptions more carefully (PR #3142)
 * Remove redundant call to preserve_fn (PR #3143)
 * Trap exceptions thrown within run_in_background (PR #3144)
@@ -46,6 +44,9 @@ Changes - Refactors:
 * Refactor request handling wrappers (PR #3203)
 * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov!
 * Refactor event storage to prepare for changes in state calculations (PR #3141)
+* Set Server header in SynapseRequest (PR #3208)
+* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178)
+* Use run_in_background in preference to preserve_fn (PR #3140)
 
 Changes - Python 3 migration:
 
-- 
cgit 1.4.1


From 47815edcfae73c5b938f8354853a09c0b80ef27e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 11 May 2018 00:17:11 +0100
Subject: ConsentResource to gather policy consent from users

Hopefully there are enough comments and docs in this that it makes sense on its
own.
---
 docs/privacy_policy_templates/README.md            |  23 +++
 docs/privacy_policy_templates/en/1.0.html          |  17 ++
 docs/privacy_policy_templates/en/success.html      |  11 ++
 synapse/app/homeserver.py                          |   9 +
 synapse/config/__init__.py                         |   6 +
 synapse/config/consent_config.py                   |  42 +++++
 synapse/config/homeserver.py                       |   8 +-
 synapse/config/key.py                              |  10 +
 synapse/http/server.py                             |  76 +++++++-
 synapse/rest/consent/__init__.py                   |   0
 synapse/rest/consent/consent_resource.py           | 210 +++++++++++++++++++++
 synapse/server.py                                  |   3 +
 synapse/storage/registration.py                    |  18 ++
 .../storage/schema/delta/48/add_user_consent.sql   |  18 ++
 14 files changed, 446 insertions(+), 5 deletions(-)
 create mode 100644 docs/privacy_policy_templates/README.md
 create mode 100644 docs/privacy_policy_templates/en/1.0.html
 create mode 100644 docs/privacy_policy_templates/en/success.html
 create mode 100644 synapse/config/consent_config.py
 create mode 100644 synapse/rest/consent/__init__.py
 create mode 100644 synapse/rest/consent/consent_resource.py
 create mode 100644 synapse/storage/schema/delta/48/add_user_consent.sql

diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md
new file mode 100644
index 0000000000..8e91c516b3
--- /dev/null
+++ b/docs/privacy_policy_templates/README.md
@@ -0,0 +1,23 @@
+If enabling the 'consent' resource in synapse, you will need some templates
+for the HTML to be served to the user. This directory contains very simple
+examples of the sort of thing that can be done.
+
+You'll need to add this sort of thing to your homeserver.yaml:
+
+```
+form_secret: <unique but arbitrary secret>
+
+user_consent:
+  template_dir: docs/privacy_policy_templates
+  default_version: 1.0
+```
+
+You should then be able to enable the `consent` resource under a `listener`
+entry. For example:
+
+```
+listeners:
+  - port: 8008
+    resources:
+      - names: [client, consent]
+```
diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html
new file mode 100644
index 0000000000..ab8666f0c3
--- /dev/null
+++ b/docs/privacy_policy_templates/en/1.0.html
@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <title>Matrix.org Privacy policy</title>
+  </head>
+  <body>
+    <p>
+      All your base are belong to us.
+    </p>
+    <form method="post" action="consent">
+      <input type="hidden" name="v" value="{{version}}"/>
+      <input type="hidden" name="u" value="{{user}}"/>
+      <input type="hidden" name="h" value="{{userhmac}}"/>
+      <input type="submit" value="Sure thing!"/>
+    </form>
+  </body>
+</html>
diff --git a/docs/privacy_policy_templates/en/success.html b/docs/privacy_policy_templates/en/success.html
new file mode 100644
index 0000000000..d55e90c94f
--- /dev/null
+++ b/docs/privacy_policy_templates/en/success.html
@@ -0,0 +1,11 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <title>Matrix.org Privacy policy</title>
+  </head>
+  <body>
+    <p>
+      Sweet.
+    </p>
+  </body>
+</html>
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a0e465d644..730271628e 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -41,6 +41,7 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \
 from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
+from synapse.rest.consent.consent_resource import ConsentResource
 from synapse.rest.key.v1.server_key_resource import LocalKey
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
@@ -182,6 +183,14 @@ class SynapseHomeServer(HomeServer):
                 "/_matrix/client/versions": client_resource,
             })
 
+        if name == "consent":
+            consent_resource = ConsentResource(self)
+            if compress:
+                consent_resource = gz_wrap(consent_resource)
+            resources.update({
+                "/_matrix/consent": consent_resource,
+            })
+
         if name == "federation":
             resources.update({
                 FEDERATION_PREFIX: TransportLayerServer(self),
diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py
index bfebb0f644..f2a5a41e92 100644
--- a/synapse/config/__init__.py
+++ b/synapse/config/__init__.py
@@ -12,3 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from ._base import ConfigError
+
+# export ConfigError if somebody does import *
+# this is largely a fudge to stop PEP8 moaning about the import
+__all__ = ["ConfigError"]
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
new file mode 100644
index 0000000000..675fce0911
--- /dev/null
+++ b/synapse/config/consent_config.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+DEFAULT_CONFIG = """\
+# User Consent configuration
+#
+# uncomment and configure if enabling the 'consent' resource under 'listeners'.
+#
+# 'template_dir' gives the location of the templates for the HTML forms.
+# This directory should contain one subdirectory per language (eg, 'en', 'fr'),
+# and each language directory should contain the policy document (named as
+# '<version>.html') and a success page (success.html).
+#
+# 'default_version' gives the version of the policy document to serve up if
+# there is no 'v' parameter.
+#
+# user_consent:
+#   template_dir: res/templates/privacy
+#   default_version: 1.0
+"""
+
+
+class ConsentConfig(Config):
+    def read_config(self, config):
+        self.consent_config = config.get("user_consent")
+
+    def default_config(self, **kwargs):
+        return DEFAULT_CONFIG
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index bf19cfee29..fb6bd3b421 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 from .tls import TlsConfig
 from .server import ServerConfig
 from .logger import LoggingConfig
@@ -37,6 +37,7 @@ from .push import PushConfig
 from .spam_checker import SpamCheckerConfig
 from .groups import GroupsConfig
 from .user_directory import UserDirectoryConfig
+from .consent_config import ConsentConfig
 
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
@@ -45,12 +46,13 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                        AppServiceConfig, KeyConfig, SAML2Config, CasConfig,
                        JWTConfig, PasswordConfig, EmailConfig,
                        WorkerConfig, PasswordAuthProviderConfig, PushConfig,
-                       SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,):
+                       SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,
+                       ConsentConfig):
     pass
 
 
 if __name__ == '__main__':
     import sys
     sys.stdout.write(
-        HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0]
+        HomeServerConfig().generate_config(sys.argv[1], sys.argv[2], True)[0]
     )
diff --git a/synapse/config/key.py b/synapse/config/key.py
index 4b8fc063d0..d1382ad9ac 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -59,14 +59,20 @@ class KeyConfig(Config):
 
         self.expire_access_token = config.get("expire_access_token", False)
 
+        # a secret which is used to calculate HMACs for form values, to stop
+        # falsification of values
+        self.form_secret = config.get("form_secret", None)
+
     def default_config(self, config_dir_path, server_name, is_generating_file=False,
                        **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
 
         if is_generating_file:
             macaroon_secret_key = random_string_with_symbols(50)
+            form_secret = '"%s"' % random_string_with_symbols(50)
         else:
             macaroon_secret_key = None
+            form_secret = 'null'
 
         return """\
         macaroon_secret_key: "%(macaroon_secret_key)s"
@@ -74,6 +80,10 @@ class KeyConfig(Config):
         # Used to enable access token expiration.
         expire_access_token: False
 
+        # a secret which is used to calculate HMACs for form values, to stop
+        # falsification of values
+        form_secret: %(form_secret)s
+
         ## Signing Keys ##
 
         # Path to the signing key to sign messages with
diff --git a/synapse/http/server.py b/synapse/http/server.py
index f29e36f490..a38209770d 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -13,7 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import cgi
+from six.moves import http_client
 
 from synapse.api.errors import (
     cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
@@ -44,6 +45,18 @@ import simplejson
 
 logger = logging.getLogger(__name__)
 
+HTML_ERROR_TEMPLATE = """<!DOCTYPE html>
+<html lang=en>
+  <head>
+    <meta charset="utf-8">
+    <title>Error {code}</title>
+  </head>
+  <body>
+     <p>{msg}</p>
+  </body>
+</html>
+"""
+
 
 def wrap_json_request_handler(h):
     """Wraps a request handler method with exception handling.
@@ -104,6 +117,65 @@ def wrap_json_request_handler(h):
     return wrap_request_handler_with_logging(wrapped_request_handler)
 
 
+def wrap_html_request_handler(h):
+    """Wraps a request handler method with exception handling.
+
+    Also adds logging as per wrap_request_handler_with_logging.
+
+    The handler method must have a signature of "handle_foo(self, request)",
+    where "self" must have a "clock" attribute (and "request" must be a
+    SynapseRequest).
+    """
+    def wrapped_request_handler(self, request):
+        d = defer.maybeDeferred(h, self, request)
+        d.addErrback(_return_html_error, request)
+        return d
+
+    return wrap_request_handler_with_logging(wrapped_request_handler)
+
+
+def _return_html_error(f, request):
+    """Sends an HTML error page corresponding to the given failure
+
+    Args:
+        f (twisted.python.failure.Failure):
+        request (twisted.web.iweb.IRequest):
+    """
+    if f.check(CodeMessageException):
+        cme = f.value
+        code = cme.code
+        msg = cme.msg
+
+        if isinstance(cme, SynapseError):
+            logger.info(
+                "%s SynapseError: %s - %s", request, code, msg
+            )
+        else:
+            logger.error(
+                "Failed handle request %r: %s",
+                request,
+                f.getTraceback().rstrip(),
+            )
+    else:
+        code = http_client.INTERNAL_SERVER_ERROR
+        msg = "Internal server error"
+
+        logger.error(
+            "Failed handle request %r: %s",
+            request,
+            f.getTraceback().rstrip(),
+        )
+
+    body = HTML_ERROR_TEMPLATE.format(
+        code=code, msg=cgi.escape(msg),
+    ).encode("utf-8")
+    request.setResponseCode(code)
+    request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+    request.setHeader(b"Content-Length", b"%i" % (len(body),))
+    request.write(body)
+    finish_request(request)
+
+
 def wrap_request_handler_with_logging(h):
     """Wraps a request handler to provide logging and metrics
 
@@ -134,7 +206,7 @@ def wrap_request_handler_with_logging(h):
                 servlet_name = self.__class__.__name__
                 with request.processing(servlet_name):
                     with PreserveLoggingContext(request_context):
-                        d = h(self, request)
+                        d = defer.maybeDeferred(h, self, request)
 
                         # record the arrival of the request *after*
                         # dispatching to the handler, so that the handler
diff --git a/synapse/rest/consent/__init__.py b/synapse/rest/consent/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
new file mode 100644
index 0000000000..d791302278
--- /dev/null
+++ b/synapse/rest/consent/consent_resource.py
@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from hashlib import sha256
+import hmac
+import logging
+from os import path
+from six.moves import http_client
+
+import jinja2
+from jinja2 import TemplateNotFound
+from twisted.internet import defer
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+
+from synapse.api.errors import NotFoundError, SynapseError, StoreError
+from synapse.config import ConfigError
+from synapse.http.server import (
+    finish_request,
+    wrap_html_request_handler,
+)
+from synapse.http.servlet import parse_string
+from synapse.types import UserID
+
+
+# language to use for the templates. TODO: figure this out from Accept-Language
+TEMPLATE_LANGUAGE = "en"
+
+logger = logging.getLogger(__name__)
+
+# use hmac.compare_digest if we have it (python 2.7.7), else just use equality
+if hasattr(hmac, "compare_digest"):
+    compare_digest = hmac.compare_digest
+else:
+    def compare_digest(a, b):
+        return a == b
+
+
+class ConsentResource(Resource):
+    """A twisted Resource to display a privacy policy and gather consent to it
+
+    When accessed via GET, returns the privacy policy via a template.
+
+    When accessed via POST, records the user's consent in the database and
+    displays a success page.
+
+    The config should include a template_dir setting which contains templates
+    for the HTML. The directory should contain one subdirectory per language
+    (eg, 'en', 'fr'), and each language directory should contain the policy
+    document (named as '<version>.html') and a success page (success.html).
+
+    Both forms take a set of parameters from the browser. For the POST form,
+    these are normally sent as form parameters (but may be query-params); for
+    GET requests they must be query params. These are:
+
+        u: the complete mxid, or the localpart of the user giving their
+           consent. Required for both GET (where it is used as an input to the
+           template) and for POST (where it is used to find the row in the db
+           to update).
+
+        h: hmac_sha256(secret, u), where 'secret' is the privacy_secret in the
+           config file. If it doesn't match, the request is 403ed.
+
+        v: the version of the privacy policy being agreed to.
+
+           For GET: optional, and defaults to whatever was set in the config
+           file. Used to choose the version of the policy to pick from the
+           templates directory.
+
+           For POST: required; gives the value to be recorded in the database
+           against the user.
+    """
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): homeserver
+        """
+        Resource.__init__(self)
+
+        self.hs = hs
+        self.store = hs.get_datastore()
+
+        # this is required by the request_handler wrapper
+        self.clock = hs.get_clock()
+
+        consent_config = hs.config.consent_config
+        if consent_config is None:
+            raise ConfigError(
+                "Consent resource is enabled but user_consent section is "
+                "missing in config file.",
+            )
+
+        # daemonize changes the cwd to /, so make the path absolute now.
+        consent_template_directory = path.abspath(
+            consent_config["template_dir"],
+        )
+        if not path.isdir(consent_template_directory):
+            raise ConfigError(
+                "Could not find template directory '%s'" % (
+                    consent_template_directory,
+                ),
+            )
+
+        loader = jinja2.FileSystemLoader(consent_template_directory)
+        self._jinja_env = jinja2.Environment(loader=loader)
+
+        self._default_consent_verison = consent_config["default_version"]
+
+        if hs.config.form_secret is None:
+            raise ConfigError(
+                "Consent resource is enabled but form_secret is not set in "
+                "config file. It should be set to an arbitrary secret string.",
+            )
+
+        self._hmac_secret = hs.config.form_secret.encode("utf-8")
+
+    def render_GET(self, request):
+        self._async_render_GET(request)
+        return NOT_DONE_YET
+
+    @wrap_html_request_handler
+    def _async_render_GET(self, request):
+        """
+        Args:
+            request (twisted.web.http.Request):
+        """
+
+        version = parse_string(request, "v",
+                               default=self._default_consent_verison)
+        username = parse_string(request, "u", required=True)
+        userhmac = parse_string(request, "h", required=True)
+
+        self._check_hash(username, userhmac)
+
+        try:
+            self._render_template(
+                request, "%s.html" % (version,),
+                user=username, userhmac=userhmac, version=version,
+            )
+        except TemplateNotFound:
+            raise NotFoundError("Unknown policy version")
+
+    def render_POST(self, request):
+        self._async_render_POST(request)
+        return NOT_DONE_YET
+
+    @wrap_html_request_handler
+    @defer.inlineCallbacks
+    def _async_render_POST(self, request):
+        """
+        Args:
+            request (twisted.web.http.Request):
+        """
+        version = parse_string(request, "v", required=True)
+        username = parse_string(request, "u", required=True)
+        userhmac = parse_string(request, "h", required=True)
+
+        self._check_hash(username, userhmac)
+
+        if username.startswith('@'):
+            qualified_user_id = username
+        else:
+            qualified_user_id = UserID(username, self.hs.hostname).to_string()
+
+        try:
+            yield self.store.user_set_consent_version(qualified_user_id, version)
+        except StoreError as e:
+            if e.code != 404:
+                raise
+            raise NotFoundError("Unknown user")
+
+        try:
+            self._render_template(request, "success.html")
+        except TemplateNotFound:
+            raise NotFoundError("success.html not found")
+
+    def _render_template(self, request, template_name, **template_args):
+        # get_template checks for ".." so we don't need to worry too much
+        # about path traversal here.
+        template_html = self._jinja_env.get_template(
+            path.join(TEMPLATE_LANGUAGE, template_name)
+        )
+        html_bytes = template_html.render(**template_args).encode("utf8")
+
+        request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+        request.setHeader(b"Content-Length", b"%i" % len(html_bytes))
+        request.write(html_bytes)
+        finish_request(request)
+
+    def _check_hash(self, userid, userhmac):
+        want_mac = hmac.new(
+            key=self._hmac_secret,
+            msg=userid,
+            digestmod=sha256,
+        ).hexdigest()
+
+        if not compare_digest(want_mac, userhmac):
+            raise SynapseError(http_client.FORBIDDEN, "HMAC incorrect")
diff --git a/synapse/server.py b/synapse/server.py
index ebdea6b0c4..21cde5b6fc 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -97,6 +97,9 @@ class HomeServer(object):
     which must be implemented by the subclass. This code may call any of the
     required "get" methods on the instance to obtain the sub-dependencies that
     one requires.
+
+    Attributes:
+        config (synapse.config.homeserver.HomeserverConfig):
     """
 
     DEPENDENCIES = [
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index a50717db2d..6ffc397861 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -286,6 +286,24 @@ class RegistrationStore(RegistrationWorkerStore,
             "user_set_password_hash", user_set_password_hash_txn
         )
 
+    def user_set_consent_version(self, user_id, consent_version):
+        """Updates the user table to record privacy policy consent
+
+        Args:
+            user_id (str): full mxid of the user to update
+            consent_version (str): version of the policy the user has consented
+                to
+
+        Raises:
+            StoreError(404) if user not found
+        """
+        return self._simple_update_one(
+            table='users',
+            keyvalues={'name': user_id, },
+            updatevalues={'consent_version': consent_version, },
+            desc="user_set_consent_version"
+        )
+
     def user_delete_access_tokens(self, user_id, except_token_id=None,
                                   device_id=None):
         """
diff --git a/synapse/storage/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/delta/48/add_user_consent.sql
new file mode 100644
index 0000000000..5237491506
--- /dev/null
+++ b/synapse/storage/schema/delta/48/add_user_consent.sql
@@ -0,0 +1,18 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* record the version of the privacy policy the user has consented to
+ */
+ALTER TABLE users ADD COLUMN consent_version TEXT;
-- 
cgit 1.4.1


From 37dbee6490dbb9cd66d887ed4d902ae8482fae2f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 15 May 2018 15:48:40 +0100
Subject: Use events_to_purge table rather than token

---
 synapse/storage/events.py | 49 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 05cde96afc..b4ae6664f0 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1871,6 +1871,10 @@ class EventsStore(EventsWorkerStore):
             "CREATE INDEX events_to_purge_should_delete"
             " ON events_to_purge(should_delete)",
         )
+        txn.execute(
+            "CREATE INDEX events_to_purge_id"
+            " ON events_to_purge(event_id)",
+        )
 
         # First ensure that we're not about to delete all the forward extremeties
         txn.execute(
@@ -1927,9 +1931,8 @@ class EventsStore(EventsWorkerStore):
         txn.execute(
             "SELECT DISTINCT e.event_id FROM events_to_purge AS e"
             " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
-            " INNER JOIN events AS e2 ON e2.event_id = ed.event_id"
-            " WHERE e2.topological_ordering >= ?",
-            (topological_ordering, )
+            " LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
+            " WHERE ep2.event_id IS NULL",
         )
         new_backwards_extrems = txn.fetchall()
 
@@ -1953,16 +1956,22 @@ class EventsStore(EventsWorkerStore):
 
         # Get all state groups that are only referenced by events that are
         # to be deleted.
-        txn.execute(
-            "SELECT state_group FROM event_to_state_groups"
-            " INNER JOIN events USING (event_id)"
-            " WHERE state_group IN ("
-            "   SELECT DISTINCT state_group FROM events_to_purge"
-            "   INNER JOIN event_to_state_groups USING (event_id)"
-            " )"
-            " GROUP BY state_group HAVING MAX(topological_ordering) < ?",
-            (topological_ordering, )
-        )
+        # This works by first getting state groups that we may want to delete,
+        # joining against event_to_state_groups to get events that use that
+        # state group, then left joining against events_to_purge again. Any
+        # state group where the left join produce *no nulls* are referenced
+        # only by events that are going to be purged.
+        txn.execute("""
+            SELECT state_group FROM
+            (
+                SELECT DISTINCT state_group FROM events_to_purge
+                INNER JOIN event_to_state_groups USING (event_id)
+            ) AS sp
+            INNER JOIN event_to_state_groups USING (state_group)
+            LEFT JOIN events_to_purge AS ep USING (event_id)
+            GROUP BY state_group
+            HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0
+        """)
 
         state_rows = txn.fetchall()
         logger.info("[purge] found %i redundant state groups", len(state_rows))
@@ -2109,10 +2118,20 @@ class EventsStore(EventsWorkerStore):
         #
         # So, let's stick it at the end so that we don't block event
         # persistence.
-        logger.info("[purge] updating room_depth")
+        txn.execute("""
+            SELECT COALESCE(MIN(depth), 0)
+            FROM event_backward_extremities AS eb
+            INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
+            INNER JOIN events AS e ON e.event_id = eg.event_id
+            WHERE eb.room_id = ?
+        """, (room_id,))
+        min_depth, = txn.fetchone()
+
+        logger.info("[purge] updating room_depth to %d", min_depth)
+
         txn.execute(
             "UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
-            (topological_ordering, room_id,)
+            (min_depth, room_id,)
         )
 
         # finally, drop the temp table. this will commit the txn in sqlite,
-- 
cgit 1.4.1


From 5f27ed75ad804ab9b5287f0deb8fd24b9a3a6232 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 15 May 2018 16:06:30 +0100
Subject: Make purge_history operate on tokens

As we're soon going to change how topological_ordering works
---
 synapse/handlers/message.py     | 12 ++++++------
 synapse/rest/client/v1/admin.py | 17 ++++++++++-------
 synapse/storage/events.py       | 17 +++++++++--------
 3 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index b793fc4df7..8343b5839d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -86,14 +86,14 @@ class MessageHandler(BaseHandler):
         # map from purge id to PurgeStatus
         self._purges_by_id = {}
 
-    def start_purge_history(self, room_id, topological_ordering,
+    def start_purge_history(self, room_id, token,
                             delete_local_events=False):
         """Start off a history purge on a room.
 
         Args:
             room_id (str): The room to purge from
 
-            topological_ordering (int): minimum topo ordering to preserve
+            token (str): topological token to delete events before
             delete_local_events (bool): True to delete local events as well as
                 remote ones
 
@@ -115,19 +115,19 @@ class MessageHandler(BaseHandler):
         self._purges_by_id[purge_id] = PurgeStatus()
         run_in_background(
             self._purge_history,
-            purge_id, room_id, topological_ordering, delete_local_events,
+            purge_id, room_id, token, delete_local_events,
         )
         return purge_id
 
     @defer.inlineCallbacks
-    def _purge_history(self, purge_id, room_id, topological_ordering,
+    def _purge_history(self, purge_id, room_id, token,
                        delete_local_events):
         """Carry out a history purge on a room.
 
         Args:
             purge_id (str): The id for this purge
             room_id (str): The room to purge from
-            topological_ordering (int): minimum topo ordering to preserve
+            token (str): topological token to delete events before
             delete_local_events (bool): True to delete local events as well as
                 remote ones
 
@@ -138,7 +138,7 @@ class MessageHandler(BaseHandler):
         try:
             with (yield self.pagination_lock.write(room_id)):
                 yield self.store.purge_history(
-                    room_id, topological_ordering, delete_local_events,
+                    room_id, token, delete_local_events,
                 )
             logger.info("[purge] complete")
             self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index efd5c9873d..282ce6be42 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -151,10 +151,11 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
             if event.room_id != room_id:
                 raise SynapseError(400, "Event is for wrong room.")
 
-            depth = event.depth
+            token = yield self.store.get_topological_token_for_event(event_id)
+
             logger.info(
-                "[purge] purging up to depth %i (event_id %s)",
-                depth, event_id,
+                "[purge] purging up to token %s (event_id %s)",
+                token, event_id,
             )
         elif 'purge_up_to_ts' in body:
             ts = body['purge_up_to_ts']
@@ -174,7 +175,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
                 )
             )
             if room_event_after_stream_ordering:
-                (_, depth, _) = room_event_after_stream_ordering
+                token = yield self.store.get_topological_token_for_event(
+                    room_event_after_stream_ordering,
+                )
             else:
                 logger.warn(
                     "[purge] purging events not possible: No event found "
@@ -187,9 +190,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
                     errcode=Codes.NOT_FOUND,
                 )
             logger.info(
-                "[purge] purging up to depth %i (received_ts %i => "
+                "[purge] purging up to token %d (received_ts %i => "
                 "stream_ordering %i)",
-                depth, ts, stream_ordering,
+                token, ts, stream_ordering,
             )
         else:
             raise SynapseError(
@@ -199,7 +202,7 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
             )
 
         purge_id = yield self.handlers.message_handler.start_purge_history(
-            room_id, depth,
+            room_id, token,
             delete_local_events=delete_local_events,
         )
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index b4ae6664f0..f65e18c1ee 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -33,7 +33,7 @@ from synapse.util.metrics import Measure
 from synapse.api.constants import EventTypes
 from synapse.api.errors import SynapseError
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
-from synapse.types import get_domain_from_id
+from synapse.types import get_domain_from_id, RoomStreamToken
 import synapse.metrics
 
 # these are only included to make the type annotations work
@@ -1803,15 +1803,14 @@ class EventsStore(EventsWorkerStore):
         return self.runInteraction("get_all_new_events", get_all_new_events_txn)
 
     def purge_history(
-        self, room_id, topological_ordering, delete_local_events,
+        self, room_id, token, delete_local_events,
     ):
         """Deletes room history before a certain point
 
         Args:
             room_id (str):
 
-            topological_ordering (int):
-                minimum topo ordering to preserve
+            token (str): A topological token to delete events before
 
             delete_local_events (bool):
                 if True, we will delete local events as well as remote ones
@@ -1821,12 +1820,12 @@ class EventsStore(EventsWorkerStore):
 
         return self.runInteraction(
             "purge_history",
-            self._purge_history_txn, room_id, topological_ordering,
+            self._purge_history_txn, room_id, token,
             delete_local_events,
         )
 
     def _purge_history_txn(
-        self, txn, room_id, topological_ordering, delete_local_events,
+        self, txn, room_id, token, delete_local_events,
     ):
         # Tables that should be pruned:
         #     event_auth
@@ -1856,6 +1855,8 @@ class EventsStore(EventsWorkerStore):
         # furthermore, we might already have the table from a previous (failed)
         # purge attempt, so let's drop the table first.
 
+        token = RoomStreamToken.parse(token)
+
         txn.execute("DROP TABLE IF EXISTS events_to_purge")
 
         txn.execute(
@@ -1888,7 +1889,7 @@ class EventsStore(EventsWorkerStore):
         rows = txn.fetchall()
         max_depth = max(row[0] for row in rows)
 
-        if max_depth <= topological_ordering:
+        if max_depth <= token.topological:
             # We need to ensure we don't delete all the events from the datanase
             # otherwise we wouldn't be able to send any events (due to not
             # having any backwards extremeties)
@@ -1904,7 +1905,7 @@ class EventsStore(EventsWorkerStore):
             should_delete_expr += " AND event_id NOT LIKE ?"
             should_delete_params += ("%:" + self.hs.hostname, )
 
-        should_delete_params += (room_id, topological_ordering)
+        should_delete_params += (room_id, token.topological)
 
         txn.execute(
             "INSERT INTO events_to_purge"
-- 
cgit 1.4.1


From 05ac15ae824cc538b869e3cc8db7af2ac22e6754 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 15 May 2018 17:01:33 +0100
Subject: Limit query load of generate_user_daily_visits

The aim is to keep track of when it was last called and only query from that point in time
---
 synapse/app/homeserver.py   | 21 ++++++----------
 synapse/storage/__init__.py | 60 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index bfc79a5e81..f25eaf9ffc 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -476,23 +476,16 @@ def run(hs):
                 " changes across releases."
             )
 
-    # def recurring_user_daily_visit_stats():
-
     def generate_user_daily_visit_stats():
         hs.get_datastore().generate_user_daily_visits()
 
-    # Since user daily stats are bucketed at midnight UTC,
-    # and user_ips.last_seen can be updated at any time, it is important to call
-    # generate_user_daily_visit_stats immediately prior to the day end. Assuming
-    # an hourly cadence, the simplist way is to allign all calls to the hour
-    # end
-    end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \
-        + datetime.timedelta(hours=1) \
-        - datetime.timedelta(seconds=10)  # Ensure method fires before day transistion
-
-    time_to_next_hour = end_of_hour - datetime.datetime.now()
-    clock.call_later(time_to_next_hour.seconds,
-                     clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000))
+    def recurring_user_daily_visit_stats():
+        clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)
+
+    # Rather than update on per session basis, batch up the requests.
+    # If you increase the loop period, the accuracy of user_daily_visits
+    # table will decrease
+    clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6949876c13..52f176a03c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -214,6 +214,9 @@ class DataStore(RoomMemberStore, RoomStore,
         self._stream_order_on_start = self.get_room_max_stream_ordering()
         self._min_stream_order_on_start = self.get_room_min_stream_ordering()
 
+        # Used in _generate_user_daily_visits to keep track of progress
+        self._last_user_visit_update = self._get_start_of_day()
+
         super(DataStore, self).__init__(db_conn, hs)
 
     def take_presence_startup_info(self):
@@ -348,27 +351,58 @@ class DataStore(RoomMemberStore, RoomStore,
 
         return self.runInteraction("count_r30_users", _count_r30_users)
 
+    def _get_start_of_day(self):
+        """
+        Returns millisecond unixtime for start of UTC day.
+        """
+        now = datetime.datetime.utcnow()
+        today_start = datetime.datetime(now.year, now.month,
+                                        now.day, tzinfo=tz.tzutc())
+        return int(time.mktime(today_start.timetuple())) * 1000
+
     def generate_user_daily_visits(self):
         """
         Generates daily visit data for use in cohort/ retention analysis
         """
         def _generate_user_daily_visits(txn):
+            logger.info("Calling _generate_user_daily_visits")
+            today_start = self._get_start_of_day()
+            a_day_in_milliseconds = 24 * 60 * 60 * 1000
 
-            # determine timestamp of the day start
-            now = datetime.datetime.utcnow()
-            today_start = datetime.datetime(now.year, now.month,
-                                            now.day, tzinfo=tz.tzutc())
-            today_start_time = int(time.mktime(today_start.timetuple())) * 1000
-            logger.info(today_start_time)
             sql = """
                 INSERT INTO user_daily_visits (user_id, device_id, timestamp)
-                SELECT user_id, device_id, ?
-                FROM user_ips AS u
-                LEFT JOIN user_daily_visits USING (user_id, device_id)
-                WHERE last_seen > ? AND timestamp IS NULL
-                GROUP BY user_id, device_id;
-                """
-            txn.execute(sql, (today_start_time, today_start_time))
+                    SELECT u.user_id, u.device_id, ?
+                    FROM user_ips AS u
+                    LEFT JOIN (
+                      SELECT user_id, device_id, timestamp FROM user_daily_visits
+                      WHERE timestamp IS ?
+                    ) udv
+                    ON u.user_id = udv.user_id AND u.device_id=udv.device_id
+                    WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL
+            """
+
+            # This means that the day has rolled over but there could still
+            # be entries from the previous day. There is an edge case
+            # where if the user logs in at 23:59 and overwrites their
+            # last_seen at 00:01 then they will not be counted in the
+            # previous day's stats - it is important that the query is run
+            # to minimise this case.
+            if today_start > self._last_user_visit_update:
+                yesterday_start = today_start - a_day_in_milliseconds
+                txn.execute(sql, (yesterday_start, yesterday_start,
+                                  self._last_user_visit_update, today_start))
+                self._last_user_visit_update = today_start
+
+            txn.execute(sql, (today_start, today_start,
+                              self._last_user_visit_update,
+                              today_start + a_day_in_milliseconds))
+            # Update _last_user_visit_update to now. The reason to do this
+            # rather just clamping to the beginning of the day is to limit
+            # the size of the join - meaning that the query can be run more
+            # frequently
+
+            now = datetime.datetime.utcnow()
+            self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000
 
         return self.runInteraction("generate_user_daily_visits",
                                    _generate_user_daily_visits)
-- 
cgit 1.4.1


From c92a8aa5780a71b5ab0227f204606004b16ff7fb Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Tue, 15 May 2018 17:31:11 +0100
Subject: pep8

---
 synapse/app/homeserver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f25eaf9ffc..41ee8b0a69 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -17,7 +17,6 @@ import gc
 import logging
 import os
 import sys
-import datetime
 
 import synapse
 import synapse.config.logger
-- 
cgit 1.4.1


From 31c2502ca8d74797f1eae553690830b8c132aed9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 16 May 2018 09:46:43 +0100
Subject: style and further contraining query

---
 synapse/storage/__init__.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 52f176a03c..6f324e075f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -368,6 +368,7 @@ class DataStore(RoomMemberStore, RoomStore,
             logger.info("Calling _generate_user_daily_visits")
             today_start = self._get_start_of_day()
             a_day_in_milliseconds = 24 * 60 * 60 * 1000
+            now = self.clock.time_msec()
 
             sql = """
                 INSERT INTO user_daily_visits (user_id, device_id, timestamp)
@@ -386,23 +387,26 @@ class DataStore(RoomMemberStore, RoomStore,
             # where if the user logs in at 23:59 and overwrites their
             # last_seen at 00:01 then they will not be counted in the
             # previous day's stats - it is important that the query is run
-            # to minimise this case.
+            # often to minimise this case.
             if today_start > self._last_user_visit_update:
                 yesterday_start = today_start - a_day_in_milliseconds
-                txn.execute(sql, (yesterday_start, yesterday_start,
-                                  self._last_user_visit_update, today_start))
+                txn.execute(sql, (
+                    yesterday_start, yesterday_start,
+                    self._last_user_visit_update, today_start
+                ))
                 self._last_user_visit_update = today_start
 
-            txn.execute(sql, (today_start, today_start,
-                              self._last_user_visit_update,
-                              today_start + a_day_in_milliseconds))
+            txn.execute(sql, (
+                today_start, today_start,
+                self._last_user_visit_update,
+                now
+            ))
             # Update _last_user_visit_update to now. The reason to do this
             # rather just clamping to the beginning of the day is to limit
             # the size of the join - meaning that the query can be run more
             # frequently
 
-            now = datetime.datetime.utcnow()
-            self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000
+            self._last_user_visit_update = now
 
         return self.runInteraction("generate_user_daily_visits",
                                    _generate_user_daily_visits)
-- 
cgit 1.4.1


From a2204cc9cc3f9edb9ae6c1f564b8f315e62d1978 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 16 May 2018 09:47:20 +0100
Subject: remove unused method recurring_user_daily_visit_stats

---
 synapse/app/homeserver.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 41ee8b0a69..a4648b84d5 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -478,9 +478,6 @@ def run(hs):
     def generate_user_daily_visit_stats():
         hs.get_datastore().generate_user_daily_visits()
 
-    def recurring_user_daily_visit_stats():
-        clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)
-
     # Rather than update on per session basis, batch up the requests.
     # If you increase the loop period, the accuracy of user_daily_visits
     # table will decrease
-- 
cgit 1.4.1


From be11a02c4f854452687f0f54492bc53b4827637d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 16 May 2018 10:45:40 +0100
Subject: remove empty line

---
 synapse/storage/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6f324e075f..4551cf8774 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -405,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore,
             # rather just clamping to the beginning of the day is to limit
             # the size of the join - meaning that the query can be run more
             # frequently
-
             self._last_user_visit_update = now
 
         return self.runInteraction("generate_user_daily_visits",
-- 
cgit 1.4.1


From c945af8799671db8fdfc47acf6b286520fb08715 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 May 2018 10:52:06 +0100
Subject: Move and rename variable

---
 synapse/storage/events.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f65e18c1ee..22699cb689 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1825,8 +1825,10 @@ class EventsStore(EventsWorkerStore):
         )
 
     def _purge_history_txn(
-        self, txn, room_id, token, delete_local_events,
+        self, txn, room_id, token_str, delete_local_events,
     ):
+        token = RoomStreamToken.parse(token_str)
+
         # Tables that should be pruned:
         #     event_auth
         #     event_backward_extremities
@@ -1855,8 +1857,6 @@ class EventsStore(EventsWorkerStore):
         # furthermore, we might already have the table from a previous (failed)
         # purge attempt, so let's drop the table first.
 
-        token = RoomStreamToken.parse(token)
-
         txn.execute("DROP TABLE IF EXISTS events_to_purge")
 
         txn.execute(
-- 
cgit 1.4.1


From 43e6e82c4d368d981f8a0015246d19c333511ace Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 May 2018 11:13:31 +0100
Subject: Comments

---
 synapse/storage/events.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 22699cb689..93ac80ee48 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1872,6 +1872,9 @@ class EventsStore(EventsWorkerStore):
             "CREATE INDEX events_to_purge_should_delete"
             " ON events_to_purge(should_delete)",
         )
+
+        # We do joins against events_to_purge for e.g. calculating state
+        # groups to purge, etc., so lets make an index.
         txn.execute(
             "CREATE INDEX events_to_purge_id"
             " ON events_to_purge(event_id)",
@@ -2119,6 +2122,11 @@ class EventsStore(EventsWorkerStore):
         #
         # So, let's stick it at the end so that we don't block event
         # persistence.
+        #
+        # We do this by calculating the minimum depth of the backwards
+        # extremities. However, the events in event_backward_extremities
+        # are ones we don't have yet so we need to look at the events that
+        # point to it via event_edges table.
         txn.execute("""
             SELECT COALESCE(MIN(depth), 0)
             FROM event_backward_extremities AS eb
-- 
cgit 1.4.1


From 680530cc7f80533b9d63ce9bed04dc28ca87ca13 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 May 2018 11:47:26 +0100
Subject: Clarify comment

---
 synapse/storage/events.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 93ac80ee48..5ebef98c4f 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1931,7 +1931,8 @@ class EventsStore(EventsWorkerStore):
         logger.info("[purge] Finding new backward extremities")
 
         # We calculate the new entries for the backward extremeties by finding
-        # all events that point to events that are to be purged
+        # events to be purged that are pointed to by events we're not going to
+        # purge.
         txn.execute(
             "SELECT DISTINCT e.event_id FROM events_to_purge AS e"
             " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
-- 
cgit 1.4.1


From 3c099219e0506f81fc333ff9d27cc3af5c365a2c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 16 May 2018 15:44:59 +0100
Subject: bump version and changelog for 0.29.0

---
 CHANGES.rst         | 4 ++++
 synapse/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 78d015db9a..b769b0f046 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,7 @@
+Changes in synapse v0.29.0 (2018-05-16)
+===========================================
+
+
 Changes in synapse v0.29.0-rc1 (2018-05-14)
 ===========================================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 69c081eb37..d94c20505e 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.29.0-rc1"
+__version__ = "0.29.0"
-- 
cgit 1.4.1


From c46367d0d755e810745cae6d0212bdaa12de7645 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 09:01:09 +0100
Subject: Move RoomCreationHandler out of synapse.handlers.Handlers

Handlers is deprecated nowadays, so let's move this out before I add a new
dependency on it.

Also fix the docstrings on create_room.
---
 synapse/handlers/__init__.py    | 5 +----
 synapse/handlers/room.py        | 8 ++++++--
 synapse/rest/client/v1/admin.py | 4 ++--
 synapse/rest/client/v1/room.py  | 5 ++---
 synapse/server.py               | 5 +++++
 synapse/server.pyi              | 3 +++
 6 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 8f8fd82eb0..d358842b3e 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -14,9 +14,7 @@
 # limitations under the License.
 
 from .register import RegistrationHandler
-from .room import (
-    RoomCreationHandler, RoomContextHandler,
-)
+from .room import RoomContextHandler
 from .message import MessageHandler
 from .federation import FederationHandler
 from .directory import DirectoryHandler
@@ -47,7 +45,6 @@ class Handlers(object):
     def __init__(self, hs):
         self.registration_handler = RegistrationHandler(hs)
         self.message_handler = MessageHandler(hs)
-        self.room_creation_handler = RoomCreationHandler(hs)
         self.federation_handler = FederationHandler(hs)
         self.directory_handler = DirectoryHandler(hs)
         self.admin_handler = AdminHandler(hs)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 8df8fcbbad..e36426de5a 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -72,10 +72,14 @@ class RoomCreationHandler(BaseHandler):
         """ Creates a new room.
 
         Args:
-            requester (Requester): The user who requested the room creation.
+            requester (synapse.types.Requester):
+                The user who requested the room creation.
             config (dict) : A dict of configuration options.
+            ratelimit (bool): set to False to disable the rate limiter
         Returns:
-            The new room ID.
+            Deferred[dict]:
+                a dict containing the keys `room_id` and, if an alias was
+                requested, `room_alias`.
         Raises:
             SynapseError if the room ID couldn't be stored, or something went
             horribly wrong.
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index efd5c9873d..646a95c9fb 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -273,8 +273,8 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(ShutdownRoomRestServlet, self).__init__(hs)
         self.store = hs.get_datastore()
-        self.handlers = hs.get_handlers()
         self.state = hs.get_state_handler()
+        self._room_creation_handler = hs.get_room_creation_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
 
@@ -296,7 +296,7 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
         message = content.get("message", self.DEFAULT_MESSAGE)
         room_name = content.get("room_name", "Content Violation Notification")
 
-        info = yield self.handlers.room_creation_handler.create_room(
+        info = yield self._room_creation_handler.create_room(
             room_creator_requester,
             config={
                 "preset": "public_chat",
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index fcf9c9ab44..0b984987ed 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -41,7 +41,7 @@ class RoomCreateRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomCreateRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self._room_creation_handler = hs.get_room_creation_handler()
 
     def register(self, http_server):
         PATTERNS = "/createRoom"
@@ -64,8 +64,7 @@ class RoomCreateRestServlet(ClientV1RestServlet):
     def on_POST(self, request):
         requester = yield self.auth.get_user_by_req(request)
 
-        handler = self.handlers.room_creation_handler
-        info = yield handler.create_room(
+        info = yield self._room_creation_handler.create_room(
             requester, self.get_room_config(request)
         )
 
diff --git a/synapse/server.py b/synapse/server.py
index 21cde5b6fc..9e6f3584b2 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -46,6 +46,7 @@ from synapse.handlers.devicemessage import DeviceMessageHandler
 from synapse.handlers.device import DeviceHandler
 from synapse.handlers.e2e_keys import E2eKeysHandler
 from synapse.handlers.presence import PresenceHandler
+from synapse.handlers.room import RoomCreationHandler
 from synapse.handlers.room_list import RoomListHandler
 from synapse.handlers.room_member import RoomMemberMasterHandler
 from synapse.handlers.room_member_worker import RoomMemberWorkerHandler
@@ -109,6 +110,7 @@ class HomeServer(object):
         'federation_server',
         'handlers',
         'auth',
+        'room_creation_handler',
         'state_handler',
         'state_resolution_handler',
         'presence_handler',
@@ -227,6 +229,9 @@ class HomeServer(object):
     def build_simple_http_client(self):
         return SimpleHttpClient(self)
 
+    def build_room_creation_handler(self):
+        return RoomCreationHandler(self)
+
     def build_state_handler(self):
         return StateHandler(self)
 
diff --git a/synapse/server.pyi b/synapse/server.pyi
index c3a9a3847b..aeda093f27 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -40,6 +40,9 @@ class HomeServer(object):
     def get_deactivate_account_handler(self) -> synapse.handlers.deactivate_account.DeactivateAccountHandler:
         pass
 
+    def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler:
+        pass
+
     def get_set_password_handler(self) -> synapse.handlers.set_password.SetPasswordHandler:
         pass
 
-- 
cgit 1.4.1


From 616da9eb1d50d9653d2cc122bedecdb0a6ca6487 Mon Sep 17 00:00:00 2001
From: rubo77 <github@r.z11.de>
Date: Wed, 16 May 2018 23:31:19 +0200
Subject: postgres.rst: Add instructions how to setup the postgres user and
 clarify the final step

---
 docs/postgres.rst | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/docs/postgres.rst b/docs/postgres.rst
index 904942ec74..296293e859 100644
--- a/docs/postgres.rst
+++ b/docs/postgres.rst
@@ -6,7 +6,13 @@ Postgres version 9.4 or later is known to work.
 Set up database
 ===============
 
-The PostgreSQL database used *must* have the correct encoding set, otherwise
+Assuming your PostgreSQL database user is called ``postgres``, create a user
+``synapse_user`` with::
+
+ su - postgres
+ createuser --pwprompt synapse_user
+
+The PostgreSQL database used *must* have the correct encoding set, otherwise it
 would not be able to store UTF8 strings. To create a database with the correct
 encoding use, e.g.::
 
@@ -46,8 +52,8 @@ As with Debian/Ubuntu, postgres support depends on the postgres python connector
 Synapse config
 ==============
 
-When you are ready to start using PostgreSQL, add the following line to your
-config file::
+When you are ready to start using PostgreSQL, edit the ``database`` section in
+your config file to match the following lines::
 
     database:
         name: psycopg2
@@ -96,9 +102,12 @@ complete, restart synapse.  For instance::
     cp homeserver.db homeserver.db.snapshot
     ./synctl start
 
-Assuming your new config file (as described in the section *Synapse config*)
-is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at
-``homeserver.db.snapshot`` then simply run::
+Copy the old config file into a new config file::
+
+    cp homeserver.yaml homeserver-postgres.yaml
+
+Edit the database section as described in the section *Synapse config* above
+and with the SQLite snapshot located at ``homeserver.db.snapshot`` simply run::
 
     synapse_port_db --sqlite-database homeserver.db.snapshot \
         --postgres-config homeserver-postgres.yaml
@@ -117,6 +126,11 @@ run::
         --postgres-config homeserver-postgres.yaml
 
 Once that has completed, change the synapse config to point at the PostgreSQL
-database configuration file ``homeserver-postgres.yaml`` (i.e. rename it to 
-``homeserver.yaml``) and restart synapse. Synapse should now be running against
-PostgreSQL.
+database configuration file ``homeserver-postgres.yaml``:
+
+    ./synctl stop
+    mv homeserver.yaml homeserver-old-sqlite.yaml 
+    mv homeserver-postgres.yaml homeserver.yaml 
+    ./synctl start
+
+Synapse should now be running against PostgreSQL.
-- 
cgit 1.4.1


From fed62e21ad2c0f622ed47283b41064e9fecaa00b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 11:34:28 +0100
Subject: Infrastructure for a server notices room

Server Notices use a special room which the user can't dismiss. They are
created on demand when some other bit of the code calls send_notice.

(This doesn't actually do much yet becuse we don't call send_notice anywhere)
---
 synapse/config/homeserver.py                     |   5 +-
 synapse/config/server_notices_config.py          |  85 ++++++++++++++++++
 synapse/handlers/register.py                     |  14 +++
 synapse/handlers/room.py                         |  16 +++-
 synapse/handlers/room_member.py                  |  40 ++++++++-
 synapse/server.py                                |   5 ++
 synapse/server.pyi                               |  12 +++
 synapse/server_notices/__init__.py               |   0
 synapse/server_notices/server_notices_manager.py | 110 +++++++++++++++++++++++
 9 files changed, 280 insertions(+), 7 deletions(-)
 create mode 100644 synapse/config/server_notices_config.py
 create mode 100644 synapse/server_notices/__init__.py
 create mode 100644 synapse/server_notices/server_notices_manager.py

diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index fb6bd3b421..1dea2ad024 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -38,6 +38,7 @@ from .spam_checker import SpamCheckerConfig
 from .groups import GroupsConfig
 from .user_directory import UserDirectoryConfig
 from .consent_config import ConsentConfig
+from .server_notices_config import ServerNoticesConfig
 
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
@@ -47,7 +48,9 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
                        JWTConfig, PasswordConfig, EmailConfig,
                        WorkerConfig, PasswordAuthProviderConfig, PushConfig,
                        SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,
-                       ConsentConfig):
+                       ConsentConfig,
+                       ServerNoticesConfig,
+                       ):
     pass
 
 
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
new file mode 100644
index 0000000000..51ddcac112
--- /dev/null
+++ b/synapse/config/server_notices_config.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from ._base import Config
+from synapse.types import UserID
+
+DEFAULT_CONFIG = """\
+# Server Notices room configuration
+#
+# Uncomment this section to enable a room which can be used to send notices
+# from the server to users. It is a special room which cannot be left; notices
+# come from a special "notices" user id.
+#
+# If you uncomment this section, you *must* define the system_mxid_localpart
+# setting, which defines the id of the user which will be used to send the
+# notices.
+#
+# It's also possible to override the room name, or the display name of the
+# "notices" user.
+#
+# server_notices:
+#   system_mxid_localpart: notices
+#   system_mxid_display_name: "Server Notices"
+#   room_name: "Server Notices"
+"""
+
+
+class ServerNoticesConfig(Config):
+    def __init__(self):
+        super(ServerNoticesConfig, self).__init__()
+
+        """The MXID to use for server notices.
+
+        None if server notices are not enabled.
+
+        type: str|None
+        """
+        self.server_notices_mxid = None
+
+        """The display name to use for the server notices user.
+
+        None if server notices are not enabled.
+
+        type: str|None
+        """
+        self.server_notices_mxid_display_name = None
+
+        """The name to use for the server notices room.
+
+        None if server notices are not enabled.
+
+        (TODO: i18n)
+
+        type: str|None
+        """
+        self.server_notices_room_name = None
+
+    def read_config(self, config):
+        c = config.get("server_notices")
+        if c is None:
+            return
+
+        mxid_localpart = c['system_mxid_localpart']
+        self.server_notices_mxid = UserID(
+            mxid_localpart, self.server_name,
+        ).to_string()
+        self.server_notices_mxid_display_name = c.get(
+            'system_mxid_display_name', 'Server Notices',
+        )
+
+        self.server_notices_room_name = c.get('room_name', "Server Notices")
+
+    def default_config(self, **kwargs):
+        return DEFAULT_CONFIG
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index f83c6b3cf8..7e52adda3c 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -34,6 +34,11 @@ logger = logging.getLogger(__name__)
 class RegistrationHandler(BaseHandler):
 
     def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
         super(RegistrationHandler, self).__init__(hs)
 
         self.auth = hs.get_auth()
@@ -49,6 +54,7 @@ class RegistrationHandler(BaseHandler):
         self._generate_user_id_linearizer = Linearizer(
             name="_generate_user_id_linearizer",
         )
+        self._server_notices_mxid = hs.config.server_notices_mxid
 
     @defer.inlineCallbacks
     def check_username(self, localpart, guest_access_token=None,
@@ -338,6 +344,14 @@ class RegistrationHandler(BaseHandler):
             yield identity_handler.bind_threepid(c, user_id)
 
     def check_user_id_not_appservice_exclusive(self, user_id, allowed_appservice=None):
+        # don't allow people to register the server notices mxid
+        if self._server_notices_mxid is not None:
+            if user_id == self._server_notices_mxid:
+                raise SynapseError(
+                    400, "This user ID is reserved.",
+                    errcode=Codes.EXCLUSIVE
+                )
+
         # valid user IDs must not clash with any user ID namespaces claimed by
         # application services.
         services = self.store.get_app_services()
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index e36426de5a..5e9fa95a2d 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -68,7 +68,8 @@ class RoomCreationHandler(BaseHandler):
         self.event_creation_handler = hs.get_event_creation_handler()
 
     @defer.inlineCallbacks
-    def create_room(self, requester, config, ratelimit=True):
+    def create_room(self, requester, config, ratelimit=True,
+                    creator_join_profile=None):
         """ Creates a new room.
 
         Args:
@@ -76,6 +77,14 @@ class RoomCreationHandler(BaseHandler):
                 The user who requested the room creation.
             config (dict) : A dict of configuration options.
             ratelimit (bool): set to False to disable the rate limiter
+
+            creator_join_profile (dict|None):
+                Set to override the displayname and avatar for the creating
+                user in this room. If unset, displayname and avatar will be
+                derived from the user's profile. If set, should contain the
+                values to go in the body of the 'join' event (typically
+                `avatar_url` and/or `displayname`.
+
         Returns:
             Deferred[dict]:
                 a dict containing the keys `room_id` and, if an alias was
@@ -180,7 +189,8 @@ class RoomCreationHandler(BaseHandler):
             initial_state=initial_state,
             creation_content=creation_content,
             room_alias=room_alias,
-            power_level_content_override=config.get("power_level_content_override", {})
+            power_level_content_override=config.get("power_level_content_override", {}),
+            creator_join_profile=creator_join_profile,
         )
 
         if "name" in config:
@@ -260,6 +270,7 @@ class RoomCreationHandler(BaseHandler):
             creation_content,
             room_alias,
             power_level_content_override,
+            creator_join_profile,
     ):
         def create(etype, content, **kwargs):
             e = {
@@ -303,6 +314,7 @@ class RoomCreationHandler(BaseHandler):
             room_id,
             "join",
             ratelimit=False,
+            content=creator_join_profile,
         )
 
         # We treat the power levels override specially as this needs to be one
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 714583f1d5..9f8975e1f2 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -17,11 +17,14 @@
 import abc
 import logging
 
+from six.moves import http_client
+
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 from twisted.internet import defer
 from unpaddedbase64 import decode_base64
 
+import synapse.server
 import synapse.types
 from synapse.api.constants import (
     EventTypes, Membership,
@@ -46,6 +49,11 @@ class RoomMemberHandler(object):
     __metaclass__ = abc.ABCMeta
 
     def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
         self.hs = hs
         self.store = hs.get_datastore()
         self.auth = hs.get_auth()
@@ -63,6 +71,7 @@ class RoomMemberHandler(object):
 
         self.clock = hs.get_clock()
         self.spam_checker = hs.get_spam_checker()
+        self._server_notices_mxid = self.config.server_notices_mxid
 
     @abc.abstractmethod
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
@@ -289,12 +298,28 @@ class RoomMemberHandler(object):
             is_blocked = yield self.store.is_room_blocked(room_id)
             if is_blocked:
                 raise SynapseError(403, "This room has been blocked on this server")
+        else:
+            if self._is_server_notice_room(room_id):
+                # we don't allow people to reject invites to, or leave, the
+                # server notice room.
+                raise SynapseError(
+                    http_client.FORBIDDEN,
+                    "You cannot leave this room",
+                )
 
-        if effective_membership_state == "invite":
+        if effective_membership_state == Membership.INVITE:
             block_invite = False
-            is_requester_admin = yield self.auth.is_server_admin(
-                requester.user,
-            )
+
+            if (self._server_notices_mxid is not None and
+                    requester.user.to_string() == self._server_notices_mxid):
+                # allow the server notices mxid to send invites
+                is_requester_admin = True
+
+            else:
+                is_requester_admin = yield self.auth.is_server_admin(
+                    requester.user,
+                )
+
             if not is_requester_admin:
                 if self.config.block_non_admin_invites:
                     logger.info(
@@ -844,6 +869,13 @@ class RoomMemberHandler(object):
 
         defer.returnValue(False)
 
+    @defer.inlineCallbacks
+    def _is_server_notice_room(self, room_id):
+        if self._server_notices_mxid is None:
+            defer.returnValue(False)
+        user_ids = yield self.store.get_users_in_room(room_id)
+        defer.returnValue(self._server_notices_mxid in user_ids)
+
 
 class RoomMemberMasterHandler(RoomMemberHandler):
     def __init__(self, hs):
diff --git a/synapse/server.py b/synapse/server.py
index 9e6f3584b2..85f54cd047 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -72,6 +72,7 @@ from synapse.rest.media.v1.media_repository import (
     MediaRepository,
     MediaRepositoryResource,
 )
+from synapse.server_notices.server_notices_manager import ServerNoticesManager
 from synapse.state import StateHandler, StateResolutionHandler
 from synapse.storage import DataStore
 from synapse.streams.events import EventSources
@@ -156,6 +157,7 @@ class HomeServer(object):
         'spam_checker',
         'room_member_handler',
         'federation_registry',
+        'server_notices_manager',
     ]
 
     def __init__(self, hostname, **kwargs):
@@ -398,6 +400,9 @@ class HomeServer(object):
     def build_federation_registry(self):
         return FederationHandlerRegistry()
 
+    def build_server_notices_manager(self):
+        return ServerNoticesManager(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 
diff --git a/synapse/server.pyi b/synapse/server.pyi
index aeda093f27..6fbe15168d 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -1,4 +1,5 @@
 import synapse.api.auth
+import synapse.config.homeserver
 import synapse.federation.transaction_queue
 import synapse.federation.transport.client
 import synapse.handlers
@@ -8,11 +9,16 @@ import synapse.handlers.device
 import synapse.handlers.e2e_keys
 import synapse.handlers.set_password
 import synapse.rest.media.v1.media_repository
+import synapse.server_notices.server_notices_manager
 import synapse.state
 import synapse.storage
 
 
 class HomeServer(object):
+    @property
+    def config(self) -> synapse.config.homeserver.HomeServerConfig:
+        pass
+
     def get_auth(self) -> synapse.api.auth.Auth:
         pass
 
@@ -43,6 +49,9 @@ class HomeServer(object):
     def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler:
         pass
 
+    def get_event_creation_handler(self) -> synapse.handlers.message.EventCreationHandler:
+        pass
+
     def get_set_password_handler(self) -> synapse.handlers.set_password.SetPasswordHandler:
         pass
 
@@ -57,3 +66,6 @@ class HomeServer(object):
 
     def get_media_repository(self) -> synapse.rest.media.v1.media_repository.MediaRepository:
         pass
+
+    def get_server_notices_manager(self) -> synapse.server_notices.server_notices_manager.ServerNoticesManager:
+        pass
diff --git a/synapse/server_notices/__init__.py b/synapse/server_notices/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
new file mode 100644
index 0000000000..0d48aeb95e
--- /dev/null
+++ b/synapse/server_notices/server_notices_manager.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.constants import EventTypes, Membership, RoomCreationPreset
+from synapse.types import create_requester
+from synapse.util.caches.descriptors import cachedInlineCallbacks
+
+logger = logging.getLogger(__name__)
+
+
+class ServerNoticesManager(object):
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+
+        self._store = hs.get_datastore()
+        self._config = hs.config
+        self._room_creation_handler = hs.get_room_creation_handler()
+        self._event_creation_handler = hs.get_event_creation_handler()
+
+    def is_enabled(self):
+        return self._config.server_notices_mxid is not None
+
+    @defer.inlineCallbacks
+    def send_notice(self, user_id, event_content):
+        room_id = yield self.get_notice_room_for_user(user_id)
+
+        system_mxid = self._config.server_notices_mxid
+        requester = create_requester(system_mxid)
+
+        logger.info("Sending server notice to %s", user_id)
+
+        yield self._event_creation_handler.create_and_send_nonmember_event(
+            requester, {
+                "type": EventTypes.Message,
+                "room_id": room_id,
+                "sender": system_mxid,
+                "content": event_content,
+            },
+            ratelimit=False,
+        )
+
+    @cachedInlineCallbacks()
+    def get_notice_room_for_user(self, user_id):
+        """Get the room for notices for a given user
+
+        If we have not yet created a notice room for this user, create it
+
+        Args:
+            user_id (str): complete user id for the user we want a room for
+
+        Returns:
+            str: room id of notice room.
+        """
+        if not self.is_enabled():
+            raise Exception("Server notices not enabled")
+
+        rooms = yield self._store.get_rooms_for_user_where_membership_is(
+            user_id, [Membership.INVITE, Membership.JOIN],
+        )
+        system_mxid = self._config.server_notices_mxid
+        for room in rooms:
+            user_ids = yield self._store.get_users_in_room(room.room_id)
+            if system_mxid in user_ids:
+                # we found a room which our user shares with the system notice
+                # user
+                logger.info("Using room %s", room.room_id)
+                defer.returnValue(room.room_id)
+
+        # apparently no existing notice room: create a new one
+        logger.info("Creating server notices room for %s", user_id)
+
+        requester = create_requester(system_mxid)
+        info = yield self._room_creation_handler.create_room(
+            requester,
+            config={
+                "preset": RoomCreationPreset.PRIVATE_CHAT,
+                "name": self._config.server_notices_room_name,
+                "power_level_content_override": {
+                    "users_default": -10,
+                },
+                "invite": (user_id,)
+            },
+            ratelimit=False,
+            creator_join_profile={
+                "displayname": self._config.server_notices_mxid_display_name,
+            },
+        )
+        room_id = info['room_id']
+
+        logger.info("Created server notices room %s for %s", room_id, user_id)
+        defer.returnValue(room_id)
-- 
cgit 1.4.1


From 1b1b47aec66e4627b9e365b4a2561cac23aa191c Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 16 May 2018 09:59:45 +0100
Subject: Reference synapse docker image and docker-compose

---
 README.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 28fbe45de6..4fe54b0c90 100644
--- a/README.rst
+++ b/README.rst
@@ -157,8 +157,9 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate the
-above in Docker at https://hub.docker.com/r/avhost/docker-matrix/tags/
+There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in `contrib/docker/README.md`.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/
 
 Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
 tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
-- 
cgit 1.4.1


From 82b0361f02b2860da572356c317de9690e88a146 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 16 May 2018 10:01:03 +0100
Subject: Document macaroon env var correctly

---
 contrib/docker/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index aed56646c2..dee5e92cab 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -111,8 +111,6 @@ variables are available for configuration:
 * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
 * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
   statistics reporting back to the Matrix project which helps us to get funding.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens
-  to the server, set this to a proper random key.
 * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
   you run your own TLS-capable reverse proxy).
 * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
@@ -132,6 +130,8 @@ Shared secrets, that will be initialized to random values if not set:
 
 * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
   registration is disable.
+* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
+  to the server.
 
 Database specific values (will use SQLite if not set):
 
-- 
cgit 1.4.1


From 450f500d0c96919a657225da63662555531531cb Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 16 May 2018 10:10:31 +0100
Subject: Note that secrets need to be retained.

---
 contrib/docker/README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index dee5e92cab..ef022f9dc1 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -1,6 +1,6 @@
 # Synapse Docker
 
-This Docker image will run Synapse as a single process. It does not provide any
+The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide any
 database server or TURN server that you should run separately.
 
 If you run a Postgres server, you should simply have it in the same Compose
@@ -37,10 +37,15 @@ then run the server:
 docker-compose up -d
 ```
 
+If secrets are not specified in the environment variables, they will be generated 
+as part of the startup. Please ensure these secrets are kept between launches of the
+Docker container, as their loss may require users to log in again.
+
 ### Manual configuration
 
 A sample ``docker-compose.yml`` is provided, including example labels for
-reverse proxying and other artifacts.
+reverse proxying and other artifacts. The docker-compose file is an example,
+please comment/uncomment sections that are not suitable for your usecase.
 
 Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
 to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
-- 
cgit 1.4.1


From a2da6de40ee52fafbb6ab1828c32abdcaf9e1865 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 17 May 2018 12:44:07 +0100
Subject: light grammar changes

---
 contrib/docker/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index ef022f9dc1..61592109cb 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -1,9 +1,9 @@
 # Synapse Docker
 
-The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide any
-database server or TURN server that you should run separately.
+The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide a
+database server or a TURN server, you should run these separately.
 
-If you run a Postgres server, you should simply have it in the same Compose
+If you run a Postgres server, you should simply include it in the same Compose
 project or set the proper environment variables and the image will automatically
 use that server.
 
@@ -37,7 +37,7 @@ then run the server:
 docker-compose up -d
 ```
 
-If secrets are not specified in the environment variables, they will be generated 
+If secrets are not specified in the environment variables, they will be generated
 as part of the startup. Please ensure these secrets are kept between launches of the
 Docker container, as their loss may require users to log in again.
 
-- 
cgit 1.4.1


From d43d480d86e8e5564643ec8d80da4ed4a0174934 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 18:22:19 +0100
Subject: Remove unused `update_external_syncs`

This method isn't used anywhere. Burninate it.
---
 synapse/handlers/presence.py | 55 --------------------------------------------
 1 file changed, 55 deletions(-)

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 585f3e4da2..91218e40e6 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -463,61 +463,6 @@ class PresenceHandler(object):
             syncing_user_ids.update(user_ids)
         return syncing_user_ids
 
-    @defer.inlineCallbacks
-    def update_external_syncs(self, process_id, syncing_user_ids):
-        """Update the syncing users for an external process
-
-        Args:
-            process_id(str): An identifier for the process the users are
-                syncing against. This allows synapse to process updates
-                as user start and stop syncing against a given process.
-            syncing_user_ids(set(str)): The set of user_ids that are
-                currently syncing on that server.
-        """
-
-        # Grab the previous list of user_ids that were syncing on that process
-        prev_syncing_user_ids = (
-            self.external_process_to_current_syncs.get(process_id, set())
-        )
-        # Grab the current presence state for both the users that are syncing
-        # now and the users that were syncing before this update.
-        prev_states = yield self.current_state_for_users(
-            syncing_user_ids | prev_syncing_user_ids
-        )
-        updates = []
-        time_now_ms = self.clock.time_msec()
-
-        # For each new user that is syncing check if we need to mark them as
-        # being online.
-        for new_user_id in syncing_user_ids - prev_syncing_user_ids:
-            prev_state = prev_states[new_user_id]
-            if prev_state.state == PresenceState.OFFLINE:
-                updates.append(prev_state.copy_and_replace(
-                    state=PresenceState.ONLINE,
-                    last_active_ts=time_now_ms,
-                    last_user_sync_ts=time_now_ms,
-                ))
-            else:
-                updates.append(prev_state.copy_and_replace(
-                    last_user_sync_ts=time_now_ms,
-                ))
-
-        # For each user that is still syncing or stopped syncing update the
-        # last sync time so that we will correctly apply the grace period when
-        # they stop syncing.
-        for old_user_id in prev_syncing_user_ids:
-            prev_state = prev_states[old_user_id]
-            updates.append(prev_state.copy_and_replace(
-                last_user_sync_ts=time_now_ms,
-            ))
-
-        yield self._update_states(updates)
-
-        # Update the last updated time for the process. We expire the entries
-        # if we don't receive an update in the given timeframe.
-        self.external_process_last_updated_ms[process_id] = self.clock.time_msec()
-        self.external_process_to_current_syncs[process_id] = syncing_user_ids
-
     @defer.inlineCallbacks
     def update_external_syncs_row(self, process_id, user_id, is_syncing, sync_time_msec):
         """Update the syncing users for an external process as a delta.
-- 
cgit 1.4.1


From 88d34053326328f3ce62d424c72ed0244f1955d4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 18:33:45 +0100
Subject: fix missing yield for server_notices_room

---
 synapse/handlers/room_member.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 9f8975e1f2..9b99f0f9d7 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -299,9 +299,10 @@ class RoomMemberHandler(object):
             if is_blocked:
                 raise SynapseError(403, "This room has been blocked on this server")
         else:
-            if self._is_server_notice_room(room_id):
-                # we don't allow people to reject invites to, or leave, the
-                # server notice room.
+            # we don't allow people to reject invites to, or leave, the
+            # server notice room.
+            is_blocked = yield self._is_server_notice_room(room_id)
+            if is_blocked:
                 raise SynapseError(
                     http_client.FORBIDDEN,
                     "You cannot leave this room",
-- 
cgit 1.4.1


From 6958459b50a3b29611f98d620d0400d245f8b60f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 17 May 2018 21:35:07 +0100
Subject: bump version, change log

---
 CHANGES.rst         | 8 +++++++-
 synapse/__init__.py | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index b769b0f046..851cb9b2d7 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,12 @@
+Changes in synapse v0.29.1 (2018-05-17)
+==========================================
+Changes:
+
+* Update docker documentation (PR #3222)
+
 Changes in synapse v0.29.0 (2018-05-16)
 ===========================================
-
+Not changes since v0.29.0-rc1
 
 Changes in synapse v0.29.0-rc1 (2018-05-14)
 ===========================================
diff --git a/synapse/__init__.py b/synapse/__init__.py
index d94c20505e..b11a1d3ea3 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.29.0"
+__version__ = "0.29.1"
-- 
cgit 1.4.1


From 8b1c856d815c8ab939cd5a272665d80a22ee5d33 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 09:15:35 +0100
Subject: Fix error in handling receipts

Fixes an error which has been happening ever since #2158 (v0.21.0-rc1):

> TypeError: argument of type 'ObservableDeferred' is not iterable

fixes #3234
---
 synapse/storage/receipts.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 2f95e7e82a..709c69a926 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -297,18 +297,22 @@ class ReceiptsWorkerStore(SQLBaseStore):
         if receipt_type != "m.read":
             return
 
-        # Returns an ObservableDeferred
+        # Returns either an ObservableDeferred or the raw result
         res = self.get_users_with_read_receipts_in_room.cache.get(
             room_id, None, update_metrics=False,
         )
 
-        if res:
-            if isinstance(res, defer.Deferred) and res.called:
+        # first handle the Deferred case
+        if isinstance(res, defer.Deferred):
+            if res.called:
                 res = res.result
-            if user_id in res:
-                # We'd only be adding to the set, so no point invalidating if the
-                # user is already there
-                return
+            else:
+                res = None
+
+        if res and user_id in res:
+            # We'd only be adding to the set, so no point invalidating if the
+            # user is already there
+            return
 
         self.get_users_with_read_receipts_in_room.invalidate((room_id,))
 
-- 
cgit 1.4.1


From d10707c8105f4cc7f079ed45960f8333c6101c86 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 11:00:55 +0100
Subject: Replace inline docstrings with "Attributes" in class docstring

---
 synapse/config/server_notices_config.py | 38 +++++++++++++--------------------
 1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index 51ddcac112..ccef8d2ec5 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -37,33 +37,25 @@ DEFAULT_CONFIG = """\
 
 
 class ServerNoticesConfig(Config):
-    def __init__(self):
-        super(ServerNoticesConfig, self).__init__()
+    """Configuration for the server notices room.
 
-        """The MXID to use for server notices.
+    Attributes:
+        server_notices_mxid (str|None):
+            The MXID to use for server notices.
+            None if server notices are not enabled.
 
-        None if server notices are not enabled.
+        server_notices_mxid_display_name (str|None):
+            The display name to use for the server notices user.
+            None if server notices are not enabled.
 
-        type: str|None
-        """
+        server_notices_room_name (str|None):
+            The name to use for the server notices room.
+            None if server notices are not enabled.
+    """
+    def __init__(self):
+        super(ServerNoticesConfig, self).__init__()
         self.server_notices_mxid = None
-
-        """The display name to use for the server notices user.
-
-        None if server notices are not enabled.
-
-        type: str|None
-        """
         self.server_notices_mxid_display_name = None
-
-        """The name to use for the server notices room.
-
-        None if server notices are not enabled.
-
-        (TODO: i18n)
-
-        type: str|None
-        """
         self.server_notices_room_name = None
 
     def read_config(self, config):
@@ -78,7 +70,7 @@ class ServerNoticesConfig(Config):
         self.server_notices_mxid_display_name = c.get(
             'system_mxid_display_name', 'Server Notices',
         )
-
+        # todo: i18n
         self.server_notices_room_name = c.get('room_name', "Server Notices")
 
     def default_config(self, **kwargs):
-- 
cgit 1.4.1


From 26305788fe23448d65d0cd0358df547e816574f2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 11:18:39 +0100
Subject: Make sure we reject attempts to invite the notices user

---
 synapse/handlers/federation.py                   | 8 ++++++++
 synapse/handlers/room_member.py                  | 7 +++++++
 synapse/server_notices/server_notices_manager.py | 5 +++++
 3 files changed, 20 insertions(+)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index f39233d846..ba3ede8024 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -81,6 +81,7 @@ class FederationHandler(BaseHandler):
         self.pusher_pool = hs.get_pusherpool()
         self.spam_checker = hs.get_spam_checker()
         self.event_creation_handler = hs.get_event_creation_handler()
+        self._server_notices_mxid = hs.config.server_notices_mxid
 
         # When joining a room we need to queue any events for that room up
         self.room_queues = {}
@@ -1180,6 +1181,13 @@ class FederationHandler(BaseHandler):
         if not self.is_mine_id(event.state_key):
             raise SynapseError(400, "The invite event must be for this server")
 
+        # block any attempts to invite the server notices mxid
+        if event.state_key == self._server_notices_mxid:
+            raise SynapseError(
+                http_client.FORBIDDEN,
+                "Cannot invite this user",
+            )
+
         event.internal_metadata.outlier = True
         event.internal_metadata.invite_from_remote = True
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 9b99f0f9d7..5263f09aaa 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -309,6 +309,13 @@ class RoomMemberHandler(object):
                 )
 
         if effective_membership_state == Membership.INVITE:
+            # block any attempts to invite the server notices mxid
+            if target.to_string() == self._server_notices_mxid:
+                raise SynapseError(
+                    http_client.FORBIDDEN,
+                    "Cannot invite this user",
+                )
+
             block_invite = False
 
             if (self._server_notices_mxid is not None and
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 0d48aeb95e..15086600ed 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -78,6 +78,11 @@ class ServerNoticesManager(object):
         )
         system_mxid = self._config.server_notices_mxid
         for room in rooms:
+            # it's worth noting that there is an asymmetry here in that we
+            # expect the user to be invited or joined, but the system user must
+            # be joined. This is kinda deliberate, in that if somebody somehow
+            # manages to invite the system user to a room, that doesn't make it
+            # the server notices room.
             user_ids = yield self._store.get_users_in_room(room.room_id)
             if system_mxid in user_ids:
                 # we found a room which our user shares with the system notice
-- 
cgit 1.4.1


From 011e1f4010776cc67eeead09c20e18da35637c00 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 11:22:12 +0100
Subject: Better docstrings

---
 synapse/server_notices/server_notices_manager.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 15086600ed..f535b9c9da 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -37,10 +37,26 @@ class ServerNoticesManager(object):
         self._event_creation_handler = hs.get_event_creation_handler()
 
     def is_enabled(self):
+        """Checks if server notices are enabled on this server.
+
+        Returns:
+            bool
+        """
         return self._config.server_notices_mxid is not None
 
     @defer.inlineCallbacks
     def send_notice(self, user_id, event_content):
+        """Send a notice to the given user
+
+        Creates the server notices room, if none exists.
+
+        Args:
+            user_id (str): mxid of user to send event to.
+            event_content (dict): content of event to send
+
+        Returns:
+            Deferrred[None]
+        """
         room_id = yield self.get_notice_room_for_user(user_id)
 
         system_mxid = self._config.server_notices_mxid
-- 
cgit 1.4.1


From 861f8a9b218eccea95ffb5c7d50fe0380beeae43 Mon Sep 17 00:00:00 2001
From: Paul Tötterman <paul.totterman@iki.fi>
Date: Fri, 18 May 2018 16:20:08 +0300
Subject: Add lxml to docker image for web previews
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Paul Tötterman <paul.totterman@iki.fi>
---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8085f3d354..565341fee3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,12 @@
 FROM docker.io/python:2-alpine3.7
 
-RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev
+RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev libxslt-dev
 
 COPY . /synapse
 
 # A wheel cache may be provided in ./cache for faster build
 RUN cd /synapse \
- && pip install --upgrade pip setuptools psycopg2 \
+ && pip install --upgrade pip setuptools psycopg2 lxml \
  && mkdir -p /synapse/cache \
  && pip install -f /synapse/cache --upgrade --process-dependency-links . \
  && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \
-- 
cgit 1.4.1


From ef466b3a131c0a6c93027a8f8194538678f920c1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 18 May 2018 15:51:21 +0100
Subject: fix psql compatability bug

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 4551cf8774..ac264b5d25 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -376,7 +376,7 @@ class DataStore(RoomMemberStore, RoomStore,
                     FROM user_ips AS u
                     LEFT JOIN (
                       SELECT user_id, device_id, timestamp FROM user_daily_visits
-                      WHERE timestamp IS ?
+                      WHERE timestamp = ?
                     ) udv
                     ON u.user_id = udv.user_id AND u.device_id=udv.device_id
                     WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL
-- 
cgit 1.4.1


From 644aac5f73dbacfe4cb47af09e804b9d1d2788a4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 18 May 2018 17:10:35 +0100
Subject: Tighter filtering for user_daily_visits

---
 synapse/storage/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index ac264b5d25..979fa22438 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -379,7 +379,11 @@ class DataStore(RoomMemberStore, RoomStore,
                       WHERE timestamp = ?
                     ) udv
                     ON u.user_id = udv.user_id AND u.device_id=udv.device_id
-                    WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL
+                    INNER JOIN users ON users.name=u.user_id
+                    WHERE last_seen > ? AND last_seen <= ?
+                    AND udv.timestamp IS NULL AND users.is_guest=0
+                    AND users.appservice_id IS NULL
+                    GROUP BY u.user_id, u.device_id
             """
 
             # This means that the day has rolled over but there could still
-- 
cgit 1.4.1


From b932b4ea256fcbfa112f006ccb2e1c10ef195c17 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 14:36:39 +0200
Subject: use repr, not str

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/http/site.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index 202a990508..75e32924e9 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -56,7 +56,7 @@ class SynapseRequest(Request):
 
     def __repr__(self):
         # We overwrite this so that we don't log ``access_token``
-        return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % (
+        return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % (
             self.__class__.__name__,
             id(self),
             self.method,
-- 
cgit 1.4.1


From aafb0f6b0d7db313ac54a8e5e933970feae4bff3 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 10:44:31 +0200
Subject: py3-ize url preview

---
 synapse/rest/media/v1/preview_url_resource.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 2839207abc..565cef2b8d 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -24,7 +24,9 @@ import shutil
 import sys
 import traceback
 import simplejson as json
-import urlparse
+
+from six.moves import urllib_parse as urlparse
+from six import string_types
 
 from twisted.web.server import NOT_DONE_YET
 from twisted.internet import defer
@@ -590,8 +592,8 @@ def _iterate_over_text(tree, *tags_to_ignore):
     # to be returned.
     elements = iter([tree])
     while True:
-        el = elements.next()
-        if isinstance(el, basestring):
+        el = next(elements)
+        if isinstance(el, string_types):
             yield el
         elif el is not None and el.tag not in tags_to_ignore:
             # el.text is the text before the first child, so we can immediately
-- 
cgit 1.4.1


From 73cbdef5f773ddd091713d71ac9c63639d7c0825 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 15 May 2018 17:55:46 +0200
Subject: fix py3 intern and remove unnecessary py3 encode

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/util/caches/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 4adae96681..329ccbb866 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -16,6 +16,9 @@
 import synapse.metrics
 import os
 
+from six.moves import intern
+import six
+
 CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
 
 metrics = synapse.metrics.get_metrics_for("synapse.util.caches")
@@ -66,7 +69,9 @@ def intern_string(string):
         return None
 
     try:
-        string = string.encode("ascii")
+        if six.PY2:
+            string = string.encode("ascii")
+
         return intern(string)
     except UnicodeEncodeError:
         return string
-- 
cgit 1.4.1


From dcc235b47d694d9ef7181b379d4e38bed2677028 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Mon, 30 Apr 2018 16:38:23 +0200
Subject: use stand-in value if maxint is not available

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/storage/_base.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2262776ab2..962cc270bc 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -30,6 +30,12 @@ import threading
 
 logger = logging.getLogger(__name__)
 
+try:
+    MAX_TXN_ID = sys.maxint - 1
+except AttributeError:
+    # python 3 does not have a maximum int value
+    MAX_TXN_ID = 2**63 - 1
+
 sql_logger = logging.getLogger("synapse.storage.SQL")
 transaction_logger = logging.getLogger("synapse.storage.txn")
 perf_logger = logging.getLogger("synapse.storage.TIME")
@@ -222,7 +228,7 @@ class SQLBaseStore(object):
 
         # We don't really need these to be unique, so lets stop it from
         # growing really large.
-        self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1)
+        self._TXN_ID = (self._TXN_ID + 1) % (MAX_TXN_ID)
 
         name = "%s-%x" % (desc, txn_id, )
 
-- 
cgit 1.4.1


From 45b55e23d34371986ee0da5be784a1f2134fd58a Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 29 Apr 2018 13:54:38 +0200
Subject: Add batch_iter to utils

There's a frequent idiom I noticed where an iterable is split up into a
number of chunks/batches. Unfortunately that method does not work with
iterators like dict.keys() in python3. This implementation works with
iterators.

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/util/__init__.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 814a7bf71b..fc11e26623 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -20,6 +20,8 @@ from twisted.internet import defer, reactor, task
 import time
 import logging
 
+from itertools import islice
+
 logger = logging.getLogger(__name__)
 
 
@@ -79,3 +81,19 @@ class Clock(object):
         except Exception:
             if not ignore_errs:
                 raise
+
+
+def batch_iter(iterable, size):
+    """batch an iterable up into tuples with a maximum size
+
+    Args:
+        iterable (iterable): the iterable to slice
+        size (int): the maximum batch size
+
+    Returns:
+        an iterator over the chunks
+    """
+    # make sure we can deal with iterables like lists too
+    sourceiter = iter(iterable)
+    # call islice until it returns an empty tuple
+    return iter(lambda: tuple(islice(sourceiter, size)), ())
-- 
cgit 1.4.1


From d9fe2b2d9dbb539da32f969b8bd752159fd5eb6f Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sun, 15 Apr 2018 21:43:35 +0200
Subject: Replace some more comparisons with six

plus a bonus b"" string I missed last time

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/events/utils.py                | 4 +++-
 synapse/events/validator.py            | 6 ++++--
 synapse/groups/groups_server.py        | 4 +++-
 synapse/http/matrixfederationclient.py | 8 +++++---
 synapse/push/push_rule_evaluator.py    | 4 +++-
 synapse/rest/client/v1/presence.py     | 8 +++++---
 6 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 824f4a42e3..29ae086786 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -20,6 +20,8 @@ from frozendict import frozendict
 
 import re
 
+from six import string_types
+
 # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
 # (?<!stuff) matches if the current position in the string is not preceded
 # by a match for 'stuff'.
@@ -277,7 +279,7 @@ def serialize_event(e, time_now_ms, as_client_event=True,
 
     if only_event_fields:
         if (not isinstance(only_event_fields, list) or
-                not all(isinstance(f, basestring) for f in only_event_fields)):
+                not all(isinstance(f, string_types) for f in only_event_fields)):
             raise TypeError("only_event_fields must be a list of strings")
         d = only_fields(d, only_event_fields)
 
diff --git a/synapse/events/validator.py b/synapse/events/validator.py
index 2f4c8a1018..e0e5bf818c 100644
--- a/synapse/events/validator.py
+++ b/synapse/events/validator.py
@@ -17,6 +17,8 @@ from synapse.types import EventID, RoomID, UserID
 from synapse.api.errors import SynapseError
 from synapse.api.constants import EventTypes, Membership
 
+from six import string_types
+
 
 class EventValidator(object):
 
@@ -49,7 +51,7 @@ class EventValidator(object):
             strings.append("state_key")
 
         for s in strings:
-            if not isinstance(getattr(event, s), basestring):
+            if not isinstance(getattr(event, s), string_types):
                 raise SynapseError(400, "Not '%s' a string type" % (s,))
 
         if event.type == EventTypes.Member:
@@ -88,5 +90,5 @@ class EventValidator(object):
         for s in keys:
             if s not in d:
                 raise SynapseError(400, "'%s' not in content" % (s,))
-            if not isinstance(d[s], basestring):
+            if not isinstance(d[s], string_types):
                 raise SynapseError(400, "Not '%s' a string type" % (s,))
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 2d95b04e0c..62d20ad130 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -20,6 +20,8 @@ from synapse.api.errors import SynapseError
 from synapse.types import GroupID, RoomID, UserID, get_domain_from_id
 from twisted.internet import defer
 
+from six import string_types
+
 logger = logging.getLogger(__name__)
 
 
@@ -431,7 +433,7 @@ class GroupsServerHandler(object):
                         "long_description"):
             if keyname in content:
                 value = content[keyname]
-                if not isinstance(value, basestring):
+                if not isinstance(value, string_types):
                     raise SynapseError(400, "%r value is not a string" % (keyname,))
                 profile[keyname] = value
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 4b2b85464d..4650f43029 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -42,6 +42,8 @@ import random
 import sys
 import urllib
 from six.moves.urllib import parse as urlparse
+from six import string_types
+
 
 logger = logging.getLogger(__name__)
 outbound_logger = logging.getLogger("synapse.http.outbound")
@@ -553,7 +555,7 @@ class MatrixFederationHttpClient(object):
 
         encoded_args = {}
         for k, vs in args.items():
-            if isinstance(vs, basestring):
+            if isinstance(vs, string_types):
                 vs = [vs]
             encoded_args[k] = [v.encode("UTF-8") for v in vs]
 
@@ -668,7 +670,7 @@ def check_content_type_is_json(headers):
         RuntimeError if the
 
     """
-    c_type = headers.getRawHeaders("Content-Type")
+    c_type = headers.getRawHeaders(b"Content-Type")
     if c_type is None:
         raise RuntimeError(
             "No Content-Type header"
@@ -685,7 +687,7 @@ def check_content_type_is_json(headers):
 def encode_query_args(args):
     encoded_args = {}
     for k, vs in args.items():
-        if isinstance(vs, basestring):
+        if isinstance(vs, string_types):
             vs = [vs]
         encoded_args[k] = [v.encode("UTF-8") for v in vs]
 
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 3601f2d365..d55efde8cc 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -21,6 +21,8 @@ from synapse.types import UserID
 from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
 from synapse.util.caches.lrucache import LruCache
 
+from six import string_types
+
 logger = logging.getLogger(__name__)
 
 
@@ -238,7 +240,7 @@ def _flatten_dict(d, prefix=[], result=None):
     if result is None:
         result = {}
     for key, value in d.items():
-        if isinstance(value, basestring):
+        if isinstance(value, string_types):
             result[".".join(prefix + [key])] = value.lower()
         elif hasattr(value, "items"):
             _flatten_dict(value, prefix=(prefix + [key]), result=result)
diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py
index 4a73813c58..647994bd53 100644
--- a/synapse/rest/client/v1/presence.py
+++ b/synapse/rest/client/v1/presence.py
@@ -23,6 +23,8 @@ from synapse.handlers.presence import format_user_presence_state
 from synapse.http.servlet import parse_json_object_from_request
 from .base import ClientV1RestServlet, client_path_patterns
 
+from six import string_types
+
 import logging
 
 logger = logging.getLogger(__name__)
@@ -71,7 +73,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet):
 
             if "status_msg" in content:
                 state["status_msg"] = content.pop("status_msg")
-                if not isinstance(state["status_msg"], basestring):
+                if not isinstance(state["status_msg"], string_types):
                     raise SynapseError(400, "status_msg must be a string.")
 
             if content:
@@ -129,7 +131,7 @@ class PresenceListRestServlet(ClientV1RestServlet):
 
         if "invite" in content:
             for u in content["invite"]:
-                if not isinstance(u, basestring):
+                if not isinstance(u, string_types):
                     raise SynapseError(400, "Bad invite value.")
                 if len(u) == 0:
                     continue
@@ -140,7 +142,7 @@ class PresenceListRestServlet(ClientV1RestServlet):
 
         if "drop" in content:
             for u in content["drop"]:
-                if not isinstance(u, basestring):
+                if not isinstance(u, string_types):
                     raise SynapseError(400, "Bad drop value.")
                 if len(u) == 0:
                     continue
-- 
cgit 1.4.1


From 933bf2dd357842b0e7c3fc7a1111d89ab52f5329 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 28 Apr 2018 13:19:12 +0200
Subject: replace some iteritems with six

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/handlers/device.py                | 14 ++++++++------
 synapse/handlers/e2e_keys.py              | 13 +++++++------
 synapse/handlers/groups_local.py          |  3 ++-
 synapse/handlers/presence.py              | 15 ++++++++-------
 synapse/handlers/sync.py                  | 14 ++++++++------
 synapse/handlers/user_directory.py        |  3 ++-
 synapse/metrics/process_collector.py      |  3 ++-
 synapse/push/bulk_push_rule_evaluator.py  | 13 +++++++------
 synapse/replication/tcp/protocol.py       |  9 +++++----
 synapse/replication/tcp/resource.py       |  3 ++-
 synapse/rest/media/v1/media_repository.py |  3 ++-
 synapse/storage/client_ips.py             |  6 ++++--
 synapse/storage/devices.py                |  9 +++++----
 synapse/storage/end_to_end_keys.py        |  6 ++++--
 synapse/storage/event_push_actions.py     |  4 +++-
 15 files changed, 69 insertions(+), 49 deletions(-)

diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index f7457a7082..31bd0e60c6 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -26,6 +26,8 @@ from ._base import BaseHandler
 
 import logging
 
+from six import itervalues, iteritems
+
 logger = logging.getLogger(__name__)
 
 
@@ -318,7 +320,7 @@ class DeviceHandler(BaseHandler):
             # The user may have left the room
             # TODO: Check if they actually did or if we were just invited.
             if room_id not in room_ids:
-                for key, event_id in current_state_ids.iteritems():
+                for key, event_id in iteritems(current_state_ids):
                     etype, state_key = key
                     if etype != EventTypes.Member:
                         continue
@@ -338,7 +340,7 @@ class DeviceHandler(BaseHandler):
             # special-case for an empty prev state: include all members
             # in the changed list
             if not event_ids:
-                for key, event_id in current_state_ids.iteritems():
+                for key, event_id in iteritems(current_state_ids):
                     etype, state_key = key
                     if etype != EventTypes.Member:
                         continue
@@ -354,10 +356,10 @@ class DeviceHandler(BaseHandler):
 
             # Check if we've joined the room? If so we just blindly add all the users to
             # the "possibly changed" users.
-            for state_dict in prev_state_ids.itervalues():
+            for state_dict in itervalues(prev_state_ids):
                 member_event = state_dict.get((EventTypes.Member, user_id), None)
                 if not member_event or member_event != current_member_id:
-                    for key, event_id in current_state_ids.iteritems():
+                    for key, event_id in iteritems(current_state_ids):
                         etype, state_key = key
                         if etype != EventTypes.Member:
                             continue
@@ -367,14 +369,14 @@ class DeviceHandler(BaseHandler):
             # If there has been any change in membership, include them in the
             # possibly changed list. We'll check if they are joined below,
             # and we're not toooo worried about spuriously adding users.
-            for key, event_id in current_state_ids.iteritems():
+            for key, event_id in iteritems(current_state_ids):
                 etype, state_key = key
                 if etype != EventTypes.Member:
                     continue
 
                 # check if this member has changed since any of the extremities
                 # at the stream_ordering, and add them to the list if so.
-                for state_dict in prev_state_ids.itervalues():
+                for state_dict in itervalues(prev_state_ids):
                     prev_event_id = state_dict.get(key, None)
                     if not prev_event_id or prev_event_id != event_id:
                         if state_key != user_id:
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 25aec624af..8a2d177539 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -19,6 +19,7 @@ import logging
 
 from canonicaljson import encode_canonical_json
 from twisted.internet import defer
+from six import iteritems
 
 from synapse.api.errors import (
     SynapseError, CodeMessageException, FederationDeniedError,
@@ -92,7 +93,7 @@ class E2eKeysHandler(object):
         remote_queries_not_in_cache = {}
         if remote_queries:
             query_list = []
-            for user_id, device_ids in remote_queries.iteritems():
+            for user_id, device_ids in iteritems(remote_queries):
                 if device_ids:
                     query_list.extend((user_id, device_id) for device_id in device_ids)
                 else:
@@ -103,9 +104,9 @@ class E2eKeysHandler(object):
                     query_list
                 )
             )
-            for user_id, devices in remote_results.iteritems():
+            for user_id, devices in iteritems(remote_results):
                 user_devices = results.setdefault(user_id, {})
-                for device_id, device in devices.iteritems():
+                for device_id, device in iteritems(devices):
                     keys = device.get("keys", None)
                     device_display_name = device.get("device_display_name", None)
                     if keys:
@@ -250,9 +251,9 @@ class E2eKeysHandler(object):
             "Claimed one-time-keys: %s",
             ",".join((
                 "%s for %s:%s" % (key_id, user_id, device_id)
-                for user_id, user_keys in json_result.iteritems()
-                for device_id, device_keys in user_keys.iteritems()
-                for key_id, _ in device_keys.iteritems()
+                for user_id, user_keys in iteritems(json_result)
+                for device_id, device_keys in iteritems(user_keys)
+                for key_id, _ in iteritems(device_keys)
             )),
         )
 
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 977993e7d4..dcae083734 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from twisted.internet import defer
+from six import iteritems
 
 from synapse.api.errors import SynapseError
 from synapse.types import get_domain_from_id
@@ -449,7 +450,7 @@ class GroupsLocalHandler(object):
 
         results = {}
         failed_results = []
-        for destination, dest_user_ids in destinations.iteritems():
+        for destination, dest_user_ids in iteritems(destinations):
             try:
                 r = yield self.transport_client.bulk_get_publicised_groups(
                     destination, list(dest_user_ids),
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 91218e40e6..b51f925220 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -25,6 +25,8 @@ The methods that define policy are:
 from twisted.internet import defer, reactor
 from contextlib import contextmanager
 
+from six import itervalues, iteritems
+
 from synapse.api.errors import SynapseError
 from synapse.api.constants import PresenceState
 from synapse.storage.presence import UserPresenceState
@@ -40,7 +42,6 @@ import synapse.metrics
 
 import logging
 
-
 logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
@@ -526,7 +527,7 @@ class PresenceHandler(object):
                 prev_state.copy_and_replace(
                     last_user_sync_ts=time_now_ms,
                 )
-                for prev_state in prev_states.itervalues()
+                for prev_state in itervalues(prev_states)
             ])
             self.external_process_last_updated_ms.pop(process_id, None)
 
@@ -549,14 +550,14 @@ class PresenceHandler(object):
             for user_id in user_ids
         }
 
-        missing = [user_id for user_id, state in states.iteritems() if not state]
+        missing = [user_id for user_id, state in iteritems(states) if not state]
         if missing:
             # There are things not in our in memory cache. Lets pull them out of
             # the database.
             res = yield self.store.get_presence_for_users(missing)
             states.update(res)
 
-            missing = [user_id for user_id, state in states.iteritems() if not state]
+            missing = [user_id for user_id, state in iteritems(states) if not state]
             if missing:
                 new = {
                     user_id: UserPresenceState.default(user_id)
@@ -1044,7 +1045,7 @@ class PresenceEventSource(object):
             defer.returnValue((updates.values(), max_token))
         else:
             defer.returnValue(([
-                s for s in updates.itervalues()
+                s for s in itervalues(updates)
                 if s.state != PresenceState.OFFLINE
             ], max_token))
 
@@ -1301,11 +1302,11 @@ def get_interested_remotes(store, states, state_handler):
     # hosts in those rooms.
     room_ids_to_states, users_to_states = yield get_interested_parties(store, states)
 
-    for room_id, states in room_ids_to_states.iteritems():
+    for room_id, states in iteritems(room_ids_to_states):
         hosts = yield state_handler.get_current_hosts_in_room(room_id)
         hosts_and_states.append((hosts, states))
 
-    for user_id, states in users_to_states.iteritems():
+    for user_id, states in iteritems(users_to_states):
         host = get_domain_from_id(user_id)
         hosts_and_states.append(([host], states))
 
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 263e42dded..d0c99c35e3 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -28,6 +28,8 @@ import collections
 import logging
 import itertools
 
+from six import itervalues, iteritems
+
 logger = logging.getLogger(__name__)
 
 
@@ -275,7 +277,7 @@ class SyncHandler(object):
                 # result returned by the event source is poor form (it might cache
                 # the object)
                 room_id = event["room_id"]
-                event_copy = {k: v for (k, v) in event.iteritems()
+                event_copy = {k: v for (k, v) in iteritems(event)
                               if k != "room_id"}
                 ephemeral_by_room.setdefault(room_id, []).append(event_copy)
 
@@ -294,7 +296,7 @@ class SyncHandler(object):
             for event in receipts:
                 room_id = event["room_id"]
                 # exclude room id, as above
-                event_copy = {k: v for (k, v) in event.iteritems()
+                event_copy = {k: v for (k, v) in iteritems(event)
                               if k != "room_id"}
                 ephemeral_by_room.setdefault(room_id, []).append(event_copy)
 
@@ -325,7 +327,7 @@ class SyncHandler(object):
                 current_state_ids = frozenset()
                 if any(e.is_state() for e in recents):
                     current_state_ids = yield self.state.get_current_state_ids(room_id)
-                    current_state_ids = frozenset(current_state_ids.itervalues())
+                    current_state_ids = frozenset(itervalues(current_state_ids))
 
                 recents = yield filter_events_for_client(
                     self.store,
@@ -382,7 +384,7 @@ class SyncHandler(object):
                 current_state_ids = frozenset()
                 if any(e.is_state() for e in loaded_recents):
                     current_state_ids = yield self.state.get_current_state_ids(room_id)
-                    current_state_ids = frozenset(current_state_ids.itervalues())
+                    current_state_ids = frozenset(itervalues(current_state_ids))
 
                 loaded_recents = yield filter_events_for_client(
                     self.store,
@@ -984,7 +986,7 @@ class SyncHandler(object):
         if since_token:
             for joined_sync in sync_result_builder.joined:
                 it = itertools.chain(
-                    joined_sync.timeline.events, joined_sync.state.itervalues()
+                    joined_sync.timeline.events, itervalues(joined_sync.state)
                 )
                 for event in it:
                     if event.type == EventTypes.Member:
@@ -1062,7 +1064,7 @@ class SyncHandler(object):
         newly_left_rooms = []
         room_entries = []
         invited = []
-        for room_id, events in mem_change_events_by_room_id.iteritems():
+        for room_id, events in iteritems(mem_change_events_by_room_id):
             non_joins = [e for e in events if e.membership != Membership.JOIN]
             has_join = len(non_joins) != len(events)
 
diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py
index 714f0195c8..cd0b7290f0 100644
--- a/synapse/handlers/user_directory.py
+++ b/synapse/handlers/user_directory.py
@@ -22,6 +22,7 @@ from synapse.util.metrics import Measure
 from synapse.util.async import sleep
 from synapse.types import get_localpart_from_id
 
+from six import iteritems
 
 logger = logging.getLogger(__name__)
 
@@ -403,7 +404,7 @@ class UserDirectoryHandler(object):
 
         if change:
             users_with_profile = yield self.state.get_current_user_in_room(room_id)
-            for user_id, profile in users_with_profile.iteritems():
+            for user_id, profile in iteritems(users_with_profile):
                 yield self._handle_new_user(room_id, user_id, profile)
         else:
             users = yield self.store.get_users_in_public_due_to_room(room_id)
diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py
index 6fec3de399..50e5b48a2b 100644
--- a/synapse/metrics/process_collector.py
+++ b/synapse/metrics/process_collector.py
@@ -15,6 +15,7 @@
 
 import os
 
+from six import iteritems
 
 TICKS_PER_SEC = 100
 BYTES_PER_PAGE = 4096
@@ -55,7 +56,7 @@ def update_resource_metrics():
             # line is PID (command) more stats go here ...
             raw_stats = line.split(") ", 1)[1].split(" ")
 
-            for (name, index) in STAT_FIELDS.iteritems():
+            for (name, index) in iteritems(STAT_FIELDS):
                 # subtract 3 from the index, because proc(5) is 1-based, and
                 # we've lost the first two fields in PID and COMMAND above
                 stats[name] = int(raw_stats[index - 3])
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 7c680659b6..2f7e77f5f5 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -30,6 +30,7 @@ from synapse.state import POWER_KEY
 
 from collections import namedtuple
 
+from six import itervalues, iteritems
 
 logger = logging.getLogger(__name__)
 
@@ -126,7 +127,7 @@ class BulkPushRuleEvaluator(object):
             )
             auth_events = yield self.store.get_events(auth_events_ids)
             auth_events = {
-                (e.type, e.state_key): e for e in auth_events.itervalues()
+                (e.type, e.state_key): e for e in itervalues(auth_events)
             }
 
         sender_level = get_user_power_level(event.sender, auth_events)
@@ -160,7 +161,7 @@ class BulkPushRuleEvaluator(object):
 
         condition_cache = {}
 
-        for uid, rules in rules_by_user.iteritems():
+        for uid, rules in iteritems(rules_by_user):
             if event.sender == uid:
                 continue
 
@@ -406,7 +407,7 @@ class RulesForRoom(object):
         # If the event is a join event then it will be in current state evnts
         # map but not in the DB, so we have to explicitly insert it.
         if event.type == EventTypes.Member:
-            for event_id in member_event_ids.itervalues():
+            for event_id in itervalues(member_event_ids):
                 if event_id == event.event_id:
                     members[event_id] = (event.state_key, event.membership)
 
@@ -414,7 +415,7 @@ class RulesForRoom(object):
             logger.debug("Found members %r: %r", self.room_id, members.values())
 
         interested_in_user_ids = set(
-            user_id for user_id, membership in members.itervalues()
+            user_id for user_id, membership in itervalues(members)
             if membership == Membership.JOIN
         )
 
@@ -426,7 +427,7 @@ class RulesForRoom(object):
         )
 
         user_ids = set(
-            uid for uid, have_pusher in if_users_with_pushers.iteritems() if have_pusher
+            uid for uid, have_pusher in iteritems(if_users_with_pushers) if have_pusher
         )
 
         logger.debug("With pushers: %r", user_ids)
@@ -447,7 +448,7 @@ class RulesForRoom(object):
         )
 
         ret_rules_by_user.update(
-            item for item in rules_by_user.iteritems() if item[0] is not None
+            item for item in iteritems(rules_by_user) if item[0] is not None
         )
 
         self.update_cache(sequence, members, ret_rules_by_user, state_group)
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index d7d38464b2..7ca1588f6a 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -68,6 +68,7 @@ import synapse.metrics
 import struct
 import fcntl
 
+from six import iterkeys, iteritems
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
@@ -392,7 +393,7 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol):
 
         if stream_name == "ALL":
             # Subscribe to all streams we're publishing to.
-            for stream in self.streamer.streams_by_name.iterkeys():
+            for stream in iterkeys(self.streamer.streams_by_name):
                 self.subscribe_to_stream(stream, token)
         else:
             self.subscribe_to_stream(stream_name, token)
@@ -498,7 +499,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
         BaseReplicationStreamProtocol.connectionMade(self)
 
         # Once we've connected subscribe to the necessary streams
-        for stream_name, token in self.handler.get_streams_to_replicate().iteritems():
+        for stream_name, token in iteritems(self.handler.get_streams_to_replicate()):
             self.replicate(stream_name, token)
 
         # Tell the server if we have any users currently syncing (should only
@@ -633,7 +634,7 @@ metrics.register_callback(
     lambda: {
         (k[0], p.name, p.conn_id): count
         for p in connected_connections
-        for k, count in p.inbound_commands_counter.counts.iteritems()
+        for k, count in iteritems(p.inbound_commands_counter.counts)
     },
     labels=["command", "name", "conn_id"],
 )
@@ -643,7 +644,7 @@ metrics.register_callback(
     lambda: {
         (k[0], p.name, p.conn_id): count
         for p in connected_connections
-        for k, count in p.outbound_commands_counter.counts.iteritems()
+        for k, count in iteritems(p.outbound_commands_counter.counts)
     },
     labels=["command", "name", "conn_id"],
 )
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index a41af4fd6c..d1c291e17d 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -26,6 +26,7 @@ from synapse.util.metrics import Measure, measure_func
 import logging
 import synapse.metrics
 
+from six import itervalues
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 stream_updates_counter = metrics.register_counter(
@@ -79,7 +80,7 @@ class ReplicationStreamer(object):
         # We only support federation stream if federation sending hase been
         # disabled on the master.
         self.streams = [
-            stream(hs) for stream in STREAMS_MAP.itervalues()
+            stream(hs) for stream in itervalues(STREAMS_MAP)
             if stream != FederationStream or not hs.config.send_federation
         ]
 
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 9800ce7581..2ac767d2dc 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -48,6 +48,7 @@ import shutil
 import cgi
 import logging
 from six.moves.urllib import parse as urlparse
+from six import iteritems
 
 logger = logging.getLogger(__name__)
 
@@ -603,7 +604,7 @@ class MediaRepository(object):
                 thumbnails[(t_width, t_height, r_type)] = r_method
 
         # Now we generate the thumbnails for each dimension, store it
-        for (t_width, t_height, t_type), t_method in thumbnails.iteritems():
+        for (t_width, t_height, t_type), t_method in iteritems(thumbnails):
             # Generate the thumbnail
             if t_method == "crop":
                 t_byte_source = yield make_deferred_yieldable(threads.deferToThread(
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index ba46907737..ce338514e8 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -22,6 +22,8 @@ from . import background_updates
 
 from synapse.util.caches import CACHE_SIZE_FACTOR
 
+from six import iteritems
+
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +101,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
     def _update_client_ips_batch_txn(self, txn, to_update):
         self.database_engine.lock_table(txn, "user_ips")
 
-        for entry in to_update.iteritems():
+        for entry in iteritems(to_update):
             (user_id, access_token, ip), (user_agent, device_id, last_seen) = entry
 
             self._simple_upsert_txn(
@@ -231,5 +233,5 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
                 "user_agent": user_agent,
                 "last_seen": last_seen,
             }
-            for (access_token, ip), (user_agent, last_seen) in results.iteritems()
+            for (access_token, ip), (user_agent, last_seen) in iteritems(results)
         ))
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 712106b83a..d149d8392e 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -21,6 +21,7 @@ from synapse.api.errors import StoreError
 from ._base import SQLBaseStore, Cache
 from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
 
+from six import itervalues, iteritems
 
 logger = logging.getLogger(__name__)
 
@@ -360,7 +361,7 @@ class DeviceStore(SQLBaseStore):
             return (now_stream_id, [])
 
         if len(query_map) >= 20:
-            now_stream_id = max(stream_id for stream_id in query_map.itervalues())
+            now_stream_id = max(stream_id for stream_id in itervalues(query_map))
 
         devices = self._get_e2e_device_keys_txn(
             txn, query_map.keys(), include_all_devices=True
@@ -373,13 +374,13 @@ class DeviceStore(SQLBaseStore):
         """
 
         results = []
-        for user_id, user_devices in devices.iteritems():
+        for user_id, user_devices in iteritems(devices):
             # The prev_id for the first row is always the last row before
             # `from_stream_id`
             txn.execute(prev_sent_id_sql, (destination, user_id, from_stream_id))
             rows = txn.fetchall()
             prev_id = rows[0][0]
-            for device_id, device in user_devices.iteritems():
+            for device_id, device in iteritems(user_devices):
                 stream_id = query_map[(user_id, device_id)]
                 result = {
                     "user_id": user_id,
@@ -483,7 +484,7 @@ class DeviceStore(SQLBaseStore):
         if devices:
             user_devices = devices[user_id]
             results = []
-            for device_id, device in user_devices.iteritems():
+            for device_id, device in iteritems(user_devices):
                 result = {
                     "device_id": device_id,
                 }
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index ff8538ddf8..b146487943 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -21,6 +21,8 @@ import simplejson as json
 
 from ._base import SQLBaseStore
 
+from six import iteritems
+
 
 class EndToEndKeyStore(SQLBaseStore):
     def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys):
@@ -81,8 +83,8 @@ class EndToEndKeyStore(SQLBaseStore):
             query_list, include_all_devices,
         )
 
-        for user_id, device_keys in results.iteritems():
-            for device_id, device_info in device_keys.iteritems():
+        for user_id, device_keys in iteritems(results):
+            for device_id, device_info in iteritems(device_keys):
                 device_info["keys"] = json.loads(device_info.pop("key_json"))
 
         defer.returnValue(results)
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index f084a5f54b..d0350ee5fe 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -22,6 +22,8 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks
 import logging
 import simplejson as json
 
+from six import iteritems
+
 logger = logging.getLogger(__name__)
 
 
@@ -420,7 +422,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
 
             txn.executemany(sql, (
                 _gen_entry(user_id, actions)
-                for user_id, actions in user_id_actions.iteritems()
+                for user_id, actions in iteritems(user_id_actions)
             ))
 
         return self.runInteraction(
-- 
cgit 1.4.1


From dfa70adc33b8f058acae8a4f4f61a83f9fab47a7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 21 May 2018 16:03:39 +0100
Subject: Add in flight request metrics

This tracks CPU and DB usage while requests are in flight, rather than
when we write the response.
---
 synapse/http/request_metrics.py | 152 +++++++++++++++++++++++++++++++++++++++-
 synapse/http/site.py            |   4 +-
 2 files changed, 154 insertions(+), 2 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 8c850bf23f..f1e9b41fde 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -98,14 +98,76 @@ response_size = metrics.register_counter(
     "response_size", labels=["method", "servlet", "tag"]
 )
 
+# In flight metrics are incremented while the requests are in flight, rather
+# than when the response was written.
+
+in_flight_requests_ru_utime = metrics.register_counter(
+    "in_flight_requests_ru_utime_seconds", labels=["method", "servlet"],
+)
+
+in_flight_requests_ru_stime = metrics.register_counter(
+    "in_flight_requests_ru_stime_seconds", labels=["method", "servlet"],
+)
+
+in_flight_requests_db_txn_count = metrics.register_counter(
+    "in_flight_requests_db_txn_count", labels=["method", "servlet"],
+)
+
+# seconds spent waiting for db txns, excluding scheduling time, when processing
+# this request
+in_flight_requests_db_txn_duration = metrics.register_counter(
+    "in_flight_requests_db_txn_duration_seconds", labels=["method", "servlet"],
+)
+
+# seconds spent waiting for a db connection, when processing this request
+in_flight_requests_db_sched_duration = metrics.register_counter(
+    "in_flight_requests_db_sched_duration_seconds", labels=["method", "servlet"]
+)
+
+_in_flight_requests_count = metrics.register_gauge(
+    "in_flight_requests_count", labels=["method", "servlet"]
+)
+
+
+# The set of all in flight requests.
+_in_flight_requests = set()
+
+
+def _collect_in_flight():
+    """Called just before metrics are collected, so we use it to update all
+    the in flight request metrics
+    """
+
+    # Map from (method, name) -> int, the number of in flight requests of that
+    # type
+    counts = {}
+
+    for rm in _in_flight_requests:
+        rm.update_metrics()
+        key = (rm.method, rm.name,)
+        counts[key] = counts.get(key, 0) + 1
+
+    for (method, name), count in counts.iteritems():
+        _in_flight_requests_count.set(count, method, name)
+
+
+metrics.register_collector(_collect_in_flight)
+
 
 class RequestMetrics(object):
-    def start(self, time_msec, name):
+    def start(self, time_msec, name, method):
         self.start = time_msec
         self.start_context = LoggingContext.current_context()
         self.name = name
+        self.method = method
+
+        self._request_stats = _RequestStats.from_context(self.start_context)
+
+        _in_flight_requests.add(self)
 
     def stop(self, time_msec, request):
+        _in_flight_requests.discard(self)
+
         context = LoggingContext.current_context()
 
         tag = ""
@@ -147,3 +209,91 @@ class RequestMetrics(object):
         )
 
         response_size.inc_by(request.sentLength, request.method, self.name, tag)
+
+        # We always call this at the end to ensure that we update the metrics
+        # regardless of whether a call to /metrics while the request was in
+        # flight.
+        self.update_metrics()
+
+    def update_metrics(self):
+        """Updates the in flight metrics with values from this request.
+        """
+
+        diff = self._request_stats.update()
+
+        in_flight_requests_ru_utime.inc_by(
+            diff.ru_utime, self.method, self.name,
+        )
+
+        in_flight_requests_ru_stime.inc_by(
+            diff.ru_stime, self.method, self.name,
+        )
+
+        in_flight_requests_db_txn_count.inc_by(
+            diff.db_txn_count, self.method, self.name,
+        )
+
+        in_flight_requests_db_txn_duration.inc_by(
+            diff.db_txn_duration_ms / 1000., self.method, self.name,
+        )
+
+        in_flight_requests_db_sched_duration.inc_by(
+            diff.db_sched_duration_ms / 1000., self.method, self.name,
+        )
+
+
+class _RequestStats(object):
+    """Keeps tracks of various metrics for an in flight request.
+    """
+
+    __slots__ = [
+        "context", "ru_utime", "ru_stime",
+        "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
+    ]
+
+    def __init__(self, context, ru_utime, ru_stime, db_txn_count,
+                 db_txn_duration_ms, db_sched_duration_ms):
+        self.context = context
+        self.ru_utime = ru_utime
+        self.ru_stime = ru_stime
+        self.db_txn_count = db_txn_count
+        self.db_txn_duration_ms = db_txn_duration_ms
+        self.db_sched_duration_ms = db_sched_duration_ms
+
+    @staticmethod
+    def from_context(context):
+        ru_utime, ru_stime = context.get_resource_usage()
+
+        return _RequestStats(
+            context,
+            ru_utime, ru_stime,
+            context.db_txn_count,
+            context.db_txn_duration_ms,
+            context.db_sched_duration_ms,
+        )
+
+    def update(self):
+        """Updates the current values and returns the difference between the
+        old and new values.
+
+        Returns:
+            _RequestStats: The difference between the old and new values
+        """
+        new = _RequestStats.from_context(self.context)
+
+        diff = _RequestStats(
+            self.context,
+            new.ru_utime - self.ru_utime,
+            new.ru_stime - self.ru_stime,
+            new.db_txn_count - self.db_txn_count,
+            new.db_txn_duration_ms - self.db_txn_duration_ms,
+            new.db_sched_duration_ms - self.db_sched_duration_ms,
+        )
+
+        self.ru_utime = new.ru_utime
+        self.ru_stime = new.ru_stime
+        self.db_txn_count = new.db_txn_count
+        self.db_txn_duration_ms = new.db_txn_duration_ms
+        self.db_sched_duration_ms = new.db_sched_duration_ms
+
+        return diff
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 202a990508..b608504225 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -85,7 +85,9 @@ class SynapseRequest(Request):
     def _started_processing(self, servlet_name):
         self.start_time = int(time.time() * 1000)
         self.request_metrics = RequestMetrics()
-        self.request_metrics.start(self.start_time, name=servlet_name)
+        self.request_metrics.start(
+            self.start_time, name=servlet_name, method=self.method,
+        )
 
         self.site.access_logger.info(
             "%s - %s - Received request: %s %s",
-- 
cgit 1.4.1


From 6d6e7288fe0200353740e03e6218a2781342f3e4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 21 May 2018 16:49:59 +0100
Subject: Stop the transaction cache caching failures

The transaction cache has some code which tries to stop it caching failures,
but if the callback function failed straight away, then things would happen
backwards and we'd end up with the failure stuck in the cache.
---
 synapse/rest/client/transactions.py    | 22 ++++++++------
 tests/rest/client/test_transactions.py | 54 ++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index fceca2edeb..93ce0f5348 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -87,19 +87,23 @@ class HttpTransactionCache(object):
 
         deferred = fn(*args, **kwargs)
 
-        # if the request fails with a Twisted failure, remove it
-        # from the transaction map. This is done to ensure that we don't
-        # cache transient errors like rate-limiting errors, etc.
+        observable = ObservableDeferred(deferred, consumeErrors=False)
+        self.transactions[txn_key] = (observable, self.clock.time_msec())
+
+        # if the request fails with an exception, remove it from the
+        # transaction map. This is done to ensure that we don't cache
+        # transient errors like rate-limiting errors, etc.
+        #
+        # (make sure we add this errback *after* adding the key above, in case
+        # the deferred has already failed and is running errbacks
+        # synchronously)
         def remove_from_map(err):
             self.transactions.pop(txn_key, None)
-            return err
+            # we deliberately do not propagate the error any further, as we
+            # expect the observers to have reported it.
+
         deferred.addErrback(remove_from_map)
 
-        # We don't add any other errbacks to the raw deferred, so we ask
-        # ObservableDeferred to swallow the error. This is fine as the error will
-        # still be reported to the observers.
-        observable = ObservableDeferred(deferred, consumeErrors=True)
-        self.transactions[txn_key] = (observable, self.clock.time_msec())
         return observable.observe()
 
     def _cleanup(self):
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index d7cea30260..b650a7772b 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -2,6 +2,8 @@ from synapse.rest.client.transactions import HttpTransactionCache
 from synapse.rest.client.transactions import CLEANUP_PERIOD_MS
 from twisted.internet import defer
 from mock import Mock, call
+
+from synapse.util.logcontext import LoggingContext
 from tests import unittest
 from tests.utils import MockClock
 
@@ -39,6 +41,58 @@ class HttpTransactionCacheTestCase(unittest.TestCase):
         # expect only a single call to do the work
         cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0)
 
+    @defer.inlineCallbacks
+    def test_does_not_cache_exceptions(self):
+        """Checks that, if the callback throws an exception, it is called again
+        for the next request.
+        """
+        called = [False]
+
+        def cb():
+            if called[0]:
+                # return a valid result the second time
+                return defer.succeed(self.mock_http_response)
+
+            called[0] = True
+            raise Exception("boo")
+
+        with LoggingContext("test") as test_context:
+            try:
+                yield self.cache.fetch_or_execute(self.mock_key, cb)
+            except Exception as e:
+                self.assertEqual(e.message, "boo")
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+            res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+            self.assertEqual(res, self.mock_http_response)
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+    @defer.inlineCallbacks
+    def test_does_not_cache_failures(self):
+        """Checks that, if the callback returns a failure, it is called again
+        for the next request.
+        """
+        called = [False]
+
+        def cb():
+            if called[0]:
+                # return a valid result the second time
+                return defer.succeed(self.mock_http_response)
+
+            called[0] = True
+            return defer.fail(Exception("boo"))
+
+        with LoggingContext("test") as test_context:
+            try:
+                yield self.cache.fetch_or_execute(self.mock_key, cb)
+            except Exception as e:
+                self.assertEqual(e.message, "boo")
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+            res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+            self.assertEqual(res, self.mock_http_response)
+            self.assertIs(LoggingContext.current_context(), test_context)
+
     @defer.inlineCallbacks
     def test_cleans_up(self):
         cb = Mock(
-- 
cgit 1.4.1


From 6e1cb54a05df3dd614acb022a665458cc1c9698f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 21 May 2018 16:58:20 +0100
Subject: Fix logcontext leak in HttpTransactionCache

ONE DAY I WILL PURGE THE WORLD OF THIS EVIL
---
 synapse/rest/client/transactions.py    | 46 ++++++++++++++++------------------
 tests/rest/client/test_transactions.py | 21 ++++++++++++++++
 2 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 93ce0f5348..20fa6678ef 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -19,6 +19,7 @@ import logging
 
 from synapse.api.auth import get_access_token_from_request
 from synapse.util.async import ObservableDeferred
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
 
@@ -80,31 +81,26 @@ class HttpTransactionCache(object):
         Returns:
             Deferred which resolves to a tuple of (response_code, response_dict).
         """
-        try:
-            return self.transactions[txn_key][0].observe()
-        except (KeyError, IndexError):
-            pass  # execute the function instead.
-
-        deferred = fn(*args, **kwargs)
-
-        observable = ObservableDeferred(deferred, consumeErrors=False)
-        self.transactions[txn_key] = (observable, self.clock.time_msec())
-
-        # if the request fails with an exception, remove it from the
-        # transaction map. This is done to ensure that we don't cache
-        # transient errors like rate-limiting errors, etc.
-        #
-        # (make sure we add this errback *after* adding the key above, in case
-        # the deferred has already failed and is running errbacks
-        # synchronously)
-        def remove_from_map(err):
-            self.transactions.pop(txn_key, None)
-            # we deliberately do not propagate the error any further, as we
-            # expect the observers to have reported it.
-
-        deferred.addErrback(remove_from_map)
-
-        return observable.observe()
+        if txn_key in self.transactions:
+            observable = self.transactions[txn_key][0]
+        else:
+            # execute the function instead.
+            deferred = run_in_background(fn, *args, **kwargs)
+
+            observable = ObservableDeferred(deferred)
+            self.transactions[txn_key] = (observable, self.clock.time_msec())
+
+            # if the request fails with an exception, remove it
+            # from the transaction map. This is done to ensure that we don't
+            # cache transient errors like rate-limiting errors, etc.
+            def remove_from_map(err):
+                self.transactions.pop(txn_key, None)
+                # we deliberately do not propagate the error any further, as we
+                # expect the observers to have reported it.
+
+            deferred.addErrback(remove_from_map)
+
+        return make_deferred_yieldable(observable.observe())
 
     def _cleanup(self):
         now = self.clock.time_msec()
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index b650a7772b..b5bc2fa255 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -3,6 +3,7 @@ from synapse.rest.client.transactions import CLEANUP_PERIOD_MS
 from twisted.internet import defer
 from mock import Mock, call
 
+from synapse.util import async
 from synapse.util.logcontext import LoggingContext
 from tests import unittest
 from tests.utils import MockClock
@@ -41,6 +42,26 @@ class HttpTransactionCacheTestCase(unittest.TestCase):
         # expect only a single call to do the work
         cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0)
 
+    @defer.inlineCallbacks
+    def test_logcontexts_with_async_result(self):
+        @defer.inlineCallbacks
+        def cb():
+            yield async.sleep(0)
+            defer.returnValue("yay")
+
+        @defer.inlineCallbacks
+        def test():
+            with LoggingContext("c") as c1:
+                res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+                self.assertIs(LoggingContext.current_context(), c1)
+                self.assertEqual(res, "yay")
+
+        # run the test twice in parallel
+        d = defer.gatherResults([test(), test()])
+        self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel)
+        yield d
+        self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel)
+
     @defer.inlineCallbacks
     def test_does_not_cache_exceptions(self):
         """Checks that, if the callback throws an exception, it is called again
-- 
cgit 1.4.1


From f258deffcb7c1fbb0389e4e67eaeb14f12c326a7 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 21 May 2018 17:01:15 -0500
Subject: remove old metrics libs

---
 synapse/metrics/metric.py            | 328 -----------------------------------
 synapse/metrics/process_collector.py | 122 -------------
 2 files changed, 450 deletions(-)
 delete mode 100644 synapse/metrics/metric.py
 delete mode 100644 synapse/metrics/process_collector.py

diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
deleted file mode 100644
index f421e7a93f..0000000000
--- a/synapse/metrics/metric.py
+++ /dev/null
@@ -1,328 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from itertools import chain
-import logging
-import re
-
-logger = logging.getLogger(__name__)
-
-
-def flatten(items):
-    """Flatten a list of lists
-
-    Args:
-        items: iterable[iterable[X]]
-
-    Returns:
-        list[X]: flattened list
-    """
-    return list(chain.from_iterable(items))
-
-
-class BaseMetric(object):
-    """Base class for metrics which report a single value per label set
-    """
-
-    def __init__(self, name, labels=[], alternative_names=[]):
-        """
-        Args:
-            name (str): principal name for this metric
-            labels (list(str)): names of the labels which will be reported
-                for this metric
-            alternative_names (iterable(str)): list of alternative names for
-                 this metric. This can be useful to provide a migration path
-                when renaming metrics.
-        """
-        self._names = [name] + list(alternative_names)
-        self.labels = labels  # OK not to clone as we never write it
-
-    def dimension(self):
-        return len(self.labels)
-
-    def is_scalar(self):
-        return not len(self.labels)
-
-    def _render_labelvalue(self, value):
-        return '"%s"' % (_escape_label_value(value),)
-
-    def _render_key(self, values):
-        if self.is_scalar():
-            return ""
-        return "{%s}" % (
-            ",".join(["%s=%s" % (k, self._render_labelvalue(v))
-                      for k, v in zip(self.labels, values)])
-        )
-
-    def _render_for_labels(self, label_values, value):
-        """Render this metric for a single set of labels
-
-        Args:
-            label_values (list[object]): values for each of the labels,
-                (which get stringified).
-            value: value of the metric at with these labels
-
-        Returns:
-            iterable[str]: rendered metric
-        """
-        rendered_labels = self._render_key(label_values)
-        return (
-            "%s%s %.12g" % (name, rendered_labels, value)
-            for name in self._names
-        )
-
-    def render(self):
-        """Render this metric
-
-        Each metric is rendered as:
-
-            name{label1="val1",label2="val2"} value
-
-        https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details
-
-        Returns:
-            iterable[str]: rendered metrics
-        """
-        raise NotImplementedError()
-
-
-class CounterMetric(BaseMetric):
-    """The simplest kind of metric; one that stores a monotonically-increasing
-    value that counts events or running totals.
-
-    Example use cases for Counters:
-    - Number of requests processed
-    - Number of items that were inserted into a queue
-    - Total amount of data that a system has processed
-    Counters can only go up (and be reset when the process restarts).
-    """
-
-    def __init__(self, *args, **kwargs):
-        super(CounterMetric, self).__init__(*args, **kwargs)
-
-        # dict[list[str]]: value for each set of label values. the keys are the
-        # label values, in the same order as the labels in self.labels.
-        #
-        # (if the metric is a scalar, the (single) key is the empty tuple).
-        self.counts = {}
-
-        # Scalar metrics are never empty
-        if self.is_scalar():
-            self.counts[()] = 0.
-
-    def inc_by(self, incr, *values):
-        if len(values) != self.dimension():
-            raise ValueError(
-                "Expected as many values to inc() as labels (%d)" % (self.dimension())
-            )
-
-        # TODO: should assert that the tag values are all strings
-
-        if values not in self.counts:
-            self.counts[values] = incr
-        else:
-            self.counts[values] += incr
-
-    def inc(self, *values):
-        self.inc_by(1, *values)
-
-    def render(self):
-        return flatten(
-            self._render_for_labels(k, self.counts[k])
-            for k in sorted(self.counts.keys())
-        )
-
-
-class GaugeMetric(BaseMetric):
-    """A metric that can go up or down
-    """
-
-    def __init__(self, *args, **kwargs):
-        super(GaugeMetric, self).__init__(*args, **kwargs)
-
-        # dict[list[str]]: value for each set of label values. the keys are the
-        # label values, in the same order as the labels in self.labels.
-        #
-        # (if the metric is a scalar, the (single) key is the empty tuple).
-        self.guages = {}
-
-    def set(self, v, *values):
-        if len(values) != self.dimension():
-            raise ValueError(
-                "Expected as many values to inc() as labels (%d)" % (self.dimension())
-            )
-
-        # TODO: should assert that the tag values are all strings
-
-        self.guages[values] = v
-
-    def render(self):
-        return flatten(
-            self._render_for_labels(k, self.guages[k])
-            for k in sorted(self.guages.keys())
-        )
-
-
-class CallbackMetric(BaseMetric):
-    """A metric that returns the numeric value returned by a callback whenever
-    it is rendered. Typically this is used to implement gauges that yield the
-    size or other state of some in-memory object by actively querying it."""
-
-    def __init__(self, name, callback, labels=[]):
-        super(CallbackMetric, self).__init__(name, labels=labels)
-
-        self.callback = callback
-
-    def render(self):
-        try:
-            value = self.callback()
-        except Exception:
-            logger.exception("Failed to render %s", self.name)
-            return ["# FAILED to render " + self.name]
-
-        if self.is_scalar():
-            return list(self._render_for_labels([], value))
-
-        return flatten(
-            self._render_for_labels(k, value[k])
-            for k in sorted(value.keys())
-        )
-
-
-class DistributionMetric(object):
-    """A combination of an event counter and an accumulator, which counts
-    both the number of events and accumulates the total value. Typically this
-    could be used to keep track of method-running times, or other distributions
-    of values that occur in discrete occurances.
-
-    TODO(paul): Try to export some heatmap-style stats?
-    """
-
-    def __init__(self, name, *args, **kwargs):
-        self.counts = CounterMetric(name + ":count", **kwargs)
-        self.totals = CounterMetric(name + ":total", **kwargs)
-
-    def inc_by(self, inc, *values):
-        self.counts.inc(*values)
-        self.totals.inc_by(inc, *values)
-
-    def render(self):
-        return self.counts.render() + self.totals.render()
-
-
-class CacheMetric(object):
-    __slots__ = (
-        "name", "cache_name", "hits", "misses", "evicted_size", "size_callback",
-    )
-
-    def __init__(self, name, size_callback, cache_name):
-        self.name = name
-        self.cache_name = cache_name
-
-        self.hits = 0
-        self.misses = 0
-        self.evicted_size = 0
-
-        self.size_callback = size_callback
-
-    def inc_hits(self):
-        self.hits += 1
-
-    def inc_misses(self):
-        self.misses += 1
-
-    def inc_evictions(self, size=1):
-        self.evicted_size += size
-
-    def render(self):
-        size = self.size_callback()
-        hits = self.hits
-        total = self.misses + self.hits
-
-        return [
-            """%s:hits{name="%s"} %d""" % (self.name, self.cache_name, hits),
-            """%s:total{name="%s"} %d""" % (self.name, self.cache_name, total),
-            """%s:size{name="%s"} %d""" % (self.name, self.cache_name, size),
-            """%s:evicted_size{name="%s"} %d""" % (
-                self.name, self.cache_name, self.evicted_size
-            ),
-        ]
-
-
-class MemoryUsageMetric(object):
-    """Keeps track of the current memory usage, using psutil.
-
-    The class will keep the current min/max/sum/counts of rss over the last
-    WINDOW_SIZE_SEC, by polling UPDATE_HZ times per second
-    """
-
-    UPDATE_HZ = 2  # number of times to get memory per second
-    WINDOW_SIZE_SEC = 30  # the size of the window in seconds
-
-    def __init__(self, hs, psutil):
-        clock = hs.get_clock()
-        self.memory_snapshots = []
-
-        self.process = psutil.Process()
-
-        clock.looping_call(self._update_curr_values, 1000 / self.UPDATE_HZ)
-
-    def _update_curr_values(self):
-        max_size = self.UPDATE_HZ * self.WINDOW_SIZE_SEC
-        self.memory_snapshots.append(self.process.memory_info().rss)
-        self.memory_snapshots[:] = self.memory_snapshots[-max_size:]
-
-    def render(self):
-        if not self.memory_snapshots:
-            return []
-
-        max_rss = max(self.memory_snapshots)
-        min_rss = min(self.memory_snapshots)
-        sum_rss = sum(self.memory_snapshots)
-        len_rss = len(self.memory_snapshots)
-
-        return [
-            "process_psutil_rss:max %d" % max_rss,
-            "process_psutil_rss:min %d" % min_rss,
-            "process_psutil_rss:total %d" % sum_rss,
-            "process_psutil_rss:count %d" % len_rss,
-        ]
-
-
-def _escape_character(m):
-    """Replaces a single character with its escape sequence.
-
-    Args:
-        m (re.MatchObject): A match object whose first group is the single
-            character to replace
-
-    Returns:
-        str
-    """
-    c = m.group(1)
-    if c == "\\":
-        return "\\\\"
-    elif c == "\"":
-        return "\\\""
-    elif c == "\n":
-        return "\\n"
-    return c
-
-
-def _escape_label_value(value):
-    """Takes a label value and escapes quotes, newlines and backslashes
-    """
-    return re.sub(r"([\n\"\\])", _escape_character, str(value))
diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py
deleted file mode 100644
index 6fec3de399..0000000000
--- a/synapse/metrics/process_collector.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-
-TICKS_PER_SEC = 100
-BYTES_PER_PAGE = 4096
-
-HAVE_PROC_STAT = os.path.exists("/proc/stat")
-HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat")
-HAVE_PROC_SELF_LIMITS = os.path.exists("/proc/self/limits")
-HAVE_PROC_SELF_FD = os.path.exists("/proc/self/fd")
-
-# Field indexes from /proc/self/stat, taken from the proc(5) manpage
-STAT_FIELDS = {
-    "utime": 14,
-    "stime": 15,
-    "starttime": 22,
-    "vsize": 23,
-    "rss": 24,
-}
-
-
-stats = {}
-
-# In order to report process_start_time_seconds we need to know the
-# machine's boot time, because the value in /proc/self/stat is relative to
-# this
-boot_time = None
-if HAVE_PROC_STAT:
-    with open("/proc/stat") as _procstat:
-        for line in _procstat:
-            if line.startswith("btime "):
-                boot_time = int(line.split()[1])
-
-
-def update_resource_metrics():
-    if HAVE_PROC_SELF_STAT:
-        global stats
-        with open("/proc/self/stat") as s:
-            line = s.read()
-            # line is PID (command) more stats go here ...
-            raw_stats = line.split(") ", 1)[1].split(" ")
-
-            for (name, index) in STAT_FIELDS.iteritems():
-                # subtract 3 from the index, because proc(5) is 1-based, and
-                # we've lost the first two fields in PID and COMMAND above
-                stats[name] = int(raw_stats[index - 3])
-
-
-def _count_fds():
-    # Not every OS will have a /proc/self/fd directory
-    if not HAVE_PROC_SELF_FD:
-        return 0
-
-    return len(os.listdir("/proc/self/fd"))
-
-
-def register_process_collector(process_metrics):
-    process_metrics.register_collector(update_resource_metrics)
-
-    if HAVE_PROC_SELF_STAT:
-        process_metrics.register_callback(
-            "cpu_user_seconds_total",
-            lambda: float(stats["utime"]) / TICKS_PER_SEC
-        )
-        process_metrics.register_callback(
-            "cpu_system_seconds_total",
-            lambda: float(stats["stime"]) / TICKS_PER_SEC
-        )
-        process_metrics.register_callback(
-            "cpu_seconds_total",
-            lambda: (float(stats["utime"] + stats["stime"])) / TICKS_PER_SEC
-        )
-
-        process_metrics.register_callback(
-            "virtual_memory_bytes",
-            lambda: int(stats["vsize"])
-        )
-        process_metrics.register_callback(
-            "resident_memory_bytes",
-            lambda: int(stats["rss"]) * BYTES_PER_PAGE
-        )
-
-        process_metrics.register_callback(
-            "start_time_seconds",
-            lambda: boot_time + int(stats["starttime"]) / TICKS_PER_SEC
-        )
-
-    if HAVE_PROC_SELF_FD:
-        process_metrics.register_callback(
-            "open_fds",
-            lambda: _count_fds()
-        )
-
-    if HAVE_PROC_SELF_LIMITS:
-        def _get_max_fds():
-            with open("/proc/self/limits") as limits:
-                for line in limits:
-                    if not line.startswith("Max open files "):
-                        continue
-                    # Line is  Max open files  $SOFT  $HARD
-                    return int(line.split()[3])
-            return None
-
-        process_metrics.register_callback(
-            "max_fds",
-            lambda: _get_max_fds()
-        )
-- 
cgit 1.4.1


From 02c1d29133a2ce3cdb5d1b23ec5edde49d813da9 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 21 May 2018 17:02:20 -0500
Subject: look at the Prometheus metrics instead

---
 synapse/app/homeserver.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 75f40fd5a4..6393804a86 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -34,8 +34,7 @@ from synapse.module_api import ModuleApi
 from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
-from synapse.metrics import register_memory_metrics
-from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
+from synapse.metrics.resource import METRICS_PREFIX
 from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \
     check_requirements
 from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX
@@ -221,7 +220,8 @@ class SynapseHomeServer(HomeServer):
             resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
 
         if name == "metrics" and self.get_config().enable_metrics:
-            resources[METRICS_PREFIX] = MetricsResource(self)
+            from prometheus_client.twisted import MetricsResource
+            resources[METRICS_PREFIX] = MetricsResource()
 
         if name == "replication":
             resources[REPLICATION_PREFIX] = ReplicationRestResource(self)
@@ -353,8 +353,6 @@ def setup(config_options):
         hs.get_datastore().start_doing_background_updates()
         hs.get_federation_client().start_get_pdu_cache()
 
-        register_memory_metrics(hs)
-
     reactor.callWhenRunning(start)
 
     return hs
-- 
cgit 1.4.1


From c60e0d5e02789217671482bc209da5ac0fd25baf Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 21 May 2018 17:03:20 -0500
Subject: don't need the resource portion

---
 synapse/metrics/resource.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/synapse/metrics/resource.py b/synapse/metrics/resource.py
index 870f400600..7996e6ab66 100644
--- a/synapse/metrics/resource.py
+++ b/synapse/metrics/resource.py
@@ -13,27 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.web.resource import Resource
-
-import synapse.metrics
-
-
 METRICS_PREFIX = "/_synapse/metrics"
-
-
-class MetricsResource(Resource):
-    isLeaf = True
-
-    def __init__(self, hs):
-        Resource.__init__(self)  # Resource is old-style, so no super()
-
-        self.hs = hs
-
-    def render_GET(self, request):
-        response = synapse.metrics.render_all()
-
-        request.setHeader("Content-Type", "text/plain")
-        request.setHeader("Content-Length", str(len(response)))
-
-        # Encode as UTF-8 (default)
-        return response.encode()
-- 
cgit 1.4.1


From df9f72d9e5fe264b86005208e0f096156eb03e4b Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 21 May 2018 19:47:37 -0500
Subject: replacing portions

---
 synapse/api/auth.py                        |   2 +-
 synapse/federation/federation_client.py    |  17 ++-
 synapse/federation/federation_server.py    |  16 ++-
 synapse/federation/send_queue.py           |   8 +-
 synapse/federation/transaction_queue.py    |  47 +++-----
 synapse/handlers/appservice.py             |  19 +--
 synapse/handlers/presence.py               |  58 ++++-----
 synapse/http/client.py                     |  20 +---
 synapse/http/matrixfederationclient.py     |  14 +--
 synapse/metrics/__init__.py                | 182 ++++++++++-------------------
 synapse/notifier.py                        |  23 ++--
 synapse/push/httppusher.py                 |  13 +--
 synapse/push/push_rule_evaluator.py        |   2 +-
 synapse/python_dependencies.py             |   1 +
 synapse/replication/tcp/protocol.py        |  88 ++++++--------
 synapse/util/caches/__init__.py            |  57 ++++++---
 synapse/util/caches/descriptors.py         |   2 +-
 synapse/util/caches/dictionary_cache.py    |   2 +-
 synapse/util/caches/expiringcache.py       |   4 +-
 synapse/util/caches/response_cache.py      |  11 +-
 synapse/util/caches/stream_change_cache.py |   2 +-
 synapse/util/metrics.py                    |  91 ++++-----------
 tests/__init__.py                          |   3 +
 23 files changed, 268 insertions(+), 414 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index f17fda6315..b052cf532b 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -57,7 +57,7 @@ class Auth(object):
         self.TOKEN_NOT_FOUND_HTTP_STATUS = 401
 
         self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
-        register_cache("token_cache", self.token_cache)
+        register_cache("cache", "token_cache", self.token_cache)
 
     @defer.inlineCallbacks
     def check_from_context(self, event, context, do_sig_check=True):
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 6163f7c466..2761ffae07 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -32,20 +32,17 @@ from synapse.federation.federation_base import (
     FederationBase,
     event_from_pdu_json,
 )
-import synapse.metrics
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.logutils import log_function
 from synapse.util.retryutils import NotRetryingDestination
 
-logger = logging.getLogger(__name__)
-
+from prometheus_client import Counter
 
-# synapse.federation.federation_client is a silly name
-metrics = synapse.metrics.get_metrics_for("synapse.federation.client")
+logger = logging.getLogger(__name__)
 
-sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"])
+sent_queries_counter = Counter("synapse_federation_client_sent_queries", "", ["type"])
 
 
 PDU_RETRY_TIME_MS = 1 * 60 * 1000
@@ -108,7 +105,7 @@ class FederationClient(FederationBase):
             a Deferred which will eventually yield a JSON object from the
             response
         """
-        sent_queries_counter.inc(query_type)
+        sent_queries_counter.labels(query_type).inc()
 
         return self.transport_layer.make_query(
             destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail,
@@ -127,7 +124,7 @@ class FederationClient(FederationBase):
             a Deferred which will eventually yield a JSON object from the
             response
         """
-        sent_queries_counter.inc("client_device_keys")
+        sent_queries_counter.labels("client_device_keys").inc()
         return self.transport_layer.query_client_keys(
             destination, content, timeout
         )
@@ -137,7 +134,7 @@ class FederationClient(FederationBase):
         """Query the device keys for a list of user ids hosted on a remote
         server.
         """
-        sent_queries_counter.inc("user_devices")
+        sent_queries_counter.labels("user_devices").inc()
         return self.transport_layer.query_user_devices(
             destination, user_id, timeout
         )
@@ -154,7 +151,7 @@ class FederationClient(FederationBase):
             a Deferred which will eventually yield a JSON object from the
             response
         """
-        sent_queries_counter.inc("client_one_time_keys")
+        sent_queries_counter.labels("client_one_time_keys").inc()
         return self.transport_layer.claim_client_keys(
             destination, content, timeout
         )
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 247ddc89d5..8211273006 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -27,12 +27,13 @@ from synapse.federation.federation_base import (
 
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
-import synapse.metrics
 from synapse.types import get_domain_from_id
 from synapse.util import async
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.logutils import log_function
 
+from prometheus_client import Counter
+
 from six import iteritems
 
 # when processing incoming transactions, we try to handle multiple rooms in
@@ -41,14 +42,11 @@ TRANSACTION_CONCURRENCY_LIMIT = 10
 
 logger = logging.getLogger(__name__)
 
-# synapse.federation.federation_server is a silly name
-metrics = synapse.metrics.get_metrics_for("synapse.federation.server")
-
-received_pdus_counter = metrics.register_counter("received_pdus")
+received_pdus_counter = Counter("synapse_federation_server_received_pdus", "")
 
-received_edus_counter = metrics.register_counter("received_edus")
+received_edus_counter = Counter("synapse_federation_server_received_edus", "")
 
-received_queries_counter = metrics.register_counter("received_queries", labels=["type"])
+received_queries_counter = Counter("synapse_federation_server_received_queries", "", ["type"])
 
 
 class FederationServer(FederationBase):
@@ -131,7 +129,7 @@ class FederationServer(FederationBase):
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
-        received_pdus_counter.inc_by(len(transaction.pdus))
+        received_pdus_counter.inc(len(transaction.pdus))
 
         pdus_by_room = {}
 
@@ -292,7 +290,7 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     def on_query_request(self, query_type, args):
-        received_queries_counter.inc(query_type)
+        received_queries_counter.labels(query_type).inc()
         resp = yield self.registry.on_query(query_type, args)
         defer.returnValue((200, resp))
 
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 0f0c687b37..e6e1888f3a 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -33,7 +33,7 @@ from .units import Edu
 
 from synapse.storage.presence import UserPresenceState
 from synapse.util.metrics import Measure
-import synapse.metrics
+from synapse.metrics import LaterGauge
 
 from blist import sorteddict
 from collections import namedtuple
@@ -45,9 +45,6 @@ from six import itervalues, iteritems
 logger = logging.getLogger(__name__)
 
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-
 class FederationRemoteSendQueue(object):
     """A drop in replacement for TransactionQueue"""
 
@@ -77,8 +74,7 @@ class FederationRemoteSendQueue(object):
         # lambda binds to the queue rather than to the name of the queue which
         # changes. ARGH.
         def register(name, queue):
-            metrics.register_callback(
-                queue_name + "_size",
+            LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), "",
                 lambda: len(queue),
             )
 
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index ded2b1871a..778924a13c 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -26,23 +26,18 @@ from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
 from synapse.util.metrics import measure_func
 from synapse.handlers.presence import format_user_presence_state, get_interested_remotes
 import synapse.metrics
+from synapse.metrics import LaterGauge
+from synapse.metrics import (
+    sent_edus_counter, sent_transactions_counter, events_processed_counter)
+
+from prometheus_client import Counter
 
 import logging
 
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-client_metrics = synapse.metrics.get_metrics_for("synapse.federation.client")
-sent_pdus_destination_dist = client_metrics.register_distribution(
-    "sent_pdu_destinations"
-)
-sent_edus_counter = client_metrics.register_counter("sent_edus")
-
-sent_transactions_counter = client_metrics.register_counter("sent_transactions")
-
-events_processed_counter = client_metrics.register_counter("events_processed")
+sent_pdus_destination_dist = Counter("synapse_federation_client_sent_pdu_destinations", "")
 
 
 class TransactionQueue(object):
@@ -69,8 +64,7 @@ class TransactionQueue(object):
         # done
         self.pending_transactions = {}
 
-        metrics.register_callback(
-            "pending_destinations",
+        LaterGauge("pending_destinations", "", [],
             lambda: len(self.pending_transactions),
         )
 
@@ -94,12 +88,12 @@ class TransactionQueue(object):
         # Map of destination -> (edu_type, key) -> Edu
         self.pending_edus_keyed_by_dest = edus_keyed = {}
 
-        metrics.register_callback(
-            "pending_pdus",
+        LaterGauge(
+            "pending_pdus", "", [],
             lambda: sum(map(len, pdus.values())),
         )
-        metrics.register_callback(
-            "pending_edus",
+        LaterGauge(
+            "pending_edus", "", [],
             lambda: (
                 sum(map(len, edus.values()))
                 + sum(map(len, presence.values()))
@@ -241,18 +235,15 @@ class TransactionQueue(object):
                     now = self.clock.time_msec()
                     ts = yield self.store.get_received_ts(events[-1].event_id)
 
-                    synapse.metrics.event_processing_lag.set(
-                        now - ts, "federation_sender",
-                    )
-                    synapse.metrics.event_processing_last_ts.set(
-                        ts, "federation_sender",
-                    )
+                    synapse.metrics.event_processing_lag.labels(
+                        "federation_sender").set(now - ts)
+                    synapse.metrics.event_processing_last_ts.labels(
+                        "federation_sender").set(ts)
 
-                events_processed_counter.inc_by(len(events))
+                events_processed_counter.inc(len(events))
 
-                synapse.metrics.event_processing_positions.set(
-                    next_token, "federation_sender",
-                )
+                synapse.metrics.event_processing_positions.labels(
+                    "federation_sender").set(next_token)
 
         finally:
             self._is_processing = False
@@ -275,7 +266,7 @@ class TransactionQueue(object):
         if not destinations:
             return
 
-        sent_pdus_destination_dist.inc_by(len(destinations))
+        sent_pdus_destination_dist.inc(len(destinations))
 
         for destination in destinations:
             self.pending_pdus_by_dest.setdefault(destination, []).append(
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index b596f098fd..a7345331af 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -21,14 +21,13 @@ from synapse.util.metrics import Measure
 from synapse.util.logcontext import (
     make_deferred_yieldable, run_in_background,
 )
+from prometheus_client import Counter
 
 import logging
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-events_processed_counter = metrics.register_counter("events_processed")
+events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "")
 
 
 def log_failure(failure):
@@ -128,18 +127,12 @@ class ApplicationServicesHandler(object):
                     now = self.clock.time_msec()
                     ts = yield self.store.get_received_ts(events[-1].event_id)
 
-                    synapse.metrics.event_processing_positions.set(
-                        upper_bound, "appservice_sender",
-                    )
+                    synapse.metrics.event_processing_positions.labels("appservice_sender").set(upper_bound)
 
-                    events_processed_counter.inc_by(len(events))
+                    events_processed_counter.inc(len(events))
 
-                    synapse.metrics.event_processing_lag.set(
-                        now - ts, "appservice_sender",
-                    )
-                    synapse.metrics.event_processing_last_ts.set(
-                        ts, "appservice_sender",
-                    )
+                    synapse.metrics.event_processing_lag.labels("appservice_sender").set(now - ts)
+                    synapse.metrics.event_processing_last_ts.labels("appservice_sender").set(ts)
             finally:
                 self.is_processing = False
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 585f3e4da2..06d937ef3a 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -36,27 +36,27 @@ from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
 from synapse.types import UserID, get_domain_from_id
-import synapse.metrics
+from synapse.metrics import LaterGauge
 
 import logging
 
+from prometheus_client import Counter
+
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
 
-notified_presence_counter = metrics.register_counter("notified_presence")
-federation_presence_out_counter = metrics.register_counter("federation_presence_out")
-presence_updates_counter = metrics.register_counter("presence_updates")
-timers_fired_counter = metrics.register_counter("timers_fired")
-federation_presence_counter = metrics.register_counter("federation_presence")
-bump_active_time_counter = metrics.register_counter("bump_active_time")
+notified_presence_counter = Counter("synapse_handler_presence_notified_presence", "")
+federation_presence_out_counter = Counter("synapse_handler_presence_federation_presence_out", "")
+presence_updates_counter = Counter("synapse_handler_presence_presence_updates", "")
+timers_fired_counter = Counter("synapse_handler_presence_timers_fired", "")
+federation_presence_counter = Counter("synapse_handler_presence_federation_presence", "")
+bump_active_time_counter = Counter("synapse_handler_presence_bump_active_time", "")
 
-get_updates_counter = metrics.register_counter("get_updates", labels=["type"])
+get_updates_counter = Counter("synapse_handler_presence_get_updates", "", ["type"])
 
-notify_reason_counter = metrics.register_counter("notify_reason", labels=["reason"])
-state_transition_counter = metrics.register_counter(
-    "state_transition", labels=["from", "to"]
+notify_reason_counter = Counter("synapse_handler_presence_notify_reason", "", ["reason"])
+state_transition_counter = Counter("synapse_handler_presence_state_transition", "", ["from", "to"]
 )
 
 
@@ -137,9 +137,9 @@ class PresenceHandler(object):
             for state in active_presence
         }
 
-        metrics.register_callback(
-            "user_to_current_state_size", lambda: len(self.user_to_current_state)
-        )
+        LaterGauge(
+            "user_to_current_state_size", "", [], lambda: len(self.user_to_current_state)
+        ).register()
 
         now = self.clock.time_msec()
         for state in active_presence:
@@ -208,7 +208,7 @@ class PresenceHandler(object):
             60 * 1000,
         )
 
-        metrics.register_callback("wheel_timer_size", lambda: len(self.wheel_timer))
+        LaterGauge("wheel_timer_size", "", [], lambda: len(self.wheel_timer)).register()
 
     @defer.inlineCallbacks
     def _on_shutdown(self):
@@ -311,10 +311,10 @@ class PresenceHandler(object):
 
             # TODO: We should probably ensure there are no races hereafter
 
-            presence_updates_counter.inc_by(len(new_states))
+            presence_updates_counter.inc(len(new_states))
 
             if to_notify:
-                notified_presence_counter.inc_by(len(to_notify))
+                notified_presence_counter.inc(len(to_notify))
                 yield self._persist_and_notify(to_notify.values())
 
             self.unpersisted_users_changes |= set(s.user_id for s in new_states)
@@ -325,7 +325,7 @@ class PresenceHandler(object):
                 if user_id not in to_notify
             }
             if to_federation_ping:
-                federation_presence_out_counter.inc_by(len(to_federation_ping))
+                federation_presence_out_counter.inc(len(to_federation_ping))
 
                 self._push_to_remotes(to_federation_ping.values())
 
@@ -363,7 +363,7 @@ class PresenceHandler(object):
                     for user_id in users_to_check
                 ]
 
-                timers_fired_counter.inc_by(len(states))
+                timers_fired_counter.inc(len(states))
 
                 changes = handle_timeouts(
                     states,
@@ -707,7 +707,7 @@ class PresenceHandler(object):
             updates.append(prev_state.copy_and_replace(**new_fields))
 
         if updates:
-            federation_presence_counter.inc_by(len(updates))
+            federation_presence_counter.inc(len(updates))
             yield self._update_states(updates)
 
     @defer.inlineCallbacks
@@ -982,28 +982,28 @@ def should_notify(old_state, new_state):
         return False
 
     if old_state.status_msg != new_state.status_msg:
-        notify_reason_counter.inc("status_msg_change")
+        notify_reason_counter.labels("status_msg_change").inc()
         return True
 
     if old_state.state != new_state.state:
-        notify_reason_counter.inc("state_change")
-        state_transition_counter.inc(old_state.state, new_state.state)
+        notify_reason_counter.labels("state_change").inc()
+        state_transition_counter.labels(old_state.state, new_state.state).inc()
         return True
 
     if old_state.state == PresenceState.ONLINE:
         if new_state.currently_active != old_state.currently_active:
-            notify_reason_counter.inc("current_active_change")
+            notify_reason_counter.labels("current_active_change").inc()
             return True
 
         if new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY:
             # Only notify about last active bumps if we're not currently acive
             if not new_state.currently_active:
-                notify_reason_counter.inc("last_active_change_online")
+                notify_reason_counter.labels("last_active_change_online").inc()
                 return True
 
     elif new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY:
         # Always notify for a transition where last active gets bumped.
-        notify_reason_counter.inc("last_active_change_not_online")
+        notify_reason_counter.labels("last_active_change_not_online").inc()
         return True
 
     return False
@@ -1077,14 +1077,14 @@ class PresenceEventSource(object):
             if changed is not None and len(changed) < 500:
                 # For small deltas, its quicker to get all changes and then
                 # work out if we share a room or they're in our presence list
-                get_updates_counter.inc("stream")
+                get_updates_counter.labels("stream").inc()
                 for other_user_id in changed:
                     if other_user_id in users_interested_in:
                         user_ids_changed.add(other_user_id)
             else:
                 # Too many possible updates. Find all users we can see and check
                 # if any of them have changed.
-                get_updates_counter.inc("full")
+                get_updates_counter.labels("full").inc()
 
                 if from_key:
                     user_ids_changed = stream_change_cache.get_entities_changed(
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 70a19d9b74..61a1d2e2b3 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -23,7 +23,6 @@ from synapse.http import cancelled_to_request_timed_out_error
 from synapse.util.async import add_timeout_to_deferred
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
-import synapse.metrics
 from synapse.http.endpoint import SpiderEndpoint
 
 from canonicaljson import encode_canonical_json
@@ -42,6 +41,7 @@ from twisted.web._newclient import ResponseDone
 
 from six import StringIO
 
+from prometheus_client import Counter
 import simplejson as json
 import logging
 import urllib
@@ -49,16 +49,8 @@ import urllib
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-outgoing_requests_counter = metrics.register_counter(
-    "requests",
-    labels=["method"],
-)
-incoming_responses_counter = metrics.register_counter(
-    "responses",
-    labels=["method", "code"],
-)
+outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"])
+incoming_responses_counter = Counter("synapse_http_client_responses", "", ["method", "code"])
 
 
 class SimpleHttpClient(object):
@@ -95,7 +87,7 @@ class SimpleHttpClient(object):
     def request(self, method, uri, *args, **kwargs):
         # A small wrapper around self.agent.request() so we can easily attach
         # counters to it
-        outgoing_requests_counter.inc(method)
+        outgoing_requests_counter.labels(method).inc()
 
         logger.info("Sending request %s %s", method, uri)
 
@@ -109,14 +101,14 @@ class SimpleHttpClient(object):
             )
             response = yield make_deferred_yieldable(request_deferred)
 
-            incoming_responses_counter.inc(method, response.code)
+            incoming_responses_counter.labels(method, response.code).inc()
             logger.info(
                 "Received response to  %s %s: %s",
                 method, uri, response.code
             )
             defer.returnValue(response)
         except Exception as e:
-            incoming_responses_counter.inc(method, "ERR")
+            incoming_responses_counter.labels(method, "ERR").inc()
             logger.info(
                 "Error sending request to  %s %s: %s %s",
                 method, uri, type(e).__name__, e.message
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 4b2b85464d..259d3884e2 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -43,19 +43,13 @@ import sys
 import urllib
 from six.moves.urllib import parse as urlparse
 
+from prometheus_client import Counter
+
 logger = logging.getLogger(__name__)
 outbound_logger = logging.getLogger("synapse.http.outbound")
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-outgoing_requests_counter = metrics.register_counter(
-    "requests",
-    labels=["method"],
-)
-incoming_responses_counter = metrics.register_counter(
-    "responses",
-    labels=["method", "code"],
-)
+outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests", "", ["method"])
+incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses", "", ["method", "code"])
 
 
 MAX_LONG_RETRIES = 10
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e3b831db67..973ba6506f 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -18,14 +18,13 @@ import functools
 import time
 import gc
 import platform
+import attr
 
-from twisted.internet import reactor
+from prometheus_client import Gauge, Histogram, Counter
+from prometheus_client.core import (
+    GaugeMetricFamily, CounterMetricFamily, REGISTRY)
 
-from .metric import (
-    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric,
-    MemoryUsageMetric, GaugeMetric,
-)
-from .process_collector import register_process_collector
+from twisted.internet import reactor
 
 
 logger = logging.getLogger(__name__)
@@ -34,149 +33,94 @@ logger = logging.getLogger(__name__)
 running_on_pypy = platform.python_implementation() == 'PyPy'
 all_metrics = []
 all_collectors = []
+all_gauges = {}
 
+@attr.s(hash=True)
+class LaterGauge(object):
 
-class Metrics(object):
-    """ A single Metrics object gives a (mutable) slice view of the all_metrics
-    dict, allowing callers to easily register new metrics that are namespaced
-    nicely."""
-
-    def __init__(self, name):
-        self.name_prefix = name
-
-    def make_subspace(self, name):
-        return Metrics("%s_%s" % (self.name_prefix, name))
-
-    def register_collector(self, func):
-        all_collectors.append(func)
-
-    def _register(self, metric_class, name, *args, **kwargs):
-        full_name = "%s_%s" % (self.name_prefix, name)
-
-        metric = metric_class(full_name, *args, **kwargs)
-
-        all_metrics.append(metric)
-        return metric
-
-    def register_counter(self, *args, **kwargs):
-        """
-        Returns:
-            CounterMetric
-        """
-        return self._register(CounterMetric, *args, **kwargs)
-
-    def register_gauge(self, *args, **kwargs):
-        """
-        Returns:
-            GaugeMetric
-        """
-        return self._register(GaugeMetric, *args, **kwargs)
+    name = attr.ib()
+    desc = attr.ib()
+    labels = attr.ib(hash=False)
+    caller = attr.ib()
 
-    def register_callback(self, *args, **kwargs):
-        """
-        Returns:
-            CallbackMetric
-        """
-        return self._register(CallbackMetric, *args, **kwargs)
+    def collect(self):
 
-    def register_distribution(self, *args, **kwargs):
-        """
-        Returns:
-            DistributionMetric
-        """
-        return self._register(DistributionMetric, *args, **kwargs)
-
-    def register_cache(self, *args, **kwargs):
-        """
-        Returns:
-            CacheMetric
-        """
-        return self._register(CacheMetric, *args, **kwargs)
+        g = GaugeMetricFamily(self.name, self.desc, self.labels)
 
+        try:
+            calls = self.caller()
+        except Exception as e:
+            print(e)
+            logger.err()
+            yield g
 
-def register_memory_metrics(hs):
-    try:
-        import psutil
-        process = psutil.Process()
-        process.memory_info().rss
-    except (ImportError, AttributeError):
-        logger.warn(
-            "psutil is not installed or incorrect version."
-            " Disabling memory metrics."
-        )
-        return
-    metric = MemoryUsageMetric(hs, psutil)
-    all_metrics.append(metric)
+        if isinstance(calls, dict):
+            for k, v in calls.items():
+                g.add_metric(k, v)
+        else:
+            g.add_metric([], calls)
 
+        yield g
 
-def get_metrics_for(pkg_name):
-    """ Returns a Metrics instance for conveniently creating metrics
-    namespaced with the given name prefix. """
+    def register(self):
+        if self.name in all_gauges.keys():
+            REGISTRY.unregister(all_gauges.pop(self.name))
 
-    # Convert a "package.name" to "package_name" because Prometheus doesn't
-    # let us use . in metric names
-    return Metrics(pkg_name.replace(".", "_"))
+        REGISTRY.register(self)
+        all_gauges[self.name] = self
 
 
-def render_all():
-    strs = []
+#
+# Python GC metrics
+#
 
-    for collector in all_collectors:
-        collector()
+gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"])
+gc_time = Histogram("python_gc_time", "Time taken to GC (ms)", ["gen"], buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000])
 
-    for metric in all_metrics:
-        try:
-            strs += metric.render()
-        except Exception:
-            strs += ["# FAILED to render"]
-            logger.exception("Failed to render metric")
+class GCCounts(object):
+    def collect(self):
+        gc_counts = gc.get_count()
 
-    strs.append("")  # to generate a final CRLF
+        cm = GaugeMetricFamily("python_gc_counts", "GC cycle counts", labels=["gen"])
+        for n, m in enumerate(gc.get_count()):
+            cm.add_metric([str(n)], m)
 
-    return "\n".join(strs)
+        yield cm
 
+REGISTRY.register(GCCounts())
 
-register_process_collector(get_metrics_for("process"))
+#
+# Twisted reactor metrics
+#
 
+tick_time = Histogram("python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (ms)", buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000])
+pending_calls_metric = Histogram("python_twisted_reactor_pending_calls", "Pending calls", buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000])
 
-python_metrics = get_metrics_for("python")
+#
+# Federation Metrics
+#
 
-gc_time = python_metrics.register_distribution("gc_time", labels=["gen"])
-gc_unreachable = python_metrics.register_counter("gc_unreachable_total", labels=["gen"])
-python_metrics.register_callback(
-    "gc_counts", lambda: {(i,): v for i, v in enumerate(gc.get_count())}, labels=["gen"]
-)
+sent_edus_counter = Counter("synapse_federation_client_sent_edus", "")
 
-reactor_metrics = get_metrics_for("python.twisted.reactor")
-tick_time = reactor_metrics.register_distribution("tick_time")
-pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
+sent_transactions_counter = Counter("synapse_federation_client_sent_transactions", "")
 
-synapse_metrics = get_metrics_for("synapse")
+events_processed_counter = Counter("synapse_federation_client_events_processed", "")
 
 # Used to track where various components have processed in the event stream,
 # e.g. federation sending, appservice sending, etc.
-event_processing_positions = synapse_metrics.register_gauge(
-    "event_processing_positions", labels=["name"],
-)
+event_processing_positions = Gauge("synapse_event_processing_positions", "", ["name"])
 
 # Used to track the current max events stream position
-event_persisted_position = synapse_metrics.register_gauge(
-    "event_persisted_position",
-)
+event_persisted_position = Gauge("synapse_event_persisted_position", "")
 
 # Used to track the received_ts of the last event processed by various
 # components
-event_processing_last_ts = synapse_metrics.register_gauge(
-    "event_processing_last_ts", labels=["name"],
-)
+event_processing_last_ts = Gauge("synapse_event_processing_last_ts", "", ["name"])
 
 # Used to track the lag processing events. This is the time difference
 # between the last processed event's received_ts and the time it was
 # finished being processed.
-event_processing_lag = synapse_metrics.register_gauge(
-    "event_processing_lag", labels=["name"],
-)
-
+event_processing_lag = Gauge("synapse_event_processing_lag", "", ["name"])
 
 def runUntilCurrentTimer(func):
 
@@ -206,8 +150,8 @@ def runUntilCurrentTimer(func):
         # since about 25% of time is actually spent running things triggered by
         # I/O events, but that is harder to capture without rewriting half the
         # reactor.
-        tick_time.inc_by(end - start)
-        pending_calls_metric.inc_by(num_pending)
+        tick_time.observe(end - start)
+        pending_calls_metric.observe(num_pending)
 
         if running_on_pypy:
             return ret
@@ -224,8 +168,8 @@ def runUntilCurrentTimer(func):
                 unreachable = gc.collect(i)
                 end = time.time() * 1000
 
-                gc_time.inc_by(end - start, i)
-                gc_unreachable.inc_by(unreachable, i)
+                gc_time.labels(i).observe(end - start)
+                gc_unreachable.labels(i).set(unreachable)
 
         return ret
 
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 8355c7d621..123e6f1840 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -28,22 +28,19 @@ from synapse.util.logcontext import PreserveLoggingContext, run_in_background
 from synapse.util.metrics import Measure
 from synapse.types import StreamToken
 from synapse.visibility import filter_events_for_client
-import synapse.metrics
+from synapse.metrics import LaterGauge
 
 from collections import namedtuple
+from prometheus_client import Counter
 
 import logging
 
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
+notified_events_counter = Counter("synapse_notifier_notified_events", "")
 
-notified_events_counter = metrics.register_counter("notified_events")
-
-users_woken_by_stream_counter = metrics.register_counter(
-    "users_woken_by_stream", labels=["stream"]
-)
+users_woken_by_stream_counter = Counter("synapse_notifier_users_woken_by_stream", "", ["stream"])
 
 
 # TODO(paul): Should be shared somewhere
@@ -108,7 +105,7 @@ class _NotifierUserStream(object):
         self.last_notified_ms = time_now_ms
         noify_deferred = self.notify_deferred
 
-        users_woken_by_stream_counter.inc(stream_key)
+        users_woken_by_stream_counter.labels(stream_key).inc()
 
         with PreserveLoggingContext():
             self.notify_deferred = ObservableDeferred(defer.Deferred())
@@ -197,14 +194,14 @@ class Notifier(object):
                 all_user_streams.add(x)
 
             return sum(stream.count_listeners() for stream in all_user_streams)
-        metrics.register_callback("listeners", count_listeners)
+        LaterGauge("listeners", "", [], count_listeners)
 
-        metrics.register_callback(
-            "rooms",
+        LaterGauge(
+            "rooms", "", [],
             lambda: count(bool, self.room_to_user_streams.values()),
         )
-        metrics.register_callback(
-            "users",
+        LaterGauge(
+            "users", "", [],
             lambda: len(self.user_to_user_stream),
         )
 
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index b077e1a446..e22088ad6f 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -20,22 +20,17 @@ from twisted.internet.error import AlreadyCalled, AlreadyCancelled
 
 from . import push_rule_evaluator
 from . import push_tools
-import synapse
 from synapse.push import PusherConfigException
 from synapse.util.logcontext import LoggingContext
 from synapse.util.metrics import Measure
 
-logger = logging.getLogger(__name__)
+from prometheus_client import Counter
 
-metrics = synapse.metrics.get_metrics_for(__name__)
+logger = logging.getLogger(__name__)
 
-http_push_processed_counter = metrics.register_counter(
-    "http_pushes_processed",
-)
+http_push_processed_counter = Counter("http_pushes_processed", "")
 
-http_push_failed_counter = metrics.register_counter(
-    "http_pushes_failed",
-)
+http_push_failed_counter = Counter("http_pushes_failed", "")
 
 
 class HttpPusher(object):
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 3601f2d365..c3e6c5c258 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -150,7 +150,7 @@ class PushRuleEvaluatorForEvent(object):
 
 # Caches (glob, word_boundary) -> regex for push. See _glob_matches
 regex_cache = LruCache(50000 * CACHE_SIZE_FACTOR)
-register_cache("regex_push_cache", regex_cache)
+register_cache("cache", "regex_push_cache", regex_cache)
 
 
 def _glob_matches(glob, value, word_boundary=False):
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 216db4d164..478c497722 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -56,6 +56,7 @@ REQUIREMENTS = {
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
     "six": ["six"],
+    "prometheus_client": ["prometheus_client"],
 }
 CONDITIONAL_REQUIREMENTS = {
     "web_client": {
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index d7d38464b2..5848f57c5e 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -60,19 +60,19 @@ from .commands import (
 )
 from .streams import STREAMS_MAP
 
+from synapse.metrics import LaterGauge
 from synapse.util.stringutils import random_string
-from synapse.metrics.metric import CounterMetric
+
+from prometheus_client import Counter
+
+from collections import defaultdict
 
 import logging
-import synapse.metrics
 import struct
 import fcntl
 
-
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-connection_close_counter = metrics.register_counter(
-    "close_reason", labels=["reason_type"],
+connection_close_counter = Counter(
+    "synapse_replication_tcp_protocol_close_reason", "", ["reason_type"],
 )
 
 
@@ -136,12 +136,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
         # The LoopingCall for sending pings.
         self._send_ping_loop = None
 
-        self.inbound_commands_counter = CounterMetric(
-            "inbound_commands", labels=["command"],
-        )
-        self.outbound_commands_counter = CounterMetric(
-            "outbound_commands", labels=["command"],
-        )
+        self.inbound_commands_counter = defaultdict(int)
+        self.outbound_commands_counter = defaultdict(int)
 
     def connectionMade(self):
         logger.info("[%s] Connection established", self.id())
@@ -201,7 +197,7 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
 
         self.last_received_command = self.clock.time_msec()
 
-        self.inbound_commands_counter.inc(cmd_name)
+        self.inbound_commands_counter[cmd_name] = self.inbound_commands_counter[cmd_name] + 1
 
         cmd_cls = COMMAND_MAP[cmd_name]
         try:
@@ -251,8 +247,7 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
             self._queue_command(cmd)
             return
 
-        self.outbound_commands_counter.inc(cmd.NAME)
-
+        self.outbound_commands_counter[cmd.NAME] = self.outbound_commands_counter[cmd.NAME] + 1
         string = "%s %s" % (cmd.NAME, cmd.to_line(),)
         if "\n" in string:
             raise Exception("Unexpected newline in command: %r", string)
@@ -317,9 +312,9 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
     def connectionLost(self, reason):
         logger.info("[%s] Replication connection closed: %r", self.id(), reason)
         if isinstance(reason, Failure):
-            connection_close_counter.inc(reason.type.__name__)
+            connection_close_counter.labels(reason.type.__name__).inc()
         else:
-            connection_close_counter.inc(reason.__class__.__name__)
+            connection_close_counter.labels(reason.__class__.__name__).inc()
 
         try:
             # Remove us from list of connections to be monitored
@@ -566,14 +561,12 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
 
 # The following simply registers metrics for the replication connections
 
-metrics.register_callback(
-    "pending_commands",
+pending_commands = LaterGauge(
+    "pending_commands", "", ["name", "conn_id"],
     lambda: {
         (p.name, p.conn_id): len(p.pending_commands)
         for p in connected_connections
-    },
-    labels=["name", "conn_id"],
-)
+    })
 
 
 def transport_buffer_size(protocol):
@@ -583,14 +576,12 @@ def transport_buffer_size(protocol):
     return 0
 
 
-metrics.register_callback(
-    "transport_send_buffer",
+transport_send_buffer = LaterGauge(
+    "synapse_replication_tcp_transport_send_buffer", "", ["name", "conn_id"],
     lambda: {
         (p.name, p.conn_id): transport_buffer_size(p)
         for p in connected_connections
-    },
-    labels=["name", "conn_id"],
-)
+    })
 
 
 def transport_kernel_read_buffer_size(protocol, read=True):
@@ -608,48 +599,37 @@ def transport_kernel_read_buffer_size(protocol, read=True):
     return 0
 
 
-metrics.register_callback(
-    "transport_kernel_send_buffer",
+tcp_transport_kernel_send_buffer = LaterGauge(
+    "synapse_replication_tcp_transport_kernel_send_buffer", "", ["name", "conn_id"],
     lambda: {
         (p.name, p.conn_id): transport_kernel_read_buffer_size(p, False)
         for p in connected_connections
-    },
-    labels=["name", "conn_id"],
-)
+    })
 
 
-metrics.register_callback(
-    "transport_kernel_read_buffer",
+tcp_transport_kernel_read_buffer = LaterGauge(
+    "synapse_replication_tcp_transport_kernel_read_buffer", "", ["name", "conn_id"],
     lambda: {
         (p.name, p.conn_id): transport_kernel_read_buffer_size(p, True)
         for p in connected_connections
-    },
-    labels=["name", "conn_id"],
-)
+    })
 
 
-metrics.register_callback(
-    "inbound_commands",
+tcp_inbound_commands = LaterGauge(
+    "synapse_replication_tcp_inbound_commands", "", ["command", "name", "conn_id"],
     lambda: {
         (k[0], p.name, p.conn_id): count
         for p in connected_connections
-        for k, count in p.inbound_commands_counter.counts.iteritems()
-    },
-    labels=["command", "name", "conn_id"],
-)
+        for k, count in p.inbound_commands_counter.items()
+    })
 
-metrics.register_callback(
-    "outbound_commands",
+tcp_outbound_commands = LaterGauge(
+    "synapse_replication_tcp_outbound_commands", "", ["command", "name", "conn_id"],
     lambda: {
         (k[0], p.name, p.conn_id): count
         for p in connected_connections
-        for k, count in p.outbound_commands_counter.counts.iteritems()
-    },
-    labels=["command", "name", "conn_id"],
-)
+        for k, count in p.outbound_commands_counter.items()
+    })
 
 # number of updates received for each RDATA stream
-inbound_rdata_count = metrics.register_counter(
-    "inbound_rdata_count",
-    labels=["stream_name"],
-)
+inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "", ["stream_name"])
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 4adae96681..438dcddf55 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -13,29 +13,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import synapse.metrics
+from prometheus_client.core import GaugeMetricFamily, REGISTRY
+
 import os
 
 CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
 
-metrics = synapse.metrics.get_metrics_for("synapse.util.caches")
-
 caches_by_name = {}
-# cache_counter = metrics.register_cache(
-#     "cache",
-#     lambda: {(name,): len(caches_by_name[name]) for name in caches_by_name.keys()},
-#     labels=["name"],
-# )
-
-
-def register_cache(name, cache):
-    caches_by_name[name] = cache
-    return metrics.register_cache(
-        "cache",
-        lambda: len(cache),
-        name,
-    )
+collectors_by_name = {}
+
+def register_cache(name, cache_name, cache):
+
+    # Check if the metric is already registered. Unregister it, if so.
+    metric_name = "synapse_util_caches_%s:%s" % (name, cache_name,)
+    if metric_name in collectors_by_name.keys():
+        REGISTRY.unregister(collectors_by_name[metric_name])
+
+    class CacheMetric(object):
+
+        hits = 0
+        misses = 0
+        evicted_size = 0
+
+        def inc_hits(self):
+            self.hits += 1
+
+        def inc_misses(self):
+            self.misses += 1
+
+        def inc_evictions(self, size=1):
+            self.evicted_size += size
+
+        def collect(self):
+            cache_size = len(cache)
+
+            gm = GaugeMetricFamily(metric_name, "", labels=["size", "hits", "misses", "total"])
+            gm.add_metric(["size"], cache_size)
+            gm.add_metric(["hits"], self.hits)
+            gm.add_metric(["misses"], self.misses)
+            gm.add_metric(["total"], self.hits + self.misses)
+            yield gm
 
+    metric = CacheMetric()
+    REGISTRY.register(metric)
+    caches_by_name[cache_name] = cache
+    collectors_by_name[metric_name] = metric
+    return metric
 
 KNOWN_KEYS = {
     key: key for key in
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 68285a7594..a4188eb099 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -80,7 +80,7 @@ class Cache(object):
         self.name = name
         self.keylen = keylen
         self.thread = None
-        self.metrics = register_cache(name, self.cache)
+        self.metrics = register_cache("descriptor", name, self.cache)
 
     def _on_evicted(self, evicted_count):
         self.metrics.inc_evictions(evicted_count)
diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py
index 1709e8b429..bdc21e348f 100644
--- a/synapse/util/caches/dictionary_cache.py
+++ b/synapse/util/caches/dictionary_cache.py
@@ -55,7 +55,7 @@ class DictionaryCache(object):
             __slots__ = []
 
         self.sentinel = Sentinel()
-        self.metrics = register_cache(name, self.cache)
+        self.metrics = register_cache("dictionary", name, self.cache)
 
     def check_thread(self):
         expected_thread = self.thread
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 0aa103eecb..ff04c91955 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -52,12 +52,12 @@ class ExpiringCache(object):
 
         self._cache = OrderedDict()
 
-        self.metrics = register_cache(cache_name, self)
-
         self.iterable = iterable
 
         self._size_estimate = 0
 
+        self.metrics = register_cache("expiring", cache_name, self)
+
     def start(self):
         if not self._expiry_ms:
             # Don't bother starting the loop if things never expire
diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 7f79333e96..a8491b42d5 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -17,7 +17,7 @@ import logging
 from twisted.internet import defer
 
 from synapse.util.async import ObservableDeferred
-from synapse.util.caches import metrics as cache_metrics
+from synapse.util.caches import register_cache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
@@ -38,15 +38,16 @@ class ResponseCache(object):
         self.timeout_sec = timeout_ms / 1000.
 
         self._name = name
-        self._metrics = cache_metrics.register_cache(
-            "response_cache",
-            size_callback=lambda: self.size(),
-            cache_name=name,
+        self._metrics = register_cache(
+            "response_cache", name, self
         )
 
     def size(self):
         return len(self.pending_result_cache)
 
+    def __len__(self):
+        return self.size()
+
     def get(self, key):
         """Look up the given key.
 
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 941d873ab8..a7fe0397fa 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -38,7 +38,7 @@ class StreamChangeCache(object):
         self._cache = sorteddict()
         self._earliest_known_stream_pos = current_stream_pos
         self.name = name
-        self.metrics = register_cache(self.name, self._cache)
+        self.metrics = register_cache("cache", self.name, self._cache)
 
         for entity, stream_pos in prefilled_cache.items():
             self.entity_has_changed(entity, stream_pos)
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index e4b5687a4b..a964286d85 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -15,8 +15,8 @@
 
 from twisted.internet import defer
 
+from prometheus_client import Counter
 from synapse.util.logcontext import LoggingContext
-import synapse.metrics
 
 from functools import wraps
 import logging
@@ -24,66 +24,21 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"])
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-
-# total number of times we have hit this block
-block_counter = metrics.register_counter(
-    "block_count",
-    labels=["block_name"],
-    alternative_names=(
-        # the following are all deprecated aliases for the same metric
-        metrics.name_prefix + x for x in (
-            "_block_timer:count",
-            "_block_ru_utime:count",
-            "_block_ru_stime:count",
-            "_block_db_txn_count:count",
-            "_block_db_txn_duration:count",
-        )
-    )
-)
-
-block_timer = metrics.register_counter(
-    "block_time_seconds",
-    labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_timer:total",
-    ),
-)
-
-block_ru_utime = metrics.register_counter(
-    "block_ru_utime_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_ru_utime:total",
-    ),
-)
-
-block_ru_stime = metrics.register_counter(
-    "block_ru_stime_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_ru_stime:total",
-    ),
-)
-
-block_db_txn_count = metrics.register_counter(
-    "block_db_txn_count", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_db_txn_count:total",
-    ),
-)
+block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"])
+
+block_ru_utime = Counter("synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"])
+
+block_ru_stime = Counter("synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"])
+
+block_db_txn_count = Counter("synapse_util_metrics_block_db_txn_count", "", ["block_name"])
 
 # seconds spent waiting for db txns, excluding scheduling time, in this block
-block_db_txn_duration = metrics.register_counter(
-    "block_db_txn_duration_seconds", labels=["block_name"],
-    alternative_names=(
-        metrics.name_prefix + "_block_db_txn_duration:total",
-    ),
-)
+block_db_txn_duration = Counter("synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"])
 
 # seconds spent waiting for a db connection, in this block
-block_db_sched_duration = metrics.register_counter(
-    "block_db_sched_duration_seconds", labels=["block_name"],
-)
+block_db_sched_duration = Counter("synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"])
 
 
 def measure_func(name):
@@ -132,8 +87,8 @@ class Measure(object):
 
         duration = self.clock.time_msec() - self.start
 
-        block_counter.inc(self.name)
-        block_timer.inc_by(duration, self.name)
+        block_counter.labels(self.name).inc()
+        block_timer.labels(self.name).inc(duration)
 
         context = LoggingContext.current_context()
 
@@ -150,19 +105,13 @@ class Measure(object):
 
         ru_utime, ru_stime = context.get_resource_usage()
 
-        block_ru_utime.inc_by(ru_utime - self.ru_utime, self.name)
-        block_ru_stime.inc_by(ru_stime - self.ru_stime, self.name)
-        block_db_txn_count.inc_by(
-            context.db_txn_count - self.db_txn_count, self.name
-        )
-        block_db_txn_duration.inc_by(
-            (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000.,
-            self.name
-        )
-        block_db_sched_duration.inc_by(
-            (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000.,
-            self.name
-        )
+        block_ru_utime.labels(self.name).inc(ru_utime - self.ru_utime)
+        block_ru_stime.labels(self.name).inc(ru_stime - self.ru_stime)
+        block_db_txn_count.labels(self.name).inc(context.db_txn_count - self.db_txn_count)
+        block_db_txn_duration.labels(self.name).inc(
+            (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000.)
+        block_db_sched_duration.labels(self.name).inc(
+            (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000.)
 
         if self.created_context:
             self.start_context.__exit__(exc_type, exc_val, exc_tb)
diff --git a/tests/__init__.py b/tests/__init__.py
index bfebb0f644..aab20e8e02 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -12,3 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from twisted.trial import util
+util.DEFAULT_TIMEOUT_DURATION = 10
-- 
cgit 1.4.1


From fcc525b0b705703fead3d703c0df62a264ff8ce8 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 21 May 2018 19:48:57 -0500
Subject: rest of the changes

---
 synapse/http/request_metrics.py          | 106 ++++-------------
 synapse/http/server.py                   |   4 +-
 synapse/push/bulk_push_rule_evaluator.py |  28 ++---
 synapse/replication/tcp/resource.py      |  30 +++--
 synapse/storage/_base.py                 |  17 ++-
 synapse/storage/events.py                |  28 ++---
 tests/metrics/__init__.py                |   0
 tests/metrics/test_metric.py             | 192 -------------------------------
 8 files changed, 68 insertions(+), 337 deletions(-)
 delete mode 100644 tests/metrics/__init__.py
 delete mode 100644 tests/metrics/test_metric.py

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 8c850bf23f..34a730d5bc 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -16,86 +16,38 @@
 
 import logging
 
-import synapse.metrics
+from prometheus_client.core import Counter, Histogram
+
 from synapse.util.logcontext import LoggingContext
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for("synapse.http.server")
 
 # total number of responses served, split by method/servlet/tag
-response_count = metrics.register_counter(
-    "response_count",
-    labels=["method", "servlet", "tag"],
-    alternative_names=(
-        # the following are all deprecated aliases for the same metric
-        metrics.name_prefix + x for x in (
-            "_requests",
-            "_response_time:count",
-            "_response_ru_utime:count",
-            "_response_ru_stime:count",
-            "_response_db_txn_count:count",
-            "_response_db_txn_duration:count",
-        )
-    )
-)
+response_count = Counter("synapse_http_server_response_count", "", ["method", "servlet", "tag"])
 
-requests_counter = metrics.register_counter(
-    "requests_received",
-    labels=["method", "servlet", ],
-)
+requests_counter = Counter("synapse_http_server_requests_received", "", ["method", "servlet"])
 
-outgoing_responses_counter = metrics.register_counter(
-    "responses",
-    labels=["method", "code"],
-)
+outgoing_responses_counter = Counter("synapse_http_server_responses", "", ["method", "code"])
 
-response_timer = metrics.register_counter(
-    "response_time_seconds",
-    labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_time:total",
-    ),
-)
+response_timer = Histogram("synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"])
 
-response_ru_utime = metrics.register_counter(
-    "response_ru_utime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_utime:total",
-    ),
-)
+response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"])
 
-response_ru_stime = metrics.register_counter(
-    "response_ru_stime_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_ru_stime:total",
-    ),
-)
+response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"])
 
-response_db_txn_count = metrics.register_counter(
-    "response_db_txn_count", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_count:total",
-    ),
-)
+response_db_txn_count = Counter("synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"])
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
-response_db_txn_duration = metrics.register_counter(
-    "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"],
-    alternative_names=(
-        metrics.name_prefix + "_response_db_txn_duration:total",
-    ),
-)
+response_db_txn_duration = Counter("synapse_http_server_response_db_txn_duration_seconds", "", ["method", "servlet", "tag"])
 
 # seconds spent waiting for a db connection, when processing this request
-response_db_sched_duration = metrics.register_counter(
-    "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"]
+response_db_sched_duration = Counter("synapse_http_request_response_db_sched_duration_seconds", "", ["method", "servlet", "tag"]
 )
 
 # size in bytes of the response written
-response_size = metrics.register_counter(
-    "response_size", labels=["method", "servlet", "tag"]
+response_size = Counter("synapse_http_request_response_size", "", ["method", "servlet", "tag"]
 )
 
 
@@ -119,31 +71,19 @@ class RequestMetrics(object):
                 )
                 return
 
-        outgoing_responses_counter.inc(request.method, str(request.code))
+        outgoing_responses_counter.labels(request.method, str(request.code)).inc()
 
-        response_count.inc(request.method, self.name, tag)
+        response_count.labels(request.method, self.name, tag).inc()
 
-        response_timer.inc_by(
-            time_msec - self.start, request.method,
-            self.name, tag
-        )
+        response_timer.labels(request.method, self.name, tag).observe(time_msec - self.start)
 
         ru_utime, ru_stime = context.get_resource_usage()
 
-        response_ru_utime.inc_by(
-            ru_utime, request.method, self.name, tag
-        )
-        response_ru_stime.inc_by(
-            ru_stime, request.method, self.name, tag
-        )
-        response_db_txn_count.inc_by(
-            context.db_txn_count, request.method, self.name, tag
-        )
-        response_db_txn_duration.inc_by(
-            context.db_txn_duration_ms / 1000., request.method, self.name, tag
-        )
-        response_db_sched_duration.inc_by(
-            context.db_sched_duration_ms / 1000., request.method, self.name, tag
-        )
-
-        response_size.inc_by(request.sentLength, request.method, self.name, tag)
+        response_ru_utime.labels(request.method, self.name, tag).inc(ru_utime)
+        response_ru_stime.labels(request.method, self.name, tag).inc(ru_stime)
+        response_db_txn_count.labels(request.method, self.name, tag).inc(context.db_txn_count)
+        response_db_txn_duration.labels(request.method, self.name, tag).inc(context.db_txn_duration_ms / 1000.)
+        response_db_sched_duration.labels(request.method, self.name, tag).inc(
+            context.db_sched_duration_ms / 1000.)
+
+        response_size.labels(request.method, self.name, tag).inc(request.sentLength)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index b6e2ae14a2..f72d986288 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -138,8 +138,8 @@ def wrap_request_handler_with_logging(h):
                         # dispatching to the handler, so that the handler
                         # can update the servlet name in the request
                         # metrics
-                        requests_counter.inc(request.method,
-                                             request.request_metrics.name)
+                        requests_counter.labels(request.method,
+                                                request.request_metrics.name).inc()
                         yield d
     return wrapped_request_handler
 
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 7c680659b6..6fcca5e260 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -22,35 +22,29 @@ from .push_rule_evaluator import PushRuleEvaluatorForEvent
 
 from synapse.event_auth import get_user_power_level
 from synapse.api.constants import EventTypes, Membership
-from synapse.metrics import get_metrics_for
-from synapse.util.caches import metrics as cache_metrics
+from synapse.util.caches import register_cache
 from synapse.util.caches.descriptors import cached
 from synapse.util.async import Linearizer
 from synapse.state import POWER_KEY
 
 from collections import namedtuple
-
+from prometheus_client import Counter
 
 logger = logging.getLogger(__name__)
 
 
 rules_by_room = {}
 
-push_metrics = get_metrics_for(__name__)
 
-push_rules_invalidation_counter = push_metrics.register_counter(
-    "push_rules_invalidation_counter"
-)
-push_rules_state_size_counter = push_metrics.register_counter(
-    "push_rules_state_size_counter"
-)
+push_rules_invalidation_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "")
+push_rules_state_size_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "")
 
 # Measures whether we use the fast path of using state deltas, or if we have to
 # recalculate from scratch
-push_rules_delta_state_cache_metric = cache_metrics.register_cache(
+push_rules_delta_state_cache_metric = register_cache(
     "cache",
-    size_callback=lambda: 0,  # Meaningless size, as this isn't a cache that stores values
-    cache_name="push_rules_delta_state_cache_metric",
+    "push_rules_delta_state_cache_metric",
+    cache=[],  # Meaningless size, as this isn't a cache that stores values
 )
 
 
@@ -64,10 +58,10 @@ class BulkPushRuleEvaluator(object):
         self.store = hs.get_datastore()
         self.auth = hs.get_auth()
 
-        self.room_push_rule_cache_metrics = cache_metrics.register_cache(
+        self.room_push_rule_cache_metrics = register_cache(
             "cache",
-            size_callback=lambda: 0,  # There's not good value for this
-            cache_name="room_push_rule_cache",
+            "room_push_rule_cache",
+            cache=[],  # Meaningless size, as this isn't a cache that stores values
         )
 
     @defer.inlineCallbacks
@@ -309,7 +303,7 @@ class RulesForRoom(object):
                 current_state_ids = context.current_state_ids
                 push_rules_delta_state_cache_metric.inc_misses()
 
-            push_rules_state_size_counter.inc_by(len(current_state_ids))
+            push_rules_state_size_counter.inc(len(current_state_ids))
 
             logger.debug(
                 "Looking for member changes in %r %r", state_group, current_state_ids
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index a41af4fd6c..0e6b1957c6 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -22,20 +22,19 @@ from .streams import STREAMS_MAP, FederationStream
 from .protocol import ServerReplicationStreamProtocol
 
 from synapse.util.metrics import Measure, measure_func
+from synapse.metrics import LaterGauge
 
 import logging
-import synapse.metrics
 
+from prometheus_client import Counter
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-stream_updates_counter = metrics.register_counter(
-    "stream_updates", labels=["stream_name"]
+stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates", "", ["stream_name"]
 )
-user_sync_counter = metrics.register_counter("user_sync")
-federation_ack_counter = metrics.register_counter("federation_ack")
-remove_pusher_counter = metrics.register_counter("remove_pusher")
-invalidate_cache_counter = metrics.register_counter("invalidate_cache")
-user_ip_cache_counter = metrics.register_counter("user_ip_cache")
+user_sync_counter = Counter("synapse_replication_tcp_resource_user_sync", "")
+federation_ack_counter = Counter("synapse_replication_tcp_resource_federation_ack", "")
+remove_pusher_counter = Counter("synapse_replication_tcp_resource_remove_pusher", "")
+invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache", "")
+user_ip_cache_counter = Counter("synapse_replication_tcp_resource_user_ip_cache", "")
 
 logger = logging.getLogger(__name__)
 
@@ -73,7 +72,8 @@ class ReplicationStreamer(object):
         # Current connections.
         self.connections = []
 
-        metrics.register_callback("total_connections", lambda: len(self.connections))
+        l = LaterGauge("synapse_replication_tcp_resource_total_connections", "", [], lambda: len(self.connections))
+        l.register()
 
         # List of streams that clients can subscribe to.
         # We only support federation stream if federation sending hase been
@@ -85,17 +85,15 @@ class ReplicationStreamer(object):
 
         self.streams_by_name = {stream.NAME: stream for stream in self.streams}
 
-        metrics.register_callback(
-            "connections_per_stream",
+        LaterGauge(
+            "synapse_replication_tcp_resource_connections_per_stream", "", ["stream_name"],
             lambda: {
                 (stream_name,): len([
                     conn for conn in self.connections
                     if stream_name in conn.replication_streams
                 ])
                 for stream_name in self.streams_by_name
-            },
-            labels=["stream_name"],
-        )
+            }).register()
 
         self.federation_sender = None
         if not hs.config.send_federation:
@@ -175,7 +173,7 @@ class ReplicationStreamer(object):
                             logger.info(
                                 "Streaming: %s -> %s", stream.NAME, updates[-1][0]
                             )
-                            stream_updates_counter.inc_by(len(updates), stream.NAME)
+                            stream_updates_counter.labels(stream.NAME).inc(len(updates))
 
                         # Some streams return multiple rows with the same stream IDs,
                         # we need to make sure they get sent out in batches. We do
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2262776ab2..d1b625dc30 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -18,8 +18,8 @@ from synapse.api.errors import StoreError
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 from synapse.util.caches.descriptors import Cache
 from synapse.storage.engines import PostgresEngine
-import synapse.metrics
 
+from prometheus_client import Histogram
 
 from twisted.internet import defer
 
@@ -34,13 +34,10 @@ sql_logger = logging.getLogger("synapse.storage.SQL")
 transaction_logger = logging.getLogger("synapse.storage.txn")
 perf_logger = logging.getLogger("synapse.storage.TIME")
 
+sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "")
 
-metrics = synapse.metrics.get_metrics_for("synapse.storage")
-
-sql_scheduling_timer = metrics.register_distribution("schedule_time")
-
-sql_query_timer = metrics.register_distribution("query_time", labels=["verb"])
-sql_txn_timer = metrics.register_distribution("transaction_time", labels=["desc"])
+sql_query_timer = Histogram("synapse_storage_query_time", "", ["verb"])
+sql_txn_timer = Histogram("synapse_storage_transaction_time", "", ["desc"])
 
 
 class LoggingTransaction(object):
@@ -117,7 +114,7 @@ class LoggingTransaction(object):
         finally:
             msecs = (time.time() * 1000) - start
             sql_logger.debug("[SQL time] {%s} %f", self.name, msecs)
-            sql_query_timer.inc_by(msecs, sql.split()[0])
+            sql_query_timer.labels(sql.split()[0]).observe(msecs)
 
 
 class PerformanceCounters(object):
@@ -287,7 +284,7 @@ class SQLBaseStore(object):
 
             self._current_txn_total_time += duration
             self._txn_perf_counters.update(desc, start, end)
-            sql_txn_timer.inc_by(duration, desc)
+            sql_txn_timer.labels(desc).observe(duration)
 
     @defer.inlineCallbacks
     def runInteraction(self, desc, func, *args, **kwargs):
@@ -349,7 +346,7 @@ class SQLBaseStore(object):
         def inner_func(conn, *args, **kwargs):
             with LoggingContext("runWithConnection") as context:
                 sched_duration_ms = time.time() * 1000 - start_time
-                sql_scheduling_timer.inc_by(sched_duration_ms)
+                sql_scheduling_timer.observe(sched_duration_ms)
                 current_context.add_database_scheduled(sched_duration_ms)
 
                 if self.database_engine.is_connection_closed(conn):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 05cde96afc..96b48cfdbb 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -40,30 +40,24 @@ import synapse.metrics
 from synapse.events import EventBase    # noqa: F401
 from synapse.events.snapshot import EventContext   # noqa: F401
 
-logger = logging.getLogger(__name__)
+from prometheus_client import Counter
 
+logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
-persist_event_counter = metrics.register_counter("persisted_events")
-event_counter = metrics.register_counter(
-    "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
-)
+persist_event_counter = Counter("synapse_storage_events_persisted_events", "")
+event_counter = Counter("synapse_storage_events_persisted_events_sep", "", ["type", "origin_type", "origin_entity"])
 
 # The number of times we are recalculating the current state
-state_delta_counter = metrics.register_counter(
-    "state_delta",
-)
+state_delta_counter = Counter("synapse_storage_events_state_delta", "")
+
 # The number of times we are recalculating state when there is only a
 # single forward extremity
-state_delta_single_event_counter = metrics.register_counter(
-    "state_delta_single_event",
-)
+state_delta_single_event_counter = Counter("synapse_storage_events_state_delta_single_event", "")
+
 # The number of times we are reculating state when we could have resonably
 # calculated the delta when we calculated the state for an event we were
 # persisting.
-state_delta_reuse_delta_counter = metrics.register_counter(
-    "state_delta_reuse_delta",
-)
+state_delta_reuse_delta_counter = Counter("synapse_storage_events_state_delta_reuse_delta", "")
 
 
 def encode_json(json_object):
@@ -445,7 +439,7 @@ class EventsStore(EventsWorkerStore):
                     state_delta_for_room=state_delta_for_room,
                     new_forward_extremeties=new_forward_extremeties,
                 )
-                persist_event_counter.inc_by(len(chunk))
+                persist_event_counter.inc(len(chunk))
                 synapse.metrics.event_persisted_position.set(
                     chunk[-1][0].internal_metadata.stream_ordering,
                 )
@@ -460,7 +454,7 @@ class EventsStore(EventsWorkerStore):
                         origin_type = "remote"
                         origin_entity = get_domain_from_id(event.sender)
 
-                    event_counter.inc(event.type, origin_type, origin_entity)
+                    event_counter.labels(event.type, origin_type, origin_entity).inc()
 
                 for room_id, new_state in current_state_for_room.iteritems():
                     self.get_current_state_ids.prefill(
diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
deleted file mode 100644
index 069c0be762..0000000000
--- a/tests/metrics/test_metric.py
+++ /dev/null
@@ -1,192 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from tests import unittest
-
-from synapse.metrics.metric import (
-    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric,
-    _escape_label_value,
-)
-
-
-class CounterMetricTestCase(unittest.TestCase):
-
-    def test_scalar(self):
-        counter = CounterMetric("scalar")
-
-        self.assertEquals(counter.render(), [
-            'scalar 0',
-        ])
-
-        counter.inc()
-
-        self.assertEquals(counter.render(), [
-            'scalar 1',
-        ])
-
-        counter.inc_by(2)
-
-        self.assertEquals(counter.render(), [
-            'scalar 3'
-        ])
-
-    def test_vector(self):
-        counter = CounterMetric("vector", labels=["method"])
-
-        # Empty counter doesn't yet know what values it has
-        self.assertEquals(counter.render(), [])
-
-        counter.inc("GET")
-
-        self.assertEquals(counter.render(), [
-            'vector{method="GET"} 1',
-        ])
-
-        counter.inc("GET")
-        counter.inc("PUT")
-
-        self.assertEquals(counter.render(), [
-            'vector{method="GET"} 2',
-            'vector{method="PUT"} 1',
-        ])
-
-
-class CallbackMetricTestCase(unittest.TestCase):
-
-    def test_scalar(self):
-        d = dict()
-
-        metric = CallbackMetric("size", lambda: len(d))
-
-        self.assertEquals(metric.render(), [
-            'size 0',
-        ])
-
-        d["key"] = "value"
-
-        self.assertEquals(metric.render(), [
-            'size 1',
-        ])
-
-    def test_vector(self):
-        vals = dict()
-
-        metric = CallbackMetric("values", lambda: vals, labels=["type"])
-
-        self.assertEquals(metric.render(), [])
-
-        # Keys have to be tuples, even if they're 1-element
-        vals[("foo",)] = 1
-        vals[("bar",)] = 2
-
-        self.assertEquals(metric.render(), [
-            'values{type="bar"} 2',
-            'values{type="foo"} 1',
-        ])
-
-
-class DistributionMetricTestCase(unittest.TestCase):
-
-    def test_scalar(self):
-        metric = DistributionMetric("thing")
-
-        self.assertEquals(metric.render(), [
-            'thing:count 0',
-            'thing:total 0',
-        ])
-
-        metric.inc_by(500)
-
-        self.assertEquals(metric.render(), [
-            'thing:count 1',
-            'thing:total 500',
-        ])
-
-    def test_vector(self):
-        metric = DistributionMetric("queries", labels=["verb"])
-
-        self.assertEquals(metric.render(), [])
-
-        metric.inc_by(300, "SELECT")
-        metric.inc_by(200, "SELECT")
-        metric.inc_by(800, "INSERT")
-
-        self.assertEquals(metric.render(), [
-            'queries:count{verb="INSERT"} 1',
-            'queries:count{verb="SELECT"} 2',
-            'queries:total{verb="INSERT"} 800',
-            'queries:total{verb="SELECT"} 500',
-        ])
-
-
-class CacheMetricTestCase(unittest.TestCase):
-
-    def test_cache(self):
-        d = dict()
-
-        metric = CacheMetric("cache", lambda: len(d), "cache_name")
-
-        self.assertEquals(metric.render(), [
-            'cache:hits{name="cache_name"} 0',
-            'cache:total{name="cache_name"} 0',
-            'cache:size{name="cache_name"} 0',
-            'cache:evicted_size{name="cache_name"} 0',
-        ])
-
-        metric.inc_misses()
-        d["key"] = "value"
-
-        self.assertEquals(metric.render(), [
-            'cache:hits{name="cache_name"} 0',
-            'cache:total{name="cache_name"} 1',
-            'cache:size{name="cache_name"} 1',
-            'cache:evicted_size{name="cache_name"} 0',
-        ])
-
-        metric.inc_hits()
-
-        self.assertEquals(metric.render(), [
-            'cache:hits{name="cache_name"} 1',
-            'cache:total{name="cache_name"} 2',
-            'cache:size{name="cache_name"} 1',
-            'cache:evicted_size{name="cache_name"} 0',
-        ])
-
-        metric.inc_evictions(2)
-
-        self.assertEquals(metric.render(), [
-            'cache:hits{name="cache_name"} 1',
-            'cache:total{name="cache_name"} 2',
-            'cache:size{name="cache_name"} 1',
-            'cache:evicted_size{name="cache_name"} 2',
-        ])
-
-
-class LabelValueEscapeTestCase(unittest.TestCase):
-    def test_simple(self):
-        string = "safjhsdlifhyskljfksdfh"
-        self.assertEqual(string, _escape_label_value(string))
-
-    def test_escape(self):
-        self.assertEqual(
-            "abc\\\"def\\nghi\\\\",
-            _escape_label_value("abc\"def\nghi\\"),
-        )
-
-    def test_sequence_of_escapes(self):
-        self.assertEqual(
-            "abc\\\"def\\nghi\\\\\\n",
-            _escape_label_value("abc\"def\nghi\\\n"),
-        )
-- 
cgit 1.4.1


From fb2806b1868baaa24d0d6e21d10f2c1f6177717e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 09:31:53 +0100
Subject: Move in_flight_requests_count to be a callback metric

---
 synapse/http/request_metrics.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index f1e9b41fde..8cb23f4d64 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -124,12 +124,8 @@ in_flight_requests_db_sched_duration = metrics.register_counter(
     "in_flight_requests_db_sched_duration_seconds", labels=["method", "servlet"]
 )
 
-_in_flight_requests_count = metrics.register_gauge(
-    "in_flight_requests_count", labels=["method", "servlet"]
-)
-
 
-# The set of all in flight requests.
+# The set of all in flight requests, set[RequestMetrics]
 _in_flight_requests = set()
 
 
@@ -138,20 +134,35 @@ def _collect_in_flight():
     the in flight request metrics
     """
 
+    for rm in _in_flight_requests:
+        rm.update_metrics()
+
+
+metrics.register_collector(_collect_in_flight)
+
+
+def _get_in_flight_counts():
+    """Returns a count of all in flight requests by (method, server_name)
+
+    Returns:
+        dict[tuple[str, str], int]
+    """
+
     # Map from (method, name) -> int, the number of in flight requests of that
     # type
     counts = {}
-
     for rm in _in_flight_requests:
-        rm.update_metrics()
         key = (rm.method, rm.name,)
         counts[key] = counts.get(key, 0) + 1
 
-    for (method, name), count in counts.iteritems():
-        _in_flight_requests_count.set(count, method, name)
+    return counts
 
 
-metrics.register_collector(_collect_in_flight)
+metrics.register_callback(
+    "in_flight_requests_count",
+    _get_in_flight_counts,
+    labels=["method", "servlet"]
+)
 
 
 class RequestMetrics(object):
-- 
cgit 1.4.1


From c435b0b44172efe485ae1401deb8adca2e774d3c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 09:34:26 +0100
Subject: Don't store context

---
 synapse/http/request_metrics.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 8cb23f4d64..5917c83958 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -230,7 +230,7 @@ class RequestMetrics(object):
         """Updates the in flight metrics with values from this request.
         """
 
-        diff = self._request_stats.update()
+        diff = self._request_stats.update(self.start_context)
 
         in_flight_requests_ru_utime.inc_by(
             diff.ru_utime, self.method, self.name,
@@ -258,13 +258,12 @@ class _RequestStats(object):
     """
 
     __slots__ = [
-        "context", "ru_utime", "ru_stime",
+        "ru_utime", "ru_stime",
         "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
     ]
 
-    def __init__(self, context, ru_utime, ru_stime, db_txn_count,
+    def __init__(self, ru_utime, ru_stime, db_txn_count,
                  db_txn_duration_ms, db_sched_duration_ms):
-        self.context = context
         self.ru_utime = ru_utime
         self.ru_stime = ru_stime
         self.db_txn_count = db_txn_count
@@ -276,24 +275,22 @@ class _RequestStats(object):
         ru_utime, ru_stime = context.get_resource_usage()
 
         return _RequestStats(
-            context,
             ru_utime, ru_stime,
             context.db_txn_count,
             context.db_txn_duration_ms,
             context.db_sched_duration_ms,
         )
 
-    def update(self):
+    def update(self, context):
         """Updates the current values and returns the difference between the
         old and new values.
 
         Returns:
             _RequestStats: The difference between the old and new values
         """
-        new = _RequestStats.from_context(self.context)
+        new = _RequestStats.from_context(context)
 
         diff = _RequestStats(
-            self.context,
             new.ru_utime - self.ru_utime,
             new.ru_stime - self.ru_stime,
             new.db_txn_count - self.db_txn_count,
-- 
cgit 1.4.1


From 020377a5504602cb5d2ce555c77bd18eefc87635 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 11:16:07 +0100
Subject: Fix logcontext resource usage tracking

---
 synapse/util/logcontext.py | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index eab9d57650..0bbda1dcaf 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -60,7 +60,7 @@ class LoggingContext(object):
     __slots__ = [
         "previous_context", "name", "ru_stime", "ru_utime",
         "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
-        "usage_start", "usage_end",
+        "usage_start",
         "main_thread", "alive",
         "request", "tag",
     ]
@@ -109,8 +109,10 @@ class LoggingContext(object):
         # ms spent waiting for db txns to be scheduled
         self.db_sched_duration_ms = 0
 
+        # If alive has the thread resource usage when the logcontext last
+        # became active.
         self.usage_start = None
-        self.usage_end = None
+
         self.main_thread = threading.current_thread()
         self.request = None
         self.tag = ""
@@ -159,7 +161,7 @@ class LoggingContext(object):
         """Restore the logging context in thread local storage to the state it
         was before this context was entered.
         Returns:
-            None to avoid suppressing any exeptions that were thrown.
+            None to avoid suppressing any exceptions that were thrown.
         """
         current = self.set_current_context(self.previous_context)
         if current is not self:
@@ -185,29 +187,43 @@ class LoggingContext(object):
 
     def start(self):
         if threading.current_thread() is not self.main_thread:
+            logger.warning("Started logcontext %s on different thread", self)
             return
 
-        if self.usage_start and self.usage_end:
-            self.ru_utime += self.usage_end.ru_utime - self.usage_start.ru_utime
-            self.ru_stime += self.usage_end.ru_stime - self.usage_start.ru_stime
-            self.usage_start = None
-            self.usage_end = None
-
+        # If we haven't already started record the thread resource usage so
+        # far
         if not self.usage_start:
             self.usage_start = get_thread_resource_usage()
 
     def stop(self):
         if threading.current_thread() is not self.main_thread:
+            logger.warning("Stopped logcontext %s on different thread", self)
             return
 
+        # When we stop lets record the resource used since we started
         if self.usage_start:
-            self.usage_end = get_thread_resource_usage()
+            usage_end = get_thread_resource_usage()
+
+            self.ru_utime += usage_end.ru_utime - self.usage_start.ru_utime
+            self.ru_stime += usage_end.ru_stime - self.usage_start.ru_stime
+
+            self.usage_start = None
+        else:
+            logger.warning("Called stop on logcontext %s without calling start", self)
 
     def get_resource_usage(self):
+        """Get CPU time used by this logcontext so far.
+
+        Returns:
+            tuple[float, float]: The user and system CPU usage in seconds
+        """
         ru_utime = self.ru_utime
         ru_stime = self.ru_stime
 
-        if self.usage_start and threading.current_thread() is self.main_thread:
+        # If we are on the correct thread and we're currently running then we
+        # can include resource usage so far.
+        is_main_thread = threading.current_thread() is self.main_thread
+        if self.alive and self.usage_start and is_main_thread:
             current = get_thread_resource_usage()
             ru_utime += current.ru_utime - self.usage_start.ru_utime
             ru_stime += current.ru_stime - self.usage_start.ru_stime
-- 
cgit 1.4.1


From 7948ecf2342d7053a6ff6daa01268b9331e1c569 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 11:39:43 +0100
Subject: Comment

---
 synapse/util/logcontext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 0bbda1dcaf..914f616312 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -200,7 +200,7 @@ class LoggingContext(object):
             logger.warning("Stopped logcontext %s on different thread", self)
             return
 
-        # When we stop lets record the resource used since we started
+        # When we stop, let's record the resource used since we started
         if self.usage_start:
             usage_end = get_thread_resource_usage()
 
-- 
cgit 1.4.1


From d14d7b8fdc67f8f784c69605f95fe136e032b5d4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 12:09:18 +0100
Subject: Rename 'version' param on user consent config

we're going to use it for the version we require too.
---
 synapse/config/consent_config.py         | 10 ++++++----
 synapse/rest/consent/consent_resource.py |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 675fce0911..45856b9e8a 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -18,19 +18,21 @@ from ._base import Config
 DEFAULT_CONFIG = """\
 # User Consent configuration
 #
-# uncomment and configure if enabling the 'consent' resource under 'listeners'.
+# Parts of this section are required if enabling the 'consent' resource under
+# 'listeners', in particular 'template_dir' and 'version'.
 #
 # 'template_dir' gives the location of the templates for the HTML forms.
 # This directory should contain one subdirectory per language (eg, 'en', 'fr'),
 # and each language directory should contain the policy document (named as
 # '<version>.html') and a success page (success.html).
 #
-# 'default_version' gives the version of the policy document to serve up if
-# there is no 'v' parameter.
+# 'version' specifies the 'current' version of the policy document. It defines
+# the version to be served by the consent resource if there is no 'v'
+# parameter.
 #
 # user_consent:
 #   template_dir: res/templates/privacy
-#   default_version: 1.0
+#   version: 1.0
 """
 
 
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index d791302278..b0536ee375 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -116,7 +116,7 @@ class ConsentResource(Resource):
         loader = jinja2.FileSystemLoader(consent_template_directory)
         self._jinja_env = jinja2.Environment(loader=loader)
 
-        self._default_consent_verison = consent_config["default_version"]
+        self._default_consent_verison = consent_config["version"]
 
         if hs.config.form_secret is None:
             raise ConfigError(
-- 
cgit 1.4.1


From 9ea219c5143de009c8e411fbb01da1da5ce50829 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 17:35:31 +0100
Subject: Send users a server notice about consent

When a user first syncs, we will send them a server notice asking them to
consent to the privacy policy if they have not already done so.
---
 synapse/config/consent_config.py                   |   8 ++
 synapse/handlers/presence.py                       |  10 +-
 synapse/replication/tcp/resource.py                |   2 +
 synapse/server.py                                  |   5 +
 synapse/server.pyi                                 |   4 +
 synapse/server_notices/consent_server_notices.py   | 101 +++++++++++++++++++++
 synapse/server_notices/server_notices_sender.py    |  58 ++++++++++++
 synapse/storage/registration.py                    |  46 ++++++++--
 .../49/add_user_consent_server_notice_sent.sql     |  20 ++++
 tests/storage/test_registration.py                 |  11 ++-
 tests/utils.py                                     |   1 +
 11 files changed, 255 insertions(+), 11 deletions(-)
 create mode 100644 synapse/server_notices/consent_server_notices.py
 create mode 100644 synapse/server_notices/server_notices_sender.py
 create mode 100644 synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 45856b9e8a..a6fbc5a058 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -30,9 +30,17 @@ DEFAULT_CONFIG = """\
 # the version to be served by the consent resource if there is no 'v'
 # parameter.
 #
+# 'server_notice_content', if enabled, will send a user a "Server Notice"
+# asking them to consent to the privacy policy. The 'server_notices' section
+# must also be configured for this to work.
+#
 # user_consent:
 #   template_dir: res/templates/privacy
 #   version: 1.0
+#   server_notice_content:
+#     msgtype: m.text
+#     body: |
+#       Pls do consent kthx
 """
 
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 91218e40e6..adc816f747 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -87,6 +87,11 @@ assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER
 class PresenceHandler(object):
 
     def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
         self.is_mine = hs.is_mine
         self.is_mine_id = hs.is_mine_id
         self.clock = hs.get_clock()
@@ -94,8 +99,8 @@ class PresenceHandler(object):
         self.wheel_timer = WheelTimer()
         self.notifier = hs.get_notifier()
         self.federation = hs.get_federation_sender()
-
         self.state = hs.get_state_handler()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
         federation_registry = hs.get_federation_registry()
 
@@ -428,6 +433,9 @@ class PresenceHandler(object):
                     last_user_sync_ts=self.clock.time_msec(),
                 )])
 
+        # send any outstanding server notices to the user.
+        yield self._server_notices_sender.on_user_syncing(user_id)
+
         @defer.inlineCallbacks
         def _end():
             try:
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index a41af4fd6c..a603c520ea 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -69,6 +69,7 @@ class ReplicationStreamer(object):
         self.presence_handler = hs.get_presence_handler()
         self.clock = hs.get_clock()
         self.notifier = hs.get_notifier()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
         # Current connections.
         self.connections = []
@@ -253,6 +254,7 @@ class ReplicationStreamer(object):
         yield self.store.insert_client_ip(
             user_id, access_token, ip, user_agent, device_id, last_seen,
         )
+        yield self._server_notices_sender.on_user_ip(user_id)
 
     def send_sync_to_all_connections(self, data):
         """Sends a SYNC command to all clients.
diff --git a/synapse/server.py b/synapse/server.py
index 85f54cd047..e7c733f2d4 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -73,6 +73,7 @@ from synapse.rest.media.v1.media_repository import (
     MediaRepositoryResource,
 )
 from synapse.server_notices.server_notices_manager import ServerNoticesManager
+from synapse.server_notices.server_notices_sender import ServerNoticesSender
 from synapse.state import StateHandler, StateResolutionHandler
 from synapse.storage import DataStore
 from synapse.streams.events import EventSources
@@ -158,6 +159,7 @@ class HomeServer(object):
         'room_member_handler',
         'federation_registry',
         'server_notices_manager',
+        'server_notices_sender',
     ]
 
     def __init__(self, hostname, **kwargs):
@@ -403,6 +405,9 @@ class HomeServer(object):
     def build_server_notices_manager(self):
         return ServerNoticesManager(self)
 
+    def build_server_notices_sender(self):
+        return ServerNoticesSender(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 
diff --git a/synapse/server.pyi b/synapse/server.pyi
index 6fbe15168d..ce28486233 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -10,6 +10,7 @@ import synapse.handlers.e2e_keys
 import synapse.handlers.set_password
 import synapse.rest.media.v1.media_repository
 import synapse.server_notices.server_notices_manager
+import synapse.server_notices.server_notices_sender
 import synapse.state
 import synapse.storage
 
@@ -69,3 +70,6 @@ class HomeServer(object):
 
     def get_server_notices_manager(self) -> synapse.server_notices.server_notices_manager.ServerNoticesManager:
         pass
+
+    def get_server_notices_sender(self) -> synapse.server_notices.server_notices_sender.ServerNoticesSender:
+        pass
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
new file mode 100644
index 0000000000..e9098aef27
--- /dev/null
+++ b/synapse/server_notices/consent_server_notices.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.config import ConfigError
+
+logger = logging.getLogger(__name__)
+
+
+class ConsentServerNotices(object):
+    """Keeps track of whether we need to send users server_notices about
+    privacy policy consent, and sends one if we do.
+    """
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+        self._server_notices_manager = hs.get_server_notices_manager()
+        self._store = hs.get_datastore()
+
+        self._current_consent_version = None
+        self._server_notice_content = None
+        self._users_in_progress = set()
+
+        consent_config = hs.config.consent_config
+        if consent_config is not None:
+            self._current_consent_version = str(consent_config["version"])
+            self._server_notice_content = consent_config.get(
+                "server_notice_content"
+            )
+
+        if self._server_notice_content is not None:
+            if not self._server_notices_manager.is_enabled():
+                raise ConfigError(
+                    "user_consent configuration requires server notices, but "
+                    "server notices are not enabled.",
+                )
+            if 'body' not in self._server_notice_content:
+                raise ConfigError(
+                    "user_consent server_notice_consent must contain a 'body' "
+                    "key.",
+                )
+
+    @defer.inlineCallbacks
+    def maybe_send_server_notice_to_user(self, user_id):
+        """Check if we need to send a notice to this user, and does so if so
+
+        Args:
+            user_id (str): user to check
+
+        Returns:
+            Deferred
+        """
+        if self._server_notice_content is None:
+            # not enabled
+            return
+
+        # make sure we don't send two messages to the same user at once
+        if user_id in self._users_in_progress:
+            return
+        self._users_in_progress.add(user_id)
+        try:
+            u = yield self._store.get_user_by_id(user_id)
+
+            if u["consent_version"] == self._current_consent_version:
+                # user has already consented
+                return
+
+            if u["consent_server_notice_sent"] == self._current_consent_version:
+                # we've already sent a notice to the user
+                return
+
+            # need to send a message
+            try:
+                yield self._server_notices_manager.send_notice(
+                    user_id, self._server_notice_content,
+                )
+                yield self._store.user_set_consent_server_notice_sent(
+                    user_id, self._current_consent_version,
+                )
+            except SynapseError as e:
+                logger.error("Error sending server notice about user consent: %s", e)
+        finally:
+            self._users_in_progress.remove(user_id)
diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py
new file mode 100644
index 0000000000..9eade85851
--- /dev/null
+++ b/synapse/server_notices/server_notices_sender.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.server_notices.consent_server_notices import ConsentServerNotices
+
+
+class ServerNoticesSender(object):
+    """A centralised place which sends server notices automatically when
+    Certain Events take place
+    """
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+        # todo: it would be nice to make this more dynamic
+        self._consent_server_notices = ConsentServerNotices(hs)
+
+    def on_user_syncing(self, user_id):
+        """Called when the user performs a sync operation.
+
+        This is only called when /sync (or /events) is called on the synapse
+        master. In a deployment with synchrotrons, on_user_ip is called
+
+        Args:
+            user_id (str): mxid of user who synced
+
+        Returns:
+            Deferred
+        """
+        return self._consent_server_notices.maybe_send_server_notice_to_user(
+            user_id,
+        )
+
+    def on_user_ip(self, user_id):
+        """Called when a worker process saw a client request.
+
+        Args:
+            user_id (str): mxid
+
+        Returns:
+            Deferred
+        """
+        return self._consent_server_notices.maybe_send_server_notice_to_user(
+            user_id,
+        )
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 8d1a01f1ee..a530e29f43 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -33,7 +33,10 @@ class RegistrationWorkerStore(SQLBaseStore):
             keyvalues={
                 "name": user_id,
             },
-            retcols=["name", "password_hash", "is_guest"],
+            retcols=[
+                "name", "password_hash", "is_guest",
+                "consent_version", "consent_server_notice_sent",
+            ],
             allow_none=True,
             desc="get_user_by_id",
         )
@@ -297,12 +300,41 @@ class RegistrationStore(RegistrationWorkerStore,
         Raises:
             StoreError(404) if user not found
         """
-        return self._simple_update_one(
-            table='users',
-            keyvalues={'name': user_id, },
-            updatevalues={'consent_version': consent_version, },
-            desc="user_set_consent_version"
-        )
+        def f(txn):
+            self._simple_update_one_txn(
+                txn,
+                table='users',
+                keyvalues={'name': user_id, },
+                updatevalues={'consent_version': consent_version, },
+            )
+            self._invalidate_cache_and_stream(
+                txn, self.get_user_by_id, (user_id,)
+            )
+        return self.runInteraction("user_set_consent_version", f)
+
+    def user_set_consent_server_notice_sent(self, user_id, consent_version):
+        """Updates the user table to record that we have sent the user a server
+        notice about privacy policy consent
+
+        Args:
+            user_id (str): full mxid of the user to update
+            consent_version (str): version of the policy we have notified the
+                user about
+
+        Raises:
+            StoreError(404) if user not found
+        """
+        def f(txn):
+            self._simple_update_one_txn(
+                txn,
+                table='users',
+                keyvalues={'name': user_id, },
+                updatevalues={'consent_server_notice_sent': consent_version, },
+            )
+            self._invalidate_cache_and_stream(
+                txn, self.get_user_by_id, (user_id,)
+            )
+        return self.runInteraction("user_set_consent_server_notice_sent", f)
 
     def user_delete_access_tokens(self, user_id, except_token_id=None,
                                   device_id=None):
diff --git a/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql
new file mode 100644
index 0000000000..14dcf18d73
--- /dev/null
+++ b/synapse/storage/schema/delta/49/add_user_consent_server_notice_sent.sql
@@ -0,0 +1,20 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* record whether we have sent a server notice about consenting to the
+ * privacy policy. Specifically records the version of the policy we sent
+ * a message about.
+ */
+ALTER TABLE users ADD COLUMN consent_server_notice_sent TEXT;
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index 7c7b164ee6..cc637dda1c 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -42,9 +42,14 @@ class RegistrationStoreTestCase(unittest.TestCase):
         yield self.store.register(self.user_id, self.tokens[0], self.pwhash)
 
         self.assertEquals(
-            # TODO(paul): Surely this field should be 'user_id', not 'name'
-            #  Additionally surely it shouldn't come in a 1-element list
-            {"name": self.user_id, "password_hash": self.pwhash, "is_guest": 0},
+            {
+                # TODO(paul): Surely this field should be 'user_id', not 'name'
+                "name": self.user_id,
+                "password_hash": self.pwhash,
+                "is_guest": 0,
+                "consent_version": None,
+                "consent_server_notice_sent": None,
+            },
             (yield self.store.get_user_by_id(self.user_id))
         )
 
diff --git a/tests/utils.py b/tests/utils.py
index c2beb5d9f7..63d8e9c640 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -63,6 +63,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.federation_rc_concurrent = 10
         config.filter_timeline_limit = 5000
         config.user_directory_search_all_users = False
+        config.consent_config = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.4.1


From d5dca9a04fbac7fea0822eaf2fd513b13449184c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 15:41:40 +0100
Subject: Move consent config parsing into ConsentConfig

turns out we need to reuse this, so it's better in the config class.
---
 synapse/config/consent_config.py                 | 16 +++++++++++++++-
 synapse/rest/consent/consent_resource.py         |  8 +++-----
 synapse/server_notices/consent_server_notices.py | 10 ++--------
 tests/utils.py                                   |  2 +-
 4 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index a6fbc5a058..8698b2993f 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -45,8 +45,22 @@ DEFAULT_CONFIG = """\
 
 
 class ConsentConfig(Config):
+    def __init__(self):
+        super(ConsentConfig, self).__init__()
+
+        self.user_consent_version = None
+        self.user_consent_template_dir = None
+        self.user_consent_server_notice_content = None
+
     def read_config(self, config):
-        self.consent_config = config.get("user_consent")
+        consent_config = config.get("user_consent")
+        if consent_config is None:
+            return
+        self.user_consent_version = str(consent_config["version"])
+        self.user_consent_template_dir = consent_config["template_dir"]
+        self.user_consent_server_notice_content = consent_config.get(
+            "server_notice_content",
+        )
 
     def default_config(self, **kwargs):
         return DEFAULT_CONFIG
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index b0536ee375..306385601c 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -95,8 +95,8 @@ class ConsentResource(Resource):
         # this is required by the request_handler wrapper
         self.clock = hs.get_clock()
 
-        consent_config = hs.config.consent_config
-        if consent_config is None:
+        self._default_consent_verison = hs.config.user_consent_version
+        if self._default_consent_verison is None:
             raise ConfigError(
                 "Consent resource is enabled but user_consent section is "
                 "missing in config file.",
@@ -104,7 +104,7 @@ class ConsentResource(Resource):
 
         # daemonize changes the cwd to /, so make the path absolute now.
         consent_template_directory = path.abspath(
-            consent_config["template_dir"],
+            hs.config.user_consent_template_dir,
         )
         if not path.isdir(consent_template_directory):
             raise ConfigError(
@@ -116,8 +116,6 @@ class ConsentResource(Resource):
         loader = jinja2.FileSystemLoader(consent_template_directory)
         self._jinja_env = jinja2.Environment(loader=loader)
 
-        self._default_consent_verison = consent_config["version"]
-
         if hs.config.form_secret is None:
             raise ConfigError(
                 "Consent resource is enabled but form_secret is not set in "
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index e9098aef27..440f6b1cd4 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -35,16 +35,10 @@ class ConsentServerNotices(object):
         self._server_notices_manager = hs.get_server_notices_manager()
         self._store = hs.get_datastore()
 
-        self._current_consent_version = None
-        self._server_notice_content = None
         self._users_in_progress = set()
 
-        consent_config = hs.config.consent_config
-        if consent_config is not None:
-            self._current_consent_version = str(consent_config["version"])
-            self._server_notice_content = consent_config.get(
-                "server_notice_content"
-            )
+        self._current_consent_version = hs.config.user_consent_version
+        self._server_notice_content = hs.config.user_consent_server_notice_content
 
         if self._server_notice_content is not None:
             if not self._server_notices_manager.is_enabled():
diff --git a/tests/utils.py b/tests/utils.py
index 63d8e9c640..9626e6fd78 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -63,7 +63,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.federation_rc_concurrent = 10
         config.filter_timeline_limit = 5000
         config.user_directory_search_all_users = False
-        config.consent_config = None
+        config.user_consent_server_notice_content = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.4.1


From 8810685df936ba60e0e9d36c9371c20f305c9126 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 10:57:56 +0100
Subject: Stub out ServerNoticesSender on the workers

... and have the sync endpoints call it directly rather than obsure indirection
via PresenceHandler
---
 synapse/handlers/events.py                         |  5 +++
 synapse/handlers/presence.py                       |  4 --
 synapse/rest/client/v2_alpha/sync.py               |  4 ++
 synapse/server.py                                  |  7 ++++
 synapse/server_notices/server_notices_sender.py    |  8 ++--
 .../server_notices/worker_server_notices_sender.py | 46 ++++++++++++++++++++++
 6 files changed, 66 insertions(+), 8 deletions(-)
 create mode 100644 synapse/server_notices/worker_server_notices_sender.py

diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index d3685fb12a..8bc642675f 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -48,6 +48,7 @@ class EventStreamHandler(BaseHandler):
 
         self.notifier = hs.get_notifier()
         self.state = hs.get_state_handler()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
     @defer.inlineCallbacks
     @log_function
@@ -58,6 +59,10 @@ class EventStreamHandler(BaseHandler):
 
         If `only_keys` is not None, events from keys will be sent down.
         """
+
+        # send any outstanding server notices to the user.
+        yield self._server_notices_sender.on_user_syncing(auth_user_id)
+
         auth_user = UserID.from_string(auth_user_id)
         presence_handler = self.hs.get_presence_handler()
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index adc816f747..500a131874 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -100,7 +100,6 @@ class PresenceHandler(object):
         self.notifier = hs.get_notifier()
         self.federation = hs.get_federation_sender()
         self.state = hs.get_state_handler()
-        self._server_notices_sender = hs.get_server_notices_sender()
 
         federation_registry = hs.get_federation_registry()
 
@@ -433,9 +432,6 @@ class PresenceHandler(object):
                     last_user_sync_ts=self.clock.time_msec(),
                 )])
 
-        # send any outstanding server notices to the user.
-        yield self._server_notices_sender.on_user_syncing(user_id)
-
         @defer.inlineCallbacks
         def _end():
             try:
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index eb91c0b293..a291cffbf1 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -85,6 +85,7 @@ class SyncRestServlet(RestServlet):
         self.clock = hs.get_clock()
         self.filtering = hs.get_filtering()
         self.presence_handler = hs.get_presence_handler()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
     @defer.inlineCallbacks
     def on_GET(self, request):
@@ -149,6 +150,9 @@ class SyncRestServlet(RestServlet):
         else:
             since_token = None
 
+        # send any outstanding server notices to the user.
+        yield self._server_notices_sender.on_user_syncing(user.to_string())
+
         affect_presence = set_presence != PresenceState.OFFLINE
 
         if affect_presence:
diff --git a/synapse/server.py b/synapse/server.py
index e7c733f2d4..58dbf78437 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -74,6 +74,9 @@ from synapse.rest.media.v1.media_repository import (
 )
 from synapse.server_notices.server_notices_manager import ServerNoticesManager
 from synapse.server_notices.server_notices_sender import ServerNoticesSender
+from synapse.server_notices.worker_server_notices_sender import (
+    WorkerServerNoticesSender,
+)
 from synapse.state import StateHandler, StateResolutionHandler
 from synapse.storage import DataStore
 from synapse.streams.events import EventSources
@@ -403,9 +406,13 @@ class HomeServer(object):
         return FederationHandlerRegistry()
 
     def build_server_notices_manager(self):
+        if self.config.worker_app:
+            raise Exception("Workers cannot send server notices")
         return ServerNoticesManager(self)
 
     def build_server_notices_sender(self):
+        if self.config.worker_app:
+            return WorkerServerNoticesSender(self)
         return ServerNoticesSender(self)
 
     def remove_pusher(self, app_id, push_key, user_id):
diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py
index 9eade85851..5d23965f34 100644
--- a/synapse/server_notices/server_notices_sender.py
+++ b/synapse/server_notices/server_notices_sender.py
@@ -31,9 +31,6 @@ class ServerNoticesSender(object):
     def on_user_syncing(self, user_id):
         """Called when the user performs a sync operation.
 
-        This is only called when /sync (or /events) is called on the synapse
-        master. In a deployment with synchrotrons, on_user_ip is called
-
         Args:
             user_id (str): mxid of user who synced
 
@@ -45,7 +42,7 @@ class ServerNoticesSender(object):
         )
 
     def on_user_ip(self, user_id):
-        """Called when a worker process saw a client request.
+        """Called on the master when a worker process saw a client request.
 
         Args:
             user_id (str): mxid
@@ -53,6 +50,9 @@ class ServerNoticesSender(object):
         Returns:
             Deferred
         """
+        # The synchrotrons use a stubbed version of ServerNoticesSender, so
+        # we check for notices to send to the user in on_user_ip as well as
+        # in on_user_syncing
         return self._consent_server_notices.maybe_send_server_notice_to_user(
             user_id,
         )
diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py
new file mode 100644
index 0000000000..25abb4ccf5
--- /dev/null
+++ b/synapse/server_notices/worker_server_notices_sender.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+
+class WorkerServerNoticesSender(object):
+    """Stub impl of ServerNoticesSender which does nothing"""
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+
+    def on_user_syncing(self, user_id):
+        """Called when the user performs a sync operation.
+
+        Args:
+            user_id (str): mxid of user who synced
+
+        Returns:
+            Deferred
+        """
+        return defer.succeed()
+
+    def on_user_ip(self, user_id):
+        """Called on the master when a worker process saw a client request.
+
+        Args:
+            user_id (str): mxid
+
+        Returns:
+            Deferred
+        """
+        raise AssertionError("on_user_ip unexpectedly called on worker")
-- 
cgit 1.4.1


From a5e2941aad9acdd5033709807bb6ddd40e4435eb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 08:56:52 +0100
Subject: Reject attempts to send event before privacy consent is given

Returns an M_CONSENT_NOT_GIVEN error (cf
https://github.com/matrix-org/matrix-doc/issues/1252) if consent is not yet
given.
---
 synapse/api/errors.py            | 30 +++++++++++++-
 synapse/api/urls.py              | 50 +++++++++++++++++++++++
 synapse/config/consent_config.py | 10 +++++
 synapse/handlers/message.py      | 86 +++++++++++++++++++++++++++++++++++++++-
 synapse/handlers/room.py         |  4 ++
 tests/utils.py                   |  1 +
 6 files changed, 179 insertions(+), 2 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index a9ff5576f3..aca804280c 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -19,6 +19,7 @@ import logging
 
 import simplejson as json
 from six import iteritems
+from six.moves import http_client
 
 logger = logging.getLogger(__name__)
 
@@ -51,6 +52,7 @@ class Codes(object):
     THREEPID_DENIED = "M_THREEPID_DENIED"
     INVALID_USERNAME = "M_INVALID_USERNAME"
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
+    CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
 
 
 class CodeMessageException(RuntimeError):
@@ -138,6 +140,32 @@ class SynapseError(CodeMessageException):
         return res
 
 
+class ConsentNotGivenError(SynapseError):
+    """The error returned to the client when the user has not consented to the
+    privacy policy.
+    """
+    def __init__(self, msg, consent_uri):
+        """Constructs a ConsentNotGivenError
+
+        Args:
+            msg (str): The human-readable error message
+            consent_url (str): The URL where the user can give their consent
+        """
+        super(ConsentNotGivenError, self).__init__(
+            code=http_client.FORBIDDEN,
+            msg=msg,
+            errcode=Codes.CONSENT_NOT_GIVEN
+        )
+        self._consent_uri = consent_uri
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            consent_uri=self._consent_uri
+        )
+
+
 class RegistrationError(SynapseError):
     """An error raised when a registration event fails."""
     pass
@@ -292,7 +320,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
 
     Args:
         msg (str): The error message.
-        code (int): The error code.
+        code (str): The error code.
         kwargs : Additional keys to add to the response.
     Returns:
         A dict representing the error response JSON.
diff --git a/synapse/api/urls.py b/synapse/api/urls.py
index 91a33a3402..bb46b5da8a 100644
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +15,12 @@
 # limitations under the License.
 
 """Contains the URL paths to prefix various aspects of the server with. """
+from hashlib import sha256
+import hmac
+
+from six.moves.urllib.parse import urlencode
+
+from synapse.config import ConfigError
 
 CLIENT_PREFIX = "/_matrix/client/api/v1"
 CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha"
@@ -25,3 +32,46 @@ SERVER_KEY_PREFIX = "/_matrix/key/v1"
 SERVER_KEY_V2_PREFIX = "/_matrix/key/v2"
 MEDIA_PREFIX = "/_matrix/media/r0"
 LEGACY_MEDIA_PREFIX = "/_matrix/media/v1"
+
+
+class ConsentURIBuilder(object):
+    def __init__(self, hs_config):
+        """
+        Args:
+            hs_config (synapse.config.homeserver.HomeServerConfig):
+        """
+        if hs_config.form_secret is None:
+            raise ConfigError(
+                "form_secret not set in config",
+            )
+        if hs_config.public_baseurl is None:
+            raise ConfigError(
+                "public_baseurl not set in config",
+            )
+
+        self._hmac_secret = hs_config.form_secret.encode("utf-8")
+        self._public_baseurl = hs_config.public_baseurl
+
+    def build_user_consent_uri(self, user_id):
+        """Build a URI which we can give to the user to do their privacy
+        policy consent
+
+        Args:
+            user_id (str): mxid or username of user
+
+        Returns
+            (str) the URI where the user can do consent
+        """
+        mac = hmac.new(
+            key=self._hmac_secret,
+            msg=user_id,
+            digestmod=sha256,
+        ).hexdigest()
+        consent_uri = "%s_matrix/consent?%s" % (
+            self._public_baseurl,
+            urlencode({
+                "u": user_id,
+                "h": mac
+            }),
+        )
+        return consent_uri
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 8698b2993f..44c4711e6c 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -34,6 +34,10 @@ DEFAULT_CONFIG = """\
 # asking them to consent to the privacy policy. The 'server_notices' section
 # must also be configured for this to work.
 #
+# 'block_events_error', if set, will block any attempts to send events
+# until the user consents to the privacy policy. The value of the setting is
+# used as the text of the error.
+#
 # user_consent:
 #   template_dir: res/templates/privacy
 #   version: 1.0
@@ -41,6 +45,8 @@ DEFAULT_CONFIG = """\
 #     msgtype: m.text
 #     body: |
 #       Pls do consent kthx
+#   block_events_error: |
+#     You can't send any messages until you consent to the privacy policy.
 """
 
 
@@ -51,6 +57,7 @@ class ConsentConfig(Config):
         self.user_consent_version = None
         self.user_consent_template_dir = None
         self.user_consent_server_notice_content = None
+        self.block_events_without_consent_error = None
 
     def read_config(self, config):
         consent_config = config.get("user_consent")
@@ -61,6 +68,9 @@ class ConsentConfig(Config):
         self.user_consent_server_notice_content = consent_config.get(
             "server_notice_content",
         )
+        self.block_events_without_consent_error = consent_config.get(
+            "block_events_error",
+        )
 
     def default_config(self, **kwargs):
         return DEFAULT_CONFIG
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 8343b5839d..c3adbc6c95 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -20,10 +20,15 @@ import sys
 from canonicaljson import encode_canonical_json
 import six
 from twisted.internet import defer, reactor
+from twisted.internet.defer import succeed
 from twisted.python.failure import Failure
 
 from synapse.api.constants import EventTypes, Membership, MAX_DEPTH
-from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.api.errors import (
+    AuthError, Codes, SynapseError,
+    ConsentNotGivenError,
+)
+from synapse.api.urls import ConsentURIBuilder
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
 from synapse.events.validator import EventValidator
@@ -431,6 +436,9 @@ class EventCreationHandler(object):
 
         self.spam_checker = hs.get_spam_checker()
 
+        if self.config.block_events_without_consent_error is not None:
+            self._consent_uri_builder = ConsentURIBuilder(self.config)
+
     @defer.inlineCallbacks
     def create_event(self, requester, event_dict, token_id=None, txn_id=None,
                      prev_events_and_hashes=None):
@@ -482,6 +490,10 @@ class EventCreationHandler(object):
                         target, e
                     )
 
+        is_exempt = yield self._is_exempt_from_privacy_policy(builder)
+        if not is_exempt:
+            yield self.assert_accepted_privacy_policy(requester)
+
         if token_id is not None:
             builder.internal_metadata.token_id = token_id
 
@@ -496,6 +508,78 @@ class EventCreationHandler(object):
 
         defer.returnValue((event, context))
 
+    def _is_exempt_from_privacy_policy(self, builder):
+        """"Determine if an event to be sent is exempt from having to consent
+        to the privacy policy
+
+        Args:
+            builder (synapse.events.builder.EventBuilder): event being created
+
+        Returns:
+            Deferred[bool]: true if the event can be sent without the user
+                consenting
+        """
+        # the only thing the user can do is join the server notices room.
+        if builder.type == EventTypes.Member:
+            membership = builder.content.get("membership", None)
+            if membership == Membership.JOIN:
+                return self._is_server_notices_room(builder.room_id)
+        return succeed(False)
+
+    @defer.inlineCallbacks
+    def _is_server_notices_room(self, room_id):
+        if self.config.server_notices_mxid is None:
+            defer.returnValue(False)
+        user_ids = yield self.store.get_users_in_room(room_id)
+        defer.returnValue(self.config.server_notices_mxid in user_ids)
+
+    @defer.inlineCallbacks
+    def assert_accepted_privacy_policy(self, requester):
+        """Check if a user has accepted the privacy policy
+
+        Called when the given user is about to do something that requires
+        privacy consent. We see if the user is exempt and otherwise check that
+        they have given consent. If they have not, a ConsentNotGiven error is
+        raised.
+
+        Args:
+            requester (synapse.types.Requester):
+                The user making the request
+
+        Returns:
+            Deferred[None]: returns normally if the user has consented or is
+                exempt
+
+        Raises:
+            ConsentNotGivenError: if the user has not given consent yet
+        """
+        if self.config.block_events_without_consent_error is None:
+            return
+
+        # exempt AS users from needing consent
+        if requester.app_service is not None:
+            return
+
+        user_id = requester.user.to_string()
+
+        # exempt the system notices user
+        if (
+            self.config.server_notices_mxid is not None and
+            user_id == self.config.server_notices_mxid
+        ):
+            return
+
+        u = yield self.store.get_user_by_id(user_id)
+        assert u is not None
+        if u["consent_version"] == self.config.user_consent_version:
+            return
+
+        consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id)
+        raise ConsentNotGivenError(
+            msg=self.config.block_events_without_consent_error,
+            consent_uri=consent_uri,
+        )
+
     @defer.inlineCallbacks
     def send_nonmember_event(self, requester, event, context, ratelimit=True):
         """
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 5e9fa95a2d..b5850db42f 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -126,6 +126,10 @@ class RoomCreationHandler(BaseHandler):
             except Exception:
                 raise SynapseError(400, "Invalid user_id: %s" % (i,))
 
+        yield self.event_creation_handler.assert_accepted_privacy_policy(
+            requester,
+        )
+
         invite_3pid_list = config.get("invite_3pid", [])
 
         visibility = config.get("visibility", None)
diff --git a/tests/utils.py b/tests/utils.py
index 9626e6fd78..262c4a5714 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -64,6 +64,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.filter_timeline_limit = 5000
         config.user_directory_search_all_users = False
         config.user_consent_server_notice_content = None
+        config.block_events_without_consent_error = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.4.1


From b5b2d5d64b2690e250fb56e851d5556c09d13f86 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 14:00:23 +0100
Subject: Fix dependency on jinja2

Delay the import of ConsentResource, so that we can get away without jinja2 if
people don't have the consent resource enabled.

Fixes #3259
---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index bceb21a8d5..caccbaa814 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -41,7 +41,6 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \
 from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
-from synapse.rest.consent.consent_resource import ConsentResource
 from synapse.rest.key.v1.server_key_resource import LocalKey
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
@@ -186,6 +185,7 @@ class SynapseHomeServer(HomeServer):
             })
 
         if name == "consent":
+            from synapse.rest.consent.consent_resource import ConsentResource
             consent_resource = ConsentResource(self)
             if compress:
                 consent_resource = gz_wrap(consent_resource)
-- 
cgit 1.4.1


From 669400e22f514efbd846cad1f2943a48d79276b7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 14:18:53 +0100
Subject: Enable auto-escaping for the consent templates

... to reduce the risk of somebody introducing an html injection attack...
---
 synapse/rest/consent/consent_resource.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 306385601c..e6a6dcbefa 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -114,7 +114,10 @@ class ConsentResource(Resource):
             )
 
         loader = jinja2.FileSystemLoader(consent_template_directory)
-        self._jinja_env = jinja2.Environment(loader=loader)
+        self._jinja_env = jinja2.Environment(
+            loader=loader,
+            autoescape=jinja2.select_autoescape(['html', 'htm', 'xml']),
+        )
 
         if hs.config.form_secret is None:
             raise ConfigError(
-- 
cgit 1.4.1


From 7b36d06a69942653f6c6fbb6dbf341a452002237 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 14:50:22 +0100
Subject: Add a 'has_consented' template var to consent forms

fixes #3260
---
 docs/privacy_policy_templates/README.md   |  2 +-
 docs/privacy_policy_templates/en/1.0.html |  6 ++++++
 synapse/rest/consent/consent_resource.py  | 17 ++++++++++++++---
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md
index 8e91c516b3..a3e6fc0986 100644
--- a/docs/privacy_policy_templates/README.md
+++ b/docs/privacy_policy_templates/README.md
@@ -9,7 +9,7 @@ form_secret: <unique but arbitrary secret>
 
 user_consent:
   template_dir: docs/privacy_policy_templates
-  default_version: 1.0
+  version: 1.0
 ```
 
 You should then be able to enable the `consent` resource under a `listener`
diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html
index ab8666f0c3..55c5e4b612 100644
--- a/docs/privacy_policy_templates/en/1.0.html
+++ b/docs/privacy_policy_templates/en/1.0.html
@@ -4,6 +4,11 @@
     <title>Matrix.org Privacy policy</title>
   </head>
   <body>
+  {% if has_consented %}
+    <p>
+      Your base already belong to us.
+    </p>
+  {% else %}
     <p>
       All your base are belong to us.
     </p>
@@ -13,5 +18,6 @@
       <input type="hidden" name="h" value="{{userhmac}}"/>
       <input type="submit" value="Sure thing!"/>
     </form>
+  {% endif %}
   </body>
 </html>
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index e6a6dcbefa..724911d1e6 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -95,8 +95,8 @@ class ConsentResource(Resource):
         # this is required by the request_handler wrapper
         self.clock = hs.get_clock()
 
-        self._default_consent_verison = hs.config.user_consent_version
-        if self._default_consent_verison is None:
+        self._default_consent_version = hs.config.user_consent_version
+        if self._default_consent_version is None:
             raise ConfigError(
                 "Consent resource is enabled but user_consent section is "
                 "missing in config file.",
@@ -132,6 +132,7 @@ class ConsentResource(Resource):
         return NOT_DONE_YET
 
     @wrap_html_request_handler
+    @defer.inlineCallbacks
     def _async_render_GET(self, request):
         """
         Args:
@@ -139,16 +140,26 @@ class ConsentResource(Resource):
         """
 
         version = parse_string(request, "v",
-                               default=self._default_consent_verison)
+                               default=self._default_consent_version)
         username = parse_string(request, "u", required=True)
         userhmac = parse_string(request, "h", required=True)
 
         self._check_hash(username, userhmac)
 
+        if username.startswith('@'):
+            qualified_user_id = username
+        else:
+            qualified_user_id = UserID(username, self.hs.hostname).to_string()
+
+        u = yield self.store.get_user_by_id(qualified_user_id)
+        if u is None:
+            raise NotFoundError("Unknown user")
+
         try:
             self._render_template(
                 request, "%s.html" % (version,),
                 user=username, userhmac=userhmac, version=version,
+                has_consented=(u["consent_version"] == version),
             )
         except TemplateNotFound:
             raise NotFoundError("Unknown policy version")
-- 
cgit 1.4.1


From cb2a2ad791155bd1e1ef58caa2ef2894a74fab08 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 16:23:39 +0100
Subject: get_domains_from_state returns list of tuples

---
 synapse/handlers/federation.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ba3ede8024..c85d194511 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -751,6 +751,16 @@ class FederationHandler(BaseHandler):
         curr_state = yield self.state_handler.get_current_state(room_id)
 
         def get_domains_from_state(state):
+            """Get joined domains from state
+
+            Args:
+                state (dict[tuple, FrozenEvent]): State map from type/state
+                    key to event.
+
+            Returns:
+                list[tuple[str, int]]: Returns a list of servers with the
+                lowest depth of their joins. Sorted by lowest depth first.
+            """
             joined_users = [
                 (state_key, int(event.depth))
                 for (e_type, state_key), event in state.items()
@@ -861,13 +871,13 @@ class FederationHandler(BaseHandler):
             likely_domains = get_domains_from_state(states[e_id])
 
             success = yield try_backfill([
-                dom for dom in likely_domains
+                dom for dom, _ in likely_domains
                 if dom not in tried_domains
             ])
             if success:
                 defer.returnValue(True)
 
-            tried_domains.update(likely_domains)
+            tried_domains.update(dom for dom, _ in likely_domains)
 
         defer.returnValue(False)
 
-- 
cgit 1.4.1


From 8f5a688d420c8f6b51826c561da9094b76fbea1e Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 10:56:03 -0500
Subject: cleanups, self-registration

---
 synapse/handlers/presence.py        |  7 ++++---
 synapse/http/request_metrics.py     | 15 +++------------
 synapse/replication/tcp/resource.py |  9 +++++----
 3 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index aca822c46a..4ee87d5714 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -142,8 +142,9 @@ class PresenceHandler(object):
         }
 
         LaterGauge(
-            "user_to_current_state_size", "", [], lambda: len(self.user_to_current_state)
-        ).register()
+            "synapse_handlers_presence_user_to_current_state_size", "", [],
+            lambda: len(self.user_to_current_state)
+        )
 
         now = self.clock.time_msec()
         for state in active_presence:
@@ -212,7 +213,7 @@ class PresenceHandler(object):
             60 * 1000,
         )
 
-        LaterGauge("wheel_timer_size", "", [], lambda: len(self.wheel_timer)).register()
+        LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [], lambda: len(self.wheel_timer))
 
     @defer.inlineCallbacks
     def _on_shutdown(self):
diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 0984870e7e..e7f1bfc4ae 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -71,24 +71,14 @@ in_flight_requests_db_sched_duration = Counter("synapse_http_request_in_flight_r
 _in_flight_requests = set()
 
 
-def _collect_in_flight():
-    """Called just before metrics are collected, so we use it to update all
-    the in flight request metrics
-    """
-
-    for rm in _in_flight_requests:
-        rm.update_metrics()
-
-
-metrics.register_collector(_collect_in_flight)
-
-
 def _get_in_flight_counts():
     """Returns a count of all in flight requests by (method, server_name)
 
     Returns:
         dict[tuple[str, str], int]
     """
+    for rm in _in_flight_requests:
+        rm.update_metrics()
 
     # Map from (method, name) -> int, the number of in flight requests of that
     # type
@@ -99,6 +89,7 @@ def _get_in_flight_counts():
 
     return counts
 
+
 LaterGauge(
     "synapse_http_request_metrics_in_flight_requests_count", "",
     ["method", "servlet"],
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index ed251c79ac..1ca30bc31a 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -73,8 +73,8 @@ class ReplicationStreamer(object):
         # Current connections.
         self.connections = []
 
-        l = LaterGauge("synapse_replication_tcp_resource_total_connections", "", [], lambda: len(self.connections))
-        l.register()
+        LaterGauge("synapse_replication_tcp_resource_total_connections", "", [],
+                   lambda: len(self.connections))
 
         # List of streams that clients can subscribe to.
         # We only support federation stream if federation sending hase been
@@ -87,14 +87,15 @@ class ReplicationStreamer(object):
         self.streams_by_name = {stream.NAME: stream for stream in self.streams}
 
         LaterGauge(
-            "synapse_replication_tcp_resource_connections_per_stream", "", ["stream_name"],
+            "synapse_replication_tcp_resource_connections_per_stream", "",
+            ["stream_name"],
             lambda: {
                 (stream_name,): len([
                     conn for conn in self.connections
                     if stream_name in conn.replication_streams
                 ])
                 for stream_name in self.streams_by_name
-            }).register()
+            })
 
         self.federation_sender = None
         if not hs.config.send_federation:
-- 
cgit 1.4.1


From 07bb9bdae82b8f1345be7c60475be3edac64b336 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 10:57:28 -0500
Subject: update gitignore to remove some files that got put on my machine

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 7acfe56d26..7940158c5b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.pyc
 .*.swp
+*~
 
 .DS_Store
 _trial_temp/
@@ -13,6 +14,7 @@ docs/build/
 cmdclient_config.json
 homeserver*.db
 homeserver*.log
+homeserver*.log.*
 homeserver*.pid
 homeserver*.yaml
 
@@ -40,6 +42,7 @@ media_store/
 *.tac
 
 build/
+venv/
 
 localhost-800*/
 static/client/register/register_config.js
-- 
cgit 1.4.1


From 0a078026eae694337cfdf8c9460d1b231b1bd04a Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 22 May 2018 17:14:06 +0100
Subject: comment typo

---
 synapse/handlers/deactivate_account.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 4eb18775e8..d58ea6c650 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -61,7 +61,7 @@ class DeactivateAccountHandler(BaseHandler):
         yield self.store.user_delete_threepids(user_id)
         yield self.store.user_set_password_hash(user_id, None)
 
-        # Add the user to a table of users penpding deactivation (ie.
+        # Add the user to a table of users pending deactivation (ie.
         # removal from all the rooms they're a member of)
         yield self.store.add_user_pending_deactivation(user_id)
 
-- 
cgit 1.4.1


From 08bfc48abf8c138b540ede6c816338c044130dec Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 17:27:03 +0100
Subject: custom error code for not leaving server notices room

---
 synapse/api/errors.py           | 1 +
 synapse/handlers/room_member.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index aca804280c..e6ad3768f0 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -53,6 +53,7 @@ class Codes(object):
     INVALID_USERNAME = "M_INVALID_USERNAME"
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
+    CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
 
 
 class CodeMessageException(RuntimeError):
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 5263f09aaa..82adfc8fdf 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -306,6 +306,7 @@ class RoomMemberHandler(object):
                 raise SynapseError(
                     http_client.FORBIDDEN,
                     "You cannot leave this room",
+                    errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM,
                 )
 
         if effective_membership_state == Membership.INVITE:
-- 
cgit 1.4.1


From ad2823ee27d7bfe766c8b84efbd0bcde02962048 Mon Sep 17 00:00:00 2001
From: hera <matrix@template.upcloud.com>
Date: Tue, 22 May 2018 16:38:46 +0000
Subject: fix synchrotron

---
 synapse/server_notices/worker_server_notices_sender.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py
index 25abb4ccf5..4a133026c3 100644
--- a/synapse/server_notices/worker_server_notices_sender.py
+++ b/synapse/server_notices/worker_server_notices_sender.py
@@ -32,7 +32,7 @@ class WorkerServerNoticesSender(object):
         Returns:
             Deferred
         """
-        return defer.succeed()
+        return defer.succeed(None)
 
     def on_user_ip(self, user_id):
         """Called on the master when a worker process saw a client request.
-- 
cgit 1.4.1


From 5494c1d71e47245b9750fc260462fe4c2ca3e273 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 18:15:21 +0100
Subject: Don't support limitless pagination

The pagination storage function supported not specifiying a limit on the
number of events returned. This was triggered when using the search or
context API with a limit of zero, which the storage function took to
mean not being limited.
---
 synapse/storage/stream.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index ea24710ad8..1701d03866 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -684,8 +684,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
                 results to only those before
             direction(char): Either 'b' or 'f' to indicate whether we are
                 paginating forwards or backwards from `from_key`.
-            limit (int): The maximum number of events to return. Zero or less
-                means no limit.
+            limit (int): The maximum number of events to return.
             event_filter (Filter|None): If provided filters the events to
                 those that match the filter.
 
@@ -694,6 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             as a list of _EventDictReturn and a token that points to the end
             of the result set.
         """
+
+        assert int(limit) >= 0
+
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
         # we have a bit of asymmetry when it comes to equalities.
@@ -723,18 +725,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
             bounds += " AND " + filter_clause
             args.extend(filter_args)
 
-        if int(limit) > 0:
-            args.append(int(limit))
-            limit_str = " LIMIT ?"
-        else:
-            limit_str = ""
+        args.append(int(limit))
 
         sql = (
             "SELECT event_id, topological_ordering, stream_ordering"
             " FROM events"
             " WHERE outlier = ? AND room_id = ? AND %(bounds)s"
             " ORDER BY topological_ordering %(order)s,"
-            " stream_ordering %(order)s %(limit)s"
+            " stream_ordering %(order)s LIMIT ?"
         ) % {
             "bounds": bounds,
             "order": order,
-- 
cgit 1.4.1


From a17e901f4dd1825d4bf21e55488939d95e1d7d80 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 18:24:32 +0100
Subject: Remove unused string formatting param

---
 synapse/storage/stream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 1701d03866..fb463c525a 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -736,7 +736,6 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
         ) % {
             "bounds": bounds,
             "order": order,
-            "limit": limit_str
         }
 
         txn.execute(sql, args)
-- 
cgit 1.4.1


From 586b66b1977e2c678e23a3b2d25a5b699b637a8e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 19:00:48 +0100
Subject: Fix that states is a dict of dicts

---
 synapse/handlers/federation.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index c85d194511..6a381f0023 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -853,10 +853,13 @@ class FederationHandler(BaseHandler):
             [resolve(room_id, [e]) for e in event_ids],
             consumeErrors=True,
         ))
+
+        # dict[str, dict[tuple, str]], a map from event_id to state map of
+        # event_ids.
         states = dict(zip(event_ids, [s.state for s in states]))
 
         state_map = yield self.store.get_events(
-            [e_id for ids in states.values() for e_id in ids],
+            [e_id for ids in states.values() for e_id in ids.itervalues()],
             get_prev_content=False
         )
         states = {
-- 
cgit 1.4.1


From e85b5a0ff717e0388179ad9664a8fdfca4b61d9a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 22 May 2018 19:02:48 +0100
Subject: Use iter* methods

---
 synapse/handlers/federation.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 6a381f0023..786c200378 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -763,7 +763,7 @@ class FederationHandler(BaseHandler):
             """
             joined_users = [
                 (state_key, int(event.depth))
-                for (e_type, state_key), event in state.items()
+                for (e_type, state_key), event in state.iteritems()
                 if e_type == EventTypes.Member
                 and event.membership == Membership.JOIN
             ]
@@ -780,7 +780,7 @@ class FederationHandler(BaseHandler):
                 except Exception:
                     pass
 
-            return sorted(joined_domains.items(), key=lambda d: d[1])
+            return sorted(joined_domains.iteritems(), key=lambda d: d[1])
 
         curr_domains = get_domains_from_state(curr_state)
 
@@ -797,7 +797,7 @@ class FederationHandler(BaseHandler):
                     yield self.backfill(
                         dom, room_id,
                         limit=100,
-                        extremities=[e for e in extremities.keys()]
+                        extremities=extremities,
                     )
                     # If this succeeded then we probably already have the
                     # appropriate stuff.
@@ -843,7 +843,7 @@ class FederationHandler(BaseHandler):
         tried_domains = set(likely_domains)
         tried_domains.add(self.server_name)
 
-        event_ids = list(extremities.keys())
+        event_ids = list(extremities.iterkeys())
 
         logger.debug("calling resolve_state_groups in _maybe_backfill")
         resolve = logcontext.preserve_fn(
@@ -859,15 +859,15 @@ class FederationHandler(BaseHandler):
         states = dict(zip(event_ids, [s.state for s in states]))
 
         state_map = yield self.store.get_events(
-            [e_id for ids in states.values() for e_id in ids.itervalues()],
+            [e_id for ids in states.itervalues() for e_id in ids.itervalues()],
             get_prev_content=False
         )
         states = {
             key: {
                 k: state_map[e_id]
-                for k, e_id in state_dict.items()
+                for k, e_id in state_dict.iteritems()
                 if e_id in state_map
-            } for key, state_dict in states.items()
+            } for key, state_dict in states.iteritems()
         }
 
         for e_id, _ in sorted_extremeties_tuple:
-- 
cgit 1.4.1


From 228f1f584ef100f18440362303775f7c860c01ca Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 15:02:38 -0500
Subject: fix the test failures

---
 synapse/replication/tcp/protocol.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 5848f57c5e..bdb110663c 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -513,7 +513,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
 
     def on_RDATA(self, cmd):
         stream_name = cmd.stream_name
-        inbound_rdata_count.inc(stream_name)
+        inbound_rdata_count.labels(stream_name).inc()
 
         try:
             row = STREAMS_MAP[stream_name].ROW_TYPE(*cmd.row)
-- 
cgit 1.4.1


From 85ba83eb5100abf02cf373d9a8d5010526facd45 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 16:28:23 -0500
Subject: fixes

---
 synapse/app/homeserver.py               |  6 +++--
 synapse/federation/transaction_queue.py |  6 ++---
 synapse/metrics/__init__.py             | 12 ++++++++--
 synapse/notifier.py                     |  6 ++---
 synapse/push/httppusher.py              |  4 ++--
 synapse/util/caches/__init__.py         | 40 ++++++++++++++++++++++++---------
 synapse/util/caches/descriptors.py      |  2 +-
 7 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a5b135193f..449bfacdb9 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -34,6 +34,7 @@ from synapse.module_api import ModuleApi
 from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
+from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX
 from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \
     check_requirements
@@ -60,6 +61,8 @@ from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
 
+from prometheus_client.twisted import MetricsResource
+
 logger = logging.getLogger("synapse.app.homeserver")
 
 
@@ -229,8 +232,7 @@ class SynapseHomeServer(HomeServer):
             resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
 
         if name == "metrics" and self.get_config().enable_metrics:
-            from prometheus_client.twisted import MetricsResource
-            resources[METRICS_PREFIX] = MetricsResource()
+            resources[METRICS_PREFIX] = MetricsResource(RegistryProxy())
 
         if name == "replication":
             resources[REPLICATION_PREFIX] = ReplicationRestResource(self)
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 778924a13c..2049351fdd 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -64,7 +64,7 @@ class TransactionQueue(object):
         # done
         self.pending_transactions = {}
 
-        LaterGauge("pending_destinations", "", [],
+        LaterGauge("synapse_federation_client_pending_destinations", "", [],
             lambda: len(self.pending_transactions),
         )
 
@@ -89,11 +89,11 @@ class TransactionQueue(object):
         self.pending_edus_keyed_by_dest = edus_keyed = {}
 
         LaterGauge(
-            "pending_pdus", "", [],
+            "synapse_federation_client_pending_pdus", "", [],
             lambda: sum(map(len, pdus.values())),
         )
         LaterGauge(
-            "pending_edus", "", [],
+            "synapse_federation_client_pending_edus", "", [],
             lambda: (
                 sum(map(len, edus.values()))
                 + sum(map(len, presence.values()))
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index ab0b921497..38408efb54 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -29,12 +29,20 @@ from twisted.internet import reactor
 
 logger = logging.getLogger(__name__)
 
-
 running_on_pypy = platform.python_implementation() == 'PyPy'
 all_metrics = []
 all_collectors = []
 all_gauges = {}
 
+
+class RegistryProxy(object):
+
+    def collect(self):
+        for metric in REGISTRY.collect():
+            if not metric.name.startswith("__"):
+                yield metric
+
+
 @attr.s(hash=True)
 class LaterGauge(object):
 
@@ -45,7 +53,7 @@ class LaterGauge(object):
 
     def collect(self):
 
-        g = GaugeMetricFamily(self.name, self.desc, self.labels)
+        g = GaugeMetricFamily(self.name, self.desc, labels=self.labels)
 
         try:
             calls = self.caller()
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 123e6f1840..40cc553918 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -194,14 +194,14 @@ class Notifier(object):
                 all_user_streams.add(x)
 
             return sum(stream.count_listeners() for stream in all_user_streams)
-        LaterGauge("listeners", "", [], count_listeners)
+        LaterGauge("synapse_notifier_listeners", "", [], count_listeners)
 
         LaterGauge(
-            "rooms", "", [],
+            "synapse_notifier_rooms", "", [],
             lambda: count(bool, self.room_to_user_streams.values()),
         )
         LaterGauge(
-            "users", "", [],
+            "synapse_notifier_users", "", [],
             lambda: len(self.user_to_user_stream),
         )
 
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index e22088ad6f..bf7ff74a1a 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -28,9 +28,9 @@ from prometheus_client import Counter
 
 logger = logging.getLogger(__name__)
 
-http_push_processed_counter = Counter("http_pushes_processed", "")
+http_push_processed_counter = Counter("synapse_http_httppusher_http_pushes_processed", "")
 
-http_push_failed_counter = Counter("http_pushes_failed", "")
+http_push_failed_counter = Counter("synapse_http_httppusher_http_pushes_failed", "")
 
 
 class HttpPusher(object):
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 438dcddf55..1c511a7072 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from prometheus_client.core import GaugeMetricFamily, REGISTRY
+from prometheus_client.core import Gauge, REGISTRY, GaugeMetricFamily
 
 import os
 
@@ -22,10 +22,20 @@ CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
 caches_by_name = {}
 collectors_by_name = {}
 
-def register_cache(name, cache_name, cache):
+cache_size = Gauge("synapse_util_caches_cache:size", "", ["name"])
+cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"])
+cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"])
+cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"])
+
+response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"])
+response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"])
+response_cache_evicted = Gauge("synapse_util_caches_response_cache:evicted_size", "", ["name"])
+response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"])
+
+def register_cache(cache_type, cache_name, cache):
 
     # Check if the metric is already registered. Unregister it, if so.
-    metric_name = "synapse_util_caches_%s:%s" % (name, cache_name,)
+    metric_name = "cache_%s_%s" % (cache_type, cache_name,)
     if metric_name in collectors_by_name.keys():
         REGISTRY.unregister(collectors_by_name[metric_name])
 
@@ -44,15 +54,22 @@ def register_cache(name, cache_name, cache):
         def inc_evictions(self, size=1):
             self.evicted_size += size
 
-        def collect(self):
-            cache_size = len(cache)
+        def describe(self):
+            return []
 
-            gm = GaugeMetricFamily(metric_name, "", labels=["size", "hits", "misses", "total"])
-            gm.add_metric(["size"], cache_size)
-            gm.add_metric(["hits"], self.hits)
-            gm.add_metric(["misses"], self.misses)
-            gm.add_metric(["total"], self.hits + self.misses)
-            yield gm
+        def collect(self):
+            if cache_type == "response_cache":
+                response_cache_size.labels(cache_name).set(len(cache))
+                response_cache_hits.labels(cache_name).set(self.hits)
+                response_cache_evicted.labels(cache_name).set(self.evicted_size)
+                response_cache_total.labels(cache_name).set(self.hits + self.misses)
+            else:
+                cache_size.labels(cache_name).set(len(cache))
+                cache_hits.labels(cache_name).set(self.hits)
+                cache_evicted.labels(cache_name).set(self.evicted_size)
+                cache_total.labels(cache_name).set(self.hits + self.misses)
+
+            yield GaugeMetricFamily("__unused", "")
 
     metric = CacheMetric()
     REGISTRY.register(metric)
@@ -60,6 +77,7 @@ def register_cache(name, cache_name, cache):
     collectors_by_name[metric_name] = metric
     return metric
 
+
 KNOWN_KEYS = {
     key: key for key in
     (
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index a4188eb099..8a9dcb2fc2 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -80,7 +80,7 @@ class Cache(object):
         self.name = name
         self.keylen = keylen
         self.thread = None
-        self.metrics = register_cache("descriptor", name, self.cache)
+        self.metrics = register_cache("cache", name, self.cache)
 
     def _on_evicted(self, evicted_count):
         self.metrics.inc_evictions(evicted_count)
-- 
cgit 1.4.1


From 4abeaedcf333ee688f8a39efe9520bc3e2e54770 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 16:45:11 -0500
Subject: tests/metrics is gone now

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 89ca6d7a19..3e114e382c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,7 @@ usedevelop=true
 commands =
     /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
     coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
-        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/metrics tests/config} \
+        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/config} \
         {env:TOXSUFFIX:}
     {env:DUMP_COVERAGE_COMMAND:coverage report -m}
 
-- 
cgit 1.4.1


From 071206304d088aac8bd0e2fff600141dae1d09b6 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 16:54:22 -0500
Subject: cleanup pep8 errors

---
 synapse/federation/federation_server.py |   5 +-
 synapse/federation/transaction_queue.py |  22 ++++--
 synapse/http/request_metrics.py         | 114 ++++++++++++++++++++++++--------
 synapse/notifier.py                     |   3 +-
 synapse/util/caches/__init__.py         |   7 +-
 5 files changed, 114 insertions(+), 37 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 8211273006..2d420a58a2 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -46,10 +46,13 @@ received_pdus_counter = Counter("synapse_federation_server_received_pdus", "")
 
 received_edus_counter = Counter("synapse_federation_server_received_edus", "")
 
-received_queries_counter = Counter("synapse_federation_server_received_queries", "", ["type"])
+received_queries_counter = Counter(
+    "synapse_federation_server_received_queries", "", ["type"]
+)
 
 
 class FederationServer(FederationBase):
+
     def __init__(self, hs):
         super(FederationServer, self).__init__(hs)
 
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 2049351fdd..53442688c8 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -28,7 +28,10 @@ from synapse.handlers.presence import format_user_presence_state, get_interested
 import synapse.metrics
 from synapse.metrics import LaterGauge
 from synapse.metrics import (
-    sent_edus_counter, sent_transactions_counter, events_processed_counter)
+    sent_edus_counter,
+    sent_transactions_counter,
+    events_processed_counter,
+)
 
 from prometheus_client import Counter
 
@@ -37,7 +40,9 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-sent_pdus_destination_dist = Counter("synapse_federation_client_sent_pdu_destinations", "")
+sent_pdus_destination_dist = Counter(
+    "synapse_federation_client_sent_pdu_destinations", ""
+)
 
 
 class TransactionQueue(object):
@@ -64,7 +69,10 @@ class TransactionQueue(object):
         # done
         self.pending_transactions = {}
 
-        LaterGauge("synapse_federation_client_pending_destinations", "", [],
+        LaterGauge(
+            "synapse_federation_client_pending_destinations",
+            "",
+            [],
             lambda: len(self.pending_transactions),
         )
 
@@ -89,11 +97,15 @@ class TransactionQueue(object):
         self.pending_edus_keyed_by_dest = edus_keyed = {}
 
         LaterGauge(
-            "synapse_federation_client_pending_pdus", "", [],
+            "synapse_federation_client_pending_pdus",
+            "",
+            [],
             lambda: sum(map(len, pdus.values())),
         )
         LaterGauge(
-            "synapse_federation_client_pending_edus", "", [],
+            "synapse_federation_client_pending_edus",
+            "",
+            [],
             lambda: (
                 sum(map(len, edus.values()))
                 + sum(map(len, presence.values()))
diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index e7f1bfc4ae..7f11b5c5a4 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -25,47 +25,87 @@ logger = logging.getLogger(__name__)
 
 
 # total number of responses served, split by method/servlet/tag
-response_count = Counter("synapse_http_server_response_count", "", ["method", "servlet", "tag"])
+response_count = Counter(
+    "synapse_http_server_response_count", "", ["method", "servlet", "tag"]
+)
 
-requests_counter = Counter("synapse_http_server_requests_received", "", ["method", "servlet"])
+requests_counter = Counter(
+    "synapse_http_server_requests_received", "", ["method", "servlet"]
+)
 
-outgoing_responses_counter = Counter("synapse_http_server_responses", "", ["method", "code"])
+outgoing_responses_counter = Counter(
+    "synapse_http_server_responses", "", ["method", "code"]
+)
 
-response_timer = Histogram("synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"])
+response_timer = Histogram(
+    "synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"]
+)
 
-response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"])
+response_ru_utime = Counter(
+    "synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"]
+)
 
-response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"])
+response_ru_stime = Counter(
+    "synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"]
+)
 
-response_db_txn_count = Counter("synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"])
+response_db_txn_count = Counter(
+    "synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"]
+)
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
-response_db_txn_duration = Counter("synapse_http_server_response_db_txn_duration_seconds", "", ["method", "servlet", "tag"])
+response_db_txn_duration = Counter(
+    "synapse_http_server_response_db_txn_duration_seconds",
+    "",
+    ["method", "servlet", "tag"],
+)
 
 # seconds spent waiting for a db connection, when processing this request
-response_db_sched_duration = Counter("synapse_http_request_response_db_sched_duration_seconds", "", ["method", "servlet", "tag"]
+response_db_sched_duration = Counter(
+    "synapse_http_request_response_db_sched_duration_seconds",
+    "",
+    ["method", "servlet", "tag"],
 )
 
 # size in bytes of the response written
-response_size = Counter("synapse_http_request_response_size", "", ["method", "servlet", "tag"]
+response_size = Counter(
+    "synapse_http_request_response_size", "", ["method", "servlet", "tag"]
 )
 
 # In flight metrics are incremented while the requests are in flight, rather
 # than when the response was written.
 
-in_flight_requests_ru_utime = Counter("synapse_http_request_in_flight_requests_ru_utime_seconds", "", ["method", "servlet"])
+in_flight_requests_ru_utime = Counter(
+    "synapse_http_request_in_flight_requests_ru_utime_seconds",
+    "",
+    ["method", "servlet"],
+)
 
-in_flight_requests_ru_stime = Counter("synapse_http_request_in_flight_requests_ru_stime_seconds", "", ["method", "servlet"])
+in_flight_requests_ru_stime = Counter(
+    "synapse_http_request_in_flight_requests_ru_stime_seconds",
+    "",
+    ["method", "servlet"],
+)
 
-in_flight_requests_db_txn_count = Counter("synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"])
+in_flight_requests_db_txn_count = Counter(
+    "synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"]
+)
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
-in_flight_requests_db_txn_duration = Counter("synapse_http_request_in_flight_requests_db_txn_duration_seconds", "", ["method", "servlet"])
+in_flight_requests_db_txn_duration = Counter(
+    "synapse_http_request_in_flight_requests_db_txn_duration_seconds",
+    "",
+    ["method", "servlet"],
+)
 
 # seconds spent waiting for a db connection, when processing this request
-in_flight_requests_db_sched_duration = Counter("synapse_http_request_in_flight_requests_db_sched_duration_seconds", "", ["method", "servlet"])
+in_flight_requests_db_sched_duration = Counter(
+    "synapse_http_request_in_flight_requests_db_sched_duration_seconds",
+    "",
+    ["method", "servlet"],
+)
 
 # The set of all in flight requests, set[RequestMetrics]
 _in_flight_requests = set()
@@ -91,9 +131,10 @@ def _get_in_flight_counts():
 
 
 LaterGauge(
-    "synapse_http_request_metrics_in_flight_requests_count", "",
+    "synapse_http_request_metrics_in_flight_requests_count",
+    "",
     ["method", "servlet"],
-    _get_in_flight_counts
+    _get_in_flight_counts,
 )
 
 
@@ -128,16 +169,23 @@ class RequestMetrics(object):
 
         response_count.labels(request.method, self.name, tag).inc()
 
-        response_timer.labels(request.method, self.name, tag).observe(time_msec - self.start)
+        response_timer.labels(request.method, self.name, tag).observe(
+            time_msec - self.start
+        )
 
         ru_utime, ru_stime = context.get_resource_usage()
 
         response_ru_utime.labels(request.method, self.name, tag).inc(ru_utime)
         response_ru_stime.labels(request.method, self.name, tag).inc(ru_stime)
-        response_db_txn_count.labels(request.method, self.name, tag).inc(context.db_txn_count)
-        response_db_txn_duration.labels(request.method, self.name, tag).inc(context.db_txn_duration_ms / 1000.)
+        response_db_txn_count.labels(request.method, self.name, tag).inc(
+            context.db_txn_count
+        )
+        response_db_txn_duration.labels(request.method, self.name, tag).inc(
+            context.db_txn_duration_ms / 1000.
+        )
         response_db_sched_duration.labels(request.method, self.name, tag).inc(
-            context.db_sched_duration_ms / 1000.)
+            context.db_sched_duration_ms / 1000.
+        )
 
         response_size.labels(request.method, self.name, tag).inc(request.sentLength)
 
@@ -154,11 +202,17 @@ class RequestMetrics(object):
         in_flight_requests_ru_utime.labels(self.method, self.name).inc(diff.ru_utime)
         in_flight_requests_ru_stime.labels(self.method, self.name).inc(diff.ru_stime)
 
-        in_flight_requests_db_txn_count.labels(self.method, self.name).inc(diff.db_txn_count)
+        in_flight_requests_db_txn_count.labels(self.method, self.name).inc(
+            diff.db_txn_count
+        )
 
-        in_flight_requests_db_txn_duration.labels(self.method, self.name).inc(diff.db_txn_duration_ms / 1000.)
+        in_flight_requests_db_txn_duration.labels(self.method, self.name).inc(
+            diff.db_txn_duration_ms / 1000.
+        )
 
-        in_flight_requests_db_sched_duration.labels(self.method, self.name).inc(diff.db_sched_duration_ms / 1000.)
+        in_flight_requests_db_sched_duration.labels(self.method, self.name).inc(
+            diff.db_sched_duration_ms / 1000.
+        )
 
 
 class _RequestStats(object):
@@ -166,12 +220,16 @@ class _RequestStats(object):
     """
 
     __slots__ = [
-        "ru_utime", "ru_stime",
-        "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
+        "ru_utime",
+        "ru_stime",
+        "db_txn_count",
+        "db_txn_duration_ms",
+        "db_sched_duration_ms",
     ]
 
-    def __init__(self, ru_utime, ru_stime, db_txn_count,
-                 db_txn_duration_ms, db_sched_duration_ms):
+    def __init__(
+        self, ru_utime, ru_stime, db_txn_count, db_txn_duration_ms, db_sched_duration_ms
+    ):
         self.ru_utime = ru_utime
         self.ru_stime = ru_stime
         self.db_txn_count = db_txn_count
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 40cc553918..6dce20a284 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -40,7 +40,8 @@ logger = logging.getLogger(__name__)
 
 notified_events_counter = Counter("synapse_notifier_notified_events", "")
 
-users_woken_by_stream_counter = Counter("synapse_notifier_users_woken_by_stream", "", ["stream"])
+users_woken_by_stream_counter = Counter(
+    "synapse_notifier_users_woken_by_stream", "", ["stream"])
 
 
 # TODO(paul): Should be shared somewhere
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 1c511a7072..e0c22df249 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -29,13 +29,16 @@ cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"])
 
 response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"])
 response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"])
-response_cache_evicted = Gauge("synapse_util_caches_response_cache:evicted_size", "", ["name"])
+response_cache_evicted = Gauge(
+    "synapse_util_caches_response_cache:evicted_size", "", ["name"]
+)
 response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"])
 
+
 def register_cache(cache_type, cache_name, cache):
 
     # Check if the metric is already registered. Unregister it, if so.
-    metric_name = "cache_%s_%s" % (cache_type, cache_name,)
+    metric_name = "cache_%s_%s" % (cache_type, cache_name)
     if metric_name in collectors_by_name.keys():
         REGISTRY.unregister(collectors_by_name[metric_name])
 
-- 
cgit 1.4.1


From 53cc2cde1f609ec34a4ce6a7c678302c65ddfe53 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 17:32:57 -0500
Subject: cleanup

---
 synapse/federation/send_queue.py         |  5 ++---
 synapse/handlers/appservice.py           |  9 ++++++---
 synapse/handlers/presence.py             | 12 ++++++++----
 synapse/http/client.py                   |  3 ++-
 synapse/http/matrixfederationclient.py   |  6 ++++--
 synapse/metrics/__init__.py              | 28 ++++++++++++++++++++++------
 synapse/push/bulk_push_rule_evaluator.py |  6 ++++--
 synapse/storage/events.py                |  9 ++++++---
 synapse/util/metrics.py                  | 15 ++++++++++-----
 9 files changed, 64 insertions(+), 29 deletions(-)

diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index e6e1888f3a..c7ed465617 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -74,9 +74,8 @@ class FederationRemoteSendQueue(object):
         # lambda binds to the queue rather than to the name of the queue which
         # changes. ARGH.
         def register(name, queue):
-            LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,), "",
-                lambda: len(queue),
-            )
+            LaterGauge("synapse_federation_send_queue_%s_size" % (queue_name,),
+                       "", lambda: len(queue))
 
         for queue_name in [
             "presence_map", "presence_changed", "keyed_edu", "keyed_edu_changed",
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index a7345331af..d9f35a5dba 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -127,12 +127,15 @@ class ApplicationServicesHandler(object):
                     now = self.clock.time_msec()
                     ts = yield self.store.get_received_ts(events[-1].event_id)
 
-                    synapse.metrics.event_processing_positions.labels("appservice_sender").set(upper_bound)
+                    synapse.metrics.event_processing_positions.labels(
+                        "appservice_sender").set(upper_bound)
 
                     events_processed_counter.inc(len(events))
 
-                    synapse.metrics.event_processing_lag.labels("appservice_sender").set(now - ts)
-                    synapse.metrics.event_processing_last_ts.labels("appservice_sender").set(ts)
+                    synapse.metrics.event_processing_lag.labels(
+                        "appservice_sender").set(now - ts)
+                    synapse.metrics.event_processing_last_ts.labels(
+                        "appservice_sender").set(ts)
             finally:
                 self.is_processing = False
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 4ee87d5714..12939aa507 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -47,7 +47,8 @@ logger = logging.getLogger(__name__)
 
 
 notified_presence_counter = Counter("synapse_handler_presence_notified_presence", "")
-federation_presence_out_counter = Counter("synapse_handler_presence_federation_presence_out", "")
+federation_presence_out_counter = Counter(
+    "synapse_handler_presence_federation_presence_out", "")
 presence_updates_counter = Counter("synapse_handler_presence_presence_updates", "")
 timers_fired_counter = Counter("synapse_handler_presence_timers_fired", "")
 federation_presence_counter = Counter("synapse_handler_presence_federation_presence", "")
@@ -55,8 +56,10 @@ bump_active_time_counter = Counter("synapse_handler_presence_bump_active_time",
 
 get_updates_counter = Counter("synapse_handler_presence_get_updates", "", ["type"])
 
-notify_reason_counter = Counter("synapse_handler_presence_notify_reason", "", ["reason"])
-state_transition_counter = Counter("synapse_handler_presence_state_transition", "", ["from", "to"]
+notify_reason_counter = Counter(
+    "synapse_handler_presence_notify_reason", "", ["reason"])
+state_transition_counter = Counter(
+    "synapse_handler_presence_state_transition", "", ["from", "to"]
 )
 
 
@@ -213,7 +216,8 @@ class PresenceHandler(object):
             60 * 1000,
         )
 
-        LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [], lambda: len(self.wheel_timer))
+        LaterGauge("synapse_handlers_presence_wheel_timer_size", "", [],
+                   lambda: len(self.wheel_timer))
 
     @defer.inlineCallbacks
     def _on_shutdown(self):
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 61a1d2e2b3..4d4eee3d64 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -50,7 +50,8 @@ import urllib
 logger = logging.getLogger(__name__)
 
 outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"])
-incoming_responses_counter = Counter("synapse_http_client_responses", "", ["method", "code"])
+incoming_responses_counter = Counter("synapse_http_client_responses", "",
+                                     ["method", "code"])
 
 
 class SimpleHttpClient(object):
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 259d3884e2..77eaa06a1a 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -48,8 +48,10 @@ from prometheus_client import Counter
 logger = logging.getLogger(__name__)
 outbound_logger = logging.getLogger("synapse.http.outbound")
 
-outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests", "", ["method"])
-incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses", "", ["method", "code"])
+outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests",
+                                    "", ["method"])
+incoming_responses_counter = Counter("synapse_http_matrixfederationclient_responses",
+                                     "", ["method", "code"])
 
 
 MAX_LONG_RETRIES = 10
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 38408efb54..bed37b5f56 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -21,15 +21,14 @@ import platform
 import attr
 
 from prometheus_client import Gauge, Histogram, Counter
-from prometheus_client.core import (
-    GaugeMetricFamily, CounterMetricFamily, REGISTRY)
+from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, REGISTRY
 
 from twisted.internet import reactor
 
 
 logger = logging.getLogger(__name__)
 
-running_on_pypy = platform.python_implementation() == 'PyPy'
+running_on_pypy = platform.python_implementation() == "PyPy"
 all_metrics = []
 all_collectors = []
 all_gauges = {}
@@ -87,9 +86,16 @@ class LaterGauge(object):
 #
 
 gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"])
-gc_time = Histogram("python_gc_time", "Time taken to GC (ms)", ["gen"], buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000])
+gc_time = Histogram(
+    "python_gc_time",
+    "Time taken to GC (ms)",
+    ["gen"],
+    buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000],
+)
+
 
 class GCCounts(object):
+
     def collect(self):
         gc_counts = gc.get_count()
 
@@ -99,14 +105,23 @@ class GCCounts(object):
 
         yield cm
 
+
 REGISTRY.register(GCCounts())
 
 #
 # Twisted reactor metrics
 #
 
-tick_time = Histogram("python_twisted_reactor_tick_time", "Tick time of the Twisted reactor (ms)", buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000])
-pending_calls_metric = Histogram("python_twisted_reactor_pending_calls", "Pending calls", buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000])
+tick_time = Histogram(
+    "python_twisted_reactor_tick_time",
+    "Tick time of the Twisted reactor (ms)",
+    buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000],
+)
+pending_calls_metric = Histogram(
+    "python_twisted_reactor_pending_calls",
+    "Pending calls",
+    buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000],
+)
 
 #
 # Federation Metrics
@@ -134,6 +149,7 @@ event_processing_last_ts = Gauge("synapse_event_processing_last_ts", "", ["name"
 # finished being processed.
 event_processing_lag = Gauge("synapse_event_processing_lag", "", ["name"])
 
+
 def runUntilCurrentTimer(func):
 
     @functools.wraps(func)
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 6fcca5e260..b0053e7f3f 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -36,8 +36,10 @@ logger = logging.getLogger(__name__)
 rules_by_room = {}
 
 
-push_rules_invalidation_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "")
-push_rules_state_size_counter = Counter("synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "")
+push_rules_invalidation_counter = Counter(
+    "synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "")
+push_rules_state_size_counter = Counter(
+    "synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "")
 
 # Measures whether we use the fast path of using state deltas, or if we have to
 # recalculate from scratch
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 00d66886ad..b96104ccae 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -45,19 +45,22 @@ from prometheus_client import Counter
 logger = logging.getLogger(__name__)
 
 persist_event_counter = Counter("synapse_storage_events_persisted_events", "")
-event_counter = Counter("synapse_storage_events_persisted_events_sep", "", ["type", "origin_type", "origin_entity"])
+event_counter = Counter("synapse_storage_events_persisted_events_sep", "",
+                        ["type", "origin_type", "origin_entity"])
 
 # The number of times we are recalculating the current state
 state_delta_counter = Counter("synapse_storage_events_state_delta", "")
 
 # The number of times we are recalculating state when there is only a
 # single forward extremity
-state_delta_single_event_counter = Counter("synapse_storage_events_state_delta_single_event", "")
+state_delta_single_event_counter = Counter(
+    "synapse_storage_events_state_delta_single_event", "")
 
 # The number of times we are reculating state when we could have resonably
 # calculated the delta when we calculated the state for an event we were
 # persisting.
-state_delta_reuse_delta_counter = Counter("synapse_storage_events_state_delta_reuse_delta", "")
+state_delta_reuse_delta_counter = Counter(
+    "synapse_storage_events_state_delta_reuse_delta", "")
 
 
 def encode_json(json_object):
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index a964286d85..424fdcb036 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -28,17 +28,22 @@ block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"])
 
 block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"])
 
-block_ru_utime = Counter("synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"])
+block_ru_utime = Counter(
+    "synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"])
 
-block_ru_stime = Counter("synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"])
+block_ru_stime = Counter(
+    "synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"])
 
-block_db_txn_count = Counter("synapse_util_metrics_block_db_txn_count", "", ["block_name"])
+block_db_txn_count = Counter(
+    "synapse_util_metrics_block_db_txn_count", "", ["block_name"])
 
 # seconds spent waiting for db txns, excluding scheduling time, in this block
-block_db_txn_duration = Counter("synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"])
+block_db_txn_duration = Counter(
+    "synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"])
 
 # seconds spent waiting for a db connection, in this block
-block_db_sched_duration = Counter("synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"])
+block_db_sched_duration = Counter(
+    "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"])
 
 
 def measure_func(name):
-- 
cgit 1.4.1


From b6063631c3c5ea4c4d118cc9f9d7e2896949ee1d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 22 May 2018 17:36:20 -0500
Subject: more cleanup

---
 synapse/metrics/__init__.py         | 4 +---
 synapse/replication/tcp/protocol.py | 9 ++++++---
 synapse/replication/tcp/resource.py | 7 ++++---
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index bed37b5f56..75117915ff 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -21,7 +21,7 @@ import platform
 import attr
 
 from prometheus_client import Gauge, Histogram, Counter
-from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, REGISTRY
+from prometheus_client.core import GaugeMetricFamily, REGISTRY
 
 from twisted.internet import reactor
 
@@ -97,8 +97,6 @@ gc_time = Histogram(
 class GCCounts(object):
 
     def collect(self):
-        gc_counts = gc.get_count()
-
         cm = GaugeMetricFamily("python_gc_counts", "GC cycle counts", labels=["gen"])
         for n, m in enumerate(gc.get_count()):
             cm.add_metric([str(n)], m)
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index bdb110663c..5995e5d553 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -197,7 +197,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
 
         self.last_received_command = self.clock.time_msec()
 
-        self.inbound_commands_counter[cmd_name] = self.inbound_commands_counter[cmd_name] + 1
+        self.inbound_commands_counter[cmd_name] = (
+            self.inbound_commands_counter[cmd_name] + 1)
 
         cmd_cls = COMMAND_MAP[cmd_name]
         try:
@@ -247,7 +248,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
             self._queue_command(cmd)
             return
 
-        self.outbound_commands_counter[cmd.NAME] = self.outbound_commands_counter[cmd.NAME] + 1
+        self.outbound_commands_counter[cmd.NAME] = (
+            self.outbound_commands_counter[cmd.NAME] + 1)
         string = "%s %s" % (cmd.NAME, cmd.to_line(),)
         if "\n" in string:
             raise Exception("Unexpected newline in command: %r", string)
@@ -632,4 +634,5 @@ tcp_outbound_commands = LaterGauge(
     })
 
 # number of updates received for each RDATA stream
-inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "", ["stream_name"])
+inbound_rdata_count = Counter("synapse_replication_tcp_inbound_rdata_count", "",
+                              ["stream_name"])
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index 1ca30bc31a..19987c06f0 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -28,12 +28,13 @@ import logging
 
 from prometheus_client import Counter
 
-stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates", "", ["stream_name"]
-)
+stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates",
+                                 "", ["stream_name"])
 user_sync_counter = Counter("synapse_replication_tcp_resource_user_sync", "")
 federation_ack_counter = Counter("synapse_replication_tcp_resource_federation_ack", "")
 remove_pusher_counter = Counter("synapse_replication_tcp_resource_remove_pusher", "")
-invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache", "")
+invalidate_cache_counter = Counter("synapse_replication_tcp_resource_invalidate_cache",
+                                   "")
 user_ip_cache_counter = Counter("synapse_replication_tcp_resource_user_ip_cache", "")
 
 logger = logging.getLogger(__name__)
-- 
cgit 1.4.1


From 0a4bca413425fee20aed2103d6886a7701ac6b09 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 May 2018 10:04:23 +0100
Subject: Use iter* methods for _filter_events_for_server

---
 synapse/handlers/federation.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ba3ede8024..c2dc1e2690 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -479,18 +479,18 @@ class FederationHandler(BaseHandler):
         # to get all state ids that we're interested in.
         event_map = yield self.store.get_events([
             e_id
-            for key_to_eid in event_to_state_ids.values()
-            for key, e_id in key_to_eid.items()
+            for key_to_eid in event_to_state_ids.itervalues()
+            for key, e_id in key_to_eid.iteritems()
             if key[0] != EventTypes.Member or check_match(key[1])
         ])
 
         event_to_state = {
             e_id: {
                 key: event_map[inner_e_id]
-                for key, inner_e_id in key_to_eid.items()
+                for key, inner_e_id in key_to_eid.iteritems()
                 if inner_e_id in event_map
             }
-            for e_id, key_to_eid in event_to_state_ids.items()
+            for e_id, key_to_eid in event_to_state_ids.iteritems()
         }
 
         def redact_disallowed(event, state):
-- 
cgit 1.4.1


From 5aaa3189d5a1ef0edc4ed09d7cc61e1a7452efed Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 May 2018 10:13:05 +0100
Subject: s/values/itervalues/

---
 synapse/handlers/federation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index c2dc1e2690..c15c8c1ed5 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -505,7 +505,7 @@ class FederationHandler(BaseHandler):
                     # membership states for the requesting server to determine
                     # if the server is either in the room or has been invited
                     # into the room.
-                    for ev in state.values():
+                    for ev in state.itervalues():
                         if ev.type != EventTypes.Member:
                             continue
                         try:
-- 
cgit 1.4.1


From e7598b666ba6c91f69f312d367017caca8f44a8f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 11:14:23 +0100
Subject: Some docs about server notices

---
 docs/server_notices.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 docs/server_notices.md

diff --git a/docs/server_notices.md b/docs/server_notices.md
new file mode 100644
index 0000000000..a37aed6792
--- /dev/null
+++ b/docs/server_notices.md
@@ -0,0 +1,53 @@
+Server Notices
+==============
+
+'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
+channel whereby server administrators can send messages to users on the server.
+
+They are used as part of the communication of Privacy Polices (see
+[privacy_policy.md]), however the intention is that they may also find a use
+for features such as "Message of the day".
+
+This is a feature specific to Synapse, but it uses standard Matrix
+communication mechanisms, so should work with any Matrix client.
+
+User experience
+---------------
+
+When the user is first sent a server notice, they will get an invitation to a
+room (typically called 'Server Notices', though this is configurable in
+`homeserver.yaml`). They will be **unable to reject** this invitation -
+attempts to do so will receive an error.
+
+Once they accept the invitation, they will see the notice message in the room
+history; it will appear to have come from the 'server notices user' (see
+below).
+
+The user is prevented from sending any messages in this room by the power
+levels. They also cannot leave it.
+
+Synapse configuration
+---------------------
+
+Server notices come from a specific user id on the server. Server
+administrators are free to choose the user id - something like `server` is
+suggested, meaning the notices will come from
+`@server:<your_server_name>`. Once the server notices user is configured, that
+user id becomes a special, privileged user, so administrators should ensure
+that **it is not already allocated**.
+
+In order to support server notices, it is necessary to add some configuration
+to the `homeserver.yaml` file. In particular, you should add a `server_notices`
+section, which should look like this:
+
+```yaml
+server_notices:
+   system_mxid_localpart: server
+   system_mxid_display_name: "Server Notices"
+   room_name: "Server Notices"
+```
+
+The only compulsory setting is `system_mxid_localpart`, which defines the user
+id of the server notices user, as above. `system_mxid_display_name` and
+`room_name` define the displayname of the system notices user, and of
+the notices room, respectively.
-- 
cgit 1.4.1


From 3601a240aaec9af7c95c11821d851fd5fa03b0c8 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 23 May 2018 12:03:23 +0100
Subject: bump version and changelog

---
 CHANGES.rst         | 36 ++++++++++++++++++++++++++++++++++++
 synapse/__init__.py |  2 +-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 851cb9b2d7..ee418abff3 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,39 @@
+Changes in synapse v0.30.0-rc1 (2018-05-23)
+==========================================
+
+This version includes support for GDPR consent collection. Users signal their
+consent through responding to a private message from a bot named
+@server:matrix.org For more information see XXXXXX
+
+GDPR Support:
+
+* ConsentResource to gather policy consent from users (PR #3213)
+* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225)
+* Infrastructure for a server notices room (PR #3232)
+* Send users a server notice about consent (PR #3236)
+* Reject attempts to send event before privacy consent is given (PR #3257)
+* Add a 'has_consented' template var to consent forms (PR #3262)
+* Fix dependency on jinja2 (PR #3263)
+
+Features:
+* Cohort analytics (PR #3163, #3241, #3251)
+* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman!
+* Add in flight request metrics (PR #3252)
+
+Changes:
+
+* Remove unused `update_external_syncs` (PR #3233)
+* Use stream rather depth ordering for push actions (PR #3212)
+* Make purge_history operate on tokens (PR #3221)
+* Don't support limitless pagination (PR #3265)
+
+Bug Fixes:
+
+* Fix logcontext resource usage tracking (PR #3258)
+* Fix error in handling receipts (PR #3235)
+* Stop the transaction cache caching failures (PR #3255)
+
+
 Changes in synapse v0.29.1 (2018-05-17)
 ==========================================
 Changes:
diff --git a/synapse/__init__.py b/synapse/__init__.py
index b11a1d3ea3..353387f154 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.29.1"
+__version__ = "0.30.0-rc1"
-- 
cgit 1.4.1


From 9e8ab0a4f44a3ec9e4b049f5571c14e333e8f0fa Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 23 May 2018 12:05:58 +0100
Subject: style

---
 CHANGES.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index ee418abff3..096fa4794a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,9 +1,9 @@
 Changes in synapse v0.30.0-rc1 (2018-05-23)
 ==========================================
 
-This version includes support for GDPR consent collection. Users signal their
-consent through responding to a private message from a bot named
-@server:matrix.org For more information see XXXXXX
+This version includes support for Privacy Notice agreement collection. More
+details live at docs/privacy_policy_templates/README.md
+
 
 GDPR Support:
 
@@ -16,6 +16,7 @@ GDPR Support:
 * Fix dependency on jinja2 (PR #3263)
 
 Features:
+
 * Cohort analytics (PR #3163, #3241, #3251)
 * Add lxml to docker image for web previews (PR #3239) Thanks to @ptman!
 * Add in flight request metrics (PR #3252)
-- 
cgit 1.4.1


From 833db2d92245a0b7cab9e16f9badc135346f0750 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 12:32:38 +0100
Subject: consent tracking docs

---
 docs/consent_tracking.md                | 152 ++++++++++++++++++++++++++++++++
 docs/privacy_policy_templates/README.md |  23 -----
 docs/server_notices.md                  |   4 +-
 3 files changed, 154 insertions(+), 25 deletions(-)
 create mode 100644 docs/consent_tracking.md
 delete mode 100644 docs/privacy_policy_templates/README.md

diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
new file mode 100644
index 0000000000..eb34749b55
--- /dev/null
+++ b/docs/consent_tracking.md
@@ -0,0 +1,152 @@
+Support in Synapse for tracking agreement to server terms and conditions
+========================================================================
+
+Synapse 0.30 introduces support for tracking whether users have agreed to the
+terms and conditions set by the administrator of a server - and blocking access
+to the server until they have.
+
+There are several parts to this functionality; each requires some specific
+configuration in `homeserver.yaml` to be enabled.
+
+Note that various parts of the configuation and this document refer to the
+"privacy policy": agreement with a privacy policy is one particular use of this
+feature, but of course adminstrators can specify other terms and conditions
+unrelated to "privacy" per se.
+
+Collecting policy agreement from a user
+---------------------------------------
+
+Synapse can be configured to serve the user a simple policy form with an
+"accept" button. Clicking "Accept" records the user's acceptance in the
+database and shows a success page.
+
+To enable this, first create templates for the policy and success pages.
+These should be stored on the local filesystem.
+
+These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
+and the `privacy_policy_templates` subdirectory of this `docs` directory gives
+examples of the sort of thing that can be done.
+
+Note that the templates must be stored under a name giving the language of the
+template - currently this must always be `en` (for "English");
+internationalisation support is intended for the future.
+
+The template for the policy itself should be versioned - for example
+`1.0.html`. The version of the policy which the user has agreed to is stored in
+the database.
+
+Once the templates are in place, make the following changes to `homeserver.yaml`:
+
+ 1. Add a `user_consent` section, which should look like:
+
+    ```yaml
+    user_consent:
+      template_dir: privacy_policy_templates
+      version: 1.0
+    ```
+
+    `template_dir` points to the directory containing the policy
+    templates. `version` defines the version of the policy which will be served
+    to the user. In the example above, Synapse will serve
+    `privacy_policy_templates/en/1.0.html`.
+
+
+ 2. Add a `form_secret` setting at the top level:
+
+
+    ```yaml
+    form_secret: "<unique secret>"
+    ```
+
+    This should be set to an arbitrary secret string (try `pwgen -y 30` to
+    generate suitable secrets).
+
+    More on what this is used for below.
+
+ 3. Add `consent` wherever the `client` resource is currently enabled in the
+    `listeners` configuration. For example:
+
+    ```
+    listeners:
+      - port: 8008
+        resources:
+          - names:
+            - client
+            - consent
+    ```
+
+
+Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this
+should be a matter of `pip install Jinja2`. On debian, try `apt-get install
+python-jinja2`.
+
+Once this is complete, and the server has been restarted, try visiting
+`https://<server>/_matrix/consent`. If configuration has been done correctly,
+this should give an error "Missing string query parameter 'u'". It is now
+possible to manually construct URIs where users can give their consent.
+
+Constructing the consent URI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It may be useful to manually construct the "consent URI" for a given user - for
+instance, in order to send them an email asking them to consent. To do this,
+take the base `https://<server>/_matrix/consent` URL and add the following
+query parameters:
+
+ * `u`: the user id of the user. This can either be a full MXID
+   (`@user:server.com`) or just the localpart (`user`).
+
+ * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is
+   possible to calculate this on the commandline with something like:
+
+   ```bash
+   echo -n '<user>' | openssl sha256 -hmac '<form_secret>'
+   ```
+
+   This should result in a URI which looks something like:
+   `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`.
+
+
+Sending users a server notice asking them to agree to the policy
+----------------------------------------------------------------
+
+It is possible to configure Synapse to send a [server
+notice](server_notices.md) to anybody who has not yet agreed to the current
+version of the policy. To do so:
+
+ * ensure that the consent resource is configured, as in the previous section
+
+ * ensure that server notices are configured, as in [server_notices.md].
+
+ * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
+   example:
+
+   ```yaml
+   user_consent:
+     server_notice_content:
+       msgtype: m.text
+       body: >-
+         Please give your consent to the privacy policy at %(consent_uri)s.
+   ```
+
+   Synapse automatically replaces the placeholder `%(consent_uri)s` with the
+   consent uri for that user.
+
+Blocking users from using the server until they agree to the policy
+-------------------------------------------------------------------
+
+Synapse can be configured to block any attempts to join rooms or send messages
+until the user has given their agreement to the policy. (Joining the server
+notices room is exempted from this).
+
+To enable this, add `block_events_error` under `user_consent`. For example:
+
+```
+user_consent:
+  block_events_error: >-
+    You can't send any messages until you consent to the privacy policy at
+    %(consent_uri)s.
+```
+
+Synapse automatically replaces the placeholder `%(consent_uri)s` with the
+consent uri for that user.
diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md
deleted file mode 100644
index a3e6fc0986..0000000000
--- a/docs/privacy_policy_templates/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-If enabling the 'consent' resource in synapse, you will need some templates
-for the HTML to be served to the user. This directory contains very simple
-examples of the sort of thing that can be done.
-
-You'll need to add this sort of thing to your homeserver.yaml:
-
-```
-form_secret: <unique but arbitrary secret>
-
-user_consent:
-  template_dir: docs/privacy_policy_templates
-  version: 1.0
-```
-
-You should then be able to enable the `consent` resource under a `listener`
-entry. For example:
-
-```
-listeners:
-  - port: 8008
-    resources:
-      - names: [client, consent]
-```
diff --git a/docs/server_notices.md b/docs/server_notices.md
index a37aed6792..5d7857d15c 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -4,8 +4,8 @@ Server Notices
 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
 channel whereby server administrators can send messages to users on the server.
 
-They are used as part of the communication of Privacy Polices (see
-[privacy_policy.md]), however the intention is that they may also find a use
+They are used as part of the communication of the server polices (see
+[consent_tracking.md]), however the intention is that they may also find a use
 for features such as "Message of the day".
 
 This is a feature specific to Synapse, but it uses standard Matrix
-- 
cgit 1.4.1


From 2574ea3dc83c58d1cb596606ac760d91022f336f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 23 May 2018 12:34:34 +0100
Subject: server_notices.md: fix link

---
 docs/server_notices.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/server_notices.md b/docs/server_notices.md
index 5d7857d15c..22bca332d5 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -5,8 +5,8 @@ Server Notices
 channel whereby server administrators can send messages to users on the server.
 
 They are used as part of the communication of the server polices (see
-[consent_tracking.md]), however the intention is that they may also find a use
-for features such as "Message of the day".
+[consent_tracking.md](consent_tracking.md)), however the intention is that 
+they may also find a use for features such as "Message of the day".
 
 This is a feature specific to Synapse, but it uses standard Matrix
 communication mechanisms, so should work with any Matrix client.
-- 
cgit 1.4.1


From 563606b8f2216f6a8ba64c4b55bf389f67c99e9e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 23 May 2018 12:37:39 +0100
Subject: consent_tracking: formatting etc

---
 docs/consent_tracking.md | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index eb34749b55..61e41468d5 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -31,9 +31,9 @@ Note that the templates must be stored under a name giving the language of the
 template - currently this must always be `en` (for "English");
 internationalisation support is intended for the future.
 
-The template for the policy itself should be versioned - for example
-`1.0.html`. The version of the policy which the user has agreed to is stored in
-the database.
+The template for the policy itself should be versioned and named according to 
+the version: for example `1.0.html`. The version of the policy which the user
+has agreed to is stored in the database.
 
 Once the templates are in place, make the following changes to `homeserver.yaml`:
 
@@ -66,7 +66,7 @@ Once the templates are in place, make the following changes to `homeserver.yaml`
  3. Add `consent` wherever the `client` resource is currently enabled in the
     `listeners` configuration. For example:
 
-    ```
+    ```yaml
     listeners:
       - port: 8008
         resources:
@@ -85,8 +85,7 @@ Once this is complete, and the server has been restarted, try visiting
 this should give an error "Missing string query parameter 'u'". It is now
 possible to manually construct URIs where users can give their consent.
 
-Constructing the consent URI
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+### Constructing the consent URI
 
 It may be useful to manually construct the "consent URI" for a given user - for
 instance, in order to send them an email asking them to consent. To do this,
@@ -116,7 +115,7 @@ version of the policy. To do so:
 
  * ensure that the consent resource is configured, as in the previous section
 
- * ensure that server notices are configured, as in [server_notices.md].
+ * ensure that server notices are configured, as in [server_notices.md](server_notices.md).
 
  * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
    example:
@@ -141,7 +140,7 @@ notices room is exempted from this).
 
 To enable this, add `block_events_error` under `user_consent`. For example:
 
-```
+```yaml
 user_consent:
   block_events_error: >-
     You can't send any messages until you consent to the privacy policy at
-- 
cgit 1.4.1


From 5ad1149f384e1a826ad73a791896d63c910d874c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 13:45:17 +0100
Subject: Notes on the manhole

---
 docs/manhole.md | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 docs/manhole.md

diff --git a/docs/manhole.md b/docs/manhole.md
new file mode 100644
index 0000000000..7375f5ad46
--- /dev/null
+++ b/docs/manhole.md
@@ -0,0 +1,43 @@
+Using the synapse manhole
+=========================
+
+The "manhole" allows server administrators to access a Python shell on a running
+Synapse installation. This is a very powerful mechanism for administration and
+debugging.
+
+To enable it, first uncomment the `manhole` listener configuration in
+`homeserver.yaml`:
+
+```yaml
+listeners:
+  - port: 9000
+    bind_addresses: ['::1', '127.0.0.1']
+    type: manhole
+```
+
+(`bind_addresses` in the above is important: it ensures that access to the
+manhole is only possible for local users).
+
+Note that this will give administrative access to synapse to **all users** with
+shell access to the server. It should therefore **not** be enabled in
+environments where untrusted users have shell access.
+
+Then restart synapse, and point an ssh client at port 9000 on localhost, using
+the username `matrix`:
+
+```bash
+ssh -p9000 matrix@localhost
+```
+
+The password is `rabbithole`.
+
+This gives a Python REPL in which `hs` gives access to the
+`synapse.server.HomeServer` object - which in turn gives access to many other
+parts of the process.
+
+As a simple example, retrieving an event from the database:
+
+```
+>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org')
+<Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
+```
-- 
cgit 1.4.1


From 052d08a6a579f4f9a98a76a3cb39c05d8d5aaa90 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 13:55:39 +0100
Subject: Using the manhole to send server notices

---
 docs/server_notices.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/server_notices.md b/docs/server_notices.md
index 22bca332d5..a96a5b88a3 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -51,3 +51,16 @@ The only compulsory setting is `system_mxid_localpart`, which defines the user
 id of the server notices user, as above. `system_mxid_display_name` and
 `room_name` define the displayname of the system notices user, and of
 the notices room, respectively.
+
+Sending notices
+---------------
+
+As of the current version of synapse, there is no convenient interface for
+sending notices (other than the automated ones sent as part of consent
+tracking).
+
+In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like:
+
+```
+>>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'})
+```
-- 
cgit 1.4.1


From 1cbb8e5a33326441ec311ce212bb62fcd6704669 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 13:58:28 +0100
Subject: fix wrapping

---
 docs/server_notices.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/server_notices.md b/docs/server_notices.md
index a96a5b88a3..9896881389 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -59,7 +59,9 @@ As of the current version of synapse, there is no convenient interface for
 sending notices (other than the automated ones sent as part of consent
 tracking).
 
-In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like:
+In the meantime, it is possible to test this feature using the manhole. Having
+gone into the manhole as described in [manhole.md](manhole.md), a notice can be
+sent with something like:
 
 ```
 >>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'})
-- 
cgit 1.4.1


From 321f02d26347fb56e1eafbba60d7faef95ae9c3f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 14:30:47 +0100
Subject: Block attempts to send server notices to remote users

---
 synapse/server_notices/server_notices_manager.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index f535b9c9da..9f76c75546 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -35,6 +35,7 @@ class ServerNoticesManager(object):
         self._config = hs.config
         self._room_creation_handler = hs.get_room_creation_handler()
         self._event_creation_handler = hs.get_event_creation_handler()
+        self._is_mine_id = hs.is_mine_id
 
     def is_enabled(self):
         """Checks if server notices are enabled on this server.
@@ -89,6 +90,9 @@ class ServerNoticesManager(object):
         if not self.is_enabled():
             raise Exception("Server notices not enabled")
 
+        assert self._is_mine_id(user_id), \
+            "Cannot send server notices to remote users"
+
         rooms = yield self._store.get_rooms_for_user_where_membership_is(
             user_id, [Membership.INVITE, Membership.JOIN],
         )
-- 
cgit 1.4.1


From cd8ab9a0d8d3e0ee305ab56788f76459596075f9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 14:43:09 +0100
Subject: mention public_baseurl

---
 docs/consent_tracking.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index 61e41468d5..66beb9263b 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -131,6 +131,11 @@ version of the policy. To do so:
    Synapse automatically replaces the placeholder `%(consent_uri)s` with the
    consent uri for that user.
 
+ * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
+   URI that clients use to connect to the server. (It is used to construct
+   `consent_uri` in the server notice.)
+
+
 Blocking users from using the server until they agree to the policy
 -------------------------------------------------------------------
 
@@ -149,3 +154,7 @@ user_consent:
 
 Synapse automatically replaces the placeholder `%(consent_uri)s` with the
 consent uri for that user.
+
+ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base
+URI that clients use to connect to the server. (It is used to construct
+`consent_uri` in the error.)
-- 
cgit 1.4.1


From 82191b08f6a45c693551983a48b49a9fc30184c4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 15:24:31 +0100
Subject: Support for putting %(consent_uri)s in messages

Make it possible to put the URI in the error message and the server notice that
get sent by the server
---
 synapse/config/consent_config.py                 | 11 ++++---
 synapse/handlers/message.py                      |  5 ++-
 synapse/server_notices/consent_server_notices.py | 41 ++++++++++++++++++++++--
 3 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 44c4711e6c..ddcd305f4c 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -43,10 +43,13 @@ DEFAULT_CONFIG = """\
 #   version: 1.0
 #   server_notice_content:
 #     msgtype: m.text
-#     body: |
-#       Pls do consent kthx
-#   block_events_error: |
-#     You can't send any messages until you consent to the privacy policy.
+#     body: >-
+#       To continue using this homeserver you must review and agree to the
+#       terms and conditions at %(consent_uri)s
+#   block_events_error: >-
+#     To continue using this homeserver you must review and agree to the
+#     terms and conditions at %(consent_uri)s
+#
 """
 
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c3adbc6c95..0886607023 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -575,8 +575,11 @@ class EventCreationHandler(object):
             return
 
         consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id)
+        msg = self.config.block_events_without_consent_error % {
+            'consent_uri': consent_uri,
+        }
         raise ConsentNotGivenError(
-            msg=self.config.block_events_without_consent_error,
+            msg=msg,
             consent_uri=consent_uri,
         )
 
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index 440f6b1cd4..a709802856 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -14,10 +14,13 @@
 # limitations under the License.
 import logging
 
+from six import (iteritems, string_types)
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
+from synapse.api.urls import ConsentURIBuilder
 from synapse.config import ConfigError
+from synapse.types import get_localpart_from_id
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +55,8 @@ class ConsentServerNotices(object):
                     "key.",
                 )
 
+            self._consent_uri_builder = ConsentURIBuilder(hs.config)
+
     @defer.inlineCallbacks
     def maybe_send_server_notice_to_user(self, user_id):
         """Check if we need to send a notice to this user, and does so if so
@@ -81,10 +86,18 @@ class ConsentServerNotices(object):
                 # we've already sent a notice to the user
                 return
 
-            # need to send a message
+            # need to send a message.
             try:
+                consent_uri = self._consent_uri_builder.build_user_consent_uri(
+                    get_localpart_from_id(user_id),
+                )
+                content = copy_with_str_subst(
+                    self._server_notice_content, {
+                        'consent_uri': consent_uri,
+                    },
+                )
                 yield self._server_notices_manager.send_notice(
-                    user_id, self._server_notice_content,
+                    user_id, content,
                 )
                 yield self._store.user_set_consent_server_notice_sent(
                     user_id, self._current_consent_version,
@@ -93,3 +106,27 @@ class ConsentServerNotices(object):
                 logger.error("Error sending server notice about user consent: %s", e)
         finally:
             self._users_in_progress.remove(user_id)
+
+
+def copy_with_str_subst(x, substitutions):
+    """Deep-copy a structure, carrying out string substitions on any strings
+
+    Args:
+        x (object): structure to be copied
+        substitutions (object): substitutions to be made - passed into the
+            string '%' operator
+
+    Returns:
+        copy of x
+    """
+    if isinstance(x, string_types):
+        return x % substitutions
+    if isinstance(x, dict):
+        return {
+            k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x)
+        }
+    if isinstance(x, (list, tuple)):
+        return [copy_with_str_subst(y) for y in x]
+
+    # assume it's uninterested and can be shallow-copied.
+    return x
-- 
cgit 1.4.1


From 3ff6f50eac86b063706f68387d3cf3e96a79a541 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 15:28:23 +0100
Subject: Use the localpart in the consent uri

... because it's shorter.
---
 synapse/handlers/message.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c3adbc6c95..7936a60af4 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -574,7 +574,9 @@ class EventCreationHandler(object):
         if u["consent_version"] == self.config.user_consent_version:
             return
 
-        consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id)
+        consent_uri = self._consent_uri_builder.build_user_consent_uri(
+            requester.user.localpart,
+        )
         raise ConsentNotGivenError(
             msg=self.config.block_events_without_consent_error,
             consent_uri=consent_uri,
-- 
cgit 1.4.1


From dda40fb55d0aa7bde6387e5e32ffed872fddea4e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 15:29:58 +0100
Subject: fix typo

---
 synapse/server_notices/server_notices_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 9f76c75546..7e4055c84a 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -56,7 +56,7 @@ class ServerNoticesManager(object):
             event_content (dict): content of event to send
 
         Returns:
-            Deferrred[None]
+            Deferred[None]
         """
         room_id = yield self.get_notice_room_for_user(user_id)
 
-- 
cgit 1.4.1


From 2df8c3139ab7be10af4aaa750bbdc4ad39209583 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 15:39:52 +0100
Subject: minor post-review tweaks

---
 docs/consent_tracking.md | 6 +++---
 docs/server_notices.md   | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index 66beb9263b..9dc0093888 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -81,9 +81,9 @@ should be a matter of `pip install Jinja2`. On debian, try `apt-get install
 python-jinja2`.
 
 Once this is complete, and the server has been restarted, try visiting
-`https://<server>/_matrix/consent`. If configuration has been done correctly,
-this should give an error "Missing string query parameter 'u'". It is now
-possible to manually construct URIs where users can give their consent.
+`https://<server>/_matrix/consent`. If correctly configured, this should give
+an error "Missing string query parameter 'u'". It is now possible to manually
+construct URIs where users can give their consent.
 
 ### Constructing the consent URI
 
diff --git a/docs/server_notices.md b/docs/server_notices.md
index 9896881389..8e18e3d95d 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -4,7 +4,7 @@ Server Notices
 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
 channel whereby server administrators can send messages to users on the server.
 
-They are used as part of the communication of the server polices (see
+They are used as part of communication of the server polices(see
 [consent_tracking.md](consent_tracking.md)), however the intention is that 
 they may also find a use for features such as "Message of the day".
 
@@ -32,7 +32,7 @@ Synapse configuration
 Server notices come from a specific user id on the server. Server
 administrators are free to choose the user id - something like `server` is
 suggested, meaning the notices will come from
-`@server:<your_server_name>`. Once the server notices user is configured, that
+`@server:<your_server_name>`. Once the Server Notices user is configured, that
 user id becomes a special, privileged user, so administrators should ensure
 that **it is not already allocated**.
 
-- 
cgit 1.4.1


From e206b2c9ace8a202bf47ec419581e19f0baa39fb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 23 May 2018 15:57:10 +0100
Subject: consent_tracking.md: clarify link

---
 docs/consent_tracking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index 9dc0093888..064eae82f7 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -24,7 +24,7 @@ To enable this, first create templates for the policy and success pages.
 These should be stored on the local filesystem.
 
 These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
-and the `privacy_policy_templates` subdirectory of this `docs` directory gives
+and [docs/privacy_policy_templates](privacy_policy_templates) gives
 examples of the sort of thing that can be done.
 
 Note that the templates must be stored under a name giving the language of the
-- 
cgit 1.4.1


From 9bf4b2bda343dcbdba7b0e9d752bc560f8e344fd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 23 May 2018 17:43:30 +0100
Subject: Allow overriding the server_notices user's avatar

probably should have done this in the first place, like @turt2live suggested.
---
 docs/server_notices.md                           |  9 ++++++---
 synapse/config/server_notices_config.py          | 15 ++++++++++++---
 synapse/server_notices/server_notices_manager.py | 17 ++++++++++++++---
 3 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/docs/server_notices.md b/docs/server_notices.md
index 8e18e3d95d..221553b24d 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -44,13 +44,16 @@ section, which should look like this:
 server_notices:
    system_mxid_localpart: server
    system_mxid_display_name: "Server Notices"
+   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
    room_name: "Server Notices"
 ```
 
 The only compulsory setting is `system_mxid_localpart`, which defines the user
-id of the server notices user, as above. `system_mxid_display_name` and
-`room_name` define the displayname of the system notices user, and of
-the notices room, respectively.
+id of the Server Notices user, as above. `room_name` defines the name of the
+room which will be created.
+
+`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
+displayname and avatar of the Server Notices user.
 
 Sending notices
 ---------------
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index ccef8d2ec5..be1d1f762c 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\
 # setting, which defines the id of the user which will be used to send the
 # notices.
 #
-# It's also possible to override the room name, or the display name of the
-# "notices" user.
+# It's also possible to override the room name, the display name of the
+# "notices" user, and the avatar for the user.
 #
 # server_notices:
 #   system_mxid_localpart: notices
 #   system_mxid_display_name: "Server Notices"
+#   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
 #   room_name: "Server Notices"
 """
 
@@ -48,6 +49,10 @@ class ServerNoticesConfig(Config):
             The display name to use for the server notices user.
             None if server notices are not enabled.
 
+        server_notices_mxid_avatar_url (str|None):
+            The display name to use for the server notices user.
+            None if server notices are not enabled.
+
         server_notices_room_name (str|None):
             The name to use for the server notices room.
             None if server notices are not enabled.
@@ -56,6 +61,7 @@ class ServerNoticesConfig(Config):
         super(ServerNoticesConfig, self).__init__()
         self.server_notices_mxid = None
         self.server_notices_mxid_display_name = None
+        self.server_notices_mxid_avatar_url = None
         self.server_notices_room_name = None
 
     def read_config(self, config):
@@ -68,7 +74,10 @@ class ServerNoticesConfig(Config):
             mxid_localpart, self.server_name,
         ).to_string()
         self.server_notices_mxid_display_name = c.get(
-            'system_mxid_display_name', 'Server Notices',
+            'system_mxid_display_name', None,
+        )
+        self.server_notices_mxid_avatar_url = c.get(
+            'system_mxid_avatar_url', None,
         )
         # todo: i18n
         self.server_notices_room_name = c.get('room_name', "Server Notices")
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 7e4055c84a..a26deace53 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -113,6 +113,19 @@ class ServerNoticesManager(object):
         # apparently no existing notice room: create a new one
         logger.info("Creating server notices room for %s", user_id)
 
+        # see if we want to override the profile info for the server user.
+        # note that if we want to override either the display name or the
+        # avatar, we have to use both.
+        join_profile = None
+        if (
+            self._config.server_notices_mxid_display_name is not None or
+            self._config.server_notices_mxid_avatar_url is not None
+        ):
+            join_profile = {
+                "displayname": self._config.server_notices_mxid_display_name,
+                "avatar_url": self._config.server_notices_mxid_avatar_url,
+            }
+
         requester = create_requester(system_mxid)
         info = yield self._room_creation_handler.create_room(
             requester,
@@ -125,9 +138,7 @@ class ServerNoticesManager(object):
                 "invite": (user_id,)
             },
             ratelimit=False,
-            creator_join_profile={
-                "displayname": self._config.server_notices_mxid_display_name,
-            },
+            creator_join_profile=join_profile,
         )
         room_id = info['room_id']
 
-- 
cgit 1.4.1


From e987079037eb89a3c5c04fe64d843a6f9e6c1718 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 23 May 2018 13:03:51 -0500
Subject: fixes

---
 synapse/federation/transaction_queue.py  |  8 ++++----
 synapse/http/request_metrics.py          | 14 +++++++-------
 synapse/push/bulk_push_rule_evaluator.py |  4 ++--
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 53442688c8..69312ec233 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -41,7 +41,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 sent_pdus_destination_dist = Counter(
-    "synapse_federation_client_sent_pdu_destinations", ""
+    "synapse_federation_transaction_queue_sent_pdu_destinations", ""
 )
 
 
@@ -70,7 +70,7 @@ class TransactionQueue(object):
         self.pending_transactions = {}
 
         LaterGauge(
-            "synapse_federation_client_pending_destinations",
+            "synapse_federation_transaction_queue_pending_destinations",
             "",
             [],
             lambda: len(self.pending_transactions),
@@ -97,13 +97,13 @@ class TransactionQueue(object):
         self.pending_edus_keyed_by_dest = edus_keyed = {}
 
         LaterGauge(
-            "synapse_federation_client_pending_pdus",
+            "synapse_federation_transaction_queue_pending_pdus",
             "",
             [],
             lambda: sum(map(len, pdus.values())),
         )
         LaterGauge(
-            "synapse_federation_client_pending_edus",
+            "synapse_federation_transaction_queue_pending_edus",
             "",
             [],
             lambda: (
diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 7f11b5c5a4..e7df494333 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -63,46 +63,46 @@ response_db_txn_duration = Counter(
 
 # seconds spent waiting for a db connection, when processing this request
 response_db_sched_duration = Counter(
-    "synapse_http_request_response_db_sched_duration_seconds",
+    "synapse_http_server_response_db_sched_duration_seconds",
     "",
     ["method", "servlet", "tag"],
 )
 
 # size in bytes of the response written
 response_size = Counter(
-    "synapse_http_request_response_size", "", ["method", "servlet", "tag"]
+    "synapse_http_server_response_size", "", ["method", "servlet", "tag"]
 )
 
 # In flight metrics are incremented while the requests are in flight, rather
 # than when the response was written.
 
 in_flight_requests_ru_utime = Counter(
-    "synapse_http_request_in_flight_requests_ru_utime_seconds",
+    "synapse_http_server_in_flight_requests_ru_utime_seconds",
     "",
     ["method", "servlet"],
 )
 
 in_flight_requests_ru_stime = Counter(
-    "synapse_http_request_in_flight_requests_ru_stime_seconds",
+    "synapse_http_server_in_flight_requests_ru_stime_seconds",
     "",
     ["method", "servlet"],
 )
 
 in_flight_requests_db_txn_count = Counter(
-    "synapse_http_request_in_flight_requests_db_txn_count", "", ["method", "servlet"]
+    "synapse_http_server_in_flight_requests_db_txn_count", "", ["method", "servlet"]
 )
 
 # seconds spent waiting for db txns, excluding scheduling time, when processing
 # this request
 in_flight_requests_db_txn_duration = Counter(
-    "synapse_http_request_in_flight_requests_db_txn_duration_seconds",
+    "synapse_http_server_in_flight_requests_db_txn_duration_seconds",
     "",
     ["method", "servlet"],
 )
 
 # seconds spent waiting for a db connection, when processing this request
 in_flight_requests_db_sched_duration = Counter(
-    "synapse_http_request_in_flight_requests_db_sched_duration_seconds",
+    "synapse_http_server_in_flight_requests_db_sched_duration_seconds",
     "",
     ["method", "servlet"],
 )
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index b0053e7f3f..da1cde417f 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -37,9 +37,9 @@ rules_by_room = {}
 
 
 push_rules_invalidation_counter = Counter(
-    "synapse_push_bulk_push_role_evaluator_push_rules_invalidation_counter", "")
+    "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", "")
 push_rules_state_size_counter = Counter(
-    "synapse_push_bulk_push_role_evaluator_push_rules_state_size_counter", "")
+    "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", "")
 
 # Measures whether we use the fast path of using state deltas, or if we have to
 # recalculate from scratch
-- 
cgit 1.4.1


From 472a5ec4e208967aaf1368ada321aa54c3cae493 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 23 May 2018 13:03:56 -0500
Subject: add back CPU metrics

---
 synapse/metrics/__init__.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 75117915ff..64ad762ff1 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -17,6 +17,7 @@ import logging
 import functools
 import time
 import gc
+import os
 import platform
 import attr
 
@@ -81,6 +82,38 @@ class LaterGauge(object):
         all_gauges[self.name] = self
 
 
+#
+# Detailed CPU metrics
+#
+
+class CPUMetrics(object):
+
+    def __init__(self):
+        ticks_per_sec = 100
+        try:
+            # Try and get the system config
+            ticks_per_sec = os.sysconf('SC_CLK_TCK')
+        except (ValueError, TypeError, AttributeError):
+            pass
+
+        self.ticks_per_sec = ticks_per_sec
+
+    def collect(self):
+
+        with open("/proc/self/stat") as s:
+            line = s.read()
+            raw_stats = line.split(") ", 1)[1].split(" ")
+
+            user = GaugeMetricFamily("process_cpu_user_seconds_total", "")
+            user.add_metric([], float(raw_stats[11]) / self.ticks_per_sec)
+            yield user
+
+            sys = GaugeMetricFamily("process_cpu_system_seconds_total", "")
+            sys.add_metric([], float(raw_stats[12]) / self.ticks_per_sec)
+            yield sys
+
+REGISTRY.register(CPUMetrics())
+
 #
 # Python GC metrics
 #
@@ -90,7 +123,8 @@ gc_time = Histogram(
     "python_gc_time",
     "Time taken to GC (ms)",
     ["gen"],
-    buckets=[1, 2, 5, 10, 25, 50, 100, 250, 500, 1000],
+    buckets=[2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 7500, 15000,
+             30000, 45000, 60000],
 )
 
 
-- 
cgit 1.4.1


From 389dac2c153f440d7ad6e9aee137ef40970b1394 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 23 May 2018 13:08:59 -0500
Subject: pepeightttt

---
 synapse/metrics/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 64ad762ff1..e33ed6c9be 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -112,6 +112,7 @@ class CPUMetrics(object):
             sys.add_metric([], float(raw_stats[12]) / self.ticks_per_sec)
             yield sys
 
+
 REGISTRY.register(CPUMetrics())
 
 #
-- 
cgit 1.4.1


From 91d95a1d8e4005ebd4805418d331a6ea8f73fdb6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 24 May 2018 14:05:07 +0100
Subject: bump version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 353387f154..5bada5e290 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.30.0-rc1"
+__version__ = "0.30.0"
-- 
cgit 1.4.1


From bdd2ed5acf7ea34092c9aa7049232ea5e8f29874 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 24 May 2018 15:02:03 +0100
Subject: update for v0.30.0

---
 CHANGES.rst | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 096fa4794a..ffadd83e4e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,11 +1,30 @@
-Changes in synapse v0.30.0-rc1 (2018-05-23)
+Changes in synapse v0.30.0 (2018-05-24)
 ==========================================
 
-This version includes support for Privacy Notice agreement collection. More
-details live at docs/privacy_policy_templates/README.md
+'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
+channel whereby server administrators can send messages to users on the server.
+
+They are used as part of communication of the server policies (see `docs/consent_tracking.md`),
+however the intention is that they may also find a use for features such
+as "Message of the day".
+
+This feature is specific to Synapse, but uses standard Matrix communication mechanisms,
+so should work with any Matrix client. For more details see ``docs/server_notices.md``
+
+Further Server Notices/Consent Tracking Support:
+
+* Allow overriding the server_notices user's avatar (#3273)
+* Use the localpart in the consent uri (#3272)
+* Support for putting %(consent_uri)s in messages (#3271)
+* Block attempts to send server notices to remote users (#3270)
+* Docs on consent bits (#3268)
 
 
-GDPR Support:
+
+Changes in synapse v0.30.0-rc1 (2018-05-23)
+==========================================
+
+Server Notices/Consent Tracking Support:
 
 * ConsentResource to gather policy consent from users (PR #3213)
 * Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225)
-- 
cgit 1.4.1


From 6f67163c6358554bfd9568e98e0ac2225436b1c1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 24 May 2018 15:04:41 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index ffadd83e4e..6e7f108f8e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -4,7 +4,7 @@ Changes in synapse v0.30.0 (2018-05-24)
 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
 channel whereby server administrators can send messages to users on the server.
 
-They are used as part of communication of the server policies (see `docs/consent_tracking.md`),
+They are used as part of communication of the server policies (see ``docs/consent_tracking.md``),
 however the intention is that they may also find a use for features such
 as "Message of the day".
 
-- 
cgit 1.4.1


From 14ca67867422d333a933b3bb0a4fd86da1656a98 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 24 May 2018 15:54:02 +0100
Subject: Update CHANGES.rst

---
 CHANGES.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 6e7f108f8e..0569b581db 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -13,11 +13,11 @@ so should work with any Matrix client. For more details see ``docs/server_notice
 
 Further Server Notices/Consent Tracking Support:
 
-* Allow overriding the server_notices user's avatar (#3273)
-* Use the localpart in the consent uri (#3272)
-* Support for putting %(consent_uri)s in messages (#3271)
-* Block attempts to send server notices to remote users (#3270)
-* Docs on consent bits (#3268)
+* Allow overriding the server_notices user's avatar (PR #3273)
+* Use the localpart in the consent uri (PR #3272)
+* Support for putting %(consent_uri)s in messages (PR #3271)
+* Block attempts to send server notices to remote users (PR #3270)
+* Docs on consent bits (PR #3268)
 
 
 
-- 
cgit 1.4.1


From 7a1af504d70e168e87f1e15ecda90179dd75ebc8 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 24 May 2018 15:59:58 +0100
Subject: Remove users from user directory on deactivate

---
 synapse/handlers/deactivate_account.py | 4 ++++
 synapse/handlers/user_directory.py     | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index d58ea6c650..c5e92f6214 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -30,6 +30,7 @@ class DeactivateAccountHandler(BaseHandler):
         self._auth_handler = hs.get_auth_handler()
         self._device_handler = hs.get_device_handler()
         self._room_member_handler = hs.get_room_member_handler()
+        self.user_directory_handler = hs.get_user_directory_handler()
 
         # Flag that indicates whether the process to part users from rooms is running
         self._user_parter_running = False
@@ -65,6 +66,9 @@ class DeactivateAccountHandler(BaseHandler):
         # removal from all the rooms they're a member of)
         yield self.store.add_user_pending_deactivation(user_id)
 
+        # delete from user directory
+        yield self.user_directory_handler.handle_user_deactivated(user_id)
+
         # Now start the process that goes through that list and
         # parts users from rooms (if it isn't already running)
         self._start_user_parting()
diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py
index 714f0195c8..2f73d664f9 100644
--- a/synapse/handlers/user_directory.py
+++ b/synapse/handlers/user_directory.py
@@ -122,6 +122,13 @@ class UserDirectoryHandler(object):
             user_id, profile.display_name, profile.avatar_url, None,
         )
 
+    @defer.inlineCallbacks
+    def handle_user_deactivated(self, user_id):
+        """Called when a user ID is deactivated
+        """
+        yield self.store.remove_from_user_dir(user_id)
+        yield self.store.remove_from_user_in_public_room(user_id)
+
     @defer.inlineCallbacks
     def _unsafe_process(self):
         # If self.pos is None then means we haven't fetched it from DB
-- 
cgit 1.4.1


From 095292304f04c0984813a261a7f6042b35cbd2dd Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 15 May 2018 17:52:41 +0200
Subject: Py3 storage/_base.py

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/storage/_base.py | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2262776ab2..51cf7b4c6c 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -27,6 +27,8 @@ import sys
 import time
 import threading
 
+from six import itervalues, iterkeys, iteritems
+from six.moves import intern, range
 
 logger = logging.getLogger(__name__)
 
@@ -137,7 +139,7 @@ class PerformanceCounters(object):
 
     def interval(self, interval_duration, limit=3):
         counters = []
-        for name, (count, cum_time) in self.current_counters.iteritems():
+        for name, (count, cum_time) in iteritems(self.current_counters):
             prev_count, prev_time = self.previous_counters.get(name, (0, 0))
             counters.append((
                 (cum_time - prev_time) / interval_duration,
@@ -543,7 +545,7 @@ class SQLBaseStore(object):
             ", ".join("%s = ?" % (k,) for k in values),
             " AND ".join("%s = ?" % (k,) for k in keyvalues)
         )
-        sqlargs = values.values() + keyvalues.values()
+        sqlargs = list(values.values()) + list(keyvalues.values())
 
         txn.execute(sql, sqlargs)
         if txn.rowcount > 0:
@@ -561,7 +563,7 @@ class SQLBaseStore(object):
             ", ".join(k for k in allvalues),
             ", ".join("?" for _ in allvalues)
         )
-        txn.execute(sql, allvalues.values())
+        txn.execute(sql, list(allvalues.values()))
         # successfully inserted
         return True
 
@@ -629,8 +631,8 @@ class SQLBaseStore(object):
         }
 
         if keyvalues:
-            sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys())
-            txn.execute(sql, keyvalues.values())
+            sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues))
+            txn.execute(sql, list(keyvalues.values()))
         else:
             txn.execute(sql)
 
@@ -694,7 +696,7 @@ class SQLBaseStore(object):
                 table,
                 " AND ".join("%s = ?" % (k, ) for k in keyvalues)
             )
-            txn.execute(sql, keyvalues.values())
+            txn.execute(sql, list(keyvalues.values()))
         else:
             sql = "SELECT %s FROM %s" % (
                 ", ".join(retcols),
@@ -725,9 +727,12 @@ class SQLBaseStore(object):
         if not iterable:
             defer.returnValue(results)
 
+        # iterables can not be sliced, so convert it to a list first
+        it_list = list(iterable)
+
         chunks = [
-            iterable[i:i + batch_size]
-            for i in xrange(0, len(iterable), batch_size)
+            it_list[i:i + batch_size]
+            for i in range(0, len(it_list), batch_size)
         ]
         for chunk in chunks:
             rows = yield self.runInteraction(
@@ -767,7 +772,7 @@ class SQLBaseStore(object):
         )
         values.extend(iterable)
 
-        for key, value in keyvalues.iteritems():
+        for key, value in iteritems(keyvalues):
             clauses.append("%s = ?" % (key,))
             values.append(value)
 
@@ -790,7 +795,7 @@ class SQLBaseStore(object):
     @staticmethod
     def _simple_update_txn(txn, table, keyvalues, updatevalues):
         if keyvalues:
-            where = "WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys())
+            where = "WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues))
         else:
             where = ""
 
@@ -802,7 +807,7 @@ class SQLBaseStore(object):
 
         txn.execute(
             update_sql,
-            updatevalues.values() + keyvalues.values()
+            list(updatevalues.values()) + list(keyvalues.values())
         )
 
         return txn.rowcount
@@ -850,7 +855,7 @@ class SQLBaseStore(object):
             " AND ".join("%s = ?" % (k,) for k in keyvalues)
         )
 
-        txn.execute(select_sql, keyvalues.values())
+        txn.execute(select_sql, list(keyvalues.values()))
 
         row = txn.fetchone()
         if not row:
@@ -888,7 +893,7 @@ class SQLBaseStore(object):
             " AND ".join("%s = ?" % (k, ) for k in keyvalues)
         )
 
-        txn.execute(sql, keyvalues.values())
+        txn.execute(sql, list(keyvalues.values()))
         if txn.rowcount == 0:
             raise StoreError(404, "No row found")
         if txn.rowcount > 1:
@@ -906,7 +911,7 @@ class SQLBaseStore(object):
             " AND ".join("%s = ?" % (k, ) for k in keyvalues)
         )
 
-        return txn.execute(sql, keyvalues.values())
+        return txn.execute(sql, list(keyvalues.values()))
 
     def _simple_delete_many(self, table, column, iterable, keyvalues, desc):
         return self.runInteraction(
@@ -938,7 +943,7 @@ class SQLBaseStore(object):
         )
         values.extend(iterable)
 
-        for key, value in keyvalues.iteritems():
+        for key, value in iteritems(keyvalues):
             clauses.append("%s = ?" % (key,))
             values.append(value)
 
@@ -978,7 +983,7 @@ class SQLBaseStore(object):
         txn.close()
 
         if cache:
-            min_val = min(cache.itervalues())
+            min_val = min(itervalues(cache))
         else:
             min_val = max_value
 
@@ -1093,7 +1098,7 @@ class SQLBaseStore(object):
                 " AND ".join("%s = ?" % (k,) for k in keyvalues),
                 " ? ASC LIMIT ? OFFSET ?"
             )
-            txn.execute(sql, keyvalues.values() + pagevalues)
+            txn.execute(sql, list(keyvalues.values()) + list(pagevalues))
         else:
             sql = "SELECT %s FROM %s ORDER BY %s" % (
                 ", ".join(retcols),
-- 
cgit 1.4.1


From 6c16a4ec1b6bc1aa3ef3bbcc080419e1b4bce5bd Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Tue, 15 May 2018 17:45:01 +0200
Subject: more iteritems

---
 synapse/storage/roommember.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 6a861943a2..7bfc3d91b5 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -30,6 +30,8 @@ from synapse.types import get_domain_from_id
 import logging
 import simplejson as json
 
+from six import itervalues, iteritems
+
 logger = logging.getLogger(__name__)
 
 
@@ -272,7 +274,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
         users_in_room = {}
         member_event_ids = [
             e_id
-            for key, e_id in current_state_ids.iteritems()
+            for key, e_id in iteritems(current_state_ids)
             if key[0] == EventTypes.Member
         ]
 
@@ -289,7 +291,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
                     users_in_room = dict(prev_res)
                     member_event_ids = [
                         e_id
-                        for key, e_id in context.delta_ids.iteritems()
+                        for key, e_id in iteritems(context.delta_ids)
                         if key[0] == EventTypes.Member
                     ]
                     for etype, state_key in context.delta_ids:
@@ -741,7 +743,7 @@ class _JoinedHostsCache(object):
             if state_entry.state_group == self.state_group:
                 pass
             elif state_entry.prev_group == self.state_group:
-                for (typ, state_key), event_id in state_entry.delta_ids.iteritems():
+                for (typ, state_key), event_id in iteritems(state_entry.delta_ids):
                     if typ != EventTypes.Member:
                         continue
 
@@ -771,7 +773,7 @@ class _JoinedHostsCache(object):
                 self.state_group = state_entry.state_group
             else:
                 self.state_group = object()
-            self._len = sum(len(v) for v in self.hosts_to_joined_users.itervalues())
+            self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users))
         defer.returnValue(frozenset(self.hosts_to_joined_users))
 
     def __len__(self):
-- 
cgit 1.4.1


From 17a70cf6e9ae016011972b4d8a97aca0c3d945d4 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Sat, 5 May 2018 22:47:18 +0200
Subject: Misc. py3 fixes

Signed-off-by: Adrian Tschira <nota@notafile.com>
---
 synapse/handlers/federation.py   | 9 +++++----
 synapse/handlers/message.py      | 9 +++++----
 synapse/storage/events_worker.py | 2 +-
 synapse/storage/filtering.py     | 2 +-
 synapse/storage/keys.py          | 2 +-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2c72beff2e..87c0615820 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -24,6 +24,7 @@ from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 import six
 from six.moves import http_client
+from six import iteritems
 from twisted.internet import defer
 from unpaddedbase64 import decode_base64
 
@@ -1388,7 +1389,7 @@ class FederationHandler(BaseHandler):
         )
 
         if state_groups:
-            _, state = state_groups.items().pop()
+            _, state = list(iteritems(state_groups)).pop()
             results = {
                 (e.type, e.state_key): e for e in state
             }
@@ -2034,7 +2035,7 @@ class FederationHandler(BaseHandler):
                 this will not be included in the current_state in the context.
         """
         state_updates = {
-            k: a.event_id for k, a in auth_events.iteritems()
+            k: a.event_id for k, a in iteritems(auth_events)
             if k != event_key
         }
         context.current_state_ids = dict(context.current_state_ids)
@@ -2044,7 +2045,7 @@ class FederationHandler(BaseHandler):
             context.delta_ids.update(state_updates)
         context.prev_state_ids = dict(context.prev_state_ids)
         context.prev_state_ids.update({
-            k: a.event_id for k, a in auth_events.iteritems()
+            k: a.event_id for k, a in iteritems(auth_events)
         })
         context.state_group = yield self.store.store_state_group(
             event.event_id,
@@ -2096,7 +2097,7 @@ class FederationHandler(BaseHandler):
 
         def get_next(it, opt=None):
             try:
-                return it.next()
+                return next(it)
             except Exception:
                 return opt
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c32b9bcae4..81cff0870e 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -19,6 +19,7 @@ import sys
 
 from canonicaljson import encode_canonical_json
 import six
+from six import string_types, itervalues, iteritems
 from twisted.internet import defer, reactor
 from twisted.internet.defer import succeed
 from twisted.python.failure import Failure
@@ -402,7 +403,7 @@ class MessageHandler(BaseHandler):
                 "avatar_url": profile.avatar_url,
                 "display_name": profile.display_name,
             }
-            for user_id, profile in users_with_profile.iteritems()
+            for user_id, profile in iteritems(users_with_profile)
         })
 
 
@@ -667,7 +668,7 @@ class EventCreationHandler(object):
 
             spam_error = self.spam_checker.check_event_for_spam(event)
             if spam_error:
-                if not isinstance(spam_error, basestring):
+                if not isinstance(spam_error, string_types):
                     spam_error = "Spam is not permitted here"
                 raise SynapseError(
                     403, spam_error, Codes.FORBIDDEN
@@ -881,7 +882,7 @@ class EventCreationHandler(object):
 
                 state_to_include_ids = [
                     e_id
-                    for k, e_id in context.current_state_ids.iteritems()
+                    for k, e_id in iteritems(context.current_state_ids)
                     if k[0] in self.hs.config.room_invite_state_types
                     or k == (EventTypes.Member, event.sender)
                 ]
@@ -895,7 +896,7 @@ class EventCreationHandler(object):
                         "content": e.content,
                         "sender": e.sender,
                     }
-                    for e in state_to_include.itervalues()
+                    for e in itervalues(state_to_include)
                 ]
 
                 invitee = UserID.from_string(event.state_key)
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index ba834854e1..32d9d00ffb 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -337,7 +337,7 @@ class EventsWorkerStore(SQLBaseStore):
     def _fetch_event_rows(self, txn, events):
         rows = []
         N = 200
-        for i in range(1 + len(events) / N):
+        for i in range(1 + len(events) // N):
             evs = events[i * N:(i + 1) * N]
             if not evs:
                 break
diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py
index 78b1e30945..2e2763126d 100644
--- a/synapse/storage/filtering.py
+++ b/synapse/storage/filtering.py
@@ -44,7 +44,7 @@ class FilteringStore(SQLBaseStore):
             desc="get_user_filter",
         )
 
-        defer.returnValue(json.loads(str(def_json).decode("utf-8")))
+        defer.returnValue(json.loads(bytes(def_json).decode("utf-8")))
 
     def add_user_filter(self, user_localpart, user_filter):
         def_json = encode_canonical_json(user_filter)
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 87aeaf71d6..0540c2b0b1 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -92,7 +92,7 @@ class KeyStore(SQLBaseStore):
 
         if verify_key_bytes:
             defer.returnValue(decode_verify_key_bytes(
-                key_id, str(verify_key_bytes)
+                key_id, bytes(verify_key_bytes)
             ))
 
     @defer.inlineCallbacks
-- 
cgit 1.4.1


From 0e617056615d5db9f95ba458b2fafbbe0103c9d8 Mon Sep 17 00:00:00 2001
From: Adrian Tschira <nota@notafile.com>
Date: Thu, 24 May 2018 20:58:21 +0200
Subject: py3-ize state.py

---
 synapse/state.py | 48 +++++++++++++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index 26093c8434..b8c27c6815 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -32,6 +32,8 @@ from frozendict import frozendict
 import logging
 import hashlib
 
+from six import iteritems, itervalues
+
 logger = logging.getLogger(__name__)
 
 
@@ -132,7 +134,7 @@ class StateHandler(object):
 
         state_map = yield self.store.get_events(state.values(), get_prev_content=False)
         state = {
-            key: state_map[e_id] for key, e_id in state.iteritems() if e_id in state_map
+            key: state_map[e_id] for key, e_id in iteritems(state) if e_id in state_map
         }
 
         defer.returnValue(state)
@@ -338,7 +340,7 @@ class StateHandler(object):
         )
 
         if len(state_groups_ids) == 1:
-            name, state_list = state_groups_ids.items().pop()
+            name, state_list = list(state_groups_ids.items()).pop()
 
             prev_group, delta_ids = yield self.store.get_state_group_delta(name)
 
@@ -378,7 +380,7 @@ class StateHandler(object):
             new_state = resolve_events_with_state_map(state_set_ids, state_map)
 
         new_state = {
-            key: state_map[ev_id] for key, ev_id in new_state.iteritems()
+            key: state_map[ev_id] for key, ev_id in iteritems(new_state)
         }
 
         return new_state
@@ -458,15 +460,15 @@ class StateResolutionHandler(object):
             # build a map from state key to the event_ids which set that state.
             # dict[(str, str), set[str])
             state = {}
-            for st in state_groups_ids.itervalues():
-                for key, e_id in st.iteritems():
+            for st in itervalues(state_groups_ids):
+                for key, e_id in iteritems(st):
                     state.setdefault(key, set()).add(e_id)
 
             # build a map from state key to the event_ids which set that state,
             # including only those where there are state keys in conflict.
             conflicted_state = {
                 k: list(v)
-                for k, v in state.iteritems()
+                for k, v in iteritems(state)
                 if len(v) > 1
             }
 
@@ -474,13 +476,13 @@ class StateResolutionHandler(object):
                 logger.info("Resolving conflicted state for %r", room_id)
                 with Measure(self.clock, "state._resolve_events"):
                     new_state = yield resolve_events_with_factory(
-                        state_groups_ids.values(),
+                        list(state_groups_ids.values()),
                         event_map=event_map,
                         state_map_factory=state_map_factory,
                     )
             else:
                 new_state = {
-                    key: e_ids.pop() for key, e_ids in state.iteritems()
+                    key: e_ids.pop() for key, e_ids in iteritems(state)
                 }
 
             with Measure(self.clock, "state.create_group_ids"):
@@ -489,8 +491,8 @@ class StateResolutionHandler(object):
                 # which will be used as a cache key for future resolutions, but
                 # not get persisted.
                 state_group = None
-                new_state_event_ids = frozenset(new_state.itervalues())
-                for sg, events in state_groups_ids.iteritems():
+                new_state_event_ids = frozenset(itervalues(new_state))
+                for sg, events in iteritems(state_groups_ids):
                     if new_state_event_ids == frozenset(e_id for e_id in events):
                         state_group = sg
                         break
@@ -501,11 +503,11 @@ class StateResolutionHandler(object):
 
                 prev_group = None
                 delta_ids = None
-                for old_group, old_ids in state_groups_ids.iteritems():
+                for old_group, old_ids in iteritems(state_groups_ids):
                     if not set(new_state) - set(old_ids):
                         n_delta_ids = {
                             k: v
-                            for k, v in new_state.iteritems()
+                            for k, v in iteritems(new_state)
                             if old_ids.get(k) != v
                         }
                         if not delta_ids or len(n_delta_ids) < len(delta_ids):
@@ -527,7 +529,7 @@ class StateResolutionHandler(object):
 
 def _ordered_events(events):
     def key_func(e):
-        return -int(e.depth), hashlib.sha1(e.event_id).hexdigest()
+        return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest()
 
     return sorted(events, key=key_func)
 
@@ -584,7 +586,7 @@ def _seperate(state_sets):
     conflicted_state = {}
 
     for state_set in state_sets[1:]:
-        for key, value in state_set.iteritems():
+        for key, value in iteritems(state_set):
             # Check if there is an unconflicted entry for the state key.
             unconflicted_value = unconflicted_state.get(key)
             if unconflicted_value is None:
@@ -640,7 +642,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
 
     needed_events = set(
         event_id
-        for event_ids in conflicted_state.itervalues()
+        for event_ids in itervalues(conflicted_state)
         for event_id in event_ids
     )
     if event_map is not None:
@@ -662,7 +664,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
         unconflicted_state, conflicted_state, state_map
     )
 
-    new_needed_events = set(auth_events.itervalues())
+    new_needed_events = set(itervalues(auth_events))
     new_needed_events -= needed_events
     if event_map is not None:
         new_needed_events -= set(event_map.iterkeys())
@@ -679,7 +681,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
 
 def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_map):
     auth_events = {}
-    for event_ids in conflicted_state.itervalues():
+    for event_ids in itervalues(conflicted_state):
         for event_id in event_ids:
             if event_id in state_map:
                 keys = event_auth.auth_types_for_event(state_map[event_id])
@@ -694,7 +696,7 @@ def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_ma
 def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ids,
                         state_map):
     conflicted_state = {}
-    for key, event_ids in conflicted_state_ds.iteritems():
+    for key, event_ids in iteritems(conflicted_state_ds):
         events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map]
         if len(events) > 1:
             conflicted_state[key] = events
@@ -703,7 +705,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_
 
     auth_events = {
         key: state_map[ev_id]
-        for key, ev_id in auth_event_ids.iteritems()
+        for key, ev_id in iteritems(auth_event_ids)
         if ev_id in state_map
     }
 
@@ -716,7 +718,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_
         raise
 
     new_state = unconflicted_state_ids
-    for key, event in resolved_state.iteritems():
+    for key, event in iteritems(resolved_state):
         new_state[key] = event.event_id
 
     return new_state
@@ -741,7 +743,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.iteritems():
+    for key, events in iteritems(conflicted_state):
         if key[0] == EventTypes.JoinRules:
             logger.debug("Resolving conflicted join rules %r", events)
             resolved_state[key] = _resolve_auth_events(
@@ -751,7 +753,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.iteritems():
+    for key, events in iteritems(conflicted_state):
         if key[0] == EventTypes.Member:
             logger.debug("Resolving conflicted member lists %r", events)
             resolved_state[key] = _resolve_auth_events(
@@ -761,7 +763,7 @@ def _resolve_state_events(conflicted_state, auth_events):
 
     auth_events.update(resolved_state)
 
-    for key, events in conflicted_state.iteritems():
+    for key, events in iteritems(conflicted_state):
         if key not in resolved_state:
             logger.debug("Resolving conflicted state %r:%r", key, events)
             resolved_state[key] = _resolve_normal_events(
-- 
cgit 1.4.1


From 757ed2725803ff6d190eb1bb8dea872345862aa3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 25 May 2018 11:07:21 +0100
Subject: Let users leave the server notice room after joining

They still can't reject invites, but we let them leave it.
---
 docs/server_notices.md          |  7 +++++--
 synapse/handlers/room_member.py | 24 ++++++++++++++----------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/docs/server_notices.md b/docs/server_notices.md
index 221553b24d..58f8776319 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -5,7 +5,7 @@ Server Notices
 channel whereby server administrators can send messages to users on the server.
 
 They are used as part of communication of the server polices(see
-[consent_tracking.md](consent_tracking.md)), however the intention is that 
+[consent_tracking.md](consent_tracking.md)), however the intention is that
 they may also find a use for features such as "Message of the day".
 
 This is a feature specific to Synapse, but it uses standard Matrix
@@ -24,7 +24,10 @@ history; it will appear to have come from the 'server notices user' (see
 below).
 
 The user is prevented from sending any messages in this room by the power
-levels. They also cannot leave it.
+levels.
+
+Having joined the room, the user can leave the room if they want. Subsequent
+server notices will then cause a new room to be created.
 
 Synapse configuration
 ---------------------
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 82adfc8fdf..f930e939e8 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -298,16 +298,6 @@ class RoomMemberHandler(object):
             is_blocked = yield self.store.is_room_blocked(room_id)
             if is_blocked:
                 raise SynapseError(403, "This room has been blocked on this server")
-        else:
-            # we don't allow people to reject invites to, or leave, the
-            # server notice room.
-            is_blocked = yield self._is_server_notice_room(room_id)
-            if is_blocked:
-                raise SynapseError(
-                    http_client.FORBIDDEN,
-                    "You cannot leave this room",
-                    errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM,
-                )
 
         if effective_membership_state == Membership.INVITE:
             # block any attempts to invite the server notices mxid
@@ -383,6 +373,20 @@ class RoomMemberHandler(object):
                 if same_sender and same_membership and same_content:
                     defer.returnValue(old_state)
 
+            # we don't allow people to reject invites to the server notice
+            # room, but they can leave it once they are joined.
+            if (
+                old_membership == Membership.INVITE and
+                effective_membership_state == Membership.LEAVE
+            ):
+                is_blocked = yield self._is_server_notice_room(room_id)
+                if is_blocked:
+                    raise SynapseError(
+                        http_client.FORBIDDEN,
+                        "You cannot reject this invite",
+                        errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM,
+                    )
+
         is_host_in_room = yield self._is_host_in_room(current_state_ids)
 
         if effective_membership_state == Membership.JOIN:
-- 
cgit 1.4.1


From ba1b1635901f148d793777ea77386dbe3412dfaf Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 25 May 2018 11:36:43 +0100
Subject: Avoid sending consent notice to guest users

we think it makes sense not to send the notices to guest users.
---
 synapse/config/consent_config.py                 | 8 +++++++-
 synapse/server_notices/consent_server_notices.py | 5 +++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index ddcd305f4c..3f29346593 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -32,7 +32,8 @@ DEFAULT_CONFIG = """\
 #
 # 'server_notice_content', if enabled, will send a user a "Server Notice"
 # asking them to consent to the privacy policy. The 'server_notices' section
-# must also be configured for this to work.
+# must also be configured for this to work. Notices will *not* be sent to
+# guest users unless 'send_server_notice_to_guests' is set to true.
 #
 # 'block_events_error', if set, will block any attempts to send events
 # until the user consents to the privacy policy. The value of the setting is
@@ -46,6 +47,7 @@ DEFAULT_CONFIG = """\
 #     body: >-
 #       To continue using this homeserver you must review and agree to the
 #       terms and conditions at %(consent_uri)s
+#   send_server_notice_to_guests: True
 #   block_events_error: >-
 #     To continue using this homeserver you must review and agree to the
 #     terms and conditions at %(consent_uri)s
@@ -60,6 +62,7 @@ class ConsentConfig(Config):
         self.user_consent_version = None
         self.user_consent_template_dir = None
         self.user_consent_server_notice_content = None
+        self.user_consent_server_notice_to_guests = False
         self.block_events_without_consent_error = None
 
     def read_config(self, config):
@@ -74,6 +77,9 @@ class ConsentConfig(Config):
         self.block_events_without_consent_error = consent_config.get(
             "block_events_error",
         )
+        self.user_consent_server_notice_to_guests = bool(consent_config.get(
+            "send_server_notice_to_guests", "False"
+        ))
 
     def default_config(self, **kwargs):
         return DEFAULT_CONFIG
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index a709802856..bb74af1af5 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -42,6 +42,7 @@ class ConsentServerNotices(object):
 
         self._current_consent_version = hs.config.user_consent_version
         self._server_notice_content = hs.config.user_consent_server_notice_content
+        self._send_to_guests = hs.config.user_consent_server_notice_to_guests
 
         if self._server_notice_content is not None:
             if not self._server_notices_manager.is_enabled():
@@ -78,6 +79,10 @@ class ConsentServerNotices(object):
         try:
             u = yield self._store.get_user_by_id(user_id)
 
+            if u["is_guest"] and not self._send_to_guests:
+                # don't send to guests
+                return
+
             if u["consent_version"] == self._current_consent_version:
                 # user has already consented
                 return
-- 
cgit 1.4.1


From 66bdae986f4bdee300aebb4ad6ebeeba48286e6b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 25 May 2018 11:42:05 +0100
Subject: Fix default for send_server_notice_to_guests

bool("False") == True...
---
 synapse/config/consent_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 3f29346593..8f6ed73328 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -78,7 +78,7 @@ class ConsentConfig(Config):
             "block_events_error",
         )
         self.user_consent_server_notice_to_guests = bool(consent_config.get(
-            "send_server_notice_to_guests", "False"
+            "send_server_notice_to_guests", False,
         ))
 
     def default_config(self, **kwargs):
-- 
cgit 1.4.1


From a2eb5db4a066148da5057b8ee844c7e4f5d888bc Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 28 May 2018 19:10:27 +1000
Subject: update metrics to be in seconds

---
 synapse/http/request_metrics.py |  6 +++---
 synapse/http/site.py            | 20 ++++++++++----------
 synapse/metrics/__init__.py     | 19 ++++++++++---------
 synapse/storage/_base.py        | 30 +++++++++++++++---------------
 4 files changed, 38 insertions(+), 37 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index e7df494333..af3067b4bb 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -38,15 +38,15 @@ outgoing_responses_counter = Counter(
 )
 
 response_timer = Histogram(
-    "synapse_http_server_response_time_seconds", "", ["method", "servlet", "tag"]
+    "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag"]
 )
 
 response_ru_utime = Counter(
-    "synapse_http_server_response_ru_utime_seconds", "", ["method", "servlet", "tag"]
+    "synapse_http_server_response_ru_utime_seconds", "sec", ["method", "servlet", "tag"]
 )
 
 response_ru_stime = Counter(
-    "synapse_http_server_response_ru_stime_seconds", "", ["method", "servlet", "tag"]
+    "synapse_http_server_response_ru_stime_seconds", "sec", ["method", "servlet", "tag"]
 )
 
 response_db_txn_count = Counter(
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 23c1b76922..60299657b9 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -83,7 +83,7 @@ class SynapseRequest(Request):
         return Request.render(self, resrc)
 
     def _started_processing(self, servlet_name):
-        self.start_time = int(time.time() * 1000)
+        self.start_time = time.time()
         self.request_metrics = RequestMetrics()
         self.request_metrics.start(
             self.start_time, name=servlet_name, method=self.method,
@@ -102,26 +102,26 @@ class SynapseRequest(Request):
             context = LoggingContext.current_context()
             ru_utime, ru_stime = context.get_resource_usage()
             db_txn_count = context.db_txn_count
-            db_txn_duration_ms = context.db_txn_duration_ms
-            db_sched_duration_ms = context.db_sched_duration_ms
+            db_txn_duration_sec = context.db_txn_duration_sec
+            db_sched_duration_sec = context.db_sched_duration_sec
         except Exception:
             ru_utime, ru_stime = (0, 0)
-            db_txn_count, db_txn_duration_ms = (0, 0)
+            db_txn_count, db_txn_duration_sec = (0, 0)
 
-        end_time = int(time.time() * 1000)
+        end_time = time.time()
 
         self.site.access_logger.info(
             "%s - %s - {%s}"
-            " Processed request: %dms (%dms, %dms) (%dms/%dms/%d)"
+            " Processed request: %.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)"
             " %sB %s \"%s %s %s\" \"%s\"",
             self.getClientIP(),
             self.site.site_tag,
             self.authenticated_entity,
             end_time - self.start_time,
-            int(ru_utime * 1000),
-            int(ru_stime * 1000),
-            db_sched_duration_ms,
-            db_txn_duration_ms,
+            ru_utime,
+            ru_stime,
+            db_sched_duration_sec,
+            db_txn_duration_sec,
             int(db_txn_count),
             self.sentLength,
             self.code,
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e33ed6c9be..0c557a43f6 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -122,10 +122,10 @@ REGISTRY.register(CPUMetrics())
 gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"])
 gc_time = Histogram(
     "python_gc_time",
-    "Time taken to GC (ms)",
+    "Time taken to GC (sec)",
     ["gen"],
-    buckets=[2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 7500, 15000,
-             30000, 45000, 60000],
+    buckets=[0.0025, 0.005, 0.01, 0.025, 0.05, 0.10, 0.25, 0.50, 1.00, 2.50,
+             5.00, 7.50, 15.00, 30.00, 45.00, 60.00],
 )
 
 
@@ -147,8 +147,9 @@ REGISTRY.register(GCCounts())
 
 tick_time = Histogram(
     "python_twisted_reactor_tick_time",
-    "Tick time of the Twisted reactor (ms)",
-    buckets=[1, 2, 5, 10, 50, 100, 250, 500, 1000, 2000],
+    "Tick time of the Twisted reactor (sec)",
+    buckets=[0.001, 0.002, 0.005, 0.001, 0.005, 0.01. 0.025, 0.05, 0.1, 0.2,
+             0.5, 1, 2, 5],
 )
 pending_calls_metric = Histogram(
     "python_twisted_reactor_pending_calls",
@@ -202,9 +203,9 @@ def runUntilCurrentTimer(func):
             num_pending += 1
 
         num_pending += len(reactor.threadCallQueue)
-        start = time.time() * 1000
+        start = time.time()
         ret = func(*args, **kwargs)
-        end = time.time() * 1000
+        end = time.time()
 
         # record the amount of wallclock time spent running pending calls.
         # This is a proxy for the actual amount of time between reactor polls,
@@ -225,9 +226,9 @@ def runUntilCurrentTimer(func):
             if threshold[i] < counts[i]:
                 logger.info("Collecting gc %d", i)
 
-                start = time.time() * 1000
+                start = time.time()
                 unreachable = gc.collect(i)
-                end = time.time() * 1000
+                end = time.time()
 
                 gc_time.labels(i).observe(end - start)
                 gc_unreachable.labels(i).set(unreachable)
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index d963af5c89..22d6257a9f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -42,10 +42,10 @@ sql_logger = logging.getLogger("synapse.storage.SQL")
 transaction_logger = logging.getLogger("synapse.storage.txn")
 perf_logger = logging.getLogger("synapse.storage.TIME")
 
-sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "")
+sql_scheduling_timer = Histogram("synapse_storage_schedule_time", "sec")
 
-sql_query_timer = Histogram("synapse_storage_query_time", "", ["verb"])
-sql_txn_timer = Histogram("synapse_storage_transaction_time", "", ["desc"])
+sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"])
+sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"])
 
 
 class LoggingTransaction(object):
@@ -110,7 +110,7 @@ class LoggingTransaction(object):
                 # Don't let logging failures stop SQL from working
                 pass
 
-        start = time.time() * 1000
+        start = time.time()
 
         try:
             return func(
@@ -120,9 +120,9 @@ class LoggingTransaction(object):
             logger.debug("[SQL FAIL] {%s} %s", self.name, e)
             raise
         finally:
-            msecs = (time.time() * 1000) - start
-            sql_logger.debug("[SQL time] {%s} %f", self.name, msecs)
-            sql_query_timer.labels(sql.split()[0]).observe(msecs)
+            secs = time.time() - start
+            sql_logger.debug("[SQL time] {%s} %f sec", self.name, secs)
+            sql_query_timer.labels(sql.split()[0]).observe(secs)
 
 
 class PerformanceCounters(object):
@@ -132,7 +132,7 @@ class PerformanceCounters(object):
 
     def update(self, key, start_time, end_time=None):
         if end_time is None:
-            end_time = time.time() * 1000
+            end_time = time.time()
         duration = end_time - start_time
         count, cum_time = self.current_counters.get(key, (0, 0))
         count += 1
@@ -222,7 +222,7 @@ class SQLBaseStore(object):
 
     def _new_transaction(self, conn, desc, after_callbacks, exception_callbacks,
                          logging_context, func, *args, **kwargs):
-        start = time.time() * 1000
+        start = time.time()
         txn_id = self._TXN_ID
 
         # We don't really need these to be unique, so lets stop it from
@@ -282,13 +282,13 @@ class SQLBaseStore(object):
             logger.debug("[TXN FAIL] {%s} %s", name, e)
             raise
         finally:
-            end = time.time() * 1000
+            end = time.time()
             duration = end - start
 
             if logging_context is not None:
                 logging_context.add_database_transaction(duration)
 
-            transaction_logger.debug("[TXN END] {%s} %f", name, duration)
+            transaction_logger.debug("[TXN END] {%s} %f sec", name, duration)
 
             self._current_txn_total_time += duration
             self._txn_perf_counters.update(desc, start, end)
@@ -349,13 +349,13 @@ class SQLBaseStore(object):
         """
         current_context = LoggingContext.current_context()
 
-        start_time = time.time() * 1000
+        start_time = time.time()
 
         def inner_func(conn, *args, **kwargs):
             with LoggingContext("runWithConnection") as context:
-                sched_duration_ms = time.time() * 1000 - start_time
-                sql_scheduling_timer.observe(sched_duration_ms)
-                current_context.add_database_scheduled(sched_duration_ms)
+                sched_duration_sec = time.time() - start_time
+                sql_scheduling_timer.observe(sched_duration_sec)
+                current_context.add_database_scheduled(sched_duration_sec)
 
                 if self.database_engine.is_connection_closed(conn):
                     logger.debug("Reconnecting closed database connection")
-- 
cgit 1.4.1


From 357c74a50f1e1588a6a3d626bddb3555452c6f56 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 28 May 2018 19:14:41 +1000
Subject: add comment about why unreg

---
 synapse/util/caches/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index d968b71e7a..183faf75a1 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -41,6 +41,8 @@ response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["n
 def register_cache(cache_type, cache_name, cache):
 
     # Check if the metric is already registered. Unregister it, if so.
+    # This usually happens during tests, as at runtime these caches are
+    # effectively singletons.
     metric_name = "cache_%s_%s" % (cache_type, cache_name)
     if metric_name in collectors_by_name.keys():
         REGISTRY.unregister(collectors_by_name[metric_name])
-- 
cgit 1.4.1


From 5c40ce37774a20b57fe726b9c3d880d089ed6f0c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 28 May 2018 19:16:09 +1000
Subject: invalid syntax :(

---
 synapse/metrics/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 0c557a43f6..371c300cac 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -148,8 +148,7 @@ REGISTRY.register(GCCounts())
 tick_time = Histogram(
     "python_twisted_reactor_tick_time",
     "Tick time of the Twisted reactor (sec)",
-    buckets=[0.001, 0.002, 0.005, 0.001, 0.005, 0.01. 0.025, 0.05, 0.1, 0.2,
-             0.5, 1, 2, 5],
+    buckets=[0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2, 5],
 )
 pending_calls_metric = Histogram(
     "python_twisted_reactor_pending_calls",
-- 
cgit 1.4.1


From 3ef5cd74a6bff9b33144cb834782e0402e6eb152 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 28 May 2018 19:39:27 +1000
Subject: update to more consistently use seconds in any metrics or logging

---
 synapse/http/request_metrics.py | 38 +++++++++++++++++++-------------------
 synapse/util/logcontext.py      | 24 ++++++++++++------------
 synapse/util/logutils.py        |  6 +++---
 synapse/util/metrics.py         |  8 ++++----
 4 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index af3067b4bb..dc06f6c443 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -139,8 +139,8 @@ LaterGauge(
 
 
 class RequestMetrics(object):
-    def start(self, time_msec, name, method):
-        self.start = time_msec
+    def start(self, time_sec, name, method):
+        self.start = time_sec
         self.start_context = LoggingContext.current_context()
         self.name = name
         self.method = method
@@ -149,7 +149,7 @@ class RequestMetrics(object):
 
         _in_flight_requests.add(self)
 
-    def stop(self, time_msec, request):
+    def stop(self, time_sec, request):
         _in_flight_requests.discard(self)
 
         context = LoggingContext.current_context()
@@ -170,7 +170,7 @@ class RequestMetrics(object):
         response_count.labels(request.method, self.name, tag).inc()
 
         response_timer.labels(request.method, self.name, tag).observe(
-            time_msec - self.start
+            time_sec - self.start
         )
 
         ru_utime, ru_stime = context.get_resource_usage()
@@ -181,10 +181,10 @@ class RequestMetrics(object):
             context.db_txn_count
         )
         response_db_txn_duration.labels(request.method, self.name, tag).inc(
-            context.db_txn_duration_ms / 1000.
+            context.db_txn_duration_sec
         )
         response_db_sched_duration.labels(request.method, self.name, tag).inc(
-            context.db_sched_duration_ms / 1000.
+            context.db_sched_duration_sec
         )
 
         response_size.labels(request.method, self.name, tag).inc(request.sentLength)
@@ -207,11 +207,11 @@ class RequestMetrics(object):
         )
 
         in_flight_requests_db_txn_duration.labels(self.method, self.name).inc(
-            diff.db_txn_duration_ms / 1000.
+            diff.db_txn_duration_sec
         )
 
         in_flight_requests_db_sched_duration.labels(self.method, self.name).inc(
-            diff.db_sched_duration_ms / 1000.
+            diff.db_sched_duration_sec
         )
 
 
@@ -223,18 +223,18 @@ class _RequestStats(object):
         "ru_utime",
         "ru_stime",
         "db_txn_count",
-        "db_txn_duration_ms",
-        "db_sched_duration_ms",
+        "db_txn_duration_sec",
+        "db_sched_duration_sec",
     ]
 
     def __init__(
-        self, ru_utime, ru_stime, db_txn_count, db_txn_duration_ms, db_sched_duration_ms
+        self, ru_utime, ru_stime, db_txn_count, db_txn_duration_sec, db_sched_duration_sec
     ):
         self.ru_utime = ru_utime
         self.ru_stime = ru_stime
         self.db_txn_count = db_txn_count
-        self.db_txn_duration_ms = db_txn_duration_ms
-        self.db_sched_duration_ms = db_sched_duration_ms
+        self.db_txn_duration_sec = db_txn_duration_sec
+        self.db_sched_duration_sec = db_sched_duration_sec
 
     @staticmethod
     def from_context(context):
@@ -243,8 +243,8 @@ class _RequestStats(object):
         return _RequestStats(
             ru_utime, ru_stime,
             context.db_txn_count,
-            context.db_txn_duration_ms,
-            context.db_sched_duration_ms,
+            context.db_txn_duration_sec,
+            context.db_sched_duration_sec,
         )
 
     def update(self, context):
@@ -260,14 +260,14 @@ class _RequestStats(object):
             new.ru_utime - self.ru_utime,
             new.ru_stime - self.ru_stime,
             new.db_txn_count - self.db_txn_count,
-            new.db_txn_duration_ms - self.db_txn_duration_ms,
-            new.db_sched_duration_ms - self.db_sched_duration_ms,
+            new.db_txn_duration_sec - self.db_txn_duration_sec,
+            new.db_sched_duration_sec - self.db_sched_duration_sec,
         )
 
         self.ru_utime = new.ru_utime
         self.ru_stime = new.ru_stime
         self.db_txn_count = new.db_txn_count
-        self.db_txn_duration_ms = new.db_txn_duration_ms
-        self.db_sched_duration_ms = new.db_sched_duration_ms
+        self.db_txn_duration_sec = new.db_txn_duration_sec
+        self.db_sched_duration_sec = new.db_sched_duration_sec
 
         return diff
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 914f616312..a58c723403 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -59,7 +59,7 @@ class LoggingContext(object):
 
     __slots__ = [
         "previous_context", "name", "ru_stime", "ru_utime",
-        "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
+        "db_txn_count", "db_txn_duration_sec", "db_sched_duration_sec",
         "usage_start",
         "main_thread", "alive",
         "request", "tag",
@@ -84,10 +84,10 @@ class LoggingContext(object):
         def stop(self):
             pass
 
-        def add_database_transaction(self, duration_ms):
+        def add_database_transaction(self, duration_sec):
             pass
 
-        def add_database_scheduled(self, sched_ms):
+        def add_database_scheduled(self, sched_sec):
             pass
 
         def __nonzero__(self):
@@ -103,11 +103,11 @@ class LoggingContext(object):
         self.ru_utime = 0.
         self.db_txn_count = 0
 
-        # ms spent waiting for db txns, excluding scheduling time
-        self.db_txn_duration_ms = 0
+        # sec spent waiting for db txns, excluding scheduling time
+        self.db_txn_duration_sec = 0
 
-        # ms spent waiting for db txns to be scheduled
-        self.db_sched_duration_ms = 0
+        # sec spent waiting for db txns to be scheduled
+        self.db_sched_duration_sec = 0
 
         # If alive has the thread resource usage when the logcontext last
         # became active.
@@ -230,18 +230,18 @@ class LoggingContext(object):
 
         return ru_utime, ru_stime
 
-    def add_database_transaction(self, duration_ms):
+    def add_database_transaction(self, duration_sec):
         self.db_txn_count += 1
-        self.db_txn_duration_ms += duration_ms
+        self.db_txn_duration_sec += duration_sec
 
-    def add_database_scheduled(self, sched_ms):
+    def add_database_scheduled(self, sched_sec):
         """Record a use of the database pool
 
         Args:
-            sched_ms (int): number of milliseconds it took us to get a
+            sched_sec (float): number of seconds it took us to get a
                 connection
         """
-        self.db_sched_duration_ms += sched_ms
+        self.db_sched_duration_sec += sched_sec
 
 
 class LoggingContextFilter(logging.Filter):
diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py
index 3a83828d25..03249c5dc8 100644
--- a/synapse/util/logutils.py
+++ b/synapse/util/logutils.py
@@ -96,7 +96,7 @@ def time_function(f):
         id = _TIME_FUNC_ID
         _TIME_FUNC_ID += 1
 
-        start = time.clock() * 1000
+        start = time.clock()
 
         try:
             _log_debug_as_f(
@@ -107,10 +107,10 @@ def time_function(f):
 
             r = f(*args, **kwargs)
         finally:
-            end = time.clock() * 1000
+            end = time.clock()
             _log_debug_as_f(
                 f,
-                "[FUNC END] {%s-%d} %f",
+                "[FUNC END] {%s-%d} %.3f sec",
                 (func_name, id, end - start,),
             )
 
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index 424fdcb036..23fc0ca20d 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -74,7 +74,7 @@ class Measure(object):
         self.created_context = False
 
     def __enter__(self):
-        self.start = self.clock.time_msec()
+        self.start = self.clock.time()
         self.start_context = LoggingContext.current_context()
         if not self.start_context:
             self.start_context = LoggingContext("Measure")
@@ -90,7 +90,7 @@ class Measure(object):
         if isinstance(exc_type, Exception) or not self.start_context:
             return
 
-        duration = self.clock.time_msec() - self.start
+        duration = self.clock.time() - self.start
 
         block_counter.labels(self.name).inc()
         block_timer.labels(self.name).inc(duration)
@@ -114,9 +114,9 @@ class Measure(object):
         block_ru_stime.labels(self.name).inc(ru_stime - self.ru_stime)
         block_db_txn_count.labels(self.name).inc(context.db_txn_count - self.db_txn_count)
         block_db_txn_duration.labels(self.name).inc(
-            (context.db_txn_duration_ms - self.db_txn_duration_ms) / 1000.)
+            context.db_txn_duration_sec - self.db_txn_duration_sec)
         block_db_sched_duration.labels(self.name).inc(
-            (context.db_sched_duration_ms - self.db_sched_duration_ms) / 1000.)
+            context.db_sched_duration_sec - self.db_sched_duration_sec)
 
         if self.created_context:
             self.start_context.__exit__(exc_type, exc_val, exc_tb)
-- 
cgit 1.4.1


From 57ad76fa4a4fb76dc1b9c7e8232b9589b9731ff6 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 28 May 2018 19:51:53 +1000
Subject: fix up tests

---
 synapse/util/metrics.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index 23fc0ca20d..1ba7d65c7c 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -62,7 +62,7 @@ class Measure(object):
     __slots__ = [
         "clock", "name", "start_context", "start", "new_context", "ru_utime",
         "ru_stime",
-        "db_txn_count", "db_txn_duration_ms", "db_sched_duration_ms",
+        "db_txn_count", "db_txn_duration_sec", "db_sched_duration_sec",
         "created_context",
     ]
 
@@ -83,8 +83,8 @@ class Measure(object):
 
         self.ru_utime, self.ru_stime = self.start_context.get_resource_usage()
         self.db_txn_count = self.start_context.db_txn_count
-        self.db_txn_duration_ms = self.start_context.db_txn_duration_ms
-        self.db_sched_duration_ms = self.start_context.db_sched_duration_ms
+        self.db_txn_duration_sec = self.start_context.db_txn_duration_sec
+        self.db_sched_duration_sec = self.start_context.db_sched_duration_sec
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if isinstance(exc_type, Exception) or not self.start_context:
-- 
cgit 1.4.1