Merge branch 'develop' of github.com:matrix-org/synapse into anoa/user_param_ui_auth anoa/user_param_ui_auth

* 'develop' of github.com:matrix-org/synapse: (369 commits)
  Add functions to `MultiWriterIdGen` used by events stream (#8164)
  Do not allow send_nonmember_event to be called with shadow-banned users. (#8158)
  Changelog fixes
  1.19.1rc1
  Make StreamIdGen `get_next` and `get_next_mult` async  (#8161)
  Wording fixes to 'name' user admin api filter (#8163)
  Fix missing double-backtick in RST document
  Search in columns 'name' and 'displayname' in the admin users endpoint (#7377)
  Add type hints for state. (#8140)
  Stop shadow-banned users from sending non-member events. (#8142)
  Allow capping a room's retention policy (#8104)
  Add healthcheck for default localhost 8008 port on /health endpoint. (#8147)
  Fix flaky shadow-ban tests. (#8152)
  Fix join ratelimiter breaking profile updates and idempotency (#8153)
  Do not apply ratelimiting on joins to appservices (#8139)
  Don't fail /submit_token requests on incorrect session ID if request_token_inhibit_3pid_errors is turned on (#7991)
  Do not apply ratelimiting on joins to appservices (#8139)
  Micro-optimisations to get_auth_chain_ids (#8132)
  Allow denying or shadow banning registrations via the spam checker (#8034)
  Stop shadow-banned users from sending invites. (#8095)
  ...

10 files changed, 297 insertions, 180 deletions

diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py
index 48da410d94..dfc1d294dc 100755
--- a/contrib/cmdclient/console.py
+++ b/contrib/cmdclient/console.py
@@ -17,9 +17,6 @@
 """ Starts a synapse client console. """
 from __future__ import print_function
 
-from twisted.internet import reactor, defer, threads
-from http import TwistedHttpClient
-
 import argparse
 import cmd
 import getpass
@@ -28,12 +25,14 @@ import shlex
 import sys
 import time
 import urllib
-import urlparse
+from http import TwistedHttpClient
 
-import nacl.signing
 import nacl.encoding
+import nacl.signing
+import urlparse
+from signedjson.sign import SignatureVerifyException, verify_signed_json
 
-from signedjson.sign import verify_signed_json, SignatureVerifyException
+from twisted.internet import defer, reactor, threads
 
 CONFIG_JSON = "cmdclient_config.json"
 
@@ -493,7 +492,7 @@ class SynapseCmd(cmd.Cmd):
         "list messages <roomid> from=END&to=START&limit=3"
         """
         args = self._parse(line, ["type", "roomid", "qp"])
-        if not "type" in args or not "roomid" in args:
+        if "type" not in args or "roomid" not in args:
             print("Must specify type and room ID.")
             return
         if args["type"] not in ["members", "messages"]:
@@ -508,7 +507,7 @@ class SynapseCmd(cmd.Cmd):
                 try:
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
-                except:
+                except Exception:
                     print("Bad query param: %s" % key_value)
                     return
 
@@ -585,7 +584,7 @@ class SynapseCmd(cmd.Cmd):
                 parsed_url = urlparse.urlparse(args["path"])
                 qp.update(urlparse.parse_qs(parsed_url.query))
                 args["path"] = parsed_url.path
-            except:
+            except Exception:
                 pass
 
         reactor.callFromThread(
@@ -610,13 +609,15 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
-        res = yield self.http_client.get_json(
-            self._url() + "/events",
-            {
-                "access_token": self._tok(),
-                "timeout": str(timeout),
-                "from": self.event_stream_token,
-            },
+        res = yield defer.ensureDeferred(
+            self.http_client.get_json(
+                self._url() + "/events",
+                {
+                    "access_token": self._tok(),
+                    "timeout": str(timeout),
+                    "from": self.event_stream_token,
+                },
+            )
         )
         print(json.dumps(res, indent=4))
 
@@ -772,10 +773,10 @@ def main(server_url, identity_server_url, username, token, config_path):
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
-            except:
+            except Exception:
                 pass
             print("Loaded config from %s" % config_path)
-    except:
+    except Exception:
         pass
 
     # Twisted-specific: Runs the command processor in Twisted's event loop
diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py
index 0e101d2be5..e2534ee584 100644
--- a/contrib/cmdclient/http.py
+++ b/contrib/cmdclient/http.py
@@ -14,14 +14,14 @@
 # limitations under the License.
 
 from __future__ import print_function
-from twisted.web.client import Agent, readBody
-from twisted.web.http_headers import Headers
-from twisted.internet import defer, reactor
-
-from pprint import pformat
 
 import json
 import urllib
+from pprint import pformat
+
+from twisted.internet import defer, reactor
+from twisted.web.client import Agent, readBody
+from twisted.web.http_headers import Headers
 
 
 class HttpClient(object):
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 17354b6610..d1ecd453db 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -50,7 +50,7 @@ services:
       - traefik.http.routers.https-synapse.tls.certResolver=le-ssl
 
   db:
-    image: docker.io/postgres:10-alpine
+    image: docker.io/postgres:12-alpine
     # Change that password, of course!
     environment:
       - POSTGRES_USER=synapse
diff --git a/contrib/experiments/test_messaging.py b/contrib/experiments/test_messaging.py
index 3bbbcfa1b4..a84ec4ecae 100644
--- a/contrib/experiments/test_messaging.py
+++ b/contrib/experiments/test_messaging.py
@@ -28,27 +28,24 @@ Currently assumes the local address is localhost:<port>
 """
 
 
-from synapse.federation import ReplicationHandler
-
-from synapse.federation.units import Pdu
-
-from synapse.util import origin_from_ucid
-
-from synapse.app.homeserver import SynapseHomeServer
-
-# from synapse.logging.utils import log_function
-
-from twisted.internet import reactor, defer
-from twisted.python import log
-
 import argparse
+import curses.wrapper
 import json
 import logging
 import os
 import re
 
 import cursesio
-import curses.wrapper
+
+from twisted.internet import defer, reactor
+from twisted.python import log
+
+from synapse.app.homeserver import SynapseHomeServer
+from synapse.federation import ReplicationHandler
+from synapse.federation.units import Pdu
+from synapse.util import origin_from_ucid
+
+# from synapse.logging.utils import log_function
 
 
 logger = logging.getLogger("example")
@@ -75,7 +72,7 @@ class InputOutput(object):
         """
 
         try:
-            m = re.match("^join (\S+)$", line)
+            m = re.match(r"^join (\S+)$", line)
             if m:
                 # The `sender` wants to join a room.
                 (room_name,) = m.groups()
@@ -84,7 +81,7 @@ class InputOutput(object):
                 # self.print_line("OK.")
                 return
 
-            m = re.match("^invite (\S+) (\S+)$", line)
+            m = re.match(r"^invite (\S+) (\S+)$", line)
             if m:
                 # `sender` wants to invite someone to a room
                 room_name, invitee = m.groups()
@@ -93,7 +90,7 @@ class InputOutput(object):
                 # self.print_line("OK.")
                 return
 
-            m = re.match("^send (\S+) (.*)$", line)
+            m = re.match(r"^send (\S+) (.*)$", line)
             if m:
                 # `sender` wants to message a room
                 room_name, body = m.groups()
@@ -102,7 +99,7 @@ class InputOutput(object):
                 # self.print_line("OK.")
                 return
 
-            m = re.match("^backfill (\S+)$", line)
+            m = re.match(r"^backfill (\S+)$", line)
             if m:
                 # we want to backfill a room
                 (room_name,) = m.groups()
@@ -201,16 +198,6 @@ class HomeServer(ReplicationHandler):
                 % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
-    # def on_state_change(self, pdu):
-    ##self.output.print_line("#%s (state) %s *** %s" %
-    ##(pdu.context, pdu.state_key, pdu.pdu_type)
-    ##)
-
-    # if "joinee" in pdu.content:
-    # self._on_join(pdu.context, pdu.content["joinee"])
-    # elif "invitee" in pdu.content:
-    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
-
     def _on_message(self, pdu):
         """ We received a message
         """
@@ -314,7 +301,7 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return [i for i in self.joined_rooms.setdefault(room_name).servers]
+        return list(self.joined_rooms.setdefault(room_name).servers)
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
@@ -334,7 +321,7 @@ def main(stdscr):
     user = args.user
     server_name = origin_from_ucid(user)
 
-    ## Set up logging ##
+    # Set up logging
 
     root_logger = logging.getLogger()
 
@@ -354,7 +341,7 @@ def main(stdscr):
     observer = log.PythonLoggingObserver()
     observer.start()
 
-    ## Set up synapse server
+    # Set up synapse server
 
     curses_stdio = cursesio.CursesStdIO(stdscr)
     input_output = InputOutput(curses_stdio, user)
@@ -368,16 +355,16 @@ def main(stdscr):
 
     input_output.set_home_server(hs)
 
-    ## Add input_output logger
+    # Add input_output logger
     io_logger = IOLoggerHandler(input_output)
     io_logger.setFormatter(formatter)
     root_logger.addHandler(io_logger)
 
-    ## Start! ##
+    # Start!
 
     try:
         port = int(server_name.split(":")[1])
-    except:
+    except Exception:
         port = 12345
 
     app_hs.get_http_server().start_listening(port)
diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 30a8681f5a..539569b5b1 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -1,7 +1,44 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "6.7.4"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "heatmap",
+      "name": "Heatmap",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    }
+  ],
   "annotations": {
     "list": [
       {
+        "$$hashKey": "object:76",
         "builtIn": 1,
         "datasource": "$datasource",
         "enable": false,
@@ -17,8 +54,8 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 1,
-  "iteration": 1591098104645,
+  "id": null,
+  "iteration": 1594646317221,
   "links": [
     {
       "asDropdown": true,
@@ -34,7 +71,7 @@
   "panels": [
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -269,7 +306,6 @@
         "show": false
       },
       "links": [],
-      "options": {},
       "reverseYBuckets": false,
       "targets": [
         {
@@ -559,7 +595,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1423,7 +1459,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1795,7 +1831,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -2531,7 +2567,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -2823,7 +2859,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -2844,7 +2880,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 33
+            "y": 6
           },
           "hiddenSeries": false,
           "id": 79,
@@ -2940,7 +2976,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 33
+            "y": 6
           },
           "hiddenSeries": false,
           "id": 83,
@@ -3038,7 +3074,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 42
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 109,
@@ -3137,7 +3173,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 42
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 111,
@@ -3223,14 +3259,14 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "",
+          "description": "Number of events queued up on the master process for processing by the federation sender",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 24
           },
           "hiddenSeries": false,
           "id": 140,
@@ -3354,6 +3390,103 @@
             "align": false,
             "alignLevel": null
           }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "description": "The number of events in the in-memory queues ",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 24
+          },
+          "hiddenSeries": false,
+          "id": 142,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "interval": "",
+              "legendFormat": "pending PDUs {{job}}-{{index}}",
+              "refId": "A"
+            },
+            {
+              "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "interval": "",
+              "legendFormat": "pending EDUs {{job}}-{{index}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "In-memory federation transmission queues",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:317",
+              "format": "short",
+              "label": "events",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "$$hashKey": "object:318",
+              "format": "short",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "title": "Federation",
@@ -3361,7 +3494,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -3567,7 +3700,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -3588,7 +3721,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 52
+            "y": 79
           },
           "hiddenSeries": false,
           "id": 48,
@@ -3682,7 +3815,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 52
+            "y": 79
           },
           "hiddenSeries": false,
           "id": 104,
@@ -3802,7 +3935,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 86
           },
           "hiddenSeries": false,
           "id": 10,
@@ -3898,7 +4031,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 59
+            "y": 86
           },
           "hiddenSeries": false,
           "id": 11,
@@ -3987,7 +4120,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -4011,7 +4144,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 80
           },
           "hiddenSeries": false,
           "id": 12,
@@ -4106,7 +4239,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 67
+            "y": 80
           },
           "hiddenSeries": false,
           "id": 26,
@@ -4201,7 +4334,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 80
+            "y": 93
           },
           "hiddenSeries": false,
           "id": 13,
@@ -4297,7 +4430,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 80
+            "y": 93
           },
           "hiddenSeries": false,
           "id": 27,
@@ -4392,7 +4525,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 93
+            "y": 106
           },
           "hiddenSeries": false,
           "id": 28,
@@ -4486,7 +4619,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 93
+            "y": 106
           },
           "hiddenSeries": false,
           "id": 25,
@@ -4572,7 +4705,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -5062,7 +5195,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -5083,7 +5216,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 66
+            "y": 121
           },
           "hiddenSeries": false,
           "id": 91,
@@ -5179,7 +5312,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 66
+            "y": 121
           },
           "hiddenSeries": false,
           "id": 21,
@@ -5271,7 +5404,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 75
+            "y": 130
           },
           "hiddenSeries": false,
           "id": 89,
@@ -5369,7 +5502,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 75
+            "y": 130
           },
           "hiddenSeries": false,
           "id": 93,
@@ -5459,7 +5592,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 84
+            "y": 139
           },
           "hiddenSeries": false,
           "id": 95,
@@ -5552,12 +5685,12 @@
             "mode": "spectrum"
           },
           "dataFormat": "tsbuckets",
-          "datasource": "Prometheus",
+          "datasource": "${DS_PROMETHEUS}",
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 84
+            "y": 139
           },
           "heatmap": {},
           "hideZeroBuckets": true,
@@ -5567,7 +5700,6 @@
             "show": true
           },
           "links": [],
-          "options": {},
           "reverseYBuckets": false,
           "targets": [
             {
@@ -5609,7 +5741,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -5630,7 +5762,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 39
+            "y": 66
           },
           "hiddenSeries": false,
           "id": 2,
@@ -5754,7 +5886,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 39
+            "y": 66
           },
           "hiddenSeries": false,
           "id": 41,
@@ -5847,7 +5979,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 46
+            "y": 73
           },
           "hiddenSeries": false,
           "id": 42,
@@ -5939,7 +6071,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 46
+            "y": 73
           },
           "hiddenSeries": false,
           "id": 43,
@@ -6031,7 +6163,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 53
+            "y": 80
           },
           "hiddenSeries": false,
           "id": 113,
@@ -6129,7 +6261,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 53
+            "y": 80
           },
           "hiddenSeries": false,
           "id": 115,
@@ -6215,7 +6347,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -6236,7 +6368,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 58
+            "y": 40
           },
           "hiddenSeries": false,
           "id": 67,
@@ -6267,7 +6399,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": " synapse_event_persisted_position{instance=\"$instance\",job=\"synapse\"}  - ignoring(index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - ignoring(instance,index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -6328,7 +6460,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 58
+            "y": 40
           },
           "hiddenSeries": false,
           "id": 71,
@@ -6362,6 +6494,7 @@
               "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
               "hide": false,
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{name}}",
               "refId": "B"
@@ -6420,7 +6553,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 67
+            "y": 49
           },
           "hiddenSeries": false,
           "id": 121,
@@ -6509,7 +6642,7 @@
     },
     {
       "collapsed": true,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -6539,7 +6672,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 41
+            "y": 86
           },
           "heatmap": {},
           "hideZeroBuckets": true,
@@ -6549,7 +6682,6 @@
             "show": true
           },
           "links": [],
-          "options": {},
           "reverseYBuckets": false,
           "targets": [
             {
@@ -6599,7 +6731,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 41
+            "y": 86
           },
           "hiddenSeries": false,
           "id": 124,
@@ -6700,7 +6832,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 49
+            "y": 94
           },
           "heatmap": {},
           "hideZeroBuckets": true,
@@ -6710,7 +6842,6 @@
             "show": true
           },
           "links": [],
-          "options": {},
           "reverseYBuckets": false,
           "targets": [
             {
@@ -6760,7 +6891,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 49
+            "y": 94
           },
           "hiddenSeries": false,
           "id": 128,
@@ -6879,7 +7010,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 57
+            "y": 102
           },
           "heatmap": {},
           "hideZeroBuckets": true,
@@ -6889,7 +7020,6 @@
             "show": true
           },
           "links": [],
-          "options": {},
           "reverseYBuckets": false,
           "targets": [
             {
@@ -6939,7 +7069,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 57
+            "y": 102
           },
           "hiddenSeries": false,
           "id": 130,
@@ -7058,7 +7188,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 65
+            "y": 110
           },
           "heatmap": {},
           "hideZeroBuckets": true,
@@ -7068,12 +7198,12 @@
             "show": true
           },
           "links": [],
-          "options": {},
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+              "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "heatmap",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{le}}",
               "refId": "A"
@@ -7118,7 +7248,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 65
+            "y": 110
           },
           "hiddenSeries": false,
           "id": 132,
@@ -7149,29 +7279,33 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)) ",
+              "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "50%",
               "refId": "A"
             },
             {
-              "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "75%",
               "refId": "B"
             },
             {
-              "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "90%",
               "refId": "C"
             },
             {
-              "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "99%",
               "refId": "D"
@@ -7181,7 +7315,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Number of state resolution performed, by number of state groups involved (quantiles)",
+          "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -7233,6 +7367,7 @@
     "list": [
       {
         "current": {
+          "selected": false,
           "text": "Prometheus",
           "value": "Prometheus"
         },
@@ -7309,14 +7444,12 @@
       },
       {
         "allValue": null,
-        "current": {
-          "text": "matrix.org",
-          "value": "matrix.org"
-        },
+        "current": {},
         "datasource": "$datasource",
         "definition": "",
         "hide": 0,
         "includeAll": false,
+        "index": -1,
         "label": null,
         "multi": false,
         "name": "instance",
@@ -7335,17 +7468,13 @@
       {
         "allFormat": "regex wildcard",
         "allValue": "",
-        "current": {
-          "text": "synapse",
-          "value": [
-            "synapse"
-          ]
-        },
+        "current": {},
         "datasource": "$datasource",
         "definition": "",
         "hide": 0,
         "hideLabel": false,
         "includeAll": true,
+        "index": -1,
         "label": "Job",
         "multi": true,
         "multiFormat": "regex values",
@@ -7366,16 +7495,13 @@
       {
         "allFormat": "regex wildcard",
         "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
+        "current": {},
         "datasource": "$datasource",
         "definition": "",
         "hide": 0,
         "hideLabel": false,
         "includeAll": true,
+        "index": -1,
         "label": "",
         "multi": true,
         "multiFormat": "regex values",
@@ -7428,5 +7554,8 @@
   "timezone": "",
   "title": "Synapse",
   "uid": "000000012",
-  "version": 29
+  "variables": {
+    "list": []
+  },
+  "version": 32
 }
\ No newline at end of file
diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py
index 92736480eb..de33fac1c7 100644
--- a/contrib/graph/graph.py
+++ b/contrib/graph/graph.py
@@ -1,5 +1,13 @@
 from __future__ import print_function
 
+import argparse
+import cgi
+import datetime
+import json
+
+import pydot
+import urllib2
+
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,15 +23,6 @@ from __future__ import print_function
 # limitations under the License.
 
 
-import sqlite3
-import pydot
-import cgi
-import json
-import datetime
-import argparse
-import urllib2
-
-
 def make_name(pdu_id, origin):
     return "%s@%s" % (pdu_id, origin)
 
@@ -33,7 +32,7 @@ def make_graph(pdus, room, filename_prefix):
     node_map = {}
 
     origins = set()
-    colors = set(("red", "green", "blue", "yellow", "purple"))
+    colors = {"red", "green", "blue", "yellow", "purple"}
 
     for pdu in pdus:
         origins.add(pdu.get("origin"))
@@ -49,7 +48,7 @@ def make_graph(pdus, room, filename_prefix):
         try:
             c = colors.pop()
             color_map[o] = c
-        except:
+        except Exception:
             print("Run out of colours!")
             color_map[o] = "black"
 
diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py
index 4619f0e3c1..0980231e4a 100644
--- a/contrib/graph/graph2.py
+++ b/contrib/graph/graph2.py
@@ -13,12 +13,13 @@
 # limitations under the License.
 
 
-import sqlite3
-import pydot
+import argparse
 import cgi
-import json
 import datetime
-import argparse
+import json
+import sqlite3
+
+import pydot
 
 from synapse.events import FrozenEvent
 from synapse.util.frozenutils import unfreeze
@@ -98,7 +99,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except:
+            except Exception:
                 end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py
index 7f9e5374a6..91db98e7ef 100644
--- a/contrib/graph/graph3.py
+++ b/contrib/graph/graph3.py
@@ -1,5 +1,15 @@
 from __future__ import print_function
 
+import argparse
+import cgi
+import datetime
+
+import pydot
+import simplejson as json
+
+from synapse.events import FrozenEvent
+from synapse.util.frozenutils import unfreeze
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,18 +25,6 @@ from __future__ import print_function
 # limitations under the License.
 
 
-import pydot
-import cgi
-import simplejson as json
-import datetime
-import argparse
-
-from synapse.events import FrozenEvent
-from synapse.util.frozenutils import unfreeze
-
-from six import string_types
-
-
 def make_graph(file_name, room_id, file_prefix, limit):
     print("Reading lines")
     with open(file_name) as f:
@@ -62,7 +60,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
                 value = "<null>"
-            elif isinstance(value, string_types):
+            elif isinstance(value, str):
                 pass
             else:
                 value = json.dumps(value)
@@ -108,7 +106,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
-            except:
+            except Exception:
                 end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
diff --git a/contrib/jitsimeetbridge/jitsimeetbridge.py b/contrib/jitsimeetbridge/jitsimeetbridge.py
index 67fb2cd1a7..69aa74bd34 100644
--- a/contrib/jitsimeetbridge/jitsimeetbridge.py
+++ b/contrib/jitsimeetbridge/jitsimeetbridge.py
@@ -12,15 +12,15 @@ npm install jquery jsdom
 """
 from __future__ import print_function
 
-import gevent
-import grequests
-from BeautifulSoup import BeautifulSoup
 import json
-import urllib
 import subprocess
 import time
 
-# ACCESS_TOKEN="" #
+import gevent
+import grequests
+from BeautifulSoup import BeautifulSoup
+
+ACCESS_TOKEN = ""
 
 MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
 MYUSERNAME = "@davetest:matrix.org"
diff --git a/contrib/scripts/kick_users.py b/contrib/scripts/kick_users.py
index f57e6e7d25..372dbd9e4f 100755
--- a/contrib/scripts/kick_users.py
+++ b/contrib/scripts/kick_users.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 from __future__ import print_function
-from argparse import ArgumentParser
+
 import json
-import requests
 import sys
 import urllib
+from argparse import ArgumentParser
+
+import requests
 
 try:
     raw_input