diff options
34 files changed, 418 insertions, 224 deletions
diff --git a/.buildkite/docker-compose.py35.pg94.yaml b/.buildkite/docker-compose.py35.pg94.yaml deleted file mode 100644 index 978aedd115..0000000000 --- a/.buildkite/docker-compose.py35.pg94.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3.1' - -services: - - postgres: - image: postgres:9.4 - environment: - POSTGRES_PASSWORD: postgres - - testenv: - image: python:3.5 - depends_on: - - postgres - env_file: .env - environment: - SYNAPSE_POSTGRES_HOST: postgres - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - working_dir: /app - volumes: - - ..:/app diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6c6229a205..20c7aab5a7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -116,23 +116,6 @@ steps: - exit_status: 2 limit: 2 - - label: ":python: 3.5 / :postgres: 9.4" - env: - TRIAL_FLAGS: "-j 4" - command: - - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" - plugins: - - docker-compose#v2.1.0: - run: testenv - config: - - .buildkite/docker-compose.py35.pg94.yaml - retry: - automatic: - - exit_status: -1 - limit: 2 - - exit_status: 2 - limit: 2 - - label: ":python: 3.5 / :postgres: 9.5" env: TRIAL_FLAGS: "-j 4" diff --git a/AUTHORS.rst b/AUTHORS.rst index 3ea18eefcb..d8b4a846d8 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -72,3 +72,6 @@ Jason Robinson <jasonr at matrix.org> Joseph Weston <joseph at weston.cloud> + Add admin API for querying HS version + +Benjamin Saunders <ben.e.saunders at gmail dot com> + * Documentation improvements diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9a283ced6e..2c44422a0e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,21 +30,20 @@ use github's pull request workflow to review the contribution, and either ask you to make any refinements needed or merge it and make them ourselves. The changes will then land on master when we next do a release. -We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI -<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All -pull requests to synapse get automatically tested by Travis and CircleCI. -If your change breaks the build, this will be shown in GitHub, so please -keep an eye on the pull request for feedback. +We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite +<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration. +Buildkite builds need to be authorised by a maintainer. If your change breaks +the build, this will be shown in GitHub, so please keep an eye on the pull +request for feedback. To run unit tests in a local development environment, you can use: -- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for - SQLite-backed Synapse on Python 2.7. -- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5. +- ``tox -e py35`` (requires tox to be installed by ``pip install tox``) + for SQLite-backed Synapse on Python 3.5. - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6. -- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7 +- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6 (requires a running local PostgreSQL with access to create databases). -- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7 +- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5 (requires Docker). Entirely self-contained, recommended if you don't want to set up PostgreSQL yourself. diff --git a/README.rst b/README.rst index 5409f0c563..13e11a5773 100644 --- a/README.rst +++ b/README.rst @@ -340,8 +340,11 @@ log lines and looking for any 'Processed request' lines which take more than a few seconds to execute. Please let us know at #synapse:matrix.org if you see this failure mode so we can help debug it, however. -Help!! Synapse eats all my RAM! -------------------------------- +Help!! Synapse is slow and eats all my RAM/CPU! +----------------------------------------------- + +First, ensure you are running the latest version of Synapse, using Python 3 +with a PostgreSQL database. Synapse's architecture is quite RAM hungry currently - we deliberately cache a lot of recent room data and metadata in RAM in order to speed up @@ -352,14 +355,29 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage in memory constrained enviroments, or increased if performance starts to degrade. +However, degraded performance due to a low cache factor, common on +machines with slow disks, often leads to explosions in memory use due +backlogged requests. In this case, reducing the cache factor will make +things worse. Instead, try increasing it drastically. 2.0 is a good +starting value. + Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant -improvement in overall amount, and especially in terms of giving back RAM -to the OS. To use it, the library must simply be put in the LD_PRELOAD -environment variable when launching Synapse. On Debian, this can be done -by installing the ``libjemalloc1`` package and adding this line to -``/etc/default/matrix-synapse``:: +improvement in overall memory use, and especially in terms of giving back +RAM to the OS. To use it, the library must simply be put in the +LD_PRELOAD environment variable when launching Synapse. On Debian, this +can be done by installing the ``libjemalloc1`` package and adding this +line to ``/etc/default/matrix-synapse``:: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 This can make a significant difference on Python 2.7 - it's unclear how much of an improvement it provides on Python 3.x. + +If you're encountering high CPU use by the Synapse process itself, you +may be affected by a bug with presence tracking that leads to a +massive excess of outgoing federation requests (see `discussion +<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics +indicate that your server is also issuing far more outgoing federation +requests than can be accounted for by your users' activity, this is a +likely cause. The misbehavior can be worked around by setting +``use_presence: false`` in the Synapse config file. diff --git a/UPGRADE.rst b/UPGRADE.rst index 6032a505c9..1fb109a218 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -49,6 +49,33 @@ returned by the Client-Server API: # configured on port 443. curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:" +Upgrading to v1.1 +================= + +Synapse 1.1 removes support for older Python and PostgreSQL versions, as +outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_. + +Minimum Python Version +---------------------- + +Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or +Python 3.7 are recommended as they have improved internal string handling, +significantly reducing memory usage. + +If you use current versions of the Matrix.org-distributed Debian packages or +Docker images, action is not required. + +If you install Synapse in a Python virtual environment, please see "Upgrading to +v0.34.0" for notes on setting up a new virtualenv under Python 3. + +Minimum PostgreSQL Version +-------------------------- + +If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above. +Please see the +`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_ +for more details on upgrading your database. + Upgrading to v1.0 ================= @@ -71,11 +98,11 @@ server in a closed federation. This can be done in one of two ways:- * Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``. See the `sample configuration file <docs/sample_config.yaml>`_ -for more details on these settings. +for more details on these settings. Email ----- -When a user requests a password reset, Synapse will send an email to the +When a user requests a password reset, Synapse will send an email to the user to confirm the request. Previous versions of Synapse delegated the job of sending this email to an diff --git a/changelog.d/4276.misc b/changelog.d/4276.misc new file mode 100644 index 0000000000..285939a4b8 --- /dev/null +++ b/changelog.d/4276.misc @@ -0,0 +1 @@ +Improve README section on performance troubleshooting. diff --git a/changelog.d/5015.misc b/changelog.d/5015.misc new file mode 100644 index 0000000000..eeec85b92c --- /dev/null +++ b/changelog.d/5015.misc @@ -0,0 +1 @@ +Add logging to 3pid invite signature verification. diff --git a/changelog.d/5446.misc b/changelog.d/5446.misc new file mode 100644 index 0000000000..e5209be0a6 --- /dev/null +++ b/changelog.d/5446.misc @@ -0,0 +1 @@ +Update Python syntax in contrib/ to Python 3. diff --git a/changelog.d/5448.removal b/changelog.d/5448.removal new file mode 100644 index 0000000000..33b9859dae --- /dev/null +++ b/changelog.d/5448.removal @@ -0,0 +1 @@ +PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. diff --git a/changelog.d/5474.feature b/changelog.d/5474.feature new file mode 100644 index 0000000000..63d9b58734 --- /dev/null +++ b/changelog.d/5474.feature @@ -0,0 +1 @@ +Allow server admins to define implementations of extra rules for allowing or denying incoming events. diff --git a/changelog.d/5477.feature b/changelog.d/5477.feature new file mode 100644 index 0000000000..63d9b58734 --- /dev/null +++ b/changelog.d/5477.feature @@ -0,0 +1 @@ +Allow server admins to define implementations of extra rules for allowing or denying incoming events. diff --git a/changelog.d/5478.misc b/changelog.d/5478.misc new file mode 100644 index 0000000000..829bb1e521 --- /dev/null +++ b/changelog.d/5478.misc @@ -0,0 +1 @@ +The demo servers talk to each other again. diff --git a/changelog.d/5490.bugfix b/changelog.d/5490.bugfix new file mode 100644 index 0000000000..4242254c53 --- /dev/null +++ b/changelog.d/5490.bugfix @@ -0,0 +1 @@ +Fix failure to start under docker with SAML support enabled. \ No newline at end of file diff --git a/changelog.d/5493.misc b/changelog.d/5493.misc new file mode 100644 index 0000000000..365e49d634 --- /dev/null +++ b/changelog.d/5493.misc @@ -0,0 +1 @@ +Track deactivated accounts in the database. diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py index 4918fa1a9a..462f146113 100755 --- a/contrib/cmdclient/console.py +++ b/contrib/cmdclient/console.py @@ -15,6 +15,7 @@ # limitations under the License. """ Starts a synapse client console. """ +from __future__ import print_function from twisted.internet import reactor, defer, threads from http import TwistedHttpClient @@ -109,7 +110,7 @@ class SynapseCmd(cmd.Cmd): by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'. """ if len(line) == 0: - print json.dumps(self.config, indent=4) + print(json.dumps(self.config, indent=4)) return try: @@ -123,8 +124,8 @@ class SynapseCmd(cmd.Cmd): ] for key, valid_vals in config_rules: if key == args["key"] and args["val"] not in valid_vals: - print "%s value must be one of %s" % (args["key"], - valid_vals) + print("%s value must be one of %s" % (args["key"], + valid_vals)) return # toggle the http client verbosity @@ -133,11 +134,11 @@ class SynapseCmd(cmd.Cmd): # assign the new config self.config[args["key"]] = args["val"] - print json.dumps(self.config, indent=4) + print(json.dumps(self.config, indent=4)) save_config(self.config) except Exception as e: - print e + print(e) def do_register(self, line): """Registers for a new account: "register <userid> <noupdate>" @@ -153,7 +154,7 @@ class SynapseCmd(cmd.Cmd): pwd = getpass.getpass("Type a password for this user: ") pwd2 = getpass.getpass("Retype the password: ") if pwd != pwd2 or len(pwd) == 0: - print "Password mismatch." + print("Password mismatch.") pwd = None else: password = pwd @@ -174,12 +175,12 @@ class SynapseCmd(cmd.Cmd): # check the registration flows url = self._url() + "/register" json_res = yield self.http_client.do_request("GET", url) - print json.dumps(json_res, indent=4) + print(json.dumps(json_res, indent=4)) passwordFlow = None for flow in json_res["flows"]: if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]): - print "Unable to register: Home server requires captcha." + print("Unable to register: Home server requires captcha.") return if flow["type"] == "m.login.password" and "stages" not in flow: passwordFlow = flow @@ -189,7 +190,7 @@ class SynapseCmd(cmd.Cmd): return json_res = yield self.http_client.do_request("POST", url, data=data) - print json.dumps(json_res, indent=4) + print(json.dumps(json_res, indent=4)) if update_config and "user_id" in json_res: self.config["user"] = json_res["user_id"] self.config["token"] = json_res["access_token"] @@ -215,7 +216,7 @@ class SynapseCmd(cmd.Cmd): reactor.callFromThread(self._do_login, user, p) #print " got %s " % p except Exception as e: - print e + print(e) @defer.inlineCallbacks def _do_login(self, user, password): @@ -227,13 +228,13 @@ class SynapseCmd(cmd.Cmd): } url = self._url() + path json_res = yield self.http_client.do_request("POST", url, data=data) - print json_res + print(json_res) if "access_token" in json_res: self.config["user"] = user self.config["token"] = json_res["access_token"] save_config(self.config) - print "Login successful." + print("Login successful.") @defer.inlineCallbacks def _check_can_login(self): @@ -242,10 +243,10 @@ class SynapseCmd(cmd.Cmd): # submitting! url = self._url() + path json_res = yield self.http_client.do_request("GET", url) - print json_res + print(json_res) if "flows" not in json_res: - print "Failed to find any login flows." + print("Failed to find any login flows.") defer.returnValue(False) flow = json_res["flows"][0] # assume first is the one we want. @@ -275,9 +276,9 @@ class SynapseCmd(cmd.Cmd): json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, headers={'Content-Type': ['application/x-www-form-urlencoded']}) - print json_res + print(json_res) if 'sid' in json_res: - print "Token sent. Your session ID is %s" % (json_res['sid']) + print("Token sent. Your session ID is %s" % (json_res['sid'])) def do_emailvalidate(self, line): """Validate and associate a third party ID @@ -297,7 +298,7 @@ class SynapseCmd(cmd.Cmd): json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, headers={'Content-Type': ['application/x-www-form-urlencoded']}) - print json_res + print(json_res) def do_3pidbind(self, line): """Validate and associate a third party ID @@ -317,7 +318,7 @@ class SynapseCmd(cmd.Cmd): json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, headers={'Content-Type': ['application/x-www-form-urlencoded']}) - print json_res + print(json_res) def do_join(self, line): """Joins a room: "join <roomid>" """ @@ -325,7 +326,7 @@ class SynapseCmd(cmd.Cmd): args = self._parse(line, ["roomid"], force_keys=True) self._do_membership_change(args["roomid"], "join", self._usr()) except Exception as e: - print e + print(e) def do_joinalias(self, line): try: @@ -333,7 +334,7 @@ class SynapseCmd(cmd.Cmd): path = "/join/%s" % urllib.quote(args["roomname"]) reactor.callFromThread(self._run_and_pprint, "POST", path, {}) except Exception as e: - print e + print(e) def do_topic(self, line): """"topic [set|get] <roomid> [<newtopic>]" @@ -343,17 +344,17 @@ class SynapseCmd(cmd.Cmd): try: args = self._parse(line, ["action", "roomid", "topic"]) if "action" not in args or "roomid" not in args: - print "Must specify set|get and a room ID." + print("Must specify set|get and a room ID.") return if args["action"].lower() not in ["set", "get"]: - print "Must specify set|get, not %s" % args["action"] + print("Must specify set|get, not %s" % args["action"]) return path = "/rooms/%s/topic" % urllib.quote(args["roomid"]) if args["action"].lower() == "set": if "topic" not in args: - print "Must specify a new topic." + print("Must specify a new topic.") return body = { "topic": args["topic"] @@ -362,7 +363,7 @@ class SynapseCmd(cmd.Cmd): elif args["action"].lower() == "get": reactor.callFromThread(self._run_and_pprint, "GET", path) except Exception as e: - print e + print(e) def do_invite(self, line): """Invite a user to a room: "invite <userid> <roomid>" """ @@ -373,7 +374,7 @@ class SynapseCmd(cmd.Cmd): reactor.callFromThread(self._do_invite, args["roomid"], user_id) except Exception as e: - print e + print(e) @defer.inlineCallbacks def _do_invite(self, roomid, userstring): @@ -393,29 +394,29 @@ class SynapseCmd(cmd.Cmd): if 'public_key' in pubKeyObj: pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder) else: - print "No public key found in pubkey response!" + print("No public key found in pubkey response!") sigValid = False if pubKey: for signame in json_res['signatures']: if signame not in TRUSTED_ID_SERVERS: - print "Ignoring signature from untrusted server %s" % (signame) + print("Ignoring signature from untrusted server %s" % (signame)) else: try: verify_signed_json(json_res, signame, pubKey) sigValid = True - print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame) + print("Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)) break except SignatureVerifyException as e: - print "Invalid signature from %s" % (signame) - print e + print("Invalid signature from %s" % (signame)) + print(e) if sigValid: - print "Resolved 3pid %s to %s" % (userstring, json_res['mxid']) + print("Resolved 3pid %s to %s" % (userstring, json_res['mxid'])) mxid = json_res['mxid'] else: - print "Got association for %s but couldn't verify signature" % (userstring) + print("Got association for %s but couldn't verify signature" % (userstring)) if not mxid: mxid = "@" + userstring + ":" + self._domain() @@ -428,7 +429,7 @@ class SynapseCmd(cmd.Cmd): args = self._parse(line, ["roomid"], force_keys=True) self._do_membership_change(args["roomid"], "leave", self._usr()) except Exception as e: - print e + print(e) def do_send(self, line): """Sends a message. "send <roomid> <body>" """ @@ -453,10 +454,10 @@ class SynapseCmd(cmd.Cmd): """ args = self._parse(line, ["type", "roomid", "qp"]) if not "type" in args or not "roomid" in args: - print "Must specify type and room ID." + print("Must specify type and room ID.") return if args["type"] not in ["members", "messages"]: - print "Unrecognised type: %s" % args["type"] + print("Unrecognised type: %s" % args["type"]) return room_id = args["roomid"] path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"]) @@ -468,7 +469,7 @@ class SynapseCmd(cmd.Cmd): key_value = key_value_str.split("=") qp[key_value[0]] = key_value[1] except: - print "Bad query param: %s" % key_value + print("Bad query param: %s" % key_value) return reactor.callFromThread(self._run_and_pprint, "GET", path, @@ -508,14 +509,14 @@ class SynapseCmd(cmd.Cmd): args = self._parse(line, ["method", "path", "data"]) # sanity check if "method" not in args or "path" not in args: - print "Must specify path and method." + print("Must specify path and method.") return args["method"] = args["method"].upper() valid_methods = ["PUT", "GET", "POST", "DELETE", "XPUT", "XGET", "XPOST", "XDELETE"] if args["method"] not in valid_methods: - print "Unsupported method: %s" % args["method"] + print("Unsupported method: %s" % args["method"]) return if "data" not in args: @@ -524,7 +525,7 @@ class SynapseCmd(cmd.Cmd): try: args["data"] = json.loads(args["data"]) except Exception as e: - print "Data is not valid JSON. %s" % e + print("Data is not valid JSON. %s" % e) return qp = {"access_token": self._tok()} @@ -553,7 +554,7 @@ class SynapseCmd(cmd.Cmd): try: timeout = int(args["timeout"]) except ValueError: - print "Timeout must be in milliseconds." + print("Timeout must be in milliseconds.") return reactor.callFromThread(self._do_event_stream, timeout) @@ -566,7 +567,7 @@ class SynapseCmd(cmd.Cmd): "timeout": str(timeout), "from": self.event_stream_token }) - print json.dumps(res, indent=4) + print(json.dumps(res, indent=4)) if "chunk" in res: for event in res["chunk"]: @@ -669,9 +670,9 @@ class SynapseCmd(cmd.Cmd): data=data, qparams=query_params) if alt_text: - print alt_text + print(alt_text) else: - print json.dumps(json_res, indent=4) + print(json.dumps(json_res, indent=4)) def save_config(config): @@ -680,16 +681,16 @@ def save_config(config): def main(server_url, identity_server_url, username, token, config_path): - print "Synapse command line client" - print "===========================" - print "Server: %s" % server_url - print "Type 'help' to get started." - print "Close this console with CTRL+C then CTRL+D." + print("Synapse command line client") + print("===========================") + print("Server: %s" % server_url) + print("Type 'help' to get started.") + print("Close this console with CTRL+C then CTRL+D.") if not username or not token: - print "- 'register <username>' - Register an account" - print "- 'stream' - Connect to the event stream" - print "- 'create <roomid>' - Create a room" - print "- 'send <roomid> <message>' - Send a message" + print("- 'register <username>' - Register an account") + print("- 'stream' - Connect to the event stream") + print("- 'create <roomid>' - Create a room") + print("- 'send <roomid> <message>' - Send a message") http_client = TwistedHttpClient() # the command line client @@ -705,7 +706,7 @@ def main(server_url, identity_server_url, username, token, config_path): http_client.verbose = "on" == syn_cmd.config["verbose"] except: pass - print "Loaded config from %s" % config_path + print("Loaded config from %s" % config_path) except: pass @@ -736,7 +737,7 @@ if __name__ == '__main__': args = parser.parse_args() if not args.server: - print "You must supply a server URL to communicate with." + print("You must supply a server URL to communicate with.") parser.print_help() sys.exit(1) diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py index c833f3f318..1bd600e148 100644 --- a/contrib/cmdclient/http.py +++ b/contrib/cmdclient/http.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function from twisted.web.client import Agent, readBody from twisted.web.http_headers import Headers from twisted.internet import defer, reactor @@ -141,15 +142,15 @@ class TwistedHttpClient(HttpClient): headers_dict["User-Agent"] = ["Synapse Cmd Client"] retries_left = 5 - print "%s to %s with headers %s" % (method, url, headers_dict) + print("%s to %s with headers %s" % (method, url, headers_dict)) if self.verbose and producer: if "password" in producer.data: temp = producer.data["password"] producer.data["password"] = "[REDACTED]" - print json.dumps(producer.data, indent=4) + print(json.dumps(producer.data, indent=4)) producer.data["password"] = temp else: - print json.dumps(producer.data, indent=4) + print(json.dumps(producer.data, indent=4)) while True: try: @@ -161,7 +162,7 @@ class TwistedHttpClient(HttpClient): ) break except Exception as e: - print "uh oh: %s" % e + print("uh oh: %s" % e) if retries_left: yield self.sleep(2 ** (5 - retries_left)) retries_left -= 1 @@ -169,8 +170,8 @@ class TwistedHttpClient(HttpClient): raise e if self.verbose: - print "Status %s %s" % (response.code, response.phrase) - print pformat(list(response.headers.getAllRawHeaders())) + print("Status %s %s" % (response.code, response.phrase)) + print(pformat(list(response.headers.getAllRawHeaders()))) defer.returnValue(response) def sleep(self, seconds): diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index afd1d446b4..e174ff5026 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,7 +49,7 @@ def make_graph(pdus, room, filename_prefix): c = colors.pop() color_map[o] = c except: - print "Run out of colours!" + print("Run out of colours!") color_map[o] = "black" graph = pydot.Dot(graph_name="Test") @@ -93,7 +94,7 @@ def make_graph(pdus, room, filename_prefix): end_name = make_name(i, o) if end_name not in node_map: - print "%s not in nodes" % end_name + print("%s not in nodes" % end_name) continue edge = pydot.Edge(node_map[start_name], node_map[end_name]) diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 7d3b4d7eb6..fe1dc81e90 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,19 +27,19 @@ from six import string_types def make_graph(file_name, room_id, file_prefix, limit): - print "Reading lines" + print("Reading lines") with open(file_name) as f: lines = f.readlines() - print "Read lines" + print("Read lines") events = [FrozenEvent(json.loads(line)) for line in lines] - print "Loaded events." + print("Loaded events.") events.sort(key=lambda e: e.depth) - print "Sorted events" + print("Sorted events") if limit: events = events[-int(limit):] @@ -55,7 +56,7 @@ def make_graph(file_name, room_id, file_prefix, limit): content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4) content = content.replace("\n", "<br/>\n") - print content + print(content) content = [] for key, value in unfreeze(event.get_dict()["content"]).items(): if value is None: @@ -74,7 +75,7 @@ def make_graph(file_name, room_id, file_prefix, limit): content = "<br/>\n".join(content) - print content + print(content) label = ( "<" @@ -102,7 +103,7 @@ def make_graph(file_name, room_id, file_prefix, limit): node_map[event.event_id] = node graph.add_node(node) - print "Created Nodes" + print("Created Nodes") for event in events: for prev_id, _ in event.prev_events: @@ -120,15 +121,15 @@ def make_graph(file_name, room_id, file_prefix, limit): edge = pydot.Edge(node_map[event.event_id], end_node) graph.add_edge(edge) - print "Created edges" + print("Created edges") graph.write('%s.dot' % file_prefix, format='raw', prog='dot') - print "Created Dot" + print("Created Dot") graph.write_svg("%s.svg" % file_prefix, prog='dot') - print "Created svg" + print("Created svg") if __name__ == "__main__": parser = argparse.ArgumentParser( diff --git a/contrib/jitsimeetbridge/jitsimeetbridge.py b/contrib/jitsimeetbridge/jitsimeetbridge.py index 15f8e1c48b..e82d1be5d2 100644 --- a/contrib/jitsimeetbridge/jitsimeetbridge.py +++ b/contrib/jitsimeetbridge/jitsimeetbridge.py @@ -8,8 +8,9 @@ we set the remote SDP at which point the stream ends. Our video never gets to the bridge. Requires: -npm install jquery jsdom +npm install jquery jsdom """ +from __future__ import print_function import gevent import grequests @@ -51,7 +52,7 @@ class TrivialMatrixClient: req = grequests.get(url) resps = grequests.map([req]) obj = json.loads(resps[0].content) - print "incoming from matrix",obj + print("incoming from matrix",obj) if 'end' not in obj: continue self.token = obj['end'] @@ -60,22 +61,22 @@ class TrivialMatrixClient: def joinRoom(self, roomId): url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token - print url + print(url) headers={ 'Content-Type': 'application/json' } req = grequests.post(url, headers=headers, data='{}') resps = grequests.map([req]) obj = json.loads(resps[0].content) - print "response: ",obj + print("response: ",obj) def sendEvent(self, roomId, evType, event): url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token - print url - print json.dumps(event) + print(url) + print(json.dumps(event)) headers={ 'Content-Type': 'application/json' } req = grequests.post(url, headers=headers, data=json.dumps(event)) resps = grequests.map([req]) obj = json.loads(resps[0].content) - print "response: ",obj + print("response: ",obj) @@ -85,31 +86,31 @@ xmppClients = {} def matrixLoop(): while True: ev = matrixCli.getEvent() - print ev + print(ev) if ev['type'] == 'm.room.member': - print 'membership event' + print('membership event') if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: roomId = ev['room_id'] - print "joining room %s" % (roomId) + print("joining room %s" % (roomId)) matrixCli.joinRoom(roomId) elif ev['type'] == 'm.room.message': if ev['room_id'] in xmppClients: - print "already have a bridge for that user, ignoring" + print("already have a bridge for that user, ignoring") continue - print "got message, connecting" + print("got message, connecting") xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) gevent.spawn(xmppClients[ev['room_id']].xmppLoop) elif ev['type'] == 'm.call.invite': - print "Incoming call" + print("Incoming call") #sdp = ev['content']['offer']['sdp'] #print "sdp: %s" % (sdp) #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) elif ev['type'] == 'm.call.answer': - print "Call answered" + print("Call answered") sdp = ev['content']['answer']['sdp'] if ev['room_id'] not in xmppClients: - print "We didn't have a call for that room" + print("We didn't have a call for that room") continue # should probably check call ID too xmppCli = xmppClients[ev['room_id']] @@ -146,7 +147,7 @@ class TrivialXmppClient: return obj def sendAnswer(self, answer): - print "sdp from matrix client",answer + print("sdp from matrix client",answer) p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) jingle, out_err = p.communicate(answer) jingle = jingle % { @@ -156,28 +157,28 @@ class TrivialXmppClient: 'responder': self.jid, 'sid': self.callsid } - print "answer jingle from sdp",jingle + print("answer jingle from sdp",jingle) res = self.sendIq(jingle) - print "reply from answer: ",res + print("reply from answer: ",res) self.ssrcs = {} jingleSoup = BeautifulSoup(jingle) for cont in jingleSoup.iq.jingle.findAll('content'): if cont.description: self.ssrcs[cont['name']] = cont.description['ssrc'] - print "my ssrcs:",self.ssrcs + print("my ssrcs:",self.ssrcs) gevent.joinall([ gevent.spawn(self.advertiseSsrcs) ]) def advertiseSsrcs(self): - time.sleep(7) - print "SSRC spammer started" + time.sleep(7) + print("SSRC spammer started") while self.running: ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] } res = self.sendIq(ssrcMsg) - print "reply from ssrc announce: ",res + print("reply from ssrc announce: ",res) time.sleep(10) @@ -186,19 +187,19 @@ class TrivialXmppClient: self.matrixCallId = time.time() res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST)) - print res + print(res) self.sid = res.body['sid'] - print "sid %s" % (self.sid) + print("sid %s" % (self.sid)) res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>") res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST)) res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>") - print res + print(res) self.jid = res.body.iq.bind.jid.string - print "jid: %s" % (self.jid) + print("jid: %s" % (self.jid)) self.shortJid = self.jid.split('-')[0] res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>") @@ -217,13 +218,13 @@ class TrivialXmppClient: if p.c and p.c.nick: u['nick'] = p.c.nick.string self.muc['users'].append(u) - print "muc: ",self.muc + print("muc: ",self.muc) # wait for stuff while True: - print "waiting..." + print("waiting...") res = self.sendIq("") - print "got from stream: ",res + print("got from stream: ",res) if res.body.iq: jingles = res.body.iq.findAll('jingle') if len(jingles): @@ -232,15 +233,15 @@ class TrivialXmppClient: elif 'type' in res.body and res.body['type'] == 'terminate': self.running = False del xmppClients[self.matrixRoom] - return + return def handleInvite(self, jingle): self.initiator = jingle['initiator'] self.callsid = jingle['sid'] p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - print "raw jingle invite",str(jingle) + print("raw jingle invite",str(jingle)) sdp, out_err = p.communicate(str(jingle)) - print "transformed remote offer sdp",sdp + print("transformed remote offer sdp",sdp) inviteEvent = { 'offer': { 'type': 'offer', @@ -252,7 +253,7 @@ class TrivialXmppClient: } matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent) -matrixCli = TrivialMatrixClient(ACCESS_TOKEN) +matrixCli = TrivialMatrixClient(ACCESS_TOKEN) # Undefined name gevent.joinall([ gevent.spawn(matrixLoop) diff --git a/contrib/scripts/kick_users.py b/contrib/scripts/kick_users.py index 5dfaec3ad0..b4a14385d0 100755 --- a/contrib/scripts/kick_users.py +++ b/contrib/scripts/kick_users.py @@ -1,10 +1,16 @@ #!/usr/bin/env python +from __future__ import print_function from argparse import ArgumentParser import json import requests import sys import urllib +try: + raw_input +except NameError: # Python 3 + raw_input = input + def _mkurl(template, kws): for key in kws: template = template.replace(key, kws[key]) @@ -13,7 +19,7 @@ def _mkurl(template, kws): def main(hs, room_id, access_token, user_id_prefix, why): if not why: why = "Automated kick." - print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix) + print("Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)) room_state_url = _mkurl( "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN", { @@ -22,13 +28,13 @@ def main(hs, room_id, access_token, user_id_prefix, why): "$TOKEN": access_token } ) - print "Getting room state => %s" % room_state_url + print("Getting room state => %s" % room_state_url) res = requests.get(room_state_url) - print "HTTP %s" % res.status_code + print("HTTP %s" % res.status_code) state_events = res.json() if "error" in state_events: - print "FATAL" - print state_events + print("FATAL") + print(state_events) return kick_list = [] @@ -44,15 +50,15 @@ def main(hs, room_id, access_token, user_id_prefix, why): kick_list.append(event["state_key"]) if len(kick_list) == 0: - print "No user IDs match the prefix '%s'" % user_id_prefix + print("No user IDs match the prefix '%s'" % user_id_prefix) return - print "The following user IDs will be kicked from %s" % room_name + print("The following user IDs will be kicked from %s" % room_name) for uid in kick_list: - print uid + print(uid) doit = raw_input("Continue? [Y]es\n") if len(doit) > 0 and doit.lower() == 'y': - print "Kicking members..." + print("Kicking members...") # encode them all kick_list = [urllib.quote(uid) for uid in kick_list] for uid in kick_list: @@ -69,14 +75,14 @@ def main(hs, room_id, access_token, user_id_prefix, why): "membership": "leave", "reason": why } - print "Kicking %s" % uid + print("Kicking %s" % uid) res = requests.put(kick_url, data=json.dumps(kick_body)) if res.status_code != 200: - print "ERROR: HTTP %s" % res.status_code + print("ERROR: HTTP %s" % res.status_code) if res.json().get("error"): - print "ERROR: JSON %s" % res.json() - - + print("ERROR: JSON %s" % res.json()) + + if __name__ == "__main__": parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.") diff --git a/demo/README b/demo/README index 0b584ceb15..0bec820ad6 100644 --- a/demo/README +++ b/demo/README @@ -1,9 +1,13 @@ +DO NOT USE THESE DEMO SERVERS IN PRODUCTION + Requires you to have done: python setup.py develop -The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. -It will also start a web server on port 8000 pointed at the webclient. +The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. + +To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs +and are configured in a highly insecure way. Do not use these configuration files in production. stop.sh will stop the synapse servers and the webclient. diff --git a/demo/start.sh b/demo/start.sh index 5c3a8fe61f..1c4f12d0bb 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -27,8 +27,70 @@ for port in 8080 8081 8082; do --config-path "$DIR/etc/$port.config" \ --report-stats no - printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config - echo 'enable_registration: true' >> $DIR/etc/$port.config + if ! grep -F "Customisation made by demo/start.sh" -q $DIR/etc/$port.config; then + printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config + + echo 'enable_registration: true' >> $DIR/etc/$port.config + + # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't + # accidentaly bork me with your fancy settings. + listeners=$(cat <<-PORTLISTENERS + # Configure server to listen on both $https_port and $port + # This overides some of the default settings above + listeners: + - port: $https_port + type: http + tls: true + resources: + - names: [client, federation] + + - port: $port + tls: false + bind_addresses: ['::1', '127.0.0.1'] + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false + PORTLISTENERS + ) + echo "${listeners}" >> $DIR/etc/$port.config + + # Disable tls for the servers + printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config + echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config + echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config + echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config + + # Set tls paths + echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config + echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config + + # Generate tls keys + openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix" + + # Ignore keys from the trusted keys server + echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config + echo 'trusted_key_servers:' >> $DIR/etc/$port.config + echo ' - server_name: "matrix.org"' >> $DIR/etc/$port.config + echo ' accept_keys_insecurely: true' >> $DIR/etc/$port.config + + # Reduce the blacklist + blacklist=$(cat <<-BLACK + # Set the blacklist so that it doesn't include 127.0.0.1 + federation_ip_range_blacklist: + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '169.254.0.0/16' + - '::1/128' + - 'fe80::/64' + - 'fc00::/7' + BLACK + ) + echo "${blacklist}" >> $DIR/etc/$port.config + fi # Check script parameters if [ $# -eq 1 ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index c35da67a2a..24921eb098 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -57,6 +57,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \ FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 +# xmlsec is required for saml support RUN apk add --no-cache --virtual .runtime_deps \ libffi \ libjpeg-turbo \ @@ -64,7 +65,8 @@ RUN apk add --no-cache --virtual .runtime_deps \ libxslt \ libpq \ zlib \ - su-exec + su-exec \ + xmlsec COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py diff --git a/docker/Dockerfile-pgtests b/docker/Dockerfile-pgtests index 7da8eeb9eb..3bfee845c6 100644 --- a/docker/Dockerfile-pgtests +++ b/docker/Dockerfile-pgtests @@ -3,10 +3,10 @@ FROM matrixdotorg/sytest:latest # The Sytest image doesn't come with python, so install that -RUN apt-get -qq install -y python python-dev python-pip +RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip # We need tox to run the tests in run_pg_tests.sh -RUN pip install tox +RUN python3 -m pip install tox ADD run_pg_tests.sh /pg_tests.sh ENTRYPOINT /pg_tests.sh diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh index e77424c41a..d18d1e4c8e 100755 --- a/docker/run_pg_tests.sh +++ b/docker/run_pg_tests.sh @@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' # Run the tests cd /src export TRIAL_FLAGS="-j 4" -tox --workdir=/tmp -e py27-postgres +tox --workdir=/tmp -e py35-postgres diff --git a/docs/postgres.rst b/docs/postgres.rst index e81e10403f..33f58e3ace 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -1,7 +1,7 @@ Using Postgres -------------- -Postgres version 9.4 or later is known to work. +Postgres version 9.5 or later is known to work. Install postgres client libraries ================================= @@ -16,7 +16,7 @@ a postgres database. * For other pre-built packages, please consult the documentation from the relevant package. -* If you installed synapse `in a virtualenv +* If you installed synapse `in a virtualenv <../INSTALL.md#installing-from-source>`_, you can install the library with:: ~/synapse/env/bin/pip install matrix-synapse[postgres] diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py index 9f98d51523..50ceeb1e8e 100644 --- a/synapse/events/third_party_rules.py +++ b/synapse/events/third_party_rules.py @@ -17,8 +17,8 @@ from twisted.internet import defer class ThirdPartyEventRules(object): - """Allows server admins to provide a Python module implementing an extra set of rules - to apply when processing events. + """Allows server admins to provide a Python module implementing an extra + set of rules to apply when processing events. This is designed to help admins of closed federations with enforcing custom behaviours. @@ -35,7 +35,10 @@ class ThirdPartyEventRules(object): module, config = hs.config.third_party_event_rules if module is not None: - self.third_party_rules = module(config=config) + self.third_party_rules = module( + config=config, + http_client=hs.get_simple_http_client(), + ) @defer.inlineCallbacks def check_event_allowed(self, event, context): @@ -46,7 +49,7 @@ class ThirdPartyEventRules(object): context (synapse.events.snapshot.EventContext): The context of the event. Returns: - defer.Deferred(bool), True if the event should be allowed, False if not. + defer.Deferred[bool]: True if the event should be allowed, False if not. """ if self.third_party_rules is None: defer.returnValue(True) @@ -60,3 +63,52 @@ class ThirdPartyEventRules(object): ret = yield self.third_party_rules.check_event_allowed(event, state_events) defer.returnValue(ret) + + @defer.inlineCallbacks + def on_create_room(self, requester, config, is_requester_admin): + """Intercept requests to create room to allow, deny or update the + request config. + + Args: + requester (Requester) + config (dict): The creation config from the client. + is_requester_admin (bool): If the requester is an admin + + Returns: + defer.Deferred + """ + + if self.third_party_rules is None: + return + + yield self.third_party_rules.on_create_room( + requester, config, is_requester_admin + ) + + @defer.inlineCallbacks + def check_threepid_can_be_invited(self, medium, address, room_id): + """Check if a provided 3PID can be invited in the given room. + + Args: + medium (str): The 3PID's medium. + address (str): The 3PID's address. + room_id (str): The room we want to invite the threepid to. + + Returns: + defer.Deferred[bool], True if the 3PID can be invited, False if not. + """ + + if self.third_party_rules is None: + defer.returnValue(True) + + state_ids = yield self.store.get_filtered_current_state_ids(room_id) + room_state_events = yield self.store.get_events(state_ids.values()) + + state_events = {} + for key, event_id in state_ids.items(): + state_events[key] = room_state_events[event_id] + + ret = yield self.third_party_rules.check_threepid_can_be_invited( + medium, address, state_events, + ) + defer.returnValue(ret) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 93e064cda3..51d7eb274b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2744,25 +2744,55 @@ class FederationHandler(BaseHandler): if not invite_event: raise AuthError(403, "Could not find invite") + logger.debug("Checking auth on event %r", event.content) + last_exception = None + # for each public key in the 3pid invite event for public_key_object in self.hs.get_auth().get_public_keys(invite_event): try: + # for each sig on the third_party_invite block of the actual invite for server, signature_block in signed["signatures"].items(): for key_name, encoded_signature in signature_block.items(): if not key_name.startswith("ed25519:"): continue - public_key = public_key_object["public_key"] - verify_key = decode_verify_key_bytes( - key_name, - decode_base64(public_key) + logger.debug( + "Attempting to verify sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, ) - verify_signed_json(signed, server, verify_key) - if "key_validity_url" in public_key_object: - yield self._check_key_revocation( - public_key, + + try: + public_key = public_key_object["public_key"] + verify_key = decode_verify_key_bytes( + key_name, + decode_base64(public_key) + ) + verify_signed_json(signed, server, verify_key) + logger.debug( + "Successfully verified sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, + ) + except Exception: + logger.info( + "Failed to verify sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, + ) + raise + try: + if "key_validity_url" in public_key_object: + yield self._check_key_revocation( + public_key, + public_key_object["key_validity_url"] + ) + except Exception: + logger.info( + "Failed to query key_validity_url %s", public_key_object["key_validity_url"] ) + raise return except Exception as e: last_exception = e diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4a17911a87..74793bab33 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -75,6 +75,10 @@ class RoomCreationHandler(BaseHandler): # linearizer to stop two upgrades happening at once self._upgrade_linearizer = Linearizer("room_upgrade_linearizer") + self._server_notices_mxid = hs.config.server_notices_mxid + + self.third_party_event_rules = hs.get_third_party_event_rules() + @defer.inlineCallbacks def upgrade_room(self, requester, old_room_id, new_version): """Replace a room with a new room with a different version @@ -470,7 +474,26 @@ class RoomCreationHandler(BaseHandler): yield self.auth.check_auth_blocking(user_id) - if not self.spam_checker.user_may_create_room(user_id): + if (self._server_notices_mxid is not None and + requester.user.to_string() == self._server_notices_mxid): + # allow the server notices mxid to create rooms + is_requester_admin = True + else: + is_requester_admin = yield self.auth.is_server_admin( + requester.user, + ) + + # Check whether the third party rules allows/changes the room create + # request. + yield self.third_party_event_rules.on_create_room( + requester, + config, + is_requester_admin=is_requester_admin, + ) + + if not is_requester_admin and not self.spam_checker.user_may_create_room( + user_id, + ): raise SynapseError(403, "You are not permitted to create rooms") if ratelimit: diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 93ac986c86..458902bb7e 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -72,6 +72,7 @@ class RoomMemberHandler(object): self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() + self.third_party_event_rules = hs.get_third_party_event_rules() self._server_notices_mxid = self.config.server_notices_mxid self._enable_lookup = hs.config.enable_3pid_lookup self.allow_per_room_profiles = self.config.allow_per_room_profiles @@ -723,6 +724,15 @@ class RoomMemberHandler(object): # can't just rely on the standard ratelimiting of events. yield self.base_handler.ratelimit(requester) + can_invite = yield self.third_party_event_rules.check_threepid_can_be_invited( + medium, address, room_id, + ) + if not can_invite: + raise SynapseError( + 403, "This third-party identifier can not be invited in this room", + Codes.FORBIDDEN, + ) + invitee = yield self._lookup_3pid( id_server, medium, address ) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 1b97ee74e3..289b6bc281 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -45,6 +45,10 @@ class PostgresEngine(object): # together. For example, version 8.1.5 will be returned as 80105 self._version = db_conn.server_version + # Are we on a supported PostgreSQL version? + if self._version < 90500: + raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") + db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) @@ -64,9 +68,9 @@ class PostgresEngine(object): @property def can_native_upsert(self): """ - Can we use native UPSERTs? This requires PostgreSQL 9.5+. + Can we use native UPSERTs? """ - return self._version >= 90500 + return True def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d36917e4d6..0b3c656e90 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -662,7 +662,7 @@ class RegistrationStore( for user in rows: if not user["count_tokens"] and not user["count_threepids"]: - self.set_user_deactivated_status_txn(txn, user["user_id"], True) + self.set_user_deactivated_status_txn(txn, user["name"], True) rows_processed_nb += 1 logger.info("Marked %d rows as deactivated", rows_processed_nb) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index ff49eaae02..10a27c207a 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -341,29 +341,7 @@ class SearchStore(BackgroundUpdateStore): for entry in entries ) - # inserts to a GIN index are normally batched up into a pending - # list, and then all committed together once the list gets to a - # certain size. The trouble with that is that postgres (pre-9.5) - # uses work_mem to determine the length of the list, and work_mem - # is typically very large. - # - # We therefore reduce work_mem while we do the insert. - # - # (postgres 9.5 uses the separate gin_pending_list_limit setting, - # so doesn't suffer the same problem, but changing work_mem will - # be harmless) - # - # Note that we don't need to worry about restoring it on - # exception, because exceptions will cause the transaction to be - # rolled back, including the effects of the SET command. - # - # Also: we use SET rather than SET LOCAL because there's lots of - # other stuff going on in this transaction, which want to have the - # normal work_mem setting. - - txn.execute("SET work_mem='256kB'") txn.executemany(sql, args) - txn.execute("RESET work_mem") elif isinstance(self.database_engine, Sqlite3Engine): sql = ( |