From 046b659ce245272eb0c38cb1ee4206b5cb9e4f0c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 17 Aug 2017 16:54:27 +0100 Subject: Improvements to the federation test client Make it read the config file, primarily. --- scripts-dev/federation_client.py | 65 +++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 7 deletions(-) mode change 100644 => 100755 scripts-dev/federation_client.py (limited to 'scripts-dev') diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py old mode 100644 new mode 100755 index d1ab42d3af..c840acb923 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -1,10 +1,30 @@ +#!/usr/bin/env python +# +# Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse import nacl.signing import json import base64 import requests import sys import srvlookup - +import yaml def encode_base64(input_bytes): """Encode bytes as a base64 string without any padding.""" @@ -120,11 +140,13 @@ def get_json(origin_name, origin_key, destination, path): origin_name, key, sig, ) authorization_headers.append(bytes(header)) - sys.stderr.write(header) - sys.stderr.write("\n") + print ("Authorization: %s" % header, file=sys.stderr) + + dest = lookup(destination, path) + print ("Requesting %s" % dest, file=sys.stderr) result = requests.get( - lookup(destination, path), + dest, headers={"Authorization": authorization_headers[0]}, verify=False, ) @@ -133,17 +155,46 @@ def get_json(origin_name, origin_key, destination, path): def main(): - origin_name, keyfile, destination, path = sys.argv[1:] + parser = argparse.ArgumentParser( + description= + "Signs and sends a federation request to a matrix homeserver", + ) + + parser.add_argument( + "-c", "--config", + type=argparse.FileType('r'), + default="homeserver.yaml", + help="Path to server config file. Used to read in server name and key " + "file", + ) + + parser.add_argument( + "-d", "--destination", + default="matrix.org", + help="name of the remote homeserver. We will do SRV lookups and " + "connect appropriately.", + ) + + parser.add_argument( + "path", + help="request path. We will add '/_matrix/federation/v1/' to this." + ) + + args = parser.parse_args() + + config = yaml.safe_load(args.config) + origin_name = config['server_name'] + keyfile = config['signing_key_path'] with open(keyfile) as f: key = read_signing_keys(f)[0] result = get_json( - origin_name, key, destination, "/_matrix/federation/v1/" + path + origin_name, key, args.destination, "/_matrix/federation/v1/" + args.path ) json.dump(result, sys.stdout) - print "" + print ("") if __name__ == "__main__": main() -- cgit 1.5.1 From a04c6bbf8f31aaafa0a67813621b85cb26179d34 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 Aug 2017 11:19:30 +0100 Subject: test federation client: Allow server-name and key-file as options so that you don't necessarily need a config file. --- scripts-dev/federation_client.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index c840acb923..82a90ef6fa 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -160,12 +160,23 @@ def main(): "Signs and sends a federation request to a matrix homeserver", ) + parser.add_argument( + "-N", "--server-name", + help="Name to give as the local homeserver. If unspecified, will be " + "read from the config file.", + ) + + parser.add_argument( + "-k", "--signing-key-path", + help="Path to the file containing the private ed25519 key to sign the " + "request with.", + ) + parser.add_argument( "-c", "--config", - type=argparse.FileType('r'), default="homeserver.yaml", - help="Path to server config file. Used to read in server name and key " - "file", + help="Path to server config file. Ignored if --server-name and " + "--signing-key-path are both given.", ) parser.add_argument( @@ -182,19 +193,28 @@ def main(): args = parser.parse_args() - config = yaml.safe_load(args.config) - origin_name = config['server_name'] - keyfile = config['signing_key_path'] + if not args.server_name or not args.signing_key_path: + read_args_from_config(args) - with open(keyfile) as f: + with open(args.signing_key_path) as f: key = read_signing_keys(f)[0] result = get_json( - origin_name, key, args.destination, "/_matrix/federation/v1/" + args.path + args.server_name, key, args.destination, "/_matrix/federation/v1/" + args.path ) json.dump(result, sys.stdout) print ("") + +def read_args_from_config(args): + with open(args.config, 'r') as fh: + config = yaml.safe_load(fh) + if not args.server_name: + args.server_name = config['server_name'] + if not args.signing_key_path: + args.signing_key_path = config['signing_key_path'] + + if __name__ == "__main__": main() -- cgit 1.5.1 From fd04968f326536fe4d4f55bd6cdd1d888b085fa3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 27 Nov 2017 10:06:59 +0000 Subject: federation_client script: Support for posting content --- scripts-dev/federation_client.py | 42 ++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 82a90ef6fa..3b28417376 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -123,15 +123,25 @@ def lookup(destination, path): except: return "https://%s:%d%s" % (destination, 8448, path) -def get_json(origin_name, origin_key, destination, path): - request_json = { - "method": "GET", + +def request_json(method, origin_name, origin_key, destination, path, content): + if method is None: + if content is None: + method = "GET" + else: + method = "POST" + + json_to_sign = { + "method": method, "uri": path, "origin": origin_name, "destination": destination, } - signed_json = sign_json(request_json, origin_key, origin_name) + if content is not None: + json_to_sign["content"] = json.loads(content) + + signed_json = sign_json(json_to_sign, origin_key, origin_name) authorization_headers = [] @@ -145,10 +155,12 @@ def get_json(origin_name, origin_key, destination, path): dest = lookup(destination, path) print ("Requesting %s" % dest, file=sys.stderr) - result = requests.get( - dest, + result = requests.request( + method=method, + url=dest, headers={"Authorization": authorization_headers[0]}, verify=False, + data=content, ) sys.stderr.write("Status Code: %d\n" % (result.status_code,)) return result.json() @@ -186,6 +198,17 @@ def main(): "connect appropriately.", ) + parser.add_argument( + "-X", "--method", + help="HTTP method to use for the request. Defaults to GET if --data is" + "unspecified, POST if it is." + ) + + parser.add_argument( + "--body", + help="Data to send as the body of the HTTP request" + ) + parser.add_argument( "path", help="request path. We will add '/_matrix/federation/v1/' to this." @@ -199,8 +222,11 @@ def main(): with open(args.signing_key_path) as f: key = read_signing_keys(f)[0] - result = get_json( - args.server_name, key, args.destination, "/_matrix/federation/v1/" + args.path + result = request_json( + args.method, + args.server_name, key, args.destination, + "/_matrix/federation/v1/" + args.path, + content=args.body, ) json.dump(result, sys.stdout) -- cgit 1.5.1 From 6c957e26f06f5806ae2e69838865ee2990a75258 Mon Sep 17 00:00:00 2001 From: rubo77 Date: Wed, 9 May 2018 00:14:01 +0200 Subject: nuke-room-from-db.sh: added postgresql option and help --- scripts-dev/nuke-room-from-db.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'scripts-dev') diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 1201d176c2..cd74d78d6a 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -6,9 +6,19 @@ ## Do not run it lightly. +set -e + +if [ "$1" == "-h" ] || [ "$1" == "" ]; then + echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run" + echo " nuke-room-from-db.sh | sqlite3 homeserver.db" + echo "or" + echo " nuke-room-from-db.sh | psql --dbname=synapse" + exit +fi + ROOMID="$1" -sqlite3 homeserver.db < Date: Wed, 9 May 2018 00:21:00 +0200 Subject: Dont nuke non-existing table event_search_content --- scripts-dev/nuke-room-from-db.sh | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts-dev') diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 1201d176c2..53617f21f1 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -29,7 +29,6 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID'; DELETE FROM state_groups_state WHERE room_id = '$ROOMID'; DELETE FROM receipts_graph WHERE room_id = '$ROOMID'; DELETE FROM receipts_linearized WHERE room_id = '$ROOMID'; -DELETE FROM event_search_content WHERE c1room_id = '$ROOMID'; DELETE FROM guest_access WHERE room_id = '$ROOMID'; DELETE FROM history_visibility WHERE room_id = '$ROOMID'; DELETE FROM room_tags WHERE room_id = '$ROOMID'; -- cgit 1.5.1 From d11b8b6b659957b28c2bbf8bc6ca060284097377 Mon Sep 17 00:00:00 2001 From: rubo77 Date: Wed, 9 May 2018 00:46:47 +0200 Subject: nuke-room-from-db.sh: nuke from table event_search too --- scripts-dev/nuke-room-from-db.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts-dev') diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh index 53617f21f1..2cfd71b0b8 100755 --- a/scripts-dev/nuke-room-from-db.sh +++ b/scripts-dev/nuke-room-from-db.sh @@ -29,6 +29,7 @@ DELETE FROM state_groups WHERE room_id = '$ROOMID'; DELETE FROM state_groups_state WHERE room_id = '$ROOMID'; DELETE FROM receipts_graph WHERE room_id = '$ROOMID'; DELETE FROM receipts_linearized WHERE room_id = '$ROOMID'; +DELETE FROM event_search WHERE room_id = '$ROOMID'; DELETE FROM guest_access WHERE room_id = '$ROOMID'; DELETE FROM history_visibility WHERE room_id = '$ROOMID'; DELETE FROM room_tags WHERE room_id = '$ROOMID'; -- cgit 1.5.1 From 96bad44f8783e6d54fd65e254072ca88031da241 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 7 Jun 2018 18:14:08 +0100 Subject: Fix federation_client to send the right Host This appears to have stopped working since matrix.org moved to cloudflare. The Host header should match the name of the server, not whatever is in the SRV record. --- scripts-dev/federation_client.py | 65 +++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 14 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 3b28417376..d2acc7654d 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -18,14 +18,22 @@ from __future__ import print_function import argparse +from urlparse import urlparse, urlunparse + import nacl.signing import json import base64 import requests import sys + +from requests.adapters import HTTPAdapter import srvlookup import yaml +# uncomment the following to enable debug logging of http requests +#from httplib import HTTPConnection +#HTTPConnection.debuglevel = 1 + def encode_base64(input_bytes): """Encode bytes as a base64 string without any padding.""" @@ -113,17 +121,6 @@ def read_signing_keys(stream): return keys -def lookup(destination, path): - if ":" in destination: - return "https://%s%s" % (destination, path) - else: - try: - srv = srvlookup.lookup("matrix", "tcp", destination)[0] - return "https://%s:%d%s" % (srv.host, srv.port, path) - except: - return "https://%s:%d%s" % (destination, 8448, path) - - def request_json(method, origin_name, origin_key, destination, path, content): if method is None: if content is None: @@ -152,13 +149,19 @@ def request_json(method, origin_name, origin_key, destination, path, content): authorization_headers.append(bytes(header)) print ("Authorization: %s" % header, file=sys.stderr) - dest = lookup(destination, path) + dest = "matrix://%s%s" % (destination, path) print ("Requesting %s" % dest, file=sys.stderr) - result = requests.request( + s = requests.Session() + s.mount("matrix://", MatrixConnectionAdapter()) + + result = s.request( method=method, url=dest, - headers={"Authorization": authorization_headers[0]}, + headers={ + "Host": destination, + "Authorization": authorization_headers[0] + }, verify=False, data=content, ) @@ -242,5 +245,39 @@ def read_args_from_config(args): args.signing_key_path = config['signing_key_path'] +class MatrixConnectionAdapter(HTTPAdapter): + @staticmethod + def lookup(s): + if s[-1] == ']': + # ipv6 literal (with no port) + return s, 8448 + + if ":" in s: + out = s.rsplit(":",1) + try: + port = int(out[1]) + except ValueError: + raise ValueError("Invalid host:port '%s'" % s) + return out[0], port + + try: + srv = srvlookup.lookup("matrix", "tcp", s)[0] + return srv.host, srv.port + except: + return s, 8448 + + def get_connection(self, url, proxies=None): + parsed = urlparse(url) + + (host, port) = self.lookup(parsed.netloc) + netloc = "%s:%d" % (host, port) + print("Connecting to %s" % (netloc,), file=sys.stderr) + url = urlunparse(( + "https", netloc, parsed.path, parsed.params, parsed.query, + parsed.fragment, + )) + return super(MatrixConnectionAdapter, self).get_connection(url, proxies) + + if __name__ == "__main__": main() -- cgit 1.5.1