diff options
author | Olivier Wilkinson (reivilibre) <oliverw@matrix.org> | 2022-03-16 14:02:58 +0000 |
---|---|---|
committer | Olivier Wilkinson (reivilibre) <oliverw@matrix.org> | 2022-03-16 14:02:58 +0000 |
commit | f5bb3887c81c0d4819a7ef3872f259e17042573b (patch) | |
tree | a7aff58cb1a4df3c19be9585782a261921fceb37 | |
parent | STASH (diff) | |
download | synapse-f5bb3887c81c0d4819a7ef3872f259e17042573b.tar.xz |
STASH
-rwxr-xr-x[-rw-r--r--] | scripts-dev/workers_setup.py | 40 | ||||
-rw-r--r-- | scripts-dev/workers_setup/homeserver.yaml.j2 | 61 | ||||
-rw-r--r-- | scripts-dev/workers_setup/worker.yaml.j2 | 18 |
3 files changed, 83 insertions, 36 deletions
diff --git a/scripts-dev/workers_setup.py b/scripts-dev/workers_setup.py index f21f8d7336..5d525d76f2 100644..100755 --- a/scripts-dev/workers_setup.py +++ b/scripts-dev/workers_setup.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Copyright 2022 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -59,7 +60,7 @@ def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]: result = [] worker_overall_num = 0 for worker_type, worker_type_count in workers: - for worker_idx in range(worker_type_count): + for worker_idx in range(1, worker_type_count + 1): worker_overall_num += 1 if worker_type == "main": worker_name = "main" @@ -73,7 +74,7 @@ def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]: return result -def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> None: +def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path, server_name: str) -> None: if target_path.exists(): print("Target path already exists. Won't overwrite.") return @@ -85,6 +86,8 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N with open(target_path.joinpath("signing.key"), "w") as fout: write_signing_keys(fout, key) + macaroon_secret_key = random_string(32) + env = Environment(loader=FileSystemLoader(dirname(__file__) + "/workers_setup")) hs_template = env.get_template("homeserver.yaml.j2") worker_template = env.get_template("worker.yaml.j2") @@ -92,6 +95,8 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N worker_dir = target_path.joinpath("workers") worker_dir.mkdir() + worker_logging_dir = target_path.joinpath("workers.logging") + worker_logging_dir.mkdir() worker_dir = worker_dir.resolve() logs_dir = target_path.joinpath("logs") @@ -102,42 +107,49 @@ def generate(worker_counts: Tuple[Tuple[str, int], ...], target_path: Path) -> N workers_by_name = {worker.name: worker for worker in all_workers} for worker in all_workers: - worker_config_path = worker_dir.joinpath(f"{worker.name}.yaml") - worker_config = worker_template.render( + log_config_path = worker_logging_dir.joinpath(f"{worker.name}.logging.yaml") + log_config = logging_template.render( worker=worker, worker_dir=worker_dir, logs_dir=logs_dir, all_workers=all_workers, workers_by_name=workers_by_name ) - with open(worker_config_path, "w") as fout: - fout.write(worker_config) + with open(log_config_path, "w") as fout: + fout.write(log_config) - log_config_path = worker_dir.joinpath(f"{worker.name}.logging.yaml") - log_config = logging_template.render( + if worker.name == "main": + # Main can't use a worker file. + continue + + worker_config_path = worker_dir.joinpath(f"{worker.name}.yaml") + worker_config = worker_template.render( worker=worker, worker_dir=worker_dir, logs_dir=logs_dir, all_workers=all_workers, workers_by_name=workers_by_name ) - with open(log_config_path, "w") as fout: - fout.write(log_config) + with open(worker_config_path, "w") as fout: + fout.write(worker_config) hs_config_path = target_path.joinpath("homeserver.yaml") hs_config = hs_template.render( all_workers=all_workers, worker_dir=worker_dir, - logs_dir=logs_dir + logs_dir=logs_dir, + server_name=server_name, + macaroon_secret_key=macaroon_secret_key ) with open(hs_config_path, "w") as fout: fout.write(hs_config) -def main(target_path: Path) -> None: - generate(DESIRED_WORKERS, target_path) +def main(target_path: Path, server_name: str) -> None: + generate(DESIRED_WORKERS, target_path, server_name) if __name__ == '__main__': target_path = Path(sys.argv[1]) - main(target_path) + server_name = sys.argv[2] + main(target_path, server_name) diff --git a/scripts-dev/workers_setup/homeserver.yaml.j2 b/scripts-dev/workers_setup/homeserver.yaml.j2 index 7ed58784f8..b2c6ebfb71 100644 --- a/scripts-dev/workers_setup/homeserver.yaml.j2 +++ b/scripts-dev/workers_setup/homeserver.yaml.j2 @@ -1,38 +1,72 @@ +server_name: {{ server_name }} + +report_stats: false + +signing_key_path: "{{ worker_dir }}/../signing.key" + +macaroon_secret_key: "{{ macaroon_secret_key }}" + +redis: + enabled: true + #host: localhost + #port: 6379 + +trusted_key_servers: [] + +database: + name: psycopg2 + args: + # Comment out user, password and host to use UNIX socket auth. + # For testing, create a database owned by your Postgres user that is logged + # in with your UNIX user + #user: "synapse" + #password: + database: "{{ server_name }}" + #host: "localhost" + cp_min: 5 + cp_max: 10 + instance_map: -{% for worker in all_workers %} +{-% for worker in all_workers %} {{ worker.name }}: host: {{ worker.ip }} port: 9090 {% endfor %} - stream_writers: +stream_writers: events: -{% for worker in all_workers %} -{% if worker.kind == "event_persister" %} +{% for worker in all_workers %-} +{-% if worker.kind == "event_persister" %} - {{ worker.name }} -{% endif %} +{% endif %-} +{-% endfor %} - typing: -{% for worker in all_workers %} -{% if worker.kind == "typing" %} +typing: +{% for worker in all_workers %-} +{-% if worker.kind == "typing" %} - {{ worker.name }} -{% endif %} +{% endif %-} +{-% endfor %} start_pushers: false pusher_instances: -{% if worker.kind == "pusher" %} +{% for worker in all_workers %-} +{-% if worker.kind == "pusher" %} - {{ worker.name }} -{% endif %} +{% endif %-} +{-% endfor %} notify_appservices: False federation_sender_instances: -{% if worker.kind == "federation_sender" %} +{% for worker in all_workers %-} +{-% if worker.kind == "federation_sender" %} - {{ worker.name }} -{% endif %} +{% endif %-} +{-% endfor %} enable_media_repo: False @@ -40,3 +74,4 @@ media_instance_running_background_jobs: "media1" update_user_directory: False +pid_file: "{{ logs_dir }}/main.pid" diff --git a/scripts-dev/workers_setup/worker.yaml.j2 b/scripts-dev/workers_setup/worker.yaml.j2 index 5c82989b57..7a83efc7df 100644 --- a/scripts-dev/workers_setup/worker.yaml.j2 +++ b/scripts-dev/workers_setup/worker.yaml.j2 @@ -3,23 +3,23 @@ worker_app: synapse.app.homeserver {% else %} worker_app: synapse.app.generic_worker -worker_name: {{ worker_name }} - # The replication listener on the main synapse process. -worker_replication_host: {{ worker_ip }} +worker_name: {{ worker.name }} +# The replication listener on the main synapse process. +worker_replication_host: {{ worker.ip }} worker_replication_http_port: 9090 - worker_listeners: +worker_listeners: - type: http port: 8080 resources: - names: - - client - - federation + - client + - federation {% if worker.kind == "media" %} - - media + - media {% endif %} -worker_log_config: '{{ workers_dir }}/{{ worker_name }}.logging.yaml' -worker_pid_file: '{{ logs_dir }}/{{ worker_name }}.pid' +worker_log_config: '{{ worker_dir }}.logging/{{ worker.name }}.logging.yaml' +worker_pid_file: '{{ logs_dir }}/{{ worker.name }}.pid' {% endif %} {% set main_worker = workers_by_name.main %} |