From 356ed0438e3081b48a29c71042620c0c68af3c25 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 19 Jul 2019 19:01:23 +1000 Subject: Speed up the PostgreSQL unit tests (#5717) --- .buildkite/pipeline.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to '.buildkite') diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7f42fad909..d5e5aeec6b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -117,8 +117,10 @@ steps: limit: 2 - label: ":python: 3.5 / :postgres: 9.5" + agents: + queue: "medium" env: - TRIAL_FLAGS: "-j 4" + TRIAL_FLAGS: "-j 8" command: - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" plugins: @@ -134,8 +136,10 @@ steps: limit: 2 - label: ":python: 3.7 / :postgres: 9.5" + agents: + queue: "medium" env: - TRIAL_FLAGS: "-j 4" + TRIAL_FLAGS: "-j 8" command: - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'" plugins: @@ -151,8 +155,10 @@ steps: limit: 2 - label: ":python: 3.7 / :postgres: 11" + agents: + queue: "medium" env: - TRIAL_FLAGS: "-j 4" + TRIAL_FLAGS: "-j 8" command: - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'" plugins: -- cgit 1.4.1 From 0d0f6d12bc84b106ac83ecf824bd722a08070b78 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 23 Jul 2019 01:05:00 +1000 Subject: Fix logging in workers (#5729) This also adds a worker blacklist. --- .buildkite/pipeline.yml | 3 ++- .buildkite/worker-blacklist | 28 ++++++++++++++++++++++++++++ changelog.d/5729.removal | 1 + synapse/config/workers.py | 1 + 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 .buildkite/worker-blacklist create mode 100644 changelog.d/5729.removal (limited to '.buildkite') diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d5e5aeec6b..c8ae1a44be 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -220,8 +220,10 @@ steps: env: POSTGRES: "1" WORKERS: "1" + BLACKLIST: "synapse-blacklist-with-workers" command: - "bash .buildkite/merge_base_branch.sh" + - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'" - "bash /synapse_sytest.sh" plugins: - docker#v3.0.1: @@ -229,7 +231,6 @@ steps: propagate-environment: true always-pull: true workdir: "/src" - soft_fail: true retry: automatic: - exit_status: -1 diff --git a/.buildkite/worker-blacklist b/.buildkite/worker-blacklist new file mode 100644 index 0000000000..a211ed7b18 --- /dev/null +++ b/.buildkite/worker-blacklist @@ -0,0 +1,28 @@ +# This file serves as a blacklist for SyTest tests that we expect will fail in +# Synapse when run under worker mode. For more details, see sytest-blacklist. + +Message history can be paginated + +m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users + +m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users + +Can re-join room if re-invited + +/upgrade creates a new room + +The only membership state included in an initial sync is for all the senders in the timeline + +Local device key changes get to remote servers + +If remote user leaves room we no longer receive device updates + +Forgotten room messages cannot be paginated + +Inbound federation can get public room list + +Members from the gap are included in gappy incr LL sync + +Leaves are present in non-gapped incremental syncs + +Old leaves are present in gapped incremental syncs \ No newline at end of file diff --git a/changelog.d/5729.removal b/changelog.d/5729.removal new file mode 100644 index 0000000000..3af5198e6b --- /dev/null +++ b/changelog.d/5729.removal @@ -0,0 +1 @@ + Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 246d72cd61..bc0fc165e3 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -31,6 +31,7 @@ class WorkerConfig(Config): self.worker_listeners = config.get("worker_listeners", []) self.worker_daemonize = config.get("worker_daemonize") self.worker_pid_file = config.get("worker_pid_file") + self.worker_log_config = config.get("worker_log_config") # The host used to connect to the main synapse self.worker_replication_host = config.get("worker_replication_host", None) -- cgit 1.4.1 From 1883223a01ee17b8813a2aca9493532bb07915d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Jul 2019 10:26:52 +0100 Subject: Mark flakey tests as blacklisted for worker mode --- .buildkite/worker-blacklist | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to '.buildkite') diff --git a/.buildkite/worker-blacklist b/.buildkite/worker-blacklist index a211ed7b18..8ed8eef1a3 100644 --- a/.buildkite/worker-blacklist +++ b/.buildkite/worker-blacklist @@ -25,4 +25,10 @@ Members from the gap are included in gappy incr LL sync Leaves are present in non-gapped incremental syncs -Old leaves are present in gapped incremental syncs \ No newline at end of file +Old leaves are present in gapped incremental syncs + +User sees updates to presence from other users in the incremental sync. + +Gapped incremental syncs include all state changes + +Old members are included in gappy incr LL sync if they start speaking -- cgit 1.4.1 From 84c6ea1af8383049333213e108941a7831b2a4fd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Jul 2019 13:04:50 +0100 Subject: Update old deps unit test to use old sqlite3 --- .buildkite/pipeline.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to '.buildkite') diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c8ae1a44be..b75269a155 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -49,14 +49,15 @@ steps: - command: - - "python -m pip install tox" + - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev" + - "python3.5 -m pip install tox" - "tox -e py35-old,codecov" label: ":python: 3.5 / SQLite / Old Deps" env: TRIAL_FLAGS: "-j 2" plugins: - docker#v3.0.1: - image: "python:3.5" + image: "ubuntu:xenial" # We use xenail to get an old sqlite and python propagate-environment: true retry: automatic: -- cgit 1.4.1 From c9456193d38ef83cef3bb76b30ef94ea28433e6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Aug 2019 13:56:26 +0100 Subject: Whitelist history visbility sytests for worker mode --- .buildkite/worker-blacklist | 4 ---- 1 file changed, 4 deletions(-) (limited to '.buildkite') diff --git a/.buildkite/worker-blacklist b/.buildkite/worker-blacklist index 8ed8eef1a3..cda5c84e94 100644 --- a/.buildkite/worker-blacklist +++ b/.buildkite/worker-blacklist @@ -3,10 +3,6 @@ Message history can be paginated -m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users - -m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users - Can re-join room if re-invited /upgrade creates a new room -- cgit 1.4.1