From 33ef86aa2515f623fa6e8657d16c918b6a6d9da5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 11 Aug 2021 19:59:57 +0100 Subject: Rename ci to .ci --- .ci/postgres-config.yaml | 19 ++++++++++++ .ci/scripts/postgres_exec.py | 31 ++++++++++++++++++++ .ci/scripts/test_old_deps.sh | 16 ++++++++++ .ci/scripts/test_synapse_port_db.sh | 57 ++++++++++++++++++++++++++++++++++++ .ci/sqlite-config.yaml | 16 ++++++++++ .ci/test_db.db | Bin 0 -> 19296256 bytes .ci/worker-blacklist | 10 +++++++ .github/workflows/tests.yml | 6 ++-- ci/postgres-config.yaml | 19 ------------ ci/scripts/postgres_exec.py | 31 -------------------- ci/scripts/test_old_deps.sh | 16 ---------- ci/scripts/test_synapse_port_db.sh | 57 ------------------------------------ ci/sqlite-config.yaml | 16 ---------- ci/test_db.db | Bin 19296256 -> 0 bytes ci/worker-blacklist | 10 ------- 15 files changed, 152 insertions(+), 152 deletions(-) create mode 100644 .ci/postgres-config.yaml create mode 100755 .ci/scripts/postgres_exec.py create mode 100755 .ci/scripts/test_old_deps.sh create mode 100755 .ci/scripts/test_synapse_port_db.sh create mode 100644 .ci/sqlite-config.yaml create mode 100644 .ci/test_db.db create mode 100644 .ci/worker-blacklist delete mode 100644 ci/postgres-config.yaml delete mode 100755 ci/scripts/postgres_exec.py delete mode 100755 ci/scripts/test_old_deps.sh delete mode 100755 ci/scripts/test_synapse_port_db.sh delete mode 100644 ci/sqlite-config.yaml delete mode 100644 ci/test_db.db delete mode 100644 ci/worker-blacklist diff --git a/.ci/postgres-config.yaml b/.ci/postgres-config.yaml new file mode 100644 index 0000000000..f5a4aecd51 --- /dev/null +++ b/.ci/postgres-config.yaml @@ -0,0 +1,19 @@ +# Configuration file used for testing the 'synapse_port_db' script. +# Tells the script to connect to the postgresql database that will be available in the +# CI's Docker setup at the point where this file is considered. +server_name: "localhost:8800" + +signing_key_path: ".ci/test.signing.key" + +report_stats: false + +database: + name: "psycopg2" + args: + user: postgres + host: localhost + password: postgres + database: synapse + +# Suppress the key server warning. +trusted_key_servers: [] diff --git a/.ci/scripts/postgres_exec.py b/.ci/scripts/postgres_exec.py new file mode 100755 index 0000000000..0f39a336d5 --- /dev/null +++ b/.ci/scripts/postgres_exec.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# Copyright 2019 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import psycopg2 + +# a very simple replacment for `psql`, to make up for the lack of the postgres client +# libraries in the synapse docker image. + +# We use "postgres" as a database because it's bound to exist and the "synapse" one +# doesn't exist yet. +db_conn = psycopg2.connect( + user="postgres", host="localhost", password="postgres", dbname="postgres" +) +db_conn.autocommit = True +cur = db_conn.cursor() +for c in sys.argv[1:]: + cur.execute(c) diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh new file mode 100755 index 0000000000..8b473936f8 --- /dev/null +++ b/.ci/scripts/test_old_deps.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# this script is run by GitHub Actions in a plain `bionic` container; it installs the +# minimal requirements for tox and hands over to the py3-old tox environment. + +set -ex + +apt-get update +apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox + +export LANG="C.UTF-8" + +# Prevent virtualenv from auto-updating pip to an incompatible version +export VIRTUALENV_NO_DOWNLOAD=1 + +exec tox -e py3-old,combine diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh new file mode 100755 index 0000000000..9ee0ad42fc --- /dev/null +++ b/.ci/scripts/test_synapse_port_db.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Test script for 'synapse_port_db'. +# - sets up synapse and deps +# - runs the port script on a prepopulated test sqlite db +# - also runs it against an new sqlite db + + +set -xe +cd `dirname $0`/../.. + +echo "--- Install dependencies" + +# Install dependencies for this test. +pip install psycopg2 coverage coverage-enable-subprocess + +# Install Synapse itself. This won't update any libraries. +pip install -e . + +echo "--- Generate the signing key" + +# Generate the server's signing key. +python -m synapse.app.homeserver --generate-keys -c ci/sqlite-config.yaml + +echo "--- Prepare test database" + +# Make sure the SQLite3 database is using the latest schema and has no pending background update. +scripts-dev/update_database --database-config ci/sqlite-config.yaml + +# Create the PostgreSQL database. +./ci/scripts/postgres_exec.py "CREATE DATABASE synapse" + +echo "+++ Run synapse_port_db against test database" +coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml + +# We should be able to run twice against the same database. +echo "+++ Run synapse_port_db a second time" +coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml + +##### + +# Now do the same again, on an empty database. + +echo "--- Prepare empty SQLite database" + +# we do this by deleting the sqlite db, and then doing the same again. +rm ci/test_db.db + +scripts-dev/update_database --database-config ci/sqlite-config.yaml + +# re-create the PostgreSQL database. +./ci/scripts/postgres_exec.py \ + "DROP DATABASE synapse" \ + "CREATE DATABASE synapse" + +echo "+++ Run synapse_port_db against empty database" +coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml diff --git a/.ci/sqlite-config.yaml b/.ci/sqlite-config.yaml new file mode 100644 index 0000000000..3373743da3 --- /dev/null +++ b/.ci/sqlite-config.yaml @@ -0,0 +1,16 @@ +# Configuration file used for testing the 'synapse_port_db' script. +# Tells the 'update_database' script to connect to the test SQLite database to upgrade its +# schema and run background updates on it. +server_name: "localhost:8800" + +signing_key_path: ".ci/test.signing.key" + +report_stats: false + +database: + name: "sqlite3" + args: + database: ".ci/test_db.db" + +# Suppress the key server warning. +trusted_key_servers: [] diff --git a/.ci/test_db.db b/.ci/test_db.db new file mode 100644 index 0000000000..a0d9f16a75 Binary files /dev/null and b/.ci/test_db.db differ diff --git a/.ci/worker-blacklist b/.ci/worker-blacklist new file mode 100644 index 0000000000..5975cb98cf --- /dev/null +++ b/.ci/worker-blacklist @@ -0,0 +1,10 @@ +# This file serves as a blacklist for SyTest tests that we expect will fail in +# Synapse when run under worker mode. For more details, see sytest-blacklist. + +Can re-join room if re-invited + +# new failures as of https://github.com/matrix-org/sytest/pull/732 +Device list doesn't change if remote server is down + +# https://buildkite.com/matrix-dot-org/synapse/builds/6134#6f67bf47-e234-474d-80e8-c6e1868b15c5 +Server correctly handles incoming m.device_list_update diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 572bc81b0f..df2e3901cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -142,7 +142,7 @@ jobs: uses: docker://ubuntu:bionic # For old python and sqlite with: workdir: /github/workspace - entrypoint: ci/scripts/test_old_deps.sh + entrypoint: .ci/scripts/test_old_deps.sh env: TRIAL_FLAGS: "--jobs=2" - name: Dump logs @@ -229,7 +229,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Prepare test blacklist - run: cat sytest-blacklist ci/worker-blacklist > synapse-blacklist-with-workers + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src @@ -278,7 +278,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - run: ci/scripts/test_synapse_port_db.sh + - run: .ci/scripts/test_synapse_port_db.sh complement: if: ${{ !failure() && !cancelled() }} diff --git a/ci/postgres-config.yaml b/ci/postgres-config.yaml deleted file mode 100644 index 511fef495d..0000000000 --- a/ci/postgres-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Configuration file used for testing the 'synapse_port_db' script. -# Tells the script to connect to the postgresql database that will be available in the -# CI's Docker setup at the point where this file is considered. -server_name: "localhost:8800" - -signing_key_path: "ci/test.signing.key" - -report_stats: false - -database: - name: "psycopg2" - args: - user: postgres - host: localhost - password: postgres - database: synapse - -# Suppress the key server warning. -trusted_key_servers: [] diff --git a/ci/scripts/postgres_exec.py b/ci/scripts/postgres_exec.py deleted file mode 100755 index 0f39a336d5..0000000000 --- a/ci/scripts/postgres_exec.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# Copyright 2019 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -import psycopg2 - -# a very simple replacment for `psql`, to make up for the lack of the postgres client -# libraries in the synapse docker image. - -# We use "postgres" as a database because it's bound to exist and the "synapse" one -# doesn't exist yet. -db_conn = psycopg2.connect( - user="postgres", host="localhost", password="postgres", dbname="postgres" -) -db_conn.autocommit = True -cur = db_conn.cursor() -for c in sys.argv[1:]: - cur.execute(c) diff --git a/ci/scripts/test_old_deps.sh b/ci/scripts/test_old_deps.sh deleted file mode 100755 index 8b473936f8..0000000000 --- a/ci/scripts/test_old_deps.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# this script is run by GitHub Actions in a plain `bionic` container; it installs the -# minimal requirements for tox and hands over to the py3-old tox environment. - -set -ex - -apt-get update -apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox - -export LANG="C.UTF-8" - -# Prevent virtualenv from auto-updating pip to an incompatible version -export VIRTUALENV_NO_DOWNLOAD=1 - -exec tox -e py3-old,combine diff --git a/ci/scripts/test_synapse_port_db.sh b/ci/scripts/test_synapse_port_db.sh deleted file mode 100755 index 9ee0ad42fc..0000000000 --- a/ci/scripts/test_synapse_port_db.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -# Test script for 'synapse_port_db'. -# - sets up synapse and deps -# - runs the port script on a prepopulated test sqlite db -# - also runs it against an new sqlite db - - -set -xe -cd `dirname $0`/../.. - -echo "--- Install dependencies" - -# Install dependencies for this test. -pip install psycopg2 coverage coverage-enable-subprocess - -# Install Synapse itself. This won't update any libraries. -pip install -e . - -echo "--- Generate the signing key" - -# Generate the server's signing key. -python -m synapse.app.homeserver --generate-keys -c ci/sqlite-config.yaml - -echo "--- Prepare test database" - -# Make sure the SQLite3 database is using the latest schema and has no pending background update. -scripts-dev/update_database --database-config ci/sqlite-config.yaml - -# Create the PostgreSQL database. -./ci/scripts/postgres_exec.py "CREATE DATABASE synapse" - -echo "+++ Run synapse_port_db against test database" -coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml - -# We should be able to run twice against the same database. -echo "+++ Run synapse_port_db a second time" -coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml - -##### - -# Now do the same again, on an empty database. - -echo "--- Prepare empty SQLite database" - -# we do this by deleting the sqlite db, and then doing the same again. -rm ci/test_db.db - -scripts-dev/update_database --database-config ci/sqlite-config.yaml - -# re-create the PostgreSQL database. -./ci/scripts/postgres_exec.py \ - "DROP DATABASE synapse" \ - "CREATE DATABASE synapse" - -echo "+++ Run synapse_port_db against empty database" -coverage run scripts/synapse_port_db --sqlite-database ci/test_db.db --postgres-config ci/postgres-config.yaml diff --git a/ci/sqlite-config.yaml b/ci/sqlite-config.yaml deleted file mode 100644 index fd5a1c1451..0000000000 --- a/ci/sqlite-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Configuration file used for testing the 'synapse_port_db' script. -# Tells the 'update_database' script to connect to the test SQLite database to upgrade its -# schema and run background updates on it. -server_name: "localhost:8800" - -signing_key_path: "ci/test.signing.key" - -report_stats: false - -database: - name: "sqlite3" - args: - database: "ci/test_db.db" - -# Suppress the key server warning. -trusted_key_servers: [] diff --git a/ci/test_db.db b/ci/test_db.db deleted file mode 100644 index a0d9f16a75..0000000000 Binary files a/ci/test_db.db and /dev/null differ diff --git a/ci/worker-blacklist b/ci/worker-blacklist deleted file mode 100644 index 5975cb98cf..0000000000 --- a/ci/worker-blacklist +++ /dev/null @@ -1,10 +0,0 @@ -# This file serves as a blacklist for SyTest tests that we expect will fail in -# Synapse when run under worker mode. For more details, see sytest-blacklist. - -Can re-join room if re-invited - -# new failures as of https://github.com/matrix-org/sytest/pull/732 -Device list doesn't change if remote server is down - -# https://buildkite.com/matrix-dot-org/synapse/builds/6134#6f67bf47-e234-474d-80e8-c6e1868b15c5 -Server correctly handles incoming m.device_list_update -- cgit 1.5.1