diff --git a/.buildkite/scripts/create_postgres_db.py b/.buildkite/scripts/create_postgres_db.py
deleted file mode 100755
index cc829db216..0000000000
--- a/.buildkite/scripts/create_postgres_db.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
- # Create a PostgresEngine.
- db_engine = create_engine({"name": "psycopg2", "args": {}})
-
- # Connect to postgres to create the base database.
- # We use "postgres" as a database because it's bound to exist and the "synapse" one
- # doesn't exist yet.
- db_conn = db_engine.module.connect(
- user="postgres", host="postgres", password="postgres", dbname="postgres"
- )
- db_conn.autocommit = True
- cur = db_conn.cursor()
- cur.execute("CREATE DATABASE synapse;")
- cur.close()
- db_conn.close()
diff --git a/.buildkite/scripts/postgres_exec.py b/.buildkite/scripts/postgres_exec.py
new file mode 100755
index 0000000000..086b391724
--- /dev/null
+++ b/.buildkite/scripts/postgres_exec.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+ user="postgres", host="postgres", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+ cur.execute(c)
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
index 8914319e38..a7e2454769 100755
--- a/.buildkite/scripts/test_synapse_port_db.sh
+++ b/.buildkite/scripts/test_synapse_port_db.sh
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
#
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
+# Test script for 'synapse_port_db'.
+# - sets up synapse and deps
+# - runs the port script on a prepopulated test sqlite db
+# - also runs it against an new sqlite db
+
set -xe
cd `dirname $0`/../..
@@ -22,15 +22,32 @@ echo "--- Generate the signing key"
# Generate the server's signing key.
python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
-echo "--- Prepare the databases"
+echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
# Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
+./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against test database"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
+#####
+
+# Now do the same again, on an empty database.
+
+echo "--- Prepare empty SQLite database"
+
+# we do this by deleting the sqlite db, and then doing the same again.
+rm .buildkite/test_db.db
+
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-echo "+++ Run synapse_port_db"
+# re-create the PostgreSQL database.
+./.buildkite/scripts/postgres_exec.py \
+ "DROP DATABASE synapse" \
+ "CREATE DATABASE synapse"
-# Run the script
+echo "+++ Run synapse_port_db against empty database"
coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 12c82ac620..e7f3be1b4e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -273,7 +273,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Patch Buildkite-specific test scripts
run: |
- sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+ sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
diff --git a/changelog.d/9930.bugfix b/changelog.d/9930.bugfix
new file mode 100644
index 0000000000..9b22ed4458
--- /dev/null
+++ b/changelog.d/9930.bugfix
@@ -0,0 +1 @@
+Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database.
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index f0c93d5226..5fb5bb35f7 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -913,10 +913,11 @@ class Porter(object):
(curr_forward_id + 1,),
)
- txn.execute(
- "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
- (curr_backward_id + 1,),
- )
+ if curr_backward_id:
+ txn.execute(
+ "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
+ (curr_backward_id + 1,),
+ )
await self.postgres_store.db_pool.runInteraction(
"_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
@@ -954,10 +955,11 @@ class Porter(object):
(curr_chain_id,),
)
- await self.postgres_store.db_pool.runInteraction(
- "_setup_event_auth_chain_id", r,
- )
-
+ if curr_chain_id is not None:
+ await self.postgres_store.db_pool.runInteraction(
+ "_setup_event_auth_chain_id",
+ r,
+ )
##############################################
|
