From c80878d22a013ed68d3929025bbd40074e66af01 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Wed, 6 Oct 2021 11:26:18 +0100 Subject: Add `--run-background-updates` option to `update_database` script. (#10954) Signed-off-by: Nick Barrett --- .ci/scripts/test_synapse_port_db.sh | 4 +- changelog.d/10954.feature | 1 + debian/changelog | 7 +++ debian/matrix-synapse-py3.links | 1 + scripts-dev/lint.sh | 2 +- scripts-dev/make_full_schema.sh | 2 +- scripts-dev/update_database | 100 ------------------------------- scripts/update_synapse_database | 116 ++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 9 files changed, 130 insertions(+), 105 deletions(-) create mode 100644 changelog.d/10954.feature delete mode 100755 scripts-dev/update_database create mode 100755 scripts/update_synapse_database diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index 2b4e5ec170..50115b3079 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -25,7 +25,7 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml echo "--- Prepare test database" # Make sure the SQLite3 database is using the latest schema and has no pending background update. -scripts-dev/update_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates # Create the PostgreSQL database. .ci/scripts/postgres_exec.py "CREATE DATABASE synapse" @@ -46,7 +46,7 @@ echo "--- Prepare empty SQLite database" # we do this by deleting the sqlite db, and then doing the same again. rm .ci/test_db.db -scripts-dev/update_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates # re-create the PostgreSQL database. .ci/scripts/postgres_exec.py \ diff --git a/changelog.d/10954.feature b/changelog.d/10954.feature new file mode 100644 index 0000000000..94dfa7175c --- /dev/null +++ b/changelog.d/10954.feature @@ -0,0 +1 @@ +Include an `update_synapse_database` script in the distribution. Contributed by @Fizzadar at Beeper. diff --git a/debian/changelog b/debian/changelog index 9e878fbc2d..8e80c78ee7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.44.0~rc2+nmu1) UNRELEASED; urgency=medium + + [ Nick @ Beeper ] + * Include an `update_synapse_database` script in the distribution. + + -- root Mon, 04 Oct 2021 13:29:26 +0000 + matrix-synapse-py3 (1.44.0) stable; urgency=medium * New synapse release 1.44.0. diff --git a/debian/matrix-synapse-py3.links b/debian/matrix-synapse-py3.links index 53e2965418..7eeba180d9 100644 --- a/debian/matrix-synapse-py3.links +++ b/debian/matrix-synapse-py3.links @@ -3,3 +3,4 @@ opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matri opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db opt/venvs/matrix-synapse/bin/synapse_review_recent_signups usr/bin/synapse_review_recent_signups opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl +opt/venvs/matrix-synapse/bin/update_synapse_database usr/bin/update_synapse_database diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 809eff166a..b6554a73c1 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -90,10 +90,10 @@ else "scripts/hash_password" "scripts/register_new_matrix_user" "scripts/synapse_port_db" + "scripts/update_synapse_database" "scripts-dev" "scripts-dev/build_debian_packages" "scripts-dev/sign_json" - "scripts-dev/update_database" "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci" ) fi diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 39bf30d258..c3c90f4ec6 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG" # Make sure the SQLite3 database is using the latest schema and has no pending background update. echo "Running db background jobs..." -scripts-dev/update_database --database-config "$SQLITE_CONFIG" +scripts/update_synapse_database --database-config --run-background-updates "$SQLITE_CONFIG" # Create the PostgreSQL database. echo "Creating postgres database..." diff --git a/scripts-dev/update_database b/scripts-dev/update_database deleted file mode 100755 index 87f709b6ed..0000000000 --- a/scripts-dev/update_database +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# Copyright 2019 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import sys - -import yaml - -from twisted.internet import defer, reactor - -import synapse -from synapse.config.homeserver import HomeServerConfig -from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.server import HomeServer -from synapse.storage import DataStore -from synapse.util.versionstring import get_version_string - -logger = logging.getLogger("update_database") - - -class MockHomeserver(HomeServer): - DATASTORE_CLASS = DataStore - - def __init__(self, config, **kwargs): - super(MockHomeserver, self).__init__( - config.server_name, reactor=reactor, config=config, **kwargs - ) - - self.version_string = "Synapse/" + get_version_string(synapse) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=( - "Updates a synapse database to the latest schema and runs background updates" - " on it." - ) - ) - parser.add_argument("-v", action="store_true") - parser.add_argument( - "--database-config", - type=argparse.FileType("r"), - required=True, - help="A database config file for either a SQLite3 database or a PostgreSQL one.", - ) - - args = parser.parse_args() - - logging_config = { - "level": logging.DEBUG if args.v else logging.INFO, - "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", - } - - logging.basicConfig(**logging_config) - - # Load, process and sanity-check the config. - hs_config = yaml.safe_load(args.database_config) - - if "database" not in hs_config: - sys.stderr.write("The configuration file must have a 'database' section.\n") - sys.exit(4) - - config = HomeServerConfig() - config.parse_config_dict(hs_config, "", "") - - # Instantiate and initialise the homeserver object. - hs = MockHomeserver(config) - - # Setup instantiates the store within the homeserver object and updates the - # DB. - hs.setup() - store = hs.get_datastore() - - async def run_background_updates(): - await store.db_pool.updates.run_background_updates(sleep=False) - # Stop the reactor to exit the script once every background update is run. - reactor.stop() - - def run(): - # Apply all background updates on the database. - defer.ensureDeferred( - run_as_background_process("background_updates", run_background_updates) - ) - - reactor.callWhenRunning(run) - - reactor.run() diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database new file mode 100755 index 0000000000..26b29b0b45 --- /dev/null +++ b/scripts/update_synapse_database @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# Copyright 2019 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import sys + +import yaml + +from twisted.internet import defer, reactor + +import synapse +from synapse.config.homeserver import HomeServerConfig +from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.server import HomeServer +from synapse.storage import DataStore +from synapse.util.versionstring import get_version_string + +logger = logging.getLogger("update_database") + + +class MockHomeserver(HomeServer): + DATASTORE_CLASS = DataStore + + def __init__(self, config, **kwargs): + super(MockHomeserver, self).__init__( + config.server_name, reactor=reactor, config=config, **kwargs + ) + + self.version_string = "Synapse/" + get_version_string(synapse) + + +def run_background_updates(hs): + store = hs.get_datastore() + + async def run_background_updates(): + await store.db_pool.updates.run_background_updates(sleep=False) + # Stop the reactor to exit the script once every background update is run. + reactor.stop() + + def run(): + # Apply all background updates on the database. + defer.ensureDeferred( + run_as_background_process("background_updates", run_background_updates) + ) + + reactor.callWhenRunning(run) + + reactor.run() + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Updates a synapse database to the latest schema and optionally runs background updates" + " on it." + ) + ) + parser.add_argument("-v", action="store_true") + parser.add_argument( + "--database-config", + type=argparse.FileType("r"), + required=True, + help="Synapse configuration file, giving the details of the database to be updated", + ) + parser.add_argument( + "--run-background-updates", + action="store_true", + required=False, + help="run background updates after upgrading the database schema", + ) + + args = parser.parse_args() + + logging_config = { + "level": logging.DEBUG if args.v else logging.INFO, + "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", + } + + logging.basicConfig(**logging_config) + + # Load, process and sanity-check the config. + hs_config = yaml.safe_load(args.database_config) + + if "database" not in hs_config: + sys.stderr.write("The configuration file must have a 'database' section.\n") + sys.exit(4) + + config = HomeServerConfig() + config.parse_config_dict(hs_config, "", "") + + # Instantiate and initialise the homeserver object. + hs = MockHomeserver(config) + + # Setup instantiates the store within the homeserver object and updates the + # DB. + hs.setup() + + if args.run_background_updates: + run_background_updates(hs) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini index 5a62ec76c2..cfe6a06942 100644 --- a/tox.ini +++ b/tox.ini @@ -41,10 +41,10 @@ lint_targets = scripts/hash_password scripts/register_new_matrix_user scripts/synapse_port_db + scripts/update_synapse_database scripts-dev scripts-dev/build_debian_packages scripts-dev/sign_json - scripts-dev/update_database stubs contrib synctl -- cgit 1.5.1