diff options
Diffstat (limited to '')
| -rw-r--r-- | changelog.d/15334.misc | 1 | ||||
| -rw-r--r-- | synapse/storage/engines/sqlite.py | 17 | ||||
| -rw-r--r-- | tests/server.py | 23 | ||||
| -rw-r--r-- | tests/unittest.py | 16 |
4 files changed, 53 insertions, 4 deletions
diff --git a/changelog.d/15334.misc b/changelog.d/15334.misc new file mode 100644 index 0000000000..0c30818ed0 --- /dev/null +++ b/changelog.d/15334.misc @@ -0,0 +1 @@ +Speed up unit tests when using SQLite3. diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 28751e89a5..ca8c59297c 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -34,6 +34,13 @@ class Sqlite3Engine(BaseDatabaseEngine[sqlite3.Connection, sqlite3.Cursor]): ":memory:", ) + # A connection to a database that has already been prepared, to use as a + # base for an in-memory connection. This is used during unit tests to + # speed up setting up the DB. + self._prepped_conn: Optional[sqlite3.Connection] = database_config.get( + "_TEST_PREPPED_CONN" + ) + if platform.python_implementation() == "PyPy": # pypy's sqlite3 module doesn't handle bytearrays, convert them # back to bytes. @@ -84,7 +91,15 @@ class Sqlite3Engine(BaseDatabaseEngine[sqlite3.Connection, sqlite3.Cursor]): # In memory databases need to be rebuilt each time. Ideally we'd # reuse the same connection as we do when starting up, but that # would involve using adbapi before we have started the reactor. - prepare_database(db_conn, self, config=None) + # + # If we have a `prepped_conn` we can use that to initialise the DB, + # otherwise we need to call `prepare_database`. + if self._prepped_conn is not None: + # Initialise the new DB from the pre-prepared DB. + assert isinstance(db_conn.conn, sqlite3.Connection) + self._prepped_conn.backup(db_conn.conn) + else: + prepare_database(db_conn, self, config=None) db_conn.create_function("rank", 1, _rank) db_conn.execute("PRAGMA foreign_keys = ON;") diff --git a/tests/server.py b/tests/server.py index bb059630fa..b52ff1c463 100644 --- a/tests/server.py +++ b/tests/server.py @@ -16,6 +16,7 @@ import json import logging import os import os.path +import sqlite3 import time import uuid import warnings @@ -79,7 +80,9 @@ from synapse.http.site import SynapseRequest from synapse.logging.context import ContextResourceUsage from synapse.server import HomeServer from synapse.storage import DataStore +from synapse.storage.database import LoggingDatabaseConnection from synapse.storage.engines import PostgresEngine, create_engine +from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util import Clock @@ -104,6 +107,10 @@ P = ParamSpec("P") # the type of thing that can be passed into `make_request` in the headers list CustomHeaderType = Tuple[Union[str, bytes], Union[str, bytes]] +# A pre-prepared SQLite DB that is used as a template when creating new SQLite +# DB each test run. This dramatically speeds up test set up when using SQLite. +PREPPED_SQLITE_DB_CONN: Optional[LoggingDatabaseConnection] = None + class TimedOutException(Exception): """ @@ -899,6 +906,22 @@ def setup_test_homeserver( "args": {"database": test_db_location, "cp_min": 1, "cp_max": 1}, } + # Check if we have set up a DB that we can use as a template. + global PREPPED_SQLITE_DB_CONN + if PREPPED_SQLITE_DB_CONN is None: + temp_engine = create_engine(database_config) + PREPPED_SQLITE_DB_CONN = LoggingDatabaseConnection( + sqlite3.connect(":memory:"), temp_engine, "PREPPED_CONN" + ) + + database = DatabaseConnectionConfig("master", database_config) + config.database.databases = [database] + prepare_database( + PREPPED_SQLITE_DB_CONN, create_engine(database_config), config + ) + + database_config["_TEST_PREPPED_CONN"] = PREPPED_SQLITE_DB_CONN + if "db_txn_limit" in kwargs: database_config["txn_limit"] = kwargs["db_txn_limit"] diff --git a/tests/unittest.py b/tests/unittest.py index f9160faa1d..8a16fd3665 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -146,6 +146,9 @@ class TestCase(unittest.TestCase): % (current_context(),) ) + # Disable GC for duration of test. See below for why. + gc.disable() + old_level = logging.getLogger().level if level is not None and old_level != level: @@ -163,12 +166,19 @@ class TestCase(unittest.TestCase): return orig() + # We want to force a GC to workaround problems with deferreds leaking + # logcontexts when they are GCed (see the logcontext docs). + # + # The easiest way to do this would be to do a full GC after each test + # run, but that is very expensive. Instead, we disable GC (above) for + # the duration of the test so that we only need to run a gen-0 GC, which + # is a lot quicker. + @around(self) def tearDown(orig: Callable[[], R]) -> R: ret = orig() - # force a GC to workaround problems with deferreds leaking logcontexts when - # they are GCed (see the logcontext docs) - gc.collect() + gc.collect(0) + gc.enable() set_current_context(SENTINEL_CONTEXT) return ret |