diff options
Diffstat (limited to '')
| -rw-r--r-- | scripts/port_from_sqlite_to_postgres.py (renamed from scripts/port_to_maria.py) | 58 | ||||
| -rwxr-xr-x | synapse/app/homeserver.py | 9 | ||||
| -rw-r--r-- | synapse/storage/engines/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/storage/engines/maria.py | 50 | ||||
| -rw-r--r-- | synapse/storage/schema/delta/16/unique_constraints.sql | 2 |
5 files changed, 26 insertions, 95 deletions
diff --git a/scripts/port_to_maria.py b/scripts/port_from_sqlite_to_postgres.py index 0d7ba92357..4b3fd9e529 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -26,7 +26,7 @@ import types import yaml -logger = logging.getLogger("port_to_maria") +logger = logging.getLogger("port_from_sqlite_to_postgres") BINARY_COLUMNS = { @@ -159,10 +159,10 @@ def chunks(n): @defer.inlineCallbacks -def handle_table(table, sqlite_store, mysql_store): +def handle_table(table, sqlite_store, postgres_store): if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. - next_chunk = yield mysql_store._simple_select_one_onecol( + next_chunk = yield postgres_store._simple_select_one_onecol( table="port_from_sqlite3", keyvalues={"table_name": table}, retcol="rowid", @@ -170,7 +170,7 @@ def handle_table(table, sqlite_store, mysql_store): ) if next_chunk is None: - yield mysql_store._simple_insert( + yield postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) @@ -183,13 +183,13 @@ def handle_table(table, sqlite_store, mysql_store): (table,) ) txn.execute("TRUNCATE %s CASCADE" % (table,)) - mysql_store._simple_insert_txn( + postgres_store._simple_insert_txn( txn, table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "delete_non_append_only", delete_all ) @@ -237,7 +237,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.database_engine.encode_parameter( + postgres_store.database_engine.encode_parameter( conv(j, col) ) for j, col in enumerate(row) @@ -245,16 +245,16 @@ def handle_table(table, sqlite_store, mysql_store): ) def ins(txn): - mysql_store.insert_many_txn(txn, table, headers[1:], rows) + postgres_store.insert_many_txn(txn, table, headers[1:], rows) - mysql_store._simple_update_one_txn( + postgres_store._simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": table}, updatevalues={"rowid": next_chunk}, ) - yield mysql_store.runInteraction("insert_many", ins) + yield postgres_store.runInteraction("insert_many", ins) else: return @@ -273,30 +273,30 @@ def setup_db(db_config, database_engine): @defer.inlineCallbacks -def main(sqlite_config, mysql_config): +def main(sqlite_config, postgress_config): try: sqlite_db_pool = adbapi.ConnectionPool( sqlite_config["name"], **sqlite_config["args"] ) - mysql_db_pool = adbapi.ConnectionPool( - mysql_config["name"], - **mysql_config["args"] + postgres_db_pool = adbapi.ConnectionPool( + postgress_config["name"], + **postgress_config["args"] ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("psycopg2") + postgres_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) - mysql_store = Store(mysql_db_pool, mysql_engine) + postgres_store = Store(postgres_db_pool, postgres_engine) - # Step 1. Set up mysql database. + # Step 1. Set up databases. logger.info("Preparing sqlite database...") setup_db(sqlite_config, sqlite_engine) - logger.info("Preparing mysql database...") - setup_db(mysql_config, mysql_engine) + logger.info("Preparing postgres database...") + setup_db(postgress_config, postgres_engine) # Step 2. Get tables. logger.info("Fetching tables...") @@ -319,7 +319,7 @@ def main(sqlite_config, mysql_config): ) try: - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "create_port_table", create_port_table ) except Exception as e: @@ -328,7 +328,7 @@ def main(sqlite_config, mysql_config): # Process tables. yield defer.gatherResults( [ - handle_table(table, sqlite_store, mysql_store) + handle_table(table, sqlite_store, postgres_store) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -336,10 +336,6 @@ def main(sqlite_config, mysql_config): consumeErrors=True, ) - # for table in ["current_state_events"]: # tables: - # if table not in ["schema_version", "applied_schema_deltas"]: - # if not table.startswith("sqlite_"): - # yield handle_table(table, sqlite_store, mysql_store) except: logger.exception("") finally: @@ -350,7 +346,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--sqlite-database") parser.add_argument( - "--mysql-config", type=argparse.FileType('r'), + "--postgres-config", type=argparse.FileType('r'), ) args = parser.parse_args() @@ -366,18 +362,12 @@ if __name__ == "__main__": }, } - mysql_config = yaml.safe_load(args.mysql_config) - # mysql_config["args"].update({ - # "sql_mode": "TRADITIONAL", - # "charset": "utf8mb4", - # "use_unicode": True, - # "collation": "utf8mb4_bin", - # }) + postgres_config = yaml.safe_load(args.postgres_config) reactor.callWhenRunning( main, sqlite_config=sqlite_config, - mysql_config=mysql_config, + postgres_config=postgres_config, ) reactor.run() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f8a33120b5..93500dd791 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -366,14 +366,7 @@ def setup(config_options): } name = db_config.get("name", None) - if name in ["MySQLdb", "mysql.connector"]: - db_config.setdefault("args", {}).update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - elif name == "psycopg2": + if name == "psycopg2": pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 548d4e1b42..eb76df7f01 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .maria import MariaEngine from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -22,7 +21,6 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, - "mysql.connector": MariaEngine, "psycopg2": PostgresEngine, } diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py deleted file mode 100644 index 90165f6849..0000000000 --- a/synapse/storage/engines/maria.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.storage import prepare_database - -import types - - -class MariaEngine(object): - def __init__(self, database_module): - self.module = database_module - - def convert_param_style(self, sql): - return sql.replace("?", "%s") - - def encode_parameter(self, param): - if isinstance(param, types.BufferType): - return bytes(param) - return param - - def on_new_connection(self, db_conn): - pass - - def prepare_database(self, db_conn): - cur = db_conn.cursor() - cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" - ) - db_conn.commit() - prepare_database(db_conn, self) - - def is_deadlock(self, error): - if isinstance(error, self.module.DatabaseError): - return error.sqlstate == "40001" and error.errno == 1213 - return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index f9fbb6b448..3604ea8427 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -1,5 +1,5 @@ --- We can use SQLite features here, since mysql support was only added in v16 +-- We can use SQLite features here, since other db support was only added in v16 -- DELETE FROM current_state_events WHERE rowid not in ( |