summary refs log tree commit diff
path: root/packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch
diff options
context:
space:
mode:
Diffstat (limited to 'packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch')
-rw-r--r--packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch235
1 files changed, 0 insertions, 235 deletions
diff --git a/packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch b/packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch
deleted file mode 100644

index ce18e13..0000000 --- a/packages/overlays/matrix-synapse/patches/0013-Check-for-CREATE-DROP-INDEX-in-schema-deltas-18440.patch +++ /dev/null
@@ -1,235 +0,0 @@ -From fa4a00a2da753a52dde582c0f56e3ea6567bd53b Mon Sep 17 00:00:00 2001 -From: Erik Johnston <erikj@element.io> -Date: Mon, 19 May 2025 11:52:05 +0100 -Subject: [PATCH 13/34] Check for `CREATE/DROP INDEX` in schema deltas (#18440) - -As these should be background updates. ---- - changelog.d/18440.misc | 1 + - scripts-dev/check_schema_delta.py | 127 +++++++++++++++++++++++------- - 2 files changed, 99 insertions(+), 29 deletions(-) - create mode 100644 changelog.d/18440.misc - -diff --git a/changelog.d/18440.misc b/changelog.d/18440.misc -new file mode 100644 -index 0000000000..6aaa6dde5c ---- /dev/null -+++ b/changelog.d/18440.misc -@@ -0,0 +1 @@ -+Add lint to ensure we don't add a `CREATE/DROP INDEX` in a schema delta. -diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py -index 467be96fdf..454784c3ae 100755 ---- a/scripts-dev/check_schema_delta.py -+++ b/scripts-dev/check_schema_delta.py -@@ -1,6 +1,8 @@ - #!/usr/bin/env python3 - - # Check that no schema deltas have been added to the wrong version. -+# -+# Also checks that schema deltas do not try and create or drop indices. - - import re - from typing import Any, Dict, List -@@ -9,6 +11,13 @@ import click - import git - - SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") -+INDEX_CREATION_REGEX = re.compile(r"CREATE .*INDEX .*ON ([a-z_]+)", flags=re.IGNORECASE) -+INDEX_DELETION_REGEX = re.compile(r"DROP .*INDEX ([a-z_]+)", flags=re.IGNORECASE) -+TABLE_CREATION_REGEX = re.compile(r"CREATE .*TABLE ([a-z_]+)", flags=re.IGNORECASE) -+ -+# The base branch we want to check against. We use the main development branch -+# on the assumption that is what we are developing against. -+DEVELOP_BRANCH = "develop" - - - @click.command() -@@ -20,6 +29,9 @@ SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") - help="Always output ANSI colours", - ) - def main(force_colors: bool) -> None: -+ # Return code. Set to non-zero when we encounter an error -+ return_code = 0 -+ - click.secho( - "+++ Checking schema deltas are in the right folder", - fg="green", -@@ -30,17 +42,17 @@ def main(force_colors: bool) -> None: - click.secho("Updating repo...") - - repo = git.Repo() -- repo.remote().fetch() -+ repo.remote().fetch(refspec=DEVELOP_BRANCH) - - click.secho("Getting current schema version...") - -- r = repo.git.show("origin/develop:synapse/storage/schema/__init__.py") -+ r = repo.git.show(f"origin/{DEVELOP_BRANCH}:synapse/storage/schema/__init__.py") - - locals: Dict[str, Any] = {} - exec(r, locals) - current_schema_version = locals["SCHEMA_VERSION"] - -- diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None) -+ diffs: List[git.Diff] = repo.remote().refs[DEVELOP_BRANCH].commit.diff(None) - - # Get the schema version of the local file to check against current schema on develop - with open("synapse/storage/schema/__init__.py") as file: -@@ -53,7 +65,7 @@ def main(force_colors: bool) -> None: - # local schema version must be +/-1 the current schema version on develop - if abs(local_schema_version - current_schema_version) != 1: - click.secho( -- "The proposed schema version has diverged more than one version from develop, please fix!", -+ f"The proposed schema version has diverged more than one version from {DEVELOP_BRANCH}, please fix!", - fg="red", - bold=True, - color=force_colors, -@@ -67,21 +79,28 @@ def main(force_colors: bool) -> None: - click.secho(f"Current schema version: {current_schema_version}") - - seen_deltas = False -- bad_files = [] -+ bad_delta_files = [] -+ changed_delta_files = [] - for diff in diffs: -- if not diff.new_file or diff.b_path is None: -+ if diff.b_path is None: -+ # We don't lint deleted files. - continue - - match = SCHEMA_FILE_REGEX.match(diff.b_path) - if not match: - continue - -+ changed_delta_files.append(diff.b_path) -+ -+ if not diff.new_file: -+ continue -+ - seen_deltas = True - - _, delta_version, _ = match.groups() - - if delta_version != str(current_schema_version): -- bad_files.append(diff.b_path) -+ bad_delta_files.append(diff.b_path) - - if not seen_deltas: - click.secho( -@@ -92,41 +111,91 @@ def main(force_colors: bool) -> None: - ) - return - -- if not bad_files: -+ if bad_delta_files: -+ bad_delta_files.sort() -+ - click.secho( -- f"All deltas are in the correct folder: {current_schema_version}!", -- fg="green", -+ "Found deltas in the wrong folder!", -+ fg="red", - bold=True, - color=force_colors, - ) -- return - -- bad_files.sort() -- -- click.secho( -- "Found deltas in the wrong folder!", -- fg="red", -- bold=True, -- color=force_colors, -- ) -+ for f in bad_delta_files: -+ click.secho( -+ f"\t{f}", -+ fg="red", -+ bold=True, -+ color=force_colors, -+ ) - -- for f in bad_files: -+ click.secho() - click.secho( -- f"\t{f}", -+ f"Please move these files to delta/{current_schema_version}/", - fg="red", - bold=True, - color=force_colors, - ) - -- click.secho() -- click.secho( -- f"Please move these files to delta/{current_schema_version}/", -- fg="red", -- bold=True, -- color=force_colors, -- ) -+ else: -+ click.secho( -+ f"All deltas are in the correct folder: {current_schema_version}!", -+ fg="green", -+ bold=True, -+ color=force_colors, -+ ) - -- click.get_current_context().exit(1) -+ # Make sure we process them in order. This sort works because deltas are numbered -+ # and delta files are also numbered in order. -+ changed_delta_files.sort() -+ -+ # Now check that we're not trying to create or drop indices. If we want to -+ # do that they should be in background updates. The exception is when we -+ # create indices on tables we've just created. -+ created_tables = set() -+ for delta_file in changed_delta_files: -+ with open(delta_file) as fd: -+ delta_lines = fd.readlines() -+ -+ for line in delta_lines: -+ # Strip SQL comments -+ line = line.split("--", maxsplit=1)[0] -+ -+ # Check and track any tables we create -+ match = TABLE_CREATION_REGEX.search(line) -+ if match: -+ table_name = match.group(1) -+ created_tables.add(table_name) -+ -+ # Check for dropping indices, these are always banned -+ match = INDEX_DELETION_REGEX.search(line) -+ if match: -+ clause = match.group() -+ -+ click.secho( -+ f"Found delta with index deletion: '{clause}' in {delta_file}\nThese should be in background updates.", -+ fg="red", -+ bold=True, -+ color=force_colors, -+ ) -+ return_code = 1 -+ -+ # Check for index creation, which is only allowed for tables we've -+ # created. -+ match = INDEX_CREATION_REGEX.search(line) -+ if match: -+ clause = match.group() -+ table_name = match.group(1) -+ if table_name not in created_tables: -+ click.secho( -+ f"Found delta with index creation: '{clause}' in {delta_file}\nThese should be in background updates.", -+ fg="red", -+ bold=True, -+ color=force_colors, -+ ) -+ return_code = 1 -+ -+ click.get_current_context().exit(return_code) - - - if __name__ == "__main__": --- -2.49.0 -