diff --git a/scripts-dev/check_locked_deps_have_sdists.py b/scripts-dev/check_locked_deps_have_sdists.py
new file mode 100755
index 0000000000..63ad99280a
--- /dev/null
+++ b/scripts-dev/check_locked_deps_have_sdists.py
@@ -0,0 +1,58 @@
+#! /usr/bin/env python
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+import tomli
+
+
+def main() -> None:
+ lockfile_path = Path(__file__).parent.parent.joinpath("poetry.lock")
+ with open(lockfile_path, "rb") as lockfile:
+ lockfile_content = tomli.load(lockfile)
+
+ # Poetry 1.3+ lockfile format:
+ # There's a `files` inline table in each [[package]]
+ packages_to_assets: Dict[str, List[Dict[str, str]]] = {
+ package["name"]: package["files"] for package in lockfile_content["package"]
+ }
+
+ success = True
+
+ for package_name, assets in packages_to_assets.items():
+ has_sdist = any(asset["file"].endswith(".tar.gz") for asset in assets)
+ if not has_sdist:
+ success = False
+ print(
+ f"Locked package {package_name!r} does not have a source distribution!",
+ file=sys.stderr,
+ )
+
+ if not success:
+ print(
+ "\nThere were some problems with the Poetry lockfile (poetry.lock).",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+ print(
+ f"Poetry lockfile OK. {len(packages_to_assets)} locked packages checked.",
+ file=sys.stderr,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 66aaa3d848..1b1761202f 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -59,6 +59,11 @@ Run the complement test suite on Synapse.
is important.
Not suitable for use in CI in case the editable environment is impure.
+ --rebuild-editable
+ Force a rebuild of the editable build of Synapse.
+ This is occasionally useful if the built-in rebuild detection with
+ --editable fails, e.g. when changing configure_workers_and_start.py.
+
For help on arguments to 'go test', run 'go help testflag'.
EOF
}
@@ -82,6 +87,9 @@ while [ $# -ge 1 ]; do
"-e"|"--editable")
use_editable_synapse=1
;;
+ "--rebuild-editable")
+ rebuild_editable_synapse=1
+ ;;
*)
# unknown arg: presumably an argument to gotest. break the loop.
break
@@ -116,7 +124,9 @@ if [ -n "$use_editable_synapse" ]; then
fi
editable_mount="$(realpath .):/editable-src:z"
- if docker inspect complement-synapse-editable &>/dev/null; then
+ if [ -n "$rebuild_editable_synapse" ]; then
+ unset skip_docker_build
+ elif docker inspect complement-synapse-editable &>/dev/null; then
# complement-synapse-editable already exists: see if we can still use it:
# - The Rust module must still be importable; it will fail to import if the Rust source has changed.
# - The Poetry lock file must be the same (otherwise we assume dependencies have changed)
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 392c509a8a..9e4ed3246e 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -112,7 +112,7 @@ python3 -m black "${files[@]}"
# Catch any common programming mistakes in Python code.
# --quiet suppresses the update check.
-ruff --quiet "${files[@]}"
+ruff --quiet --fix "${files[@]}"
# Catch any common programming mistakes in Rust code.
#
diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh
index e2bc1640bb..473f54772a 100755
--- a/scripts-dev/make_full_schema.sh
+++ b/scripts-dev/make_full_schema.sh
@@ -19,7 +19,8 @@ usage() {
echo "-c"
echo " CI mode. Prints every command that the script runs."
echo "-o <path>"
- echo " Directory to output full schema files to."
+ echo " Directory to output full schema files to. You probably want to use"
+ echo " '-o synapse/storage/schema'"
echo "-n <schema number>"
echo " Schema number for the new snapshot. Used to set the location of files within "
echo " the output directory, mimicking that of synapse/storage/schemas."
@@ -27,6 +28,11 @@ usage() {
echo "-h"
echo " Display this help text."
echo ""
+ echo ""
+ echo "You probably want to invoke this with something like"
+ echo " docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:11-alpine"
+ echo " echo postgres | scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema"
+ echo ""
echo " NB: make sure to run this against the *oldest* supported version of postgres,"
echo " or else pg_dump might output non-backwards-compatible syntax."
}
@@ -189,7 +195,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
echo "Running db background jobs..."
-synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
+poetry run python synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
# Create the PostgreSQL database.
echo "Creating postgres databases..."
@@ -198,7 +204,7 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_MAIN_DB_NAM
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_STATE_DB_NAME"
echo "Running db background jobs..."
-synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
+poetry run python synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
echo "Dropping unwanted db tables..."
@@ -293,4 +299,12 @@ pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owne
pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
+if [[ "$OUTPUT_DIR" == *synapse/storage/schema ]]; then
+ echo "Updating contrib/datagrip symlinks..."
+ ln -sf "../../synapse/storage/schema/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/common.sql"
+ ln -sf "../../synapse/storage/schema/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/main.sql"
+ ln -sf "../../synapse/storage/schema/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/state.sql"
+else
+ echo "Not updating contrib/datagrip symlinks (unknown output directory)"
+fi
echo "Done! Files dumped to: $OUTPUT_DIR"
|
