diff --git a/changelog.d/14548.misc b/changelog.d/14548.misc
new file mode 100644
index 0000000000..416332015c
--- /dev/null
+++ b/changelog.d/14548.misc
@@ -0,0 +1 @@
+Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively.
\ No newline at end of file
diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index 0c2d4f3047..faf7f2cef8 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -1,6 +1,7 @@
# syntax=docker/dockerfile:1
ARG SYNAPSE_VERSION=latest
+ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
# first of all, we create a base image with an nginx which we can copy into the
# target image. For repeated rebuilds, this is much faster than apt installing
@@ -23,7 +24,7 @@ FROM debian:bullseye-slim AS deps_base
FROM redis:6-bullseye AS redis_base
# now build the final image, based on the the regular Synapse docker image
-FROM matrixdotorg/synapse:$SYNAPSE_VERSION
+FROM $FROM
# Install supervisord with pip instead of apt, to avoid installing a second
# copy of python.
diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index c0935c99a8..be1aa1c55e 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -7,8 +7,9 @@
# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
ARG SYNAPSE_VERSION=latest
+ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
-FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
+FROM $FROM
# First of all, we copy postgres server from the official postgres image,
# since for repeated rebuilds, this is much faster than apt installing
# postgres each time.
diff --git a/docker/editable.Dockerfile b/docker/editable.Dockerfile
new file mode 100644
index 0000000000..0e8cf2e712
--- /dev/null
+++ b/docker/editable.Dockerfile
@@ -0,0 +1,75 @@
+# syntax=docker/dockerfile:1
+# This dockerfile builds an editable install of Synapse.
+#
+# Used by `complement.sh`. Not suitable for production use.
+
+ARG PYTHON_VERSION=3.9
+
+###
+### Stage 0: generate requirements.txt
+###
+# We hardcode the use of Debian bullseye here because this could change upstream
+# and other Dockerfiles used for testing are expecting bullseye.
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
+
+# Install Rust and other dependencies (stolen from normal Dockerfile)
+# install the OS build deps
+RUN \
+ --mount=type=cache,target=/var/cache/apt,sharing=locked \
+ --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update -qq && apt-get install -yqq \
+ build-essential \
+ libffi-dev \
+ libjpeg-dev \
+ libpq-dev \
+ libssl-dev \
+ libwebp-dev \
+ libxml++2.6-dev \
+ libxslt1-dev \
+ openssl \
+ zlib1g-dev \
+ git \
+ curl \
+ gosu \
+ libjpeg62-turbo \
+ libpq5 \
+ libwebp6 \
+ xmlsec1 \
+ libjemalloc2 \
+ && rm -rf /var/lib/apt/lists/*
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+RUN mkdir /rust /cargo
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
+
+
+# Make a base copy of the editable source tree, so that we have something to
+# install and build now — even though it's going to be covered up by a mount
+# at runtime.
+COPY synapse /editable-src/synapse/
+COPY rust /editable-src/rust/
+# ... and what we need to `pip install`.
+COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/
+
+RUN pip install poetry
+RUN poetry config virtualenvs.create false
+RUN cd /editable-src && poetry install --extras all
+
+# Make copies of useful things for inspection:
+# - the Rust module (must be copied to the editable source tree before startup)
+# - poetry.lock is useful for checking if dependencies have changed.
+RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak
+RUN cp /editable-src/poetry.lock /poetry.lock.bak
+
+
+### Extra setup from original Dockerfile
+COPY ./docker/start.py /start.py
+COPY ./docker/conf /conf
+
+EXPOSE 8008/tcp 8009/tcp 8448/tcp
+
+ENTRYPOINT ["/start.py"]
+
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
+ CMD curl -fSs http://localhost:8008/health || exit 1
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 7744b47097..8741ba3e34 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -53,6 +53,12 @@ Run the complement test suite on Synapse.
Only build the Docker images. Don't actually run Complement.
Conflicts with -f/--fast.
+ -e, --editable
+ Use an editable build of Synapse, rebuilding the image if necessary.
+ This is suitable for use in development where a fast turn-around time
+ is important.
+ Not suitable for use in CI in case the editable environment is impure.
+
For help on arguments to 'go test', run 'go help testflag'.
EOF
}
@@ -73,6 +79,9 @@ while [ $# -ge 1 ]; do
"--build-only")
skip_complement_run=1
;;
+ "-e"|"--editable")
+ use_editable_synapse=1
+ ;;
*)
# unknown arg: presumably an argument to gotest. break the loop.
break
@@ -96,25 +105,76 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then
echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
fi
+if [ -n "$use_editable_synapse" ]; then
+ if [[ -e synapse/synapse_rust.abi3.so ]]; then
+ # In an editable install, back up the host's compiled Rust module to prevent
+ # inconvenience; the container will overwrite the module with its own copy.
+ mv -n synapse/synapse_rust.abi3.so synapse/synapse_rust.abi3.so~host
+ # And restore it on exit:
+ synapse_pkg=`realpath synapse`
+ trap "mv -f '$synapse_pkg/synapse_rust.abi3.so~host' '$synapse_pkg/synapse_rust.abi3.so'" EXIT
+ fi
+
+ editable_mount="$(realpath .):/editable-src:z"
+ if docker inspect complement-synapse-editable &>/dev/null; then
+ # complement-synapse-editable already exists: see if we can still use it:
+ # - The Rust module must still be importable; it will fail to import if the Rust source has changed.
+ # - The Poetry lock file must be the same (otherwise we assume dependencies have changed)
+
+ # First set up the module in the right place for an editable installation.
+ docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
+
+ if (docker run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \
+ && docker run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then
+ skip_docker_build=1
+ else
+ echo "Editable Synapse image is stale. Will rebuild."
+ unset skip_docker_build
+ fi
+ fi
+fi
+
if [ -z "$skip_docker_build" ]; then
- # Build the base Synapse image from the local checkout
- echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
- docker build -t matrixdotorg/synapse \
- --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
- --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
- -f "docker/Dockerfile" .
- echo_if_github "::endgroup::"
-
- # Build the workers docker image (from the base Synapse image we just built).
- echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
- docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
- echo_if_github "::endgroup::"
-
- # Build the unified Complement image (from the worker Synapse image we just built).
- echo_if_github "::group::Build Docker image: complement/Dockerfile"
- docker build -t complement-synapse \
- -f "docker/complement/Dockerfile" "docker/complement"
- echo_if_github "::endgroup::"
+ if [ -n "$use_editable_synapse" ]; then
+
+ # Build a special image designed for use in development with editable
+ # installs.
+ docker build -t synapse-editable \
+ -f "docker/editable.Dockerfile" .
+
+ docker build -t synapse-workers-editable \
+ --build-arg FROM=synapse-editable \
+ -f "docker/Dockerfile-workers" .
+
+ docker build -t complement-synapse-editable \
+ --build-arg FROM=synapse-workers-editable \
+ -f "docker/complement/Dockerfile" "docker/complement"
+
+ # Prepare the Rust module
+ docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
+
+ else
+
+ # Build the base Synapse image from the local checkout
+ echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
+ docker build -t matrixdotorg/synapse \
+ --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
+ --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
+ -f "docker/Dockerfile" .
+ echo_if_github "::endgroup::"
+
+ # Build the workers docker image (from the base Synapse image we just built).
+ echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
+ docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+ echo_if_github "::endgroup::"
+
+ # Build the unified Complement image (from the worker Synapse image we just built).
+ echo_if_github "::group::Build Docker image: complement/Dockerfile"
+ docker build -t complement-synapse \
+ -f "docker/complement/Dockerfile" "docker/complement"
+ echo_if_github "::endgroup::"
+
+ fi
fi
if [ -n "$skip_complement_run" ]; then
@@ -123,6 +183,10 @@ if [ -n "$skip_complement_run" ]; then
fi
export COMPLEMENT_BASE_IMAGE=complement-synapse
+if [ -n "$use_editable_synapse" ]; then
+ export COMPLEMENT_BASE_IMAGE=complement-synapse-editable
+ export COMPLEMENT_HOST_MOUNTS="$editable_mount"
+fi
extra_test_args=()
|
