Merge branch 'develop' into dmr/pyproject-poetry

Signed-off-by: Dan Callahan <danc@element.io>

1 files changed, 55 insertions, 35 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 132ac56d24..bedf09b1ba 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -14,31 +14,59 @@
 #    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
 #
 
-ARG PYTHON_VERSION=3.9
-
-FROM docker.io/python:${PYTHON_VERSION}-slim as base
-
-###
-### Stage 0: builder
-###
-
 # Irritatingly, there is no blessed guide on how to distribute an application with its
-# poetry-managed environment in a docker image. For a while,
-# `poetry export | pip install -r /dev/stdin` seemed plausible but is limited by bugs
+# poetry-managed environment in a docker image. We have opted for
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
 # in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
-# This is inspired from:
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
 #     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
 #     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
-FROM base as builder
+
+
+
+ARG PYTHON_VERSION=3.9
+
+###
+### Stage 0: generate requirements.txt
+###
+FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for pip (below, for apt and poetry), to improve
+# Here we use it to set up a cache for apt (and below for pip), to improve
 # rebuild speeds on slow connections.
-# We install poetry as --user so that it doesn't end up in the system-wide python
-# installation. That gets copied later into the runtime image.
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update && apt-get install -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+# We install poetry in its own build stage to avoid its dependencies conflicting with
+# synapse's dependencies.
+# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
+# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
+#    https://github.com/python-poetry/poetry/pull/5156 and
+#    https://github.com/python-poetry/poetry/issues/5141 ;
+# without it, we generate a requirements.txt with incorrect environment markers,
+# which causes necessary packages to be omitted when we `pip install`.
+#
+# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
+# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user poetry==1.1.12
+  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
+
+WORKDIR /synapse
+
+# Copy just what we need to run `poetry export`...
+COPY pyproject.toml poetry.lock README.rst /synapse/
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+###
+### Stage 1: builder
+###
+FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
 RUN \
@@ -58,33 +86,25 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /synapse
-
-# Copy just what we need to run `poetry install`
-COPY pyproject.toml poetry.lock README.rst /synapse/
-
-# Install to the Python installation which hosts `pip`. In this case, it's the system
-# Python.
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true \
-    POETRY_VIRTUALENVS_CREATE=true \
-    POETRY_HOME=/opt/poetry
 # To speed up rebuilds, install all of the dependencies before we copy over
 # the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
-RUN --mount=type=cache,target=/opt/poetry/artifacts \
-    --mount=type=cache,target=/opt/poetry/.cache/pypoetry/cache \
-  /root/.local/bin/poetry install --no-dev --no-root --no-interaction --no-ansi --extras all
+#
+# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
+COPY --from=requirements /synapse/requirements.txt /synapse/
+RUN --mount=type=cache,target=/root/.cache/pip \
+  pip install --prefix="/install" --no-warn-script-location -r /synapse/requirements.txt
 
-# Copy over the synapse source code.
+# Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
+# ... and what we need to `pip install`.
+COPY pyproject.toml poetry.lock README.rst /synapse/
 
-# Install the synapse package itself, by omitting the --no-root argument
-RUN --mount=type=cache,target=/opt/poetry/artifacts \
-    --mount=type=cache,target=/opt/poetry/cache \
-  /root/.local/bin/poetry install --no-dev --no-interaction --no-ansi --extras all
+# Install the synapse package itself.
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
-### Stage 1: runtime
+### Stage 2: runtime
 ###
 
 FROM base