diff options
Diffstat (limited to '')
| -rw-r--r-- | .dockerignore | 6 | ||||
| -rw-r--r-- | changelog.d/12385.docker | 1 | ||||
| -rw-r--r-- | docker/Dockerfile | 78 | ||||
| -rwxr-xr-x | docker/start.py | 10 |
4 files changed, 69 insertions, 26 deletions
diff --git a/.dockerignore b/.dockerignore index 434231fce9..a236760cf1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,12 @@ # things to include !docker !synapse -!MANIFEST.in !README.rst +!pyproject.toml +!poetry.lock + +# TODO: remove these once we have moved over to using poetry-core in pyproject.toml +!MANIFEST.in !setup.py **/__pycache__ diff --git a/changelog.d/12385.docker b/changelog.d/12385.docker new file mode 100644 index 0000000000..abe2127ea0 --- /dev/null +++ b/changelog.d/12385.docker @@ -0,0 +1 @@ +Bundle locked versions of dependencies into the Docker image. \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 24b5515eb9..6009da7db7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,20 +14,61 @@ # DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 . # +# Irritatingly, there is no blessed guide on how to distribute an application with its +# poetry-managed environment in a docker image. We have opted for +# `poetry export | pip install -r /dev/stdin`, but there are known bugs in +# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released. +# In case we get bitten by those bugs in the future, the recommendations here might +# be useful: +# https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865 +# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc + + + ARG PYTHON_VERSION=3.9 ### -### Stage 0: builder +### Stage 0: generate requirements.txt ### -FROM docker.io/python:${PYTHON_VERSION}-slim as builder +FROM docker.io/python:${PYTHON_VERSION}-slim as requirements -# install the OS build deps -# # RUN --mount is specific to buildkit and is documented at # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for apt, to improve rebuild speeds on -# slow connections. +# Here we use it to set up a cache for apt (and below for pip), to improve +# rebuild speeds on slow connections. +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y git \ + && rm -rf /var/lib/apt/lists/* + +# We install poetry in its own build stage to avoid its dependencies conflicting with +# synapse's dependencies. +# We use a specific commit from poetry's master branch instead of our usual 1.1.12, +# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to +# https://github.com/python-poetry/poetry/pull/5156 and +# https://github.com/python-poetry/poetry/issues/5141 ; +# without it, we generate a requirements.txt with incorrect environment markers, +# which causes necessary packages to be omitted when we `pip install`. # +# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also +# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export). +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5 + +WORKDIR /synapse + +# Copy just what we need to run `poetry export`... +COPY pyproject.toml poetry.lock README.rst /synapse/ + +RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt + +### +### Stage 1: builder +### +FROM docker.io/python:${PYTHON_VERSION}-slim as builder + +# install the OS build deps RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -45,30 +86,27 @@ RUN \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -# Copy just what we need to pip install -COPY MANIFEST.in README.rst setup.py /synapse/ -COPY synapse/__init__.py /synapse/synapse/__init__.py -COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py - # To speed up rebuilds, install all of the dependencies before we copy over -# the whole synapse project so that we this layer in the Docker cache can be +# the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source # -# This is aiming at installing the `install_requires` and `extras_require` from `setup.py` +# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. +COPY --from=requirements /synapse/requirements.txt /synapse/ RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix="/install" --no-warn-script-location \ - /synapse[all] + pip install --prefix="/install" --no-warn-script-location -r /synapse/requirements.txt -# Copy over the rest of the project +# Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ +# ... and what we need to `pip install`. +# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy +# pyproject.toml here, ditching setup.py and MANIFEST.in. +COPY setup.py MANIFEST.in README.rst /synapse/ -# Install the synapse package itself and all of its children packages. -# -# This is aiming at installing only the `packages=find_packages(...)` from `setup.py +# Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse ### -### Stage 1: runtime +### Stage 2: runtime ### FROM docker.io/python:${PYTHON_VERSION}-slim diff --git a/docker/start.py b/docker/start.py index ec9eeb49ae..ac62bbc8ba 100755 --- a/docker/start.py +++ b/docker/start.py @@ -108,7 +108,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): # Hopefully we already have a signing key, but generate one if not. args = [ - "python", + sys.executable, "-m", "synapse.app.homeserver", "--config-path", @@ -158,7 +158,7 @@ def run_generate_config(environ, ownership): # generate the main config file, and a signing key. args = [ - "python", + sys.executable, "-m", "synapse.app.homeserver", "--server-name", @@ -175,7 +175,7 @@ def run_generate_config(environ, ownership): "--open-private-ports", ] # log("running %s" % (args, )) - os.execv("/usr/local/bin/python", args) + os.execv(sys.executable, args) def main(args, environ): @@ -254,12 +254,12 @@ running with 'migrate_config'. See the README for more details. log("Starting synapse with args " + " ".join(args)) - args = ["python"] + args + args = [sys.executable] + args if ownership is not None: args = ["gosu", ownership] + args os.execve("/usr/sbin/gosu", args, environ) else: - os.execve("/usr/local/bin/python", args, environ) + os.execve(sys.executable, args, environ) if __name__ == "__main__": |