From b5effc72016021cc38f8d7949420d9246787fe11 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 12:43:04 +0100 Subject: Update trial old deps CI to use poetry 1.2.0 (#13707) --- .ci/scripts/prepare_old_deps.sh | 64 +++++++++++++++++++++++++++++++ .ci/scripts/test_old_deps.sh | 83 ----------------------------------------- .github/workflows/tests.yml | 45 ++++++++++++++++++---- changelog.d/13707.misc | 1 + 4 files changed, 103 insertions(+), 90 deletions(-) create mode 100755 .ci/scripts/prepare_old_deps.sh delete mode 100755 .ci/scripts/test_old_deps.sh create mode 100644 changelog.d/13707.misc diff --git a/.ci/scripts/prepare_old_deps.sh b/.ci/scripts/prepare_old_deps.sh new file mode 100755 index 0000000000..7e4f060b17 --- /dev/null +++ b/.ci/scripts/prepare_old_deps.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# this script is run by GitHub Actions in a plain `focal` container; it +# - installs the minimal system requirements, and poetry; +# - patches the project definition file to refer to old versions only; +# - creates a venv with these old versions using poetry; and finally +# - invokes `trial` to run the tests with old deps. + +set -ex + +# Prevent virtualenv from auto-updating pip to an incompatible version +export VIRTUALENV_NO_DOWNLOAD=1 + +# TODO: in the future, we could use an implementation of +# https://github.com/python-poetry/poetry/issues/3527 +# https://github.com/pypa/pip/issues/8085 +# to select the lowest possible versions, rather than resorting to this sed script. + +# Patch the project definitions in-place: +# - Replace all lower and tilde bounds with exact bounds +# - Replace all caret bounds---but not the one that defines the supported Python version! +# - Delete all lines referring to psycopg2 --- so no testing of postgres support. +# - Use pyopenssl 17.0, which is the oldest version that works with +# a `cryptography` compiled against OpenSSL 1.1. +# - Omit systemd: we're not logging to journal here. + +sed -i \ + -e "s/[~>]=/==/g" \ + -e '/^python = "^/!s/\^/==/g' \ + -e "/psycopg2/d" \ + -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \ + -e '/systemd/d' \ + pyproject.toml + +# Use poetry to do the installation. This ensures that the versions are all mutually +# compatible (as far the package metadata declares, anyway); pip's package resolver +# is more lax. +# +# Rather than `poetry install --no-dev`, we drop all dev dependencies from the +# toml file. This means we don't have to ensure compatibility between old deps and +# dev tools. + +pip install toml wheel + +REMOVE_DEV_DEPENDENCIES=" +import toml +with open('pyproject.toml', 'r') as f: + data = toml.loads(f.read()) + +del data['tool']['poetry']['dev-dependencies'] + +with open('pyproject.toml', 'w') as f: + toml.dump(data, f) +" +python3 -c "$REMOVE_DEV_DEPENDENCIES" + +pip install poetry==1.2.0 +poetry lock + +echo "::group::Patched pyproject.toml" +cat pyproject.toml +echo "::endgroup::" +echo "::group::Lockfile after patch" +cat poetry.lock +echo "::endgroup::" diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh deleted file mode 100755 index 478c8d639a..0000000000 --- a/.ci/scripts/test_old_deps.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# this script is run by GitHub Actions in a plain `focal` container; it -# - installs the minimal system requirements, and poetry; -# - patches the project definition file to refer to old versions only; -# - creates a venv with these old versions using poetry; and finally -# - invokes `trial` to run the tests with old deps. - -# Prevent tzdata from asking for user input -export DEBIAN_FRONTEND=noninteractive - -set -ex - -apt-get update -apt-get install -y \ - python3 python3-dev python3-pip python3-venv pipx \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - -export LANG="C.UTF-8" - -# Prevent virtualenv from auto-updating pip to an incompatible version -export VIRTUALENV_NO_DOWNLOAD=1 - -# TODO: in the future, we could use an implementation of -# https://github.com/python-poetry/poetry/issues/3527 -# https://github.com/pypa/pip/issues/8085 -# to select the lowest possible versions, rather than resorting to this sed script. - -# Patch the project definitions in-place: -# - Replace all lower and tilde bounds with exact bounds -# - Replace all caret bounds---but not the one that defines the supported Python version! -# - Delete all lines referring to psycopg2 --- so no testing of postgres support. -# - Use pyopenssl 17.0, which is the oldest version that works with -# a `cryptography` compiled against OpenSSL 1.1. -# - Omit systemd: we're not logging to journal here. - -# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints -# We don't use these yet, but IIRC they are the default bound used when you `poetry add`. -# The sed expression 's/\^/==/g' ought to do the trick. But it would also change -# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps -# runs on 3.8 (#12343). - -sed -i \ - -e "s/[~>]=/==/g" \ - -e '/^python = "^/!s/\^/==/g' \ - -e "/psycopg2/d" \ - -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \ - -e '/systemd/d' \ - pyproject.toml - -# Use poetry to do the installation. This ensures that the versions are all mutually -# compatible (as far the package metadata declares, anyway); pip's package resolver -# is more lax. -# -# Rather than `poetry install --no-dev`, we drop all dev dependencies from the -# toml file. This means we don't have to ensure compatibility between old deps and -# dev tools. - -pip install --user toml - -REMOVE_DEV_DEPENDENCIES=" -import toml -with open('pyproject.toml', 'r') as f: - data = toml.loads(f.read()) - -del data['tool']['poetry']['dev-dependencies'] - -with open('pyproject.toml', 'w') as f: - toml.dump(data, f) -" -python3 -c "$REMOVE_DEV_DEPENDENCIES" - -pipx install poetry==1.1.14 -~/.local/bin/poetry lock - -echo "::group::Patched pyproject.toml" -cat pyproject.toml -echo "::endgroup::" -echo "::group::Lockfile after patch" -cat poetry.lock -echo "::endgroup::" - -~/.local/bin/poetry install -E "all test" -~/.local/bin/poetry run trial --jobs=2 tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc1de2893c..16fb4b43e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -135,16 +135,47 @@ jobs: # Note: sqlite only; no postgres if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Test with old deps - uses: docker://ubuntu:focal # For old python and sqlite - # Note: focal seems to be using 3.8, but the oldest is 3.7? - # See https://github.com/matrix-org/synapse/issues/12343 + + # There aren't wheels for some of the older deps, so we need to install + # their build dependencies + - run: | + sudo apt-get -qq install build-essential libffi-dev python-dev \ + libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev + + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + + # Calculating the old-deps actually takes a bunch of time, so we cache the + # pyproject.toml / poetry.lock. We need to cache pyproject.toml as + # otherwise the `poetry install` step will error due to the poetry.lock + # file being outdated. + # + # This caches the output of `Prepare old deps`, which should generate the + # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. + - uses: actions/cache@v3 + id: cache-poetry-old-deps + name: Cache poetry.lock with: - workdir: /github/workspace - entrypoint: .ci/scripts/test_old_deps.sh + path: | + poetry.lock + pyproject.toml + key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} + - name: Prepare old deps + if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' + run: .ci/scripts/prepare_old_deps.sh + + # We only now install poetry so that `setup-python-poetry` caches the + # right poetry.lock's dependencies. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: '3.7' + extras: "all test" + + - run: poetry run trial -j 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/13707.misc b/changelog.d/13707.misc new file mode 100644 index 0000000000..e72c322d2e --- /dev/null +++ b/changelog.d/13707.misc @@ -0,0 +1 @@ +Update trial old deps CI to use poetry 1.2.0. -- cgit 1.4.1