diff options
| -rwxr-xr-x | .ci/scripts/test_old_deps.sh | 71 | ||||
| -rw-r--r-- | .github/workflows/tests.yml | 5 | ||||
| -rw-r--r-- | changelog.d/12407.misc | 1 | ||||
| -rw-r--r-- | tox.ini | 22 |
4 files changed, 70 insertions, 29 deletions
diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh index b2859f7522..769ca4517e 100755 --- a/.ci/scripts/test_old_deps.sh +++ b/.ci/scripts/test_old_deps.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -# this script is run by GitHub Actions in a plain `focal` container; it installs the -# minimal requirements for tox and hands over to the py3-old tox environment. +# this script is run by GitHub Actions in a plain `focal` container; it +# - installs the minimal system requirements, and poetry; +# - patches the project definition file to refer to old versions only; +# - creates a venv with these old versions using poetry; and finally +# - invokes `trial` to run the tests with old deps. # Prevent tzdata from asking for user input export DEBIAN_FRONTEND=noninteractive @@ -9,12 +12,70 @@ set -ex apt-get update apt-get install -y \ - python3 python3-dev python3-pip python3-venv \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev + python3 python3-dev python3-pip python3-venv pipx \ + libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev export LANG="C.UTF-8" # Prevent virtualenv from auto-updating pip to an incompatible version export VIRTUALENV_NO_DOWNLOAD=1 -exec tox -e py3-old +# TODO: in the future, we could use an implementation of +# https://github.com/python-poetry/poetry/issues/3527 +# https://github.com/pypa/pip/issues/8085 +# to select the lowest possible versions, rather than resorting to this sed script. + +# Patch the project definitions in-place: +# - Replace all lower and tilde bounds with exact bounds +# - Make the pyopenssl 17.0, which is the oldest version that works with +# a `cryptography` compiled against OpenSSL 1.1. +# - Delete all lines referring to psycopg2 --- so no testing of postgres support. +# - Omit systemd: we're not logging to journal here. + +# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints +# We don't use these yet, but IIRC they are the default bound used when you `poetry add`. +# The sed expression 's/\^/==/g' ought to do the trick. But it would also change +# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps +# runs on 3.8 (#12343). + +sed -i \ + -e "s/[~>]=/==/g" \ + -e "/psycopg2/d" \ + -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \ + -e '/systemd/d' \ + pyproject.toml + +# Use poetry to do the installation. This ensures that the versions are all mutually +# compatible (as far the package metadata declares, anyway); pip's package resolver +# is more lax. +# +# Rather than `poetry install --no-dev`, we drop all dev dependencies from the +# toml file. This means we don't have to ensure compatibility between old deps and +# dev tools. + +pip install --user toml + +REMOVE_DEV_DEPENDENCIES=" +import toml +with open('pyproject.toml', 'r') as f: + data = toml.loads(f.read()) + +del data['tool']['poetry']['dev-dependencies'] + +with open('pyproject.toml', 'w') as f: + toml.dump(data, f) +" +python3 -c "$REMOVE_DEV_DEPENDENCIES" + +pipx install poetry==1.1.12 +~/.local/bin/poetry lock + +echo "::group::Patched pyproject.toml" +cat pyproject.toml +echo "::endgroup::" +echo "::group::Lockfile after patch" +cat poetry.lock +echo "::endgroup::" + +~/.local/bin/poetry install -E "all test" +~/.local/bin/poetry run trial --jobs=2 tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0fbffd159c..5a98f61932 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -128,6 +128,7 @@ jobs: || true trial-olddeps: + # Note: sqlite only; no postgres if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest @@ -135,11 +136,11 @@ jobs: - uses: actions/checkout@v2 - name: Test with old deps uses: docker://ubuntu:focal # For old python and sqlite + # Note: focal seems to be using 3.8, but the oldest is 3.7? + # See https://github.com/matrix-org/synapse/issues/12343 with: workdir: /github/workspace entrypoint: .ci/scripts/test_old_deps.sh - env: - TRIAL_FLAGS: "--jobs=2" - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/12407.misc b/changelog.d/12407.misc new file mode 100644 index 0000000000..896da9b74d --- /dev/null +++ b/changelog.d/12407.misc @@ -0,0 +1 @@ +Run the olddeps CI job using Poetry. \ No newline at end of file diff --git a/tox.ini b/tox.ini index f1c1bcd822..b4ce400edf 100644 --- a/tox.ini +++ b/tox.ini @@ -105,29 +105,7 @@ commands = # ) usedevelop=true -# A test suite for the oldest supported versions of Python libraries, to catch -# any uses of APIs not available in them. -[testenv:py3-old] -skip_install = true -usedevelop = false -deps = - Automat == 0.8.0 - lxml - # markupsafe 2.1 introduced a change that breaks Jinja 2.x. Since we depend on - # Jinja >= 2.9, it means this test suite will fail if markupsafe >= 2.1 is installed. - markupsafe < 2.1 - {[base]deps} - -commands = - # Make all greater-thans equals so we test the oldest version of our direct - # dependencies, but make the pyopenssl 17.0, which can work against an - # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis). - /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install' - - # Install Synapse itself. This won't update any libraries. - pip install -e ".[test]" - {[testenv]commands} [testenv:benchmark] deps = |