Try to fix olddeps

3 files changed, 54 insertions, 31 deletions

diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh
index b2859f7522..02040986b8 100755
--- a/.ci/scripts/test_old_deps.sh
+++ b/.ci/scripts/test_old_deps.sh
@@ -9,12 +9,57 @@ set -ex
 
 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+# I'd prefer to use something like this
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# rather than this sed script. But that's an Opinion.
+
+# patch the project definitions in-place
+# replace all lower bounds with exact bounds
+# but make the pyopenssl 17.0, which can work against an
+# OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
+# delete all lines referring to psycopg2 --- so no testing of postgres support
+# Omit systemd: we're not logging to journal here.
+
+sed -i-backup \
+   -e "s/[~>]=/==/g" \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/psycopg2/d' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial -j2 tests
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 214b42fe77..41770f263d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -115,18 +115,18 @@ jobs:
           || true
 
   trial-olddeps:
+    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
+#    needs: linting-done
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
         uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -149,7 +149,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
-        extras: [""]
+        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
diff --git a/tox.ini b/tox.ini
index 8f85d59f42..d3b03e2660 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,13 +2,13 @@
 envlist = py37, py38, py39, py310
 
 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208
-minversion = 2.3.2
+minversion = 3.3.0
 
 # the tox-venv plugin makes tox use python's built-in `venv` module rather than
 # the legacy `virtualenv` tool. `virtualenv` embeds its own `pip`, `setuptools`,
 # etc, and ends up being rather unreliable.
 requires = tox-venv
-
+isolated_build = true
 [base]
 deps =
     python-subunit
@@ -92,29 +92,7 @@ commands =
 # )
 usedevelop=true
 
-# A test suite for the oldest supported versions of Python libraries, to catch
-# any uses of APIs not available in them.
-[testenv:py3-old]
-skip_install = true
-usedevelop = false
-deps =
-    Automat == 0.8.0
-    lxml
-    # markupsafe 2.1 introduced a change that breaks Jinja 2.x. Since we depend on
-    # Jinja >= 2.9, it means this test suite will fail if markupsafe >= 2.1 is installed.
-    markupsafe < 2.1
-    {[base]deps}
-
-commands =
-    # Make all greater-thans equals so we test the oldest version of our direct
-    # dependencies, but make the pyopenssl 17.0, which can work against an
-    # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
-    /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install'
-
-    # Install Synapse itself. This won't update any libraries.
-    pip install -e ".[test]"
 
-    {[testenv]commands}
 
 [testenv:benchmark]
 deps =