Drop support for and remove references to EOL Python 3.6 (#11683)

* remove reference in comments to python3.6

* upgrade tox python env in script

* bump python version in example for completeness

* upgrade python version requirement in setup doc

* upgrade necessary python version in __init__.py

* upgrade python version in setup.py

* newsfragment

* drops refs to bionic and replace with focal

* bump refs to postgres 9.6 to 10

* fix hanging ci

* try installing tzdata first

* revert change made in b979f336

* ignore new random mypy error while debugging other error

* fix lint error for temporary workaround

* revert change to install list

* try passing env var

* export debian frontend var?

* move line and add comment

* bump pillow dependency

* bump lxml depenency

* install libjpeg-dev for pillow

* bump automat version to one compatible with py3.8

* add libwebp for pillow

* bump twisted trunk python version

* change suffix of newsfragment

* remove redundant python 3.7 checks

* lint

13 files changed, 25 insertions, 28 deletions

diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh
index 8b473936f8..a54aa86fbc 100755
--- a/.ci/scripts/test_old_deps.sh
+++ b/.ci/scripts/test_old_deps.sh
@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
-
-# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# this script is run by GitHub Actions in a plain `focal` container; it installs the
 # minimal requirements for tox and hands over to the py3-old tox environment.
 
+# Prevent tzdata from asking for user input
+export DEBIAN_FRONTEND=noninteractive
+
 set -ex
 
 apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4f58069702..e47671102e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -141,7 +141,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
-        uses: docker://ubuntu:bionic # For old python and sqlite
+        uses: docker://ubuntu:focal # For old python and sqlite
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
@@ -213,15 +213,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: bionic
+          - sytest-tag: focal
 
-          - sytest-tag: bionic
+          - sytest-tag: focal
             postgres: postgres
 
           - sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: bionic
+          - sytest-tag: focal
             postgres: multi-postgres
             workers: workers
 
diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml
index e974ac7aba..fb9d46b7bf 100644
--- a/.github/workflows/twisted_trunk.yml
+++ b/.github/workflows/twisted_trunk.yml
@@ -25,7 +25,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
       - run: .ci/patch_for_twisted_trunk.sh
       - run: pip install tox
       - run: tox -e py
diff --git a/changelog.d/11683.removal b/changelog.d/11683.removal
new file mode 100644
index 0000000000..b1f048f7f5
--- /dev/null
+++ b/changelog.d/11683.removal
@@ -0,0 +1 @@
+Drop support for Python 3.6, which is EOL.
\ No newline at end of file
diff --git a/docker/Dockerfile-pgtests b/docker/Dockerfile-pgtests
index 92b804d193..b94484ea7f 100644
--- a/docker/Dockerfile-pgtests
+++ b/docker/Dockerfile-pgtests
@@ -1,6 +1,6 @@
 # Use the Sytest image that comes with a lot of the build dependencies
 # pre-installed
-FROM matrixdotorg/sytest:bionic
+FROM matrixdotorg/sytest:focal
 
 # The Sytest image doesn't come with python, so install that
 RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh
index 58e2177d34..b22b6ef16b 100755
--- a/docker/run_pg_tests.sh
+++ b/docker/run_pg_tests.sh
@@ -16,4 +16,4 @@ sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/dat
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py36-postgres "$@"
+tox --workdir=./.tox-pg-container -e py37-postgres "$@"
diff --git a/docs/admin_api/version_api.md b/docs/admin_api/version_api.md
index efb4a0c0f7..27977de0d3 100644
--- a/docs/admin_api/version_api.md
+++ b/docs/admin_api/version_api.md
@@ -16,6 +16,6 @@ It returns a JSON body like the following:
 ```json
 {
     "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-    "python_version": "3.6.8"
+    "python_version": "3.7.8"
 }
 ```
diff --git a/setup.py b/setup.py
index e618ff898b..d0511c767f 100755
--- a/setup.py
+++ b/setup.py
@@ -150,7 +150,7 @@ setup(
     zip_safe=False,
     long_description=long_description,
     long_description_content_type="text/x-rst",
-    python_requires="~=3.6",
+    python_requires="~=3.7",
     entry_points={
         "console_scripts": [
             "synapse_homeserver = synapse.app.homeserver:main",
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 3d0d165f48..4ef8728018 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -21,8 +21,8 @@ import os
 import sys
 
 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 6):
-    print("Synapse requires Python 3.6 or above.")
+if sys.version_info < (3, 7):
+    print("Synapse requires Python 3.7 or above.")
     sys.exit(1)
 
 # Twisted and canonicaljson will fail to import when this file is executed to
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 579adbbca0..e5ee03b79f 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -16,7 +16,6 @@ import atexit
 import gc
 import logging
 import os
-import platform
 import signal
 import socket
 import sys
@@ -468,16 +467,12 @@ async def start(hs: "HomeServer") -> None:
     # everything currently allocated are things that will be used for the
     # rest of time. Doing so means less work each GC (hopefully).
     #
-    # This only works on Python 3.7
-    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
-        gc.collect()
-        gc.freeze()
+    gc.collect()
+    gc.freeze()
 
     # Speed up shutdowns by freezing all allocated objects. This moves everything
     # into the permanent generation and excludes them from the final GC.
-    # Unfortunately only works on Python 3.7
-    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
-        atexit.register(gc.freeze)
+    atexit.register(gc.freeze)
 
 
 def setup_sentry(hs: "HomeServer") -> None:
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index d844fbb3b3..22b4606ae0 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -70,7 +70,7 @@ REQUIREMENTS = [
     "pyasn1>=0.1.9",
     "pyasn1-modules>=0.0.7",
     "bcrypt>=3.1.0",
-    "pillow>=4.3.0",
+    "pillow>=5.4.0",
     "sortedcontainers>=1.4.4",
     "pymacaroons>=0.13.0",
     "msgpack>=0.5.2",
@@ -107,7 +107,7 @@ CONDITIONAL_REQUIREMENTS = {
     # `systemd.journal.JournalHandler`, as is documented in
     # `contrib/systemd/log_config.yaml`.
     "systemd": ["systemd-python>=231"],
-    "url_preview": ["lxml>=3.5.0"],
+    "url_preview": ["lxml>=4.2.0"],
     "sentry": ["sentry-sdk>=0.7.2"],
     "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"],
     "jwt": ["pyjwt>=1.6.4"],
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 30f948a0f7..b3d71f661c 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -46,8 +46,8 @@ class PostgresEngine(BaseDatabaseEngine):
         self._version = db_conn.server_version
 
         # Are we on a supported PostgreSQL version?
-        if not allow_outdated_version and self._version < 90600:
-            raise RuntimeError("Synapse requires PostgreSQL 9.6 or above.")
+        if not allow_outdated_version and self._version < 100000:
+            raise RuntimeError("Synapse requires PostgreSQL 10 or above.")
 
         with db_conn.cursor() as txn:
             txn.execute("SHOW SERVER_ENCODING")
diff --git a/tox.ini b/tox.ini
index 2ffca14b22..32679e9106 100644
--- a/tox.ini
+++ b/tox.ini
@@ -117,8 +117,7 @@ usedevelop=true
 skip_install = true
 usedevelop = false
 deps =
-    # Old automat version for Twisted
-    Automat == 0.3.0
+    Automat == 0.8.0
     lxml
     {[base]deps}