Clean up tox.ini (#8963)

... and disable coverage tracking for mypy and friends.

3 files changed, 34 insertions, 19 deletions

diff --git a/changelog.d/8963.misc b/changelog.d/8963.misc
new file mode 100644
index 0000000000..495d89e8ee
--- /dev/null
+++ b/changelog.d/8963.misc
@@ -0,0 +1 @@
+Clean up tox.ini file; disable coverage checking for non-test runs.
diff --git a/mypy.ini b/mypy.ini
index 1904204025..0518d3f1af 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -7,6 +7,11 @@ show_error_codes = True
 show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
+
+# To find all folders that pass mypy you run:
+#
+#   find synapse/* -type d -not -name __pycache__ -exec bash -c "mypy '{}' > /dev/null"  \; -print
+
 files =
   scripts-dev/sign_json,
   synapse/api,
diff --git a/tox.ini b/tox.ini
index c232676826..8e8b495292 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,9 @@ deps =
     python-subunit
     junitxml
     coverage
-    coverage-enable-subprocess
+
+    # this is pinned since it's a bit of an obscure package.
+    coverage-enable-subprocess==1.0
 
     # cyptography 2.2 requires setuptools >= 18.5
     #
@@ -23,29 +25,37 @@ deps =
     # install the "enum34" dependency of cryptography.
     pip>=10
 
-setenv =
-    PYTHONDONTWRITEBYTECODE = no_byte_code
-    COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
-
 [testenv]
 deps =
     {[base]deps}
 extras = all, test
 
-whitelist_externals =
-    sh
-
 setenv =
-    {[base]setenv}
+    # use a postgres db for tox environments with "-postgres" in the name
+    # (see https://tox.readthedocs.io/en/3.20.1/config.html#factors-and-factor-conditional-settings)
     postgres: SYNAPSE_POSTGRES = 1
+
+    # this is used by .coveragerc to refer to the top of our tree.
     TOP={toxinidir}
 
 passenv = *
 
 commands =
-    /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
-    # Add this so that coverage will run on subprocesses
-    {envbindir}/coverage run "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+    # the "env" invocation enables coverage checking for sub-processes. This is
+    # particularly important when running trial with `-j`, since that will make
+    # it run tests in a subprocess, whose coverage would otherwise not be
+    # tracked.  (It also makes an explicit `coverage run` command redundant.)
+    #
+    # (See https://coverage.readthedocs.io/en/coverage-5.3/subprocess.html.
+    # Note that the `coverage.process_startup()` call is done by
+    # `coverage-enable-subprocess`.)
+    #
+    # we use "env" rather than putting a value in `setenv` so that it is not
+    # inherited by other tox environments.
+    #
+    # keep this in sync with the copy in `testenv:py35-old`.
+    #
+    /usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
 
 # As of twisted 16.4, trial tries to import the tests as a package (previously
 # it loaded the files explicitly), which means they need to be on the
@@ -80,10 +90,9 @@ deps =
 
     lxml
     coverage
-    coverage-enable-subprocess
+    coverage-enable-subprocess==1.0
 
 commands =
-    /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
     # Make all greater-thans equals so we test the oldest version of our direct
     # dependencies, but make the pyopenssl 17.0, which can work against an
     # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
@@ -92,7 +101,11 @@ commands =
     # Install Synapse itself. This won't update any libraries.
     pip install -e ".[test]"
 
-    {envbindir}/coverage run "{envbindir}/trial"  {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+    # we have to duplicate the command from `testenv` rather than refer to it
+    # as `{[testenv]commands}`, because we run on ubuntu xenial, which has
+    # tox 2.3.1, and https://github.com/tox-dev/tox/issues/208.
+    #
+    /usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
 
 [testenv:benchmark]
 deps =
@@ -157,7 +170,3 @@ deps =
     {[base]deps}
 extras = all,mypy
 commands = mypy
-
-# To find all folders that pass mypy you run:
-#
-#   find synapse/* -type d -not -name __pycache__ -exec bash -c "mypy '{}' > /dev/null"  \; -print