Merge remote-tracking branch 'origin/develop' into erikj/cache_memory_usage

8 files changed, 84 insertions, 59 deletions

diff --git a/changelog.d/9610.docker b/changelog.d/9610.docker
new file mode 100644
index 0000000000..056252a669
--- /dev/null
+++ b/changelog.d/9610.docker
@@ -0,0 +1 @@
+Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project).
diff --git a/changelog.d/9698.misc b/changelog.d/9698.misc
new file mode 100644
index 0000000000..d199e846c5
--- /dev/null
+++ b/changelog.d/9698.misc
@@ -0,0 +1 @@
+Suppress "CryptographyDeprecationWarning: int_from_bytes is deprecated".
diff --git a/changelog.d/9699.bugfix b/changelog.d/9699.bugfix
new file mode 100644
index 0000000000..e871825b33
--- /dev/null
+++ b/changelog.d/9699.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index a442b34598..5b7bf02776 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -25,42 +25,40 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
-        build-essential \
-        libffi-dev \
-        libjpeg-dev \
-        libpq-dev \
-        libssl-dev \
-        libwebp-dev \
-        libxml++2.6-dev \
-        libxslt1-dev \
-        openssl \
-        rustc \
-        zlib1g-dev \
-        && rm -rf /var/lib/apt/lists/*
-
-# Build dependencies that are not available as wheels, to speed up rebuilds
-RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
-        frozendict \
-        jaeger-client \
-        opentracing \
-        # Match the version constraints of Synapse
-        "prometheus_client>=0.4.0" \
-        psycopg2 \
-        pycparser \
-        pyrsistent \
-        pyyaml \
-        simplejson \
-        threadloop \
-        thrift
-
-# now install synapse and all of the python deps to /install.
-COPY synapse /synapse/synapse/
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    openssl \
+    rustc \
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy just what we need to pip install
 COPY scripts /synapse/scripts/
 COPY MANIFEST.in README.rst setup.py synctl /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 
+# To speed up rebuilds, install all of the dependencies before we copy over
+# the whole synapse project so that we this layer in the Docker cache can be
+# used while you develop on the source
+#
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
 RUN pip install --prefix="/install" --no-warn-script-location \
-        /synapse[all]
+    /synapse[all]
+
+# Copy over the rest of the project
+COPY synapse /synapse/synapse/
+
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
 ### Stage 1: runtime
@@ -69,16 +67,16 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
 RUN apt-get update && apt-get install -y \
-        curl \
-        gosu \
-        libjpeg62-turbo \
-        libpq5 \
-        libwebp6 \
-        xmlsec1 \
-        libjemalloc2 \
-        libssl-dev \
-        openssl \
-        && rm -rf /var/lib/apt/lists/*
+    curl \
+    gosu \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp6 \
+    xmlsec1 \
+    libjemalloc2 \
+    libssl-dev \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -91,4 +89,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 ENTRYPOINT ["/start.py"]
 
 HEALTHCHECK --interval=1m --timeout=5s \
-        CMD curl -fSs http://localhost:8008/health || exit 1
+    CMD curl -fSs http://localhost:8008/health || exit 1
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 43b1f1e94b..3912c8994c 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -21,8 +21,10 @@ import signal
 import socket
 import sys
 import traceback
+import warnings
 from typing import Awaitable, Callable, Iterable
 
+from cryptography.utils import CryptographyDeprecationWarning
 from typing_extensions import NoReturn
 
 from twisted.internet import defer, error, reactor
@@ -195,6 +197,25 @@ def listen_metrics(bind_addresses, port):
         start_http_server(port, addr=host, registry=RegistryProxy)
 
 
+def listen_manhole(bind_addresses: Iterable[str], port: int, manhole_globals: dict):
+    # twisted.conch.manhole 21.1.0 uses "int_from_bytes", which produces a confusing
+    # warning. It's fixed by https://github.com/twisted/twisted/pull/1522), so
+    # suppress the warning for now.
+    warnings.filterwarnings(
+        action="ignore",
+        category=CryptographyDeprecationWarning,
+        message="int_from_bytes is deprecated",
+    )
+
+    from synapse.util.manhole import manhole
+
+    listen_tcp(
+        bind_addresses,
+        port,
+        manhole(username="matrix", password="rabbithole", globals=manhole_globals),
+    )
+
+
 def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
     """
     Create a TCP socket for a port and several addresses
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index caef394e1d..6139881dbb 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -147,7 +147,6 @@ from synapse.storage.databases.main.user_directory import UserDirectoryStore
 from synapse.types import ReadReceipt
 from synapse.util.async_helpers import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.manhole import manhole
 from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger("synapse.app.generic_worker")
@@ -640,12 +639,8 @@ class GenericWorkerServer(HomeServer):
             if listener.type == "http":
                 self._listen_http(listener)
             elif listener.type == "manhole":
-                _base.listen_tcp(
-                    listener.bind_addresses,
-                    listener.port,
-                    manhole(
-                        username="matrix", password="rabbithole", globals={"hs": self}
-                    ),
+                _base.listen_manhole(
+                    listener.bind_addresses, listener.port, manhole_globals={"hs": self}
                 )
             elif listener.type == "metrics":
                 if not self.get_config().enable_metrics:
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 244657cb88..3bfe9d507f 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -67,7 +67,6 @@ from synapse.storage import DataStore
 from synapse.storage.engines import IncorrectDatabaseSetup
 from synapse.storage.prepare_database import UpgradeDatabaseException
 from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
 from synapse.util.versionstring import get_version_string
 
@@ -288,12 +287,8 @@ class SynapseHomeServer(HomeServer):
             if listener.type == "http":
                 self._listening_services.extend(self._listener_http(config, listener))
             elif listener.type == "manhole":
-                listen_tcp(
-                    listener.bind_addresses,
-                    listener.port,
-                    manhole(
-                        username="matrix", password="rabbithole", globals={"hs": self}
-                    ),
+                _base.listen_manhole(
+                    listener.bind_addresses, listener.port, manhole_globals={"hs": self}
                 )
             elif listener.type == "replication":
                 services = listen_tcp(
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 14ddaed026..2a1c925ee8 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import logging
 from typing import List, Set
 
@@ -101,7 +102,7 @@ CONDITIONAL_REQUIREMENTS = {
         "txacme>=0.9.2",
         # txacme depends on eliot. Eliot 1.8.0 is incompatible with
         # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418
-        'eliot<1.8.0;python_version<"3.5.3"',
+        "eliot<1.8.0;python_version<'3.5.3'",
     ],
     "saml2": [
         # pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749)
@@ -131,6 +132,18 @@ for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
         ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
 
 
+# ensure there are no double-quote characters in any of the deps (otherwise the
+# 'pip install' incantation in DependencyException will break)
+for dep in itertools.chain(
+    REQUIREMENTS,
+    *CONDITIONAL_REQUIREMENTS.values(),
+):
+    if '"' in dep:
+        raise Exception(
+            "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,)
+        )
+
+
 def list_requirements():
     return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
 
@@ -150,7 +163,7 @@ class DependencyException(Exception):
     @property
     def dependencies(self):
         for i in self.args[0]:
-            yield "'" + i + "'"
+            yield '"' + i + '"'
 
 
 def check_requirements(for_feature=None):