diff --git a/changelog.d/9610.docker b/changelog.d/9610.docker
new file mode 100644
index 0000000000..056252a669
--- /dev/null
+++ b/changelog.d/9610.docker
@@ -0,0 +1 @@
+Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project).
diff --git a/changelog.d/9698.misc b/changelog.d/9698.misc
new file mode 100644
index 0000000000..d199e846c5
--- /dev/null
+++ b/changelog.d/9698.misc
@@ -0,0 +1 @@
+Suppress "CryptographyDeprecationWarning: int_from_bytes is deprecated".
diff --git a/changelog.d/9699.bugfix b/changelog.d/9699.bugfix
new file mode 100644
index 0000000000..e871825b33
--- /dev/null
+++ b/changelog.d/9699.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index a442b34598..5b7bf02776 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -25,42 +25,40 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
# install the OS build deps
RUN apt-get update && apt-get install -y \
- build-essential \
- libffi-dev \
- libjpeg-dev \
- libpq-dev \
- libssl-dev \
- libwebp-dev \
- libxml++2.6-dev \
- libxslt1-dev \
- openssl \
- rustc \
- zlib1g-dev \
- && rm -rf /var/lib/apt/lists/*
-
-# Build dependencies that are not available as wheels, to speed up rebuilds
-RUN pip install --prefix="/install" --no-warn-script-location \
- cryptography \
- frozendict \
- jaeger-client \
- opentracing \
- # Match the version constraints of Synapse
- "prometheus_client>=0.4.0" \
- psycopg2 \
- pycparser \
- pyrsistent \
- pyyaml \
- simplejson \
- threadloop \
- thrift
-
-# now install synapse and all of the python deps to /install.
-COPY synapse /synapse/synapse/
+ build-essential \
+ libffi-dev \
+ libjpeg-dev \
+ libpq-dev \
+ libssl-dev \
+ libwebp-dev \
+ libxml++2.6-dev \
+ libxslt1-dev \
+ openssl \
+ rustc \
+ zlib1g-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy just what we need to pip install
COPY scripts /synapse/scripts/
COPY MANIFEST.in README.rst setup.py synctl /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
+# To speed up rebuilds, install all of the dependencies before we copy over
+# the whole synapse project so that we this layer in the Docker cache can be
+# used while you develop on the source
+#
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
RUN pip install --prefix="/install" --no-warn-script-location \
- /synapse[all]
+ /synapse[all]
+
+# Copy over the rest of the project
+COPY synapse /synapse/synapse/
+
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
###
### Stage 1: runtime
@@ -69,16 +67,16 @@ RUN pip install --prefix="/install" --no-warn-script-location \
FROM docker.io/python:${PYTHON_VERSION}-slim
RUN apt-get update && apt-get install -y \
- curl \
- gosu \
- libjpeg62-turbo \
- libpq5 \
- libwebp6 \
- xmlsec1 \
- libjemalloc2 \
- libssl-dev \
- openssl \
- && rm -rf /var/lib/apt/lists/*
+ curl \
+ gosu \
+ libjpeg62-turbo \
+ libpq5 \
+ libwebp6 \
+ xmlsec1 \
+ libjemalloc2 \
+ libssl-dev \
+ openssl \
+ && rm -rf /var/lib/apt/lists/*
COPY --from=builder /install /usr/local
COPY ./docker/start.py /start.py
@@ -91,4 +89,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
ENTRYPOINT ["/start.py"]
HEALTHCHECK --interval=1m --timeout=5s \
- CMD curl -fSs http://localhost:8008/health || exit 1
+ CMD curl -fSs http://localhost:8008/health || exit 1
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 43b1f1e94b..3912c8994c 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -21,8 +21,10 @@ import signal
import socket
import sys
import traceback
+import warnings
from typing import Awaitable, Callable, Iterable
+from cryptography.utils import CryptographyDeprecationWarning
from typing_extensions import NoReturn
from twisted.internet import defer, error, reactor
@@ -195,6 +197,25 @@ def listen_metrics(bind_addresses, port):
start_http_server(port, addr=host, registry=RegistryProxy)
+def listen_manhole(bind_addresses: Iterable[str], port: int, manhole_globals: dict):
+ # twisted.conch.manhole 21.1.0 uses "int_from_bytes", which produces a confusing
+ # warning. It's fixed by https://github.com/twisted/twisted/pull/1522), so
+ # suppress the warning for now.
+ warnings.filterwarnings(
+ action="ignore",
+ category=CryptographyDeprecationWarning,
+ message="int_from_bytes is deprecated",
+ )
+
+ from synapse.util.manhole import manhole
+
+ listen_tcp(
+ bind_addresses,
+ port,
+ manhole(username="matrix", password="rabbithole", globals=manhole_globals),
+ )
+
+
def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
"""
Create a TCP socket for a port and several addresses
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index caef394e1d..6139881dbb 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -147,7 +147,6 @@ from synapse.storage.databases.main.user_directory import UserDirectoryStore
from synapse.types import ReadReceipt
from synapse.util.async_helpers import Linearizer
from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string
logger = logging.getLogger("synapse.app.generic_worker")
@@ -640,12 +639,8 @@ class GenericWorkerServer(HomeServer):
if listener.type == "http":
self._listen_http(listener)
elif listener.type == "manhole":
- _base.listen_tcp(
- listener.bind_addresses,
- listener.port,
- manhole(
- username="matrix", password="rabbithole", globals={"hs": self}
- ),
+ _base.listen_manhole(
+ listener.bind_addresses, listener.port, manhole_globals={"hs": self}
)
elif listener.type == "metrics":
if not self.get_config().enable_metrics:
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 244657cb88..3bfe9d507f 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -67,7 +67,6 @@ from synapse.storage import DataStore
from synapse.storage.engines import IncorrectDatabaseSetup
from synapse.storage.prepare_database import UpgradeDatabaseException
from synapse.util.httpresourcetree import create_resource_tree
-from synapse.util.manhole import manhole
from synapse.util.module_loader import load_module
from synapse.util.versionstring import get_version_string
@@ -288,12 +287,8 @@ class SynapseHomeServer(HomeServer):
if listener.type == "http":
self._listening_services.extend(self._listener_http(config, listener))
elif listener.type == "manhole":
- listen_tcp(
- listener.bind_addresses,
- listener.port,
- manhole(
- username="matrix", password="rabbithole", globals={"hs": self}
- ),
+ _base.listen_manhole(
+ listener.bind_addresses, listener.port, manhole_globals={"hs": self}
)
elif listener.type == "replication":
services = listen_tcp(
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 14ddaed026..2a1c925ee8 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import itertools
import logging
from typing import List, Set
@@ -101,7 +102,7 @@ CONDITIONAL_REQUIREMENTS = {
"txacme>=0.9.2",
# txacme depends on eliot. Eliot 1.8.0 is incompatible with
# python 3.5.2, as per https://github.com/itamarst/eliot/issues/418
- 'eliot<1.8.0;python_version<"3.5.3"',
+ "eliot<1.8.0;python_version<'3.5.3'",
],
"saml2": [
# pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749)
@@ -131,6 +132,18 @@ for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
+# ensure there are no double-quote characters in any of the deps (otherwise the
+# 'pip install' incantation in DependencyException will break)
+for dep in itertools.chain(
+ REQUIREMENTS,
+ *CONDITIONAL_REQUIREMENTS.values(),
+):
+ if '"' in dep:
+ raise Exception(
+ "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,)
+ )
+
+
def list_requirements():
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
@@ -150,7 +163,7 @@ class DependencyException(Exception):
@property
def dependencies(self):
for i in self.args[0]:
- yield "'" + i + "'"
+ yield '"' + i + '"'
def check_requirements(for_feature=None):
|
