summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--changelog.d/12088.misc1
-rw-r--r--synapse/app/__init__.py6
-rw-r--r--synapse/app/homeserver.py2
-rw-r--r--synapse/config/cache.py2
-rw-r--r--synapse/config/metrics.py2
-rw-r--r--synapse/config/oidc.py2
-rw-r--r--synapse/config/redis.py2
-rw-r--r--synapse/config/repository.py2
-rw-r--r--synapse/config/saml2.py2
-rw-r--r--synapse/config/tracer.py2
-rw-r--r--synapse/python_dependencies.py107
-rw-r--r--synapse/util/check_dependencies.py127
-rw-r--r--tests/util/test_check_dependencies.py95
13 files changed, 237 insertions, 115 deletions
diff --git a/changelog.d/12088.misc b/changelog.d/12088.misc
new file mode 100644
index 0000000000..ce4213650c
--- /dev/null
+++ b/changelog.d/12088.misc
@@ -0,0 +1 @@
+Inspect application dependencies using `importlib.metadata` or its backport.
\ No newline at end of file
diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py
index ee51480a9e..334c3d2c17 100644
--- a/synapse/app/__init__.py
+++ b/synapse/app/__init__.py
@@ -15,13 +15,13 @@ import logging
 import sys
 from typing import Container
 
-from synapse import python_dependencies  # noqa: E402
+from synapse.util import check_dependencies
 
 logger = logging.getLogger(__name__)
 
 try:
-    python_dependencies.check_requirements()
-except python_dependencies.DependencyException as e:
+    check_dependencies.check_requirements()
+except check_dependencies.DependencyException as e:
     sys.stderr.writelines(
         e.message  # noqa: B306, DependencyException.message is a property
     )
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b9931001c2..a6789a840e 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -59,7 +59,6 @@ from synapse.http.server import (
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
-from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
@@ -70,6 +69,7 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
 from synapse.storage import DataStore
+from synapse.util.check_dependencies import check_requirements
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.module_loader import load_module
 
diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 387ac6d115..9a68da9c33 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -20,7 +20,7 @@ from typing import Callable, Dict, Optional
 
 import attr
 
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
 
 from ._base import Config, ConfigError
 
diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py
index 1cc26e7578..f62292ecf6 100644
--- a/synapse/config/metrics.py
+++ b/synapse/config/metrics.py
@@ -15,7 +15,7 @@
 
 import attr
 
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
 
 from ._base import Config, ConfigError
 
diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py
index e783b11315..f7e4f9ef22 100644
--- a/synapse/config/oidc.py
+++ b/synapse/config/oidc.py
@@ -20,11 +20,11 @@ import attr
 
 from synapse.config._util import validate_config
 from synapse.config.sso import SsoAttributeRequirement
-from synapse.python_dependencies import DependencyException, check_requirements
 from synapse.types import JsonDict
 from synapse.util.module_loader import load_module
 from synapse.util.stringutils import parse_and_validate_mxc_uri
 
+from ..util.check_dependencies import DependencyException, check_requirements
 from ._base import Config, ConfigError, read_file
 
 DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
diff --git a/synapse/config/redis.py b/synapse/config/redis.py
index 33104af734..bdb1aac3a2 100644
--- a/synapse/config/redis.py
+++ b/synapse/config/redis.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from synapse.config._base import Config
-from synapse.python_dependencies import check_requirements
+from synapse.util.check_dependencies import check_requirements
 
 
 class RedisConfig(Config):
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 1980351e77..0a0d901bfb 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -20,8 +20,8 @@ from urllib.request import getproxies_environment  # type: ignore
 import attr
 
 from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
-from synapse.python_dependencies import DependencyException, check_requirements
 from synapse.types import JsonDict
+from synapse.util.check_dependencies import DependencyException, check_requirements
 from synapse.util.module_loader import load_module
 
 from ._base import Config, ConfigError
diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py
index ec9d9f65e7..43c456d5c6 100644
--- a/synapse/config/saml2.py
+++ b/synapse/config/saml2.py
@@ -17,8 +17,8 @@ import logging
 from typing import Any, List, Set
 
 from synapse.config.sso import SsoAttributeRequirement
-from synapse.python_dependencies import DependencyException, check_requirements
 from synapse.types import JsonDict
+from synapse.util.check_dependencies import DependencyException, check_requirements
 from synapse.util.module_loader import load_module, load_python_module
 
 from ._base import Config, ConfigError
diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py
index 21b9a88353..7aff618ea6 100644
--- a/synapse/config/tracer.py
+++ b/synapse/config/tracer.py
@@ -14,7 +14,7 @@
 
 from typing import Set
 
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
 
 from ._base import Config, ConfigError
 
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index f43fbb5842..8f48a33936 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -17,14 +17,7 @@
 
 import itertools
 import logging
-from typing import List, Set
-
-from pkg_resources import (
-    DistributionNotFound,
-    Requirement,
-    VersionConflict,
-    get_provider,
-)
+from typing import Set
 
 logger = logging.getLogger(__name__)
 
@@ -90,6 +83,8 @@ REQUIREMENTS = [
     # ijson 3.1.4 fixes a bug with "." in property names
     "ijson>=3.1.4",
     "matrix-common~=1.1.0",
+    # For runtime introspection of our dependencies
+    "packaging~=21.3",
 ]
 
 CONDITIONAL_REQUIREMENTS = {
@@ -144,102 +139,6 @@ def list_requirements():
     return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
 
 
-class DependencyException(Exception):
-    @property
-    def message(self):
-        return "\n".join(
-            [
-                "Missing Requirements: %s" % (", ".join(self.dependencies),),
-                "To install run:",
-                "    pip install --upgrade --force %s" % (" ".join(self.dependencies),),
-                "",
-            ]
-        )
-
-    @property
-    def dependencies(self):
-        for i in self.args[0]:
-            yield '"' + i + '"'
-
-
-def check_requirements(for_feature=None):
-    deps_needed = []
-    errors = []
-
-    if for_feature:
-        reqs = CONDITIONAL_REQUIREMENTS[for_feature]
-    else:
-        reqs = REQUIREMENTS
-
-    for dependency in reqs:
-        try:
-            _check_requirement(dependency)
-        except VersionConflict as e:
-            deps_needed.append(dependency)
-            errors.append(
-                "Needed %s, got %s==%s"
-                % (
-                    dependency,
-                    e.dist.project_name,  # type: ignore[attr-defined] # noqa
-                    e.dist.version,  # type: ignore[attr-defined] # noqa
-                )
-            )
-        except DistributionNotFound:
-            deps_needed.append(dependency)
-            if for_feature:
-                errors.append(
-                    "Needed %s for the '%s' feature but it was not installed"
-                    % (dependency, for_feature)
-                )
-            else:
-                errors.append("Needed %s but it was not installed" % (dependency,))
-
-    if not for_feature:
-        # Check the optional dependencies are up to date. We allow them to not be
-        # installed.
-        OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
-
-        for dependency in OPTS:
-            try:
-                _check_requirement(dependency)
-            except VersionConflict as e:
-                deps_needed.append(dependency)
-                errors.append(
-                    "Needed optional %s, got %s==%s"
-                    % (
-                        dependency,
-                        e.dist.project_name,  # type: ignore[attr-defined] # noqa
-                        e.dist.version,  # type: ignore[attr-defined] # noqa
-                    )
-                )
-            except DistributionNotFound:
-                # If it's not found, we don't care
-                pass
-
-    if deps_needed:
-        for err in errors:
-            logging.error(err)
-
-        raise DependencyException(deps_needed)
-
-
-def _check_requirement(dependency_string):
-    """Parses a dependency string, and checks if the specified requirement is installed
-
-    Raises:
-        VersionConflict if the requirement is installed, but with the the wrong version
-        DistributionNotFound if nothing is found to provide the requirement
-    """
-    req = Requirement.parse(dependency_string)
-
-    # first check if the markers specify that this requirement needs installing
-    if req.marker is not None and not req.marker.evaluate():
-        # not required for this environment
-        return
-
-    get_provider(req)
-
-
 if __name__ == "__main__":
     import sys
 
diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py
new file mode 100644
index 0000000000..3a1f6b3c75
--- /dev/null
+++ b/synapse/util/check_dependencies.py
@@ -0,0 +1,127 @@
+import logging
+from typing import Iterable, NamedTuple, Optional
+
+from packaging.requirements import Requirement
+
+DISTRIBUTION_NAME = "matrix-synapse"
+
+try:
+    from importlib import metadata
+except ImportError:
+    import importlib_metadata as metadata  # type: ignore[no-redef]
+
+
+class DependencyException(Exception):
+    @property
+    def message(self) -> str:
+        return "\n".join(
+            [
+                "Missing Requirements: %s" % (", ".join(self.dependencies),),
+                "To install run:",
+                "    pip install --upgrade --force %s" % (" ".join(self.dependencies),),
+                "",
+            ]
+        )
+
+    @property
+    def dependencies(self) -> Iterable[str]:
+        for i in self.args[0]:
+            yield '"' + i + '"'
+
+
+EXTRAS = set(metadata.metadata(DISTRIBUTION_NAME).get_all("Provides-Extra"))
+
+
+class Dependency(NamedTuple):
+    requirement: Requirement
+    must_be_installed: bool
+
+
+def _generic_dependencies() -> Iterable[Dependency]:
+    """Yield pairs (requirement, must_be_installed)."""
+    requirements = metadata.requires(DISTRIBUTION_NAME)
+    assert requirements is not None
+    for raw_requirement in requirements:
+        req = Requirement(raw_requirement)
+        # https://packaging.pypa.io/en/latest/markers.html#usage notes that
+        #   > Evaluating an extra marker with no environment is an error
+        # so we pass in a dummy empty extra value here.
+        must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
+        yield Dependency(req, must_be_installed)
+
+
+def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
+    """Yield additional dependencies needed for a given `extra`."""
+    requirements = metadata.requires(DISTRIBUTION_NAME)
+    assert requirements is not None
+    for raw_requirement in requirements:
+        req = Requirement(raw_requirement)
+        # Exclude mandatory deps by only selecting deps needed with this extra.
+        if (
+            req.marker is not None
+            and req.marker.evaluate({"extra": extra})
+            and not req.marker.evaluate({"extra": ""})
+        ):
+            yield Dependency(req, True)
+
+
+def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
+    if extra:
+        return f"Need {requirement.name} for {extra}, but it is not installed"
+    else:
+        return f"Need {requirement.name}, but it is not installed"
+
+
+def _incorrect_version(
+    requirement: Requirement, got: str, extra: Optional[str] = None
+) -> str:
+    if extra:
+        return f"Need {requirement} for {extra}, but got {requirement.name}=={got}"
+    else:
+        return f"Need {requirement}, but got {requirement.name}=={got}"
+
+
+def check_requirements(extra: Optional[str] = None) -> None:
+    """Check Synapse's dependencies are present and correctly versioned.
+
+    If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
+    `pip install matrix-synapse[saml2]`).
+
+    If `extra` is None, this function checks that
+    - all mandatory dependencies are installed and correctly versioned, and
+    - each optional dependency that's installed is correctly versioned.
+
+    If `extra` is not None, this function checks that
+    - the dependencies needed for that extra are installed and correctly versioned.
+
+    :raises DependencyException: if a dependency is missing or incorrectly versioned.
+    :raises ValueError: if this extra does not exist.
+    """
+    # First work out which dependencies are required, and which are optional.
+    if extra is None:
+        dependencies = _generic_dependencies()
+    elif extra in EXTRAS:
+        dependencies = _dependencies_for_extra(extra)
+    else:
+        raise ValueError(f"Synapse does not provide the feature '{extra}'")
+
+    deps_unfulfilled = []
+    errors = []
+
+    for (requirement, must_be_installed) in dependencies:
+        try:
+            dist: metadata.Distribution = metadata.distribution(requirement.name)
+        except metadata.PackageNotFoundError:
+            if must_be_installed:
+                deps_unfulfilled.append(requirement.name)
+                errors.append(_not_installed(requirement, extra))
+        else:
+            if not requirement.specifier.contains(dist.version):
+                deps_unfulfilled.append(requirement.name)
+                errors.append(_incorrect_version(requirement, dist.version, extra))
+
+    if deps_unfulfilled:
+        for err in errors:
+            logging.error(err)
+
+        raise DependencyException(deps_unfulfilled)
diff --git a/tests/util/test_check_dependencies.py b/tests/util/test_check_dependencies.py
new file mode 100644
index 0000000000..3c07252252
--- /dev/null
+++ b/tests/util/test_check_dependencies.py
@@ -0,0 +1,95 @@
+from contextlib import contextmanager
+from typing import Generator, Optional
+from unittest.mock import patch
+
+from synapse.util.check_dependencies import (
+    DependencyException,
+    check_requirements,
+    metadata,
+)
+
+from tests.unittest import TestCase
+
+
+class DummyDistribution(metadata.Distribution):
+    def __init__(self, version: str):
+        self._version = version
+
+    @property
+    def version(self):
+        return self._version
+
+    def locate_file(self, path):
+        raise NotImplementedError()
+
+    def read_text(self, filename):
+        raise NotImplementedError()
+
+
+old = DummyDistribution("0.1.2")
+new = DummyDistribution("1.2.3")
+
+# could probably use stdlib TestCase --- no need for twisted here
+
+
+class TestDependencyChecker(TestCase):
+    @contextmanager
+    def mock_installed_package(
+        self, distribution: Optional[DummyDistribution]
+    ) -> Generator[None, None, None]:
+        """Pretend that looking up any distribution yields the given `distribution`."""
+
+        def mock_distribution(name: str):
+            if distribution is None:
+                raise metadata.PackageNotFoundError
+            else:
+                return distribution
+
+        with patch(
+            "synapse.util.check_dependencies.metadata.distribution",
+            mock_distribution,
+        ):
+            yield
+
+    def test_mandatory_dependency(self) -> None:
+        """Complain if a required package is missing or old."""
+        with patch(
+            "synapse.util.check_dependencies.metadata.requires",
+            return_value=["dummypkg >= 1"],
+        ):
+            with self.mock_installed_package(None):
+                self.assertRaises(DependencyException, check_requirements)
+            with self.mock_installed_package(old):
+                self.assertRaises(DependencyException, check_requirements)
+            with self.mock_installed_package(new):
+                # should not raise
+                check_requirements()
+
+    def test_generic_check_of_optional_dependency(self) -> None:
+        """Complain if an optional package is old."""
+        with patch(
+            "synapse.util.check_dependencies.metadata.requires",
+            return_value=["dummypkg >= 1; extra == 'cool-extra'"],
+        ):
+            with self.mock_installed_package(None):
+                # should not raise
+                check_requirements()
+            with self.mock_installed_package(old):
+                self.assertRaises(DependencyException, check_requirements)
+            with self.mock_installed_package(new):
+                # should not raise
+                check_requirements()
+
+    def test_check_for_extra_dependencies(self) -> None:
+        """Complain if a package required for an extra is missing or old."""
+        with patch(
+            "synapse.util.check_dependencies.metadata.requires",
+            return_value=["dummypkg >= 1; extra == 'cool-extra'"],
+        ), patch("synapse.util.check_dependencies.EXTRAS", {"cool-extra"}):
+            with self.mock_installed_package(None):
+                self.assertRaises(DependencyException, check_requirements, "cool-extra")
+            with self.mock_installed_package(old):
+                self.assertRaises(DependencyException, check_requirements, "cool-extra")
+            with self.mock_installed_package(new):
+                # should not raise
+                check_requirements()
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-09-20 13:27:31 +0000