diff --git a/changelog.d/12088.misc b/changelog.d/12088.misc
new file mode 100644
index 0000000000..ce4213650c
--- /dev/null
+++ b/changelog.d/12088.misc
@@ -0,0 +1 @@
+Inspect application dependencies using `importlib.metadata` or its backport.
\ No newline at end of file
diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py
index ee51480a9e..334c3d2c17 100644
--- a/synapse/app/__init__.py
+++ b/synapse/app/__init__.py
@@ -15,13 +15,13 @@ import logging
import sys
from typing import Container
-from synapse import python_dependencies # noqa: E402
+from synapse.util import check_dependencies
logger = logging.getLogger(__name__)
try:
- python_dependencies.check_requirements()
-except python_dependencies.DependencyException as e:
+ check_dependencies.check_requirements()
+except check_dependencies.DependencyException as e:
sys.stderr.writelines(
e.message # noqa: B306, DependencyException.message is a property
)
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b9931001c2..a6789a840e 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -59,7 +59,6 @@ from synapse.http.server import (
from synapse.http.site import SynapseSite
from synapse.logging.context import LoggingContext
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
-from synapse.python_dependencies import check_requirements
from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
from synapse.rest import ClientRestResource
@@ -70,6 +69,7 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree
from synapse.rest.well_known import well_known_resource
from synapse.server import HomeServer
from synapse.storage import DataStore
+from synapse.util.check_dependencies import check_requirements
from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.module_loader import load_module
diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 387ac6d115..9a68da9c33 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -20,7 +20,7 @@ from typing import Callable, Dict, Optional
import attr
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError
diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py
index 1cc26e7578..f62292ecf6 100644
--- a/synapse/config/metrics.py
+++ b/synapse/config/metrics.py
@@ -15,7 +15,7 @@
import attr
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError
diff --git a/synapse/config/oidc.py b/synapse/config/oidc.py
index e783b11315..f7e4f9ef22 100644
--- a/synapse/config/oidc.py
+++ b/synapse/config/oidc.py
@@ -20,11 +20,11 @@ import attr
from synapse.config._util import validate_config
from synapse.config.sso import SsoAttributeRequirement
-from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.module_loader import load_module
from synapse.util.stringutils import parse_and_validate_mxc_uri
+from ..util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError, read_file
DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
diff --git a/synapse/config/redis.py b/synapse/config/redis.py
index 33104af734..bdb1aac3a2 100644
--- a/synapse/config/redis.py
+++ b/synapse/config/redis.py
@@ -13,7 +13,7 @@
# limitations under the License.
from synapse.config._base import Config
-from synapse.python_dependencies import check_requirements
+from synapse.util.check_dependencies import check_requirements
class RedisConfig(Config):
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 1980351e77..0a0d901bfb 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -20,8 +20,8 @@ from urllib.request import getproxies_environment # type: ignore
import attr
from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
-from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
+from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module
from ._base import Config, ConfigError
diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py
index ec9d9f65e7..43c456d5c6 100644
--- a/synapse/config/saml2.py
+++ b/synapse/config/saml2.py
@@ -17,8 +17,8 @@ import logging
from typing import Any, List, Set
from synapse.config.sso import SsoAttributeRequirement
-from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
+from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module, load_python_module
from ._base import Config, ConfigError
diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py
index 21b9a88353..7aff618ea6 100644
--- a/synapse/config/tracer.py
+++ b/synapse/config/tracer.py
@@ -14,7 +14,7 @@
from typing import Set
-from synapse.python_dependencies import DependencyException, check_requirements
+from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index f43fbb5842..8f48a33936 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -17,14 +17,7 @@
import itertools
import logging
-from typing import List, Set
-
-from pkg_resources import (
- DistributionNotFound,
- Requirement,
- VersionConflict,
- get_provider,
-)
+from typing import Set
logger = logging.getLogger(__name__)
@@ -90,6 +83,8 @@ REQUIREMENTS = [
# ijson 3.1.4 fixes a bug with "." in property names
"ijson>=3.1.4",
"matrix-common~=1.1.0",
+ # For runtime introspection of our dependencies
+ "packaging~=21.3",
]
CONDITIONAL_REQUIREMENTS = {
@@ -144,102 +139,6 @@ def list_requirements():
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
-class DependencyException(Exception):
- @property
- def message(self):
- return "\n".join(
- [
- "Missing Requirements: %s" % (", ".join(self.dependencies),),
- "To install run:",
- " pip install --upgrade --force %s" % (" ".join(self.dependencies),),
- "",
- ]
- )
-
- @property
- def dependencies(self):
- for i in self.args[0]:
- yield '"' + i + '"'
-
-
-def check_requirements(for_feature=None):
- deps_needed = []
- errors = []
-
- if for_feature:
- reqs = CONDITIONAL_REQUIREMENTS[for_feature]
- else:
- reqs = REQUIREMENTS
-
- for dependency in reqs:
- try:
- _check_requirement(dependency)
- except VersionConflict as e:
- deps_needed.append(dependency)
- errors.append(
- "Needed %s, got %s==%s"
- % (
- dependency,
- e.dist.project_name, # type: ignore[attr-defined] # noqa
- e.dist.version, # type: ignore[attr-defined] # noqa
- )
- )
- except DistributionNotFound:
- deps_needed.append(dependency)
- if for_feature:
- errors.append(
- "Needed %s for the '%s' feature but it was not installed"
- % (dependency, for_feature)
- )
- else:
- errors.append("Needed %s but it was not installed" % (dependency,))
-
- if not for_feature:
- # Check the optional dependencies are up to date. We allow them to not be
- # installed.
- OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
-
- for dependency in OPTS:
- try:
- _check_requirement(dependency)
- except VersionConflict as e:
- deps_needed.append(dependency)
- errors.append(
- "Needed optional %s, got %s==%s"
- % (
- dependency,
- e.dist.project_name, # type: ignore[attr-defined] # noqa
- e.dist.version, # type: ignore[attr-defined] # noqa
- )
- )
- except DistributionNotFound:
- # If it's not found, we don't care
- pass
-
- if deps_needed:
- for err in errors:
- logging.error(err)
-
- raise DependencyException(deps_needed)
-
-
-def _check_requirement(dependency_string):
- """Parses a dependency string, and checks if the specified requirement is installed
-
- Raises:
- VersionConflict if the requirement is installed, but with the the wrong version
- DistributionNotFound if nothing is found to provide the requirement
- """
- req = Requirement.parse(dependency_string)
-
- # first check if the markers specify that this requirement needs installing
- if req.marker is not None and not req.marker.evaluate():
- # not required for this environment
- return
-
- get_provider(req)
-
-
if __name__ == "__main__":
import sys
diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py
new file mode 100644
index 0000000000..3a1f6b3c75
--- /dev/null
+++ b/synapse/util/check_dependencies.py
@@ -0,0 +1,127 @@
+import logging
+from typing import Iterable, NamedTuple, Optional
+
+from packaging.requirements import Requirement
+
+DISTRIBUTION_NAME = "matrix-synapse"
+
+try:
+ from importlib import metadata
+except ImportError:
+ import importlib_metadata as metadata # type: ignore[no-redef]
+
+
+class DependencyException(Exception):
+ @property
+ def message(self) -> str:
+ return "\n".join(
+ [
+ "Missing Requirements: %s" % (", ".join(self.dependencies),),
+ "To install run:",
+ " pip install --upgrade --force %s" % (" ".join(self.dependencies),),
+ "",
+ ]
+ )
+
+ @property
+ def dependencies(self) -> Iterable[str]:
+ for i in self.args[0]:
+ yield '"' + i + '"'
+
+
+EXTRAS = set(metadata.metadata(DISTRIBUTION_NAME).get_all("Provides-Extra"))
+
+
+class Dependency(NamedTuple):
+ requirement: Requirement
+ must_be_installed: bool
+
+
+def _generic_dependencies() -> Iterable[Dependency]:
+ """Yield pairs (requirement, must_be_installed)."""
+ requirements = metadata.requires(DISTRIBUTION_NAME)
+ assert requirements is not None
+ for raw_requirement in requirements:
+ req = Requirement(raw_requirement)
+ # https://packaging.pypa.io/en/latest/markers.html#usage notes that
+ # > Evaluating an extra marker with no environment is an error
+ # so we pass in a dummy empty extra value here.
+ must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
+ yield Dependency(req, must_be_installed)
+
+
+def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
+ """Yield additional dependencies needed for a given `extra`."""
+ requirements = metadata.requires(DISTRIBUTION_NAME)
+ assert requirements is not None
+ for raw_requirement in requirements:
+ req = Requirement(raw_requirement)
+ # Exclude mandatory deps by only selecting deps needed with this extra.
+ if (
+ req.marker is not None
+ and req.marker.evaluate({"extra": extra})
+ and not req.marker.evaluate({"extra": ""})
+ ):
+ yield Dependency(req, True)
+
+
+def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
+ if extra:
+ return f"Need {requirement.name} for {extra}, but it is not installed"
+ else:
+ return f"Need {requirement.name}, but it is not installed"
+
+
+def _incorrect_version(
+ requirement: Requirement, got: str, extra: Optional[str] = None
+) -> str:
+ if extra:
+ return f"Need {requirement} for {extra}, but got {requirement.name}=={got}"
+ else:
+ return f"Need {requirement}, but got {requirement.name}=={got}"
+
+
+def check_requirements(extra: Optional[str] = None) -> None:
+ """Check Synapse's dependencies are present and correctly versioned.
+
+ If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
+ `pip install matrix-synapse[saml2]`).
+
+ If `extra` is None, this function checks that
+ - all mandatory dependencies are installed and correctly versioned, and
+ - each optional dependency that's installed is correctly versioned.
+
+ If `extra` is not None, this function checks that
+ - the dependencies needed for that extra are installed and correctly versioned.
+
+ :raises DependencyException: if a dependency is missing or incorrectly versioned.
+ :raises ValueError: if this extra does not exist.
+ """
+ # First work out which dependencies are required, and which are optional.
+ if extra is None:
+ dependencies = _generic_dependencies()
+ elif extra in EXTRAS:
+ dependencies = _dependencies_for_extra(extra)
+ else:
+ raise ValueError(f"Synapse does not provide the feature '{extra}'")
+
+ deps_unfulfilled = []
+ errors = []
+
+ for (requirement, must_be_installed) in dependencies:
+ try:
+ dist: metadata.Distribution = metadata.distribution(requirement.name)
+ except metadata.PackageNotFoundError:
+ if must_be_installed:
+ deps_unfulfilled.append(requirement.name)
+ errors.append(_not_installed(requirement, extra))
+ else:
+ if not requirement.specifier.contains(dist.version):
+ deps_unfulfilled.append(requirement.name)
+ errors.append(_incorrect_version(requirement, dist.version, extra))
+
+ if deps_unfulfilled:
+ for err in errors:
+ logging.error(err)
+
+ raise DependencyException(deps_unfulfilled)
diff --git a/tests/util/test_check_dependencies.py b/tests/util/test_check_dependencies.py
new file mode 100644
index 0000000000..3c07252252
--- /dev/null
+++ b/tests/util/test_check_dependencies.py
@@ -0,0 +1,95 @@
+from contextlib import contextmanager
+from typing import Generator, Optional
+from unittest.mock import patch
+
+from synapse.util.check_dependencies import (
+ DependencyException,
+ check_requirements,
+ metadata,
+)
+
+from tests.unittest import TestCase
+
+
+class DummyDistribution(metadata.Distribution):
+ def __init__(self, version: str):
+ self._version = version
+
+ @property
+ def version(self):
+ return self._version
+
+ def locate_file(self, path):
+ raise NotImplementedError()
+
+ def read_text(self, filename):
+ raise NotImplementedError()
+
+
+old = DummyDistribution("0.1.2")
+new = DummyDistribution("1.2.3")
+
+# could probably use stdlib TestCase --- no need for twisted here
+
+
+class TestDependencyChecker(TestCase):
+ @contextmanager
+ def mock_installed_package(
+ self, distribution: Optional[DummyDistribution]
+ ) -> Generator[None, None, None]:
+ """Pretend that looking up any distribution yields the given `distribution`."""
+
+ def mock_distribution(name: str):
+ if distribution is None:
+ raise metadata.PackageNotFoundError
+ else:
+ return distribution
+
+ with patch(
+ "synapse.util.check_dependencies.metadata.distribution",
+ mock_distribution,
+ ):
+ yield
+
+ def test_mandatory_dependency(self) -> None:
+ """Complain if a required package is missing or old."""
+ with patch(
+ "synapse.util.check_dependencies.metadata.requires",
+ return_value=["dummypkg >= 1"],
+ ):
+ with self.mock_installed_package(None):
+ self.assertRaises(DependencyException, check_requirements)
+ with self.mock_installed_package(old):
+ self.assertRaises(DependencyException, check_requirements)
+ with self.mock_installed_package(new):
+ # should not raise
+ check_requirements()
+
+ def test_generic_check_of_optional_dependency(self) -> None:
+ """Complain if an optional package is old."""
+ with patch(
+ "synapse.util.check_dependencies.metadata.requires",
+ return_value=["dummypkg >= 1; extra == 'cool-extra'"],
+ ):
+ with self.mock_installed_package(None):
+ # should not raise
+ check_requirements()
+ with self.mock_installed_package(old):
+ self.assertRaises(DependencyException, check_requirements)
+ with self.mock_installed_package(new):
+ # should not raise
+ check_requirements()
+
+ def test_check_for_extra_dependencies(self) -> None:
+ """Complain if a package required for an extra is missing or old."""
+ with patch(
+ "synapse.util.check_dependencies.metadata.requires",
+ return_value=["dummypkg >= 1; extra == 'cool-extra'"],
+ ), patch("synapse.util.check_dependencies.EXTRAS", {"cool-extra"}):
+ with self.mock_installed_package(None):
+ self.assertRaises(DependencyException, check_requirements, "cool-extra")
+ with self.mock_installed_package(old):
+ self.assertRaises(DependencyException, check_requirements, "cool-extra")
+ with self.mock_installed_package(new):
+ # should not raise
+ check_requirements()
|
