diff options
| -rw-r--r-- | .dockerignore | 4 | ||||
| -rw-r--r-- | .github/workflows/tests.yml | 2 | ||||
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | MANIFEST.in | 54 | ||||
| -rw-r--r-- | changelog.d/12337.feature | 1 | ||||
| -rw-r--r-- | changelog.d/12478.misc | 1 | ||||
| -rw-r--r-- | docker/Dockerfile | 6 | ||||
| -rw-r--r-- | mypy.ini | 1 | ||||
| -rw-r--r-- | pyproject.toml | 4 | ||||
| -rw-r--r-- | setup.cfg | 9 | ||||
| -rwxr-xr-x | setup.py | 183 | ||||
| -rw-r--r-- | synapse/__init__.py | 4 | ||||
| -rw-r--r-- | synapse/python_dependencies.py | 151 |
13 files changed, 11 insertions, 412 deletions
diff --git a/.dockerignore b/.dockerignore index a236760cf1..7809863ef3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,4 @@ !pyproject.toml !poetry.lock -# TODO: remove these once we have moved over to using poetry-core in pyproject.toml -!MANIFEST.in -!setup.py - **/__pycache__ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7946b76dba..cad4cb6d77 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install -e . + - run: pip install . - run: scripts-dev/generate_sample_config.sh --check - run: scripts-dev/config-lint.sh diff --git a/.gitignore b/.gitignore index c011cd27a4..e58affb241 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ _trial_temp*/ .DS_Store __pycache__/ -# We do want the poetry lockfile. TODO: is there a good reason for ignoring -# '*.lock' above? If not, let's nuke it. +# We do want the poetry lockfile. !poetry.lock # stuff that is likely to exist when you run a server locally diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d744c090ac..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,54 +0,0 @@ -include LICENSE -include VERSION -include *.rst -include *.md -include demo/README -include demo/demo.tls.dh -include demo/*.py -include demo/*.sh - -include synapse/py.typed -recursive-include synapse/storage *.sql -recursive-include synapse/storage *.sql.postgres -recursive-include synapse/storage *.sql.sqlite -recursive-include synapse/storage *.py -recursive-include synapse/storage *.txt -recursive-include synapse/storage *.md - -recursive-include docs * -recursive-include scripts-dev * -recursive-include synapse *.pyi -recursive-include tests *.py -recursive-include tests *.pem -recursive-include tests *.p8 -recursive-include tests *.crt -recursive-include tests *.key - -recursive-include synapse/res * -recursive-include synapse/static *.css -recursive-include synapse/static *.gif -recursive-include synapse/static *.html -recursive-include synapse/static *.js - -exclude .codecov.yml -exclude .coveragerc -exclude .dockerignore -exclude .editorconfig -exclude Dockerfile -exclude mypy.ini -exclude sytest-blacklist -exclude test_postgresql.sh - -include book.toml -include pyproject.toml -recursive-include changelog.d * - -include .flake8 -prune .circleci -prune .github -prune .ci -prune contrib -prune debian -prune demo/etc -prune docker -prune stubs diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature new file mode 100644 index 0000000000..6c4444c707 --- /dev/null +++ b/changelog.d/12337.feature @@ -0,0 +1 @@ +Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc new file mode 100644 index 0000000000..061a604a1e --- /dev/null +++ b/changelog.d/12478.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f87702cc8..4523c60645 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ WORKDIR /synapse # Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ +COPY pyproject.toml poetry.lock /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy -# pyproject.toml here, ditching setup.py and MANIFEST.in. -COPY setup.py MANIFEST.in README.rst /synapse/ +COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse diff --git a/mypy.ini b/mypy.ini index 5246f987c0..b2c3b1524c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -13,7 +13,6 @@ no_implicit_optional = True files = docker/, scripts-dev/, - setup.py, synapse/, tests/ diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3f..c04857b791 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,5 +280,5 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6213f3265b..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[check-manifest] -ignore = - .git-blame-ignore-revs - contrib - contrib/* - docs/* - pylint.cfg - tox.ini - diff --git a/setup.py b/setup.py deleted file mode 100755 index ecd30247ed..0000000000 --- a/setup.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2017-2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Any, Dict - -from setuptools import Command, find_packages, setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -# Some notes on `setup.py test`: -# -# Once upon a time we used to try to make `setup.py test` run `tox` to run the -# tests. That's a bad idea for three reasons: -# -# 1: `setup.py test` is supposed to find out whether the tests work in the -# *current* environmentt, not whatever tox sets up. -# 2: Empirically, trying to install tox during the test run wasn't working ("No -# module named virtualenv"). -# 3: The tox documentation advises against it[1]. -# -# Even further back in time, we used to use setuptools_trial [2]. That has its -# own set of issues: for instance, it requires installation of Twisted to build -# an sdist (because the recommended mode of usage is to add it to -# `setup_requires`). That in turn means that in order to successfully run tox -# you have to have the python header files installed for whichever version of -# python tox uses (which is python3 on recent ubuntus, for example). -# -# So, for now at least, we stick with what appears to be the convention among -# Twisted projects, and don't attempt to do anything when someone runs -# `setup.py test`; instead we direct people to run `trial` directly if they -# care. -# -# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command -# [2]: https://pypi.python.org/pypi/setuptools_trial -class TestCommand(Command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print( - """Synapse's tests cannot be run via setup.py. To run them, try: - PYTHONPATH="." trial tests -""" - ) - - -def read_file(path_segments): - """Read a file from the package. Takes a list of strings to join to - make the path""" - file_path = os.path.join(here, *path_segments) - with open(file_path) as f: - return f.read() - - -def exec_file(path_segments): - """Execute a single python file to get the variables defined in it""" - result: Dict[str, Any] = {} - code = read_file(path_segments) - exec(code, result) - return result - - -version = exec_file(("synapse", "__init__.py"))["__version__"] -dependencies = exec_file(("synapse", "python_dependencies.py")) -long_description = read_file(("README.rst",)) - -REQUIREMENTS = dependencies["REQUIREMENTS"] -CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"] -ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"] - -# Make `pip install matrix-synapse[all]` install all the optional dependencies. -CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) - -# Developer dependencies should not get included in "all". -# -# We pin black so that our tests don't start failing on new releases. -CONDITIONAL_REQUIREMENTS["lint"] = [ - "isort==5.7.0", - "black==22.3.0", - "flake8-comprehensions", - "flake8-bugbear==21.3.2", - "flake8", -] - -CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.931", - "mypy-zope==0.3.5", - "types-bleach>=4.1.0", - "types-jsonschema>=3.2.0", - "types-opentracing>=2.4.2", - "types-Pillow>=8.3.4", - "types-psycopg2>=2.9.9", - "types-pyOpenSSL>=20.0.7", - "types-PyYAML>=5.4.10", - "types-requests>=2.26.0", - "types-setuptools>=57.4.0", -] - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# parameterized_class decorator was introduced in parameterized 0.7.0 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"] - -CONDITIONAL_REQUIREMENTS["dev"] = ( - CONDITIONAL_REQUIREMENTS["lint"] - + CONDITIONAL_REQUIREMENTS["mypy"] - + CONDITIONAL_REQUIREMENTS["test"] - + [ - # The following are used by the release script - "click==8.1.0", - "redbaron==0.9.2", - "GitPython==3.1.14", - "commonmark==0.9.1", - "pygithub==1.55", - # The following are executed as commands by the release script. - "twine", - "towncrier", - ] -) - -setup( - name="matrix-synapse", - version=version, - packages=find_packages(exclude=["tests", "tests.*"]), - description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=REQUIREMENTS, - extras_require=CONDITIONAL_REQUIREMENTS, - include_package_data=True, - zip_safe=False, - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires="~=3.7", - entry_points={ - "console_scripts": [ - # Application - "synapse_homeserver = synapse.app.homeserver:main", - "synapse_worker = synapse.app.generic_worker:main", - "synctl = synapse._scripts.synctl:main", - # Scripts - "export_signing_key = synapse._scripts.export_signing_key:main", - "generate_config = synapse._scripts.generate_config:main", - "generate_log_config = synapse._scripts.generate_log_config:main", - "generate_signing_key = synapse._scripts.generate_signing_key:main", - "hash_password = synapse._scripts.hash_password:main", - "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main", - "synapse_port_db = synapse._scripts.synapse_port_db:main", - "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main", - "update_synapse_database = synapse._scripts.update_synapse_database:main", - ] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Communications :: Chat", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - cmdclass={"test": TestCommand}, -) diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d..1613941759 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import json import os import sys +from matrix_common.versionstring import get_distribution_version_string + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -68,7 +70,7 @@ try: except ImportError: pass -__version__ = "1.57.0" +__version__ = get_distribution_version_string("matrix-synapse") if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index ec199a161d..0000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import logging -from typing import Set - -logger = logging.getLogger(__name__) - - -# REQUIREMENTS is a simple list of requirement specifiers[1], and must be -# installed. It is passed to setup() as install_requires in setup.py. -# -# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict -# of lists. The dict key is the optional dependency name and can be passed to -# pip when installing. The list is a series of requirement specifiers[1] to be -# installed when that optional dependency requirement is specified. It is passed -# to setup() as extras_require in setup.py -# -# Note that these both represent runtime dependencies (and the versions -# installed are checked at runtime). -# -# Also note that we replicate these constraints in the Synapse Dockerfile while -# pre-installing dependencies. If these constraints are updated here, the same -# change should be made in the Dockerfile. -# -# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. - -REQUIREMENTS = [ - # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 - "jsonschema>=3.0.0", - # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 - "frozendict>=1,!=2.1.2", - "unpaddedbase64>=1.1.0", - "canonicaljson>=1.4.0", - # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0", - "pynacl>=1.2.1", - # validating SSL certs for IP addresses requires service_identity 18.1. - "service_identity>=18.1.0", - # Twisted 18.9 introduces some logger improvements that the structured - # logger utilises - "Twisted[tls]>=18.9.0", - "treq>=15.1", - # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0", - "pyyaml>=3.11", - "pyasn1>=0.1.9", - "pyasn1-modules>=0.0.7", - "bcrypt>=3.1.0", - "pillow>=5.4.0", - "sortedcontainers>=1.4.4", - "pymacaroons>=0.13.0", - "msgpack>=0.5.2", - "phonenumbers>=8.2.0", - # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - "prometheus_client>=0.4.0", - # we use `order`, which arrived in attrs 19.2.0. - # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.2.0,!=21.1.0", - "netaddr>=0.7.18", - # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not - # end up with a broken installation, with recent MarkupSafe but old Jinja, we - # add a lower bound to the Jinja2 dependency. - "Jinja2>=3.0", - "bleach>=1.4.3", - # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. - "typing-extensions>=3.10.0", - # We enforce that we have a `cryptography` version that bundles an `openssl` - # with the latest security patches. - "cryptography>=3.4.7", - # ijson 3.1.4 fixes a bug with "." in property names - "ijson>=3.1.4", - "matrix-common~=1.1.0", - # We need packaging.requirements.Requirement, added in 16.1. - "packaging>=16.1", - # At the time of writing, we only use functions from the version `importlib.metadata` - # which shipped in Python 3.8. This corresponds to version 1.4 of the backport. - "importlib_metadata>=1.4 ; python_version < '3.8'", -] - -CONDITIONAL_REQUIREMENTS = { - "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", - ], - "saml2": [ - "pysaml2>=4.5.0", - ], - "oidc": ["authlib>=0.14.0"], - # systemd-python is necessary for logging to the systemd journal via - # `systemd.journal.JournalHandler`, as is documented in - # `contrib/systemd/log_config.yaml`. - "systemd": ["systemd-python>=231"], - "url_preview": ["lxml>=4.2.0"], - "sentry": ["sentry-sdk>=0.7.2"], - "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], - "jwt": ["pyjwt>=1.6.4"], - # hiredis is not a *strict* dependency, but it makes things much faster. - # (if it is not installed, we fall back to slow code.) - "redis": ["txredisapi>=1.4.7", "hiredis"], - # Required to use experimental `caches.track_memory_usage` config option. - "cache_memory": ["pympler"], -} - -ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() - -for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): - # Exclude systemd as it's a system-based requirement. - # Exclude lint as it's a dev-based requirement. - if name not in ["systemd"]: - ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS - - -# ensure there are no double-quote characters in any of the deps (otherwise the -# 'pip install' incantation in DependencyException will break) -for dep in itertools.chain( - REQUIREMENTS, - *CONDITIONAL_REQUIREMENTS.values(), -): - if '"' in dep: - raise Exception( - "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) - ) - - -def list_requirements(): - return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) - - -if __name__ == "__main__": - import sys - - sys.stdout.writelines(req + "\n" for req in list_requirements()) |