From c4e29b6908ac8ae57b5e9a3e7662ad638b61e94a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 26 Aug 2022 14:29:31 +0100 Subject: Improve documentation around user registration (#13640) Update a bunch of the documentation for user registration, add some cross links, etc. --- docs/setup/installation.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'docs/setup') diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 260e50577b..bb78b3267a 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -506,9 +506,13 @@ email will be disabled. ### Registering a user -The easiest way to create a new user is to do so from a client like [Element](https://element.io/). +One way to create a new user is to do so from a client like +[Element](https://element.io/). This requires registration to be enabled via +the +[`enable_registration`](../usage/configuration/config_documentation.md#enable_registration) +setting. -Alternatively, you can do so from the command line. This can be done as follows: +Alternatively, you can create new users from the command line. This can be done as follows: 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was installed via a prebuilt package, `register_new_matrix_user` should already be @@ -520,7 +524,7 @@ Alternatively, you can do so from the command line. This can be done as follows: ``` 2. Run the following command: ```sh - register_new_matrix_user -c homeserver.yaml http://localhost:8008 + register_new_matrix_user -c homeserver.yaml ``` This will prompt you to add details for the new user, and will then connect to @@ -533,12 +537,13 @@ Make admin [no]: Success! ``` -This process uses a setting `registration_shared_secret` in -`homeserver.yaml`, which is shared between Synapse itself and the -`register_new_matrix_user` script. It doesn't matter what it is (a random -value is generated by `--generate-config`), but it should be kept secret, as -anyone with knowledge of it can register users, including admin accounts, -on your server even if `enable_registration` is `false`. +This process uses a setting +[`registration_shared_secret`](../usage/configuration/config_documentation.md#registration_shared_secret), +which is shared between Synapse itself and the `register_new_matrix_user` +script. It doesn't matter what it is (a random value is generated by +`--generate-config`), but it should be kept secret, as anyone with knowledge of +it can register users, including admin accounts, on your server even if +`enable_registration` is `false`. ### Setting up a TURN server -- cgit 1.5.1 From c9b7e9735508bb148c6ad59c433d71e5b8b360ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 19:01:37 +0100 Subject: Add a stub Rust crate (#12595) --- .dockerignore | 4 ++ .github/workflows/release-artifacts.yml | 65 +++++++++++++++++++++++++++++++-- .github/workflows/tests.yml | 19 +++++++++- .gitignore | 7 ++++ Cargo.toml | 5 +++ build_rust.py | 20 ++++++++++ changelog.d/12595.misc | 1 + debian/build_virtualenv | 7 +++- debian/changelog | 4 ++ debian/rules | 2 + docker/Dockerfile | 14 ++++++- docker/Dockerfile-dhvirtualenv | 10 +++++ docs/deprecation_policy.md | 13 +++++++ docs/development/contributing_guide.md | 10 ++++- docs/setup/installation.md | 4 ++ mypy.ini | 6 ++- poetry.lock | 35 +++++++++++++++++- pyproject.toml | 39 +++++++++++++++++++- rust/Cargo.toml | 21 +++++++++++ rust/src/lib.rs | 15 ++++++++ stubs/synapse/__init__.pyi | 0 stubs/synapse/synapse_rust.pyi | 1 + tests/test_rust.py | 11 ++++++ 23 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 Cargo.toml create mode 100644 build_rust.py create mode 100644 changelog.d/12595.misc create mode 100644 rust/Cargo.toml create mode 100644 rust/src/lib.rs create mode 100644 stubs/synapse/__init__.pyi create mode 100644 stubs/synapse/synapse_rust.pyi create mode 100644 tests/test_rust.py (limited to 'docs/setup') diff --git a/.dockerignore b/.dockerignore index 7809863ef3..8eb1e4df8a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,12 @@ # things to include !docker !synapse +!rust !README.rst !pyproject.toml !poetry.lock +!build_rust.py + +rust/target **/__pycache__ diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index ed4fc6179d..0708d631cd 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -15,7 +15,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - + permissions: contents: write @@ -89,9 +89,67 @@ jobs: name: debs path: debs/* + build-wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, macos-10.15] + is_pr: + - ${{ startsWith(github.ref, 'refs/pull/') }} + + exclude: + # Don't build macos wheels on PR CI. + - is_pr: true + os: "macos-10.15" + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v3 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0 + + # Only build a single wheel in CI. + - name: Set env vars. + run: | + echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV + if: startsWith(github.ref, 'refs/pull/') + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + # Skip testing for platforms which various libraries don't have wheels + # for, and so need extra build deps. + CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx* + + - uses: actions/upload-artifact@v3 + with: + name: Wheel + path: ./wheelhouse/*.whl + build-sdist: - name: "Build pypi distribution files" - uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1" + name: Build sdist + runs-on: ubuntu-latest + if: ${{ !startsWith(github.ref, 'refs/pull/') }} + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - run: pip install build + + - name: Build sdist + run: python -m build --sdist + + - uses: actions/upload-artifact@v2 + with: + name: Sdist + path: dist/*.tar.gz + # if it's a tag, create a release and attach the artifacts to it attach-assets: @@ -99,6 +157,7 @@ jobs: if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} needs: - build-debs + - build-wheels - build-sdist runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 16fb4b43e2..5f96bdfa7f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -139,6 +139,12 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + # There aren't wheels for some of the older deps, so we need to install # their build dependencies - run: | @@ -175,7 +181,7 @@ jobs: python-version: '3.7' extras: "all test" - - run: poetry run trial -j 2 tests + - run: poetry run trial -j2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} @@ -247,6 +253,11 @@ jobs: - uses: actions/checkout@v2 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src @@ -353,6 +364,12 @@ jobs: with: path: synapse + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh diff --git a/.gitignore b/.gitignore index e58affb241..31a60bb7bd 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,10 @@ book/ # complement /complement-* /master.tar.gz + +# rust +/target/ +/synapse/*.so + +# Poetry will create a setup.py, which we don't want to include. +/setup.py diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..de141bdee9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +# We make the whole Synapse folder a workspace so that we can run `cargo` +# commands from the root (rather than having to cd into rust/). + +[workspace] +members = ["rust"] diff --git a/build_rust.py b/build_rust.py new file mode 100644 index 0000000000..5c5e557ee8 --- /dev/null +++ b/build_rust.py @@ -0,0 +1,20 @@ +# A build script for poetry that adds the rust extension. + +import os +from typing import Any, Dict + +from setuptools_rust import Binding, RustExtension + + +def build(setup_kwargs: Dict[str, Any]) -> None: + original_project_dir = os.path.dirname(os.path.realpath(__file__)) + cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml") + + extension = RustExtension( + target="synapse.synapse_rust", + path=cargo_toml_path, + binding=Binding.PyO3, + py_limited_api=True, + ) + setup_kwargs.setdefault("rust_extensions", []).append(extension) + setup_kwargs["zip_safe"] = False diff --git a/changelog.d/12595.misc b/changelog.d/12595.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/12595.misc @@ -0,0 +1 @@ +Add a stub Rust crate. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index ed916ac97a..dd97e888ba 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -61,7 +61,7 @@ dh_virtualenv \ --extras="all,systemd,test" \ --requirements="exported_requirements.txt" -PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" +PACKAGE_BUILD_DIR="$(pwd)/debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python" @@ -78,9 +78,14 @@ case "$DEB_BUILD_OPTIONS" in cp -r tests "$tmpdir" + # To avoid pulling in the unbuilt Synapse in the local directory + pushd / + PYTHONPATH="$tmpdir" \ "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests + popd + ;; esac diff --git a/debian/changelog b/debian/changelog index 2b7b329b6b..bd2d56e738 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,11 +12,15 @@ matrix-synapse-py3 (1.66.0) stable; urgency=medium matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium + [ Jörg Behrmann ] * Update debhelper to compatibility level 12. * Drop the preinst script stopping synapse. * Allocate a group for the system user. * Change dpkg-statoverride to --force-statoverride-add. + [ Erik Johnston ] + * Disable `dh_auto_configure` as it broke during Rust build. + -- Jörg Behrmann Tue, 23 Aug 2022 17:17:00 +0100 matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium diff --git a/debian/rules b/debian/rules index 3b79d56074..914d068f2a 100755 --- a/debian/rules +++ b/debian/rules @@ -12,6 +12,8 @@ override_dh_installsystemd: # we don't really want to strip the symbols from our object files. override_dh_strip: +override_dh_auto_configure: + # many libraries pulled from PyPI have allocatable sections after # non-allocatable ones on which dwz errors out. For those without the issue the # gains are only marginal diff --git a/docker/Dockerfile b/docker/Dockerfile index b87d263cff..a057bf397b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -92,11 +92,20 @@ RUN \ libxml++2.6-dev \ libxslt1-dev \ openssl \ - rustc \ zlib1g-dev \ git \ + curl \ && rm -rf /var/lib/apt/lists/* + +# Install rust and ensure its in the PATH +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + # To speed up rebuilds, install all of the dependencies before we copy over # the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source @@ -108,8 +117,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ +COPY rust /synapse/rust/ # ... and what we need to `pip install`. -COPY pyproject.toml README.rst /synapse/ +COPY pyproject.toml README.rst build_rust.py /synapse/ # Repeat of earlier build argument declaration, as this is a new build stage. ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index fbc1d2346f..ca3a259081 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -72,6 +72,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \ && env DEBIAN_FRONTEND=noninteractive apt-get install \ -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ build-essential \ + curl \ debhelper \ devscripts \ libsystemd-dev \ @@ -85,6 +86,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \ libpq-dev \ xmlsec1 +# Install rust and ensure it's in the PATH +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + + COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb / # install dhvirtualenv. Update the apt cache again first, in case we got a diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 359dac07c3..b8a46e3d60 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -18,6 +18,12 @@ documented at [https://endoflife.date/python](https://endoflife.date/python) and [https://endoflife.date/postgresql](https://endoflife.date/postgresql). +A Rust compiler is required to build Synapse from source. For any given release +the minimum required version may be bumped up to a recent Rust version, and so +people building from source should ensure they can fetch recent versions of Rust +(e.g. by using [rustup](https://rustup.rs/)). + + Context ------- @@ -31,3 +37,10 @@ long process. By following the upstream support life cycles Synapse can ensure that its dependencies continue to get security patches, while not requiring system admins to constantly update their platform dependencies to the latest versions. + +For Rust, the situation is a bit different given that a) the Rust foundation +does not generally support older Rust versions, and b) the library ecosystem +generally bump their minimum support Rust versions frequently. In general, the +Synapse team will try to avoid updating the dependency on Rust to the absolute +latest version, but introducing a formal policy is hard given the constraints of +the ecosystem. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 4e1df51164..cb0d727efa 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -28,6 +28,9 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/). +A recent version of the Rust compiler is needed to build the native modules. The +easiest way of installing the latest version is to use [rustup](https://rustup.rs/). + # 3. Get the source. @@ -114,6 +117,11 @@ Some documentation also exists in [Synapse's GitHub Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily contributed to by community authors. +When changes are made to any Rust code then you must call either `poetry install` +or `maturin develop` (if installed) to rebuild the Rust code. Using [`maturin`](https://github.com/PyO3/maturin) +is quicker than `poetry install`, so is recommended when making frequent +changes to the Rust code. + # 8. Test, test, test! @@ -195,7 +203,7 @@ The database file can then be inspected with: sqlite3 _trial_temp/test.db ``` -Note that the database file is cleared at the beginning of each test run. Thus it +Note that the database file is cleared at the beginning of each test run. Thus it will always only contain the data generated by the *last run test*. Though generally when debugging, one is only running a single test anyway. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index bb78b3267a..90737520ba 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -196,6 +196,10 @@ System requirements: - Python 3.7 or later, up to Python 3.10. - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org +If building on an uncommon architecture for which pre-built wheels are +unavailable, you will need to have a recent Rust compiler installed. The easiest +way of installing the latest version is to use [rustup](https://rustup.rs/). + To install the Synapse homeserver run: ```sh diff --git a/mypy.ini b/mypy.ini index e2034e411f..64f9097206 100644 --- a/mypy.ini +++ b/mypy.ini @@ -16,7 +16,8 @@ files = docker/, scripts-dev/, synapse/, - tests/ + tests/, + build_rust.py # Note: Better exclusion syntax coming in mypy > 0.910 # https://github.com/python/mypy/pull/11329 @@ -181,3 +182,6 @@ ignore_missing_imports = True [mypy-incremental.*] ignore_missing_imports = True + +[mypy-setuptools_rust.*] +ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 44df7d395c..cdc69f8ea9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1035,6 +1035,18 @@ python-versions = ">=3.6" cryptography = ">=2.0" jeepney = ">=0.6" +[[package]] +name = "semantic-version" +version = "2.10.0" +description = "A library implementing the 'SemVer' scheme." +category = "main" +optional = false +python-versions = ">=2.7" + +[package.extras] +dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1)", "coverage", "flake8", "nose2", "readme-renderer (<25.0)", "tox", "wheel", "zest.releaser[recommended]"] +doc = ["Sphinx", "sphinx-rtd-theme"] + [[package]] name = "sentry-sdk" version = "1.5.11" @@ -1099,6 +1111,19 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "setuptools-rust" +version = "1.5.1" +description = "Setuptools Rust extension plugin" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +semantic-version = ">=2.8.2,<3" +setuptools = ">=62.4" +typing-extensions = ">=3.7.4.3" + [[package]] name = "signedjson" version = "1.1.4" @@ -1600,7 +1625,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "0df36bf75561fef340a7af704ed379b235f07a7d4a231aaccec5e7afb87159ca" +content-hash = "79cfa09d59f9f8b5ef24318fb860df1915f54328692aa56d04331ecbdd92a8cb" [metadata.files] attrs = [ @@ -2472,6 +2497,10 @@ secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] +semantic-version = [ + {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, + {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, +] sentry-sdk = [ {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, @@ -2484,6 +2513,10 @@ setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, ] +setuptools-rust = [ + {file = "setuptools-rust-1.5.1.tar.gz", hash = "sha256:0e05e456645d59429cb1021370aede73c0760e9360bbfdaaefb5bced530eb9d7"}, + {file = "setuptools_rust-1.5.1-py3-none-any.whl", hash = "sha256:306b236ff3aa5229180e58292610d0c2c51bb488191122d2fc559ae4caeb7d5e"}, +] signedjson = [ {file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, {file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, diff --git a/pyproject.toml b/pyproject.toml index 8b2b5060b1..7cc9de5bc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ include_trailing_comma = true combine_as_imports = true skip_gitignore = true +[tool.maturin] +manifest-path = "rust/Cargo.toml" + [tool.poetry] name = "matrix-synapse" version = "1.66.0" @@ -82,8 +85,17 @@ include = [ { path = "sytest-blacklist", format = "sdist" }, { path = "tests", format = "sdist" }, { path = "UPGRADE.rst", format = "sdist" }, + { path = "Cargo.toml", format = "sdist" }, + { path = "rust/Cargo.toml", format = "sdist" }, + { path = "rust/Cargo.lock", format = "sdist" }, + { path = "rust/src/**", format = "sdist" }, +] +exclude = [ + { path = "synapse/*.so", format = "sdist"} ] +build = "build_rust.py" + [tool.poetry.scripts] synapse_homeserver = "synapse.app.homeserver:main" synapse_worker = "synapse.app.generic_worker:main" @@ -161,6 +173,15 @@ importlib_metadata = { version = ">=1.4", python = "<3.8" } # This is the most recent version of Pydantic with available on common distros. pydantic = ">=1.7.4" +# This is for building the rust components during "poetry install", which +# currently ignores the `build-system.requires` directive (c.f. +# https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and +# `poetry build` do the right thing without this explicit dependency. +# +# This isn't really a dev-dependency, as `poetry install --no-dev` will fail, +# but the alternative is to add it to the main list of deps where it isn't +# needed. +setuptools_rust = ">=1.3" # Optional Dependencies @@ -285,5 +306,21 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.0.0", "setuptools_rust>=1.3"] build-backend = "poetry.core.masonry.api" + + +[tool.cibuildwheel] +# Skip unsupported platforms (by us or by Rust). +skip = "cp36* *-musllinux_i686" + +# We need a rust compiler +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y" +environment= { PATH = "$PATH:$HOME/.cargo/bin" } + +# For some reason if we don't manually clean the build directory we +# can end up polluting the next build with a .so that is for the wrong +# Python version. +before-build = "rm -rf {project}/build" +build-frontend = "build" +test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print(sum_as_string(1, 2))'" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000000..0a9760cafc --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,21 @@ +[package] +# We name the package `synapse` so that things like logging have the right +# logging target. +name = "synapse" + +# dummy version. See pyproject.toml for the Synapse's version number. +version = "0.1.0" + +edition = "2021" +rust-version = "1.61.0" + +[lib] +name = "synapse" +crate-type = ["cdylib"] + +[package.metadata.maturin] +# This is where we tell maturin where to place the built library. +name = "synapse.synapse_rust" + +[dependencies] +pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000000..fc4eb39154 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,15 @@ +use pyo3::prelude::*; + +/// Formats the sum of two numbers as string. +#[pyfunction] +#[pyo3(text_signature = "(a, b, /)")] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// The entry point for defining the Python module. +#[pymodule] +fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) +} diff --git a/stubs/synapse/__init__.pyi b/stubs/synapse/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/stubs/synapse/synapse_rust.pyi b/stubs/synapse/synapse_rust.pyi new file mode 100644 index 0000000000..5b51ba05d7 --- /dev/null +++ b/stubs/synapse/synapse_rust.pyi @@ -0,0 +1 @@ +def sum_as_string(a: int, b: int) -> str: ... diff --git a/tests/test_rust.py b/tests/test_rust.py new file mode 100644 index 0000000000..55d8b6b28c --- /dev/null +++ b/tests/test_rust.py @@ -0,0 +1,11 @@ +from synapse.synapse_rust import sum_as_string + +from tests import unittest + + +class RustTestCase(unittest.TestCase): + """Basic tests to ensure that we can call into Rust code.""" + + def test_basic(self): + result = sum_as_string(1, 2) + self.assertEqual("3", result) -- cgit 1.5.1 From 41df25bbbd29caeb539269a436fbe6bc57fad93c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:01:21 +0100 Subject: installation.md: require libpq on M1 macs (#13480) --- changelog.d/13480.doc | 1 + docs/setup/installation.md | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13480.doc (limited to 'docs/setup') diff --git a/changelog.d/13480.doc b/changelog.d/13480.doc new file mode 100644 index 0000000000..ae5df16367 --- /dev/null +++ b/changelog.d/13480.doc @@ -0,0 +1 @@ +Note that `libpq` is required on ARM-based Macs. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 90737520ba..96833effc6 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -303,9 +303,10 @@ You may need to install the latest Xcode developer tools: xcode-select --install ``` -On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh): +On ARM-based Macs you may need to install libjpeg and libpq. +You can use Homebrew (https://brew.sh): ```sh - brew install jpeg + brew install jpeg libpq ``` On macOS Catalina (10.15) you may need to explicitly install OpenSSL -- cgit 1.5.1 From f34b0bc26273f5ddb5c0ada6dbe52f862d0e4b5c Mon Sep 17 00:00:00 2001 From: enterprisey Date: Mon, 26 Sep 2022 10:26:59 -0400 Subject: Update NixOS module URL (#13818) * Update NixOS module URL * Create 13818.doc --- changelog.d/13818.doc | 1 + docs/setup/installation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13818.doc (limited to 'docs/setup') diff --git a/changelog.d/13818.doc b/changelog.d/13818.doc new file mode 100644 index 0000000000..16b31f5071 --- /dev/null +++ b/changelog.d/13818.doc @@ -0,0 +1 @@ +Update URL for the NixOS module for Synapse. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 96833effc6..dcd8f17c5e 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -181,7 +181,7 @@ doas pkg_add synapse #### NixOS Robin Lambertz has packaged Synapse for NixOS at: - + ### Installing as a Python module from PyPI -- cgit 1.5.1 From 64dd8a9c6e43beed32fa1c94fda1d3d80957fef6 Mon Sep 17 00:00:00 2001 From: sando38 <90323876+sando38@users.noreply.github.com> Date: Mon, 14 Nov 2022 18:55:10 +0100 Subject: Include additional TURN server example into documentation (#14293) * Include eturnal TURN server configuration example and moving specific configuration examples into sub folders. * Update docs/turn-howto.md Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> * Update docs/setup/turn/coturn.md Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> * Update docs/setup/turn/eturnal.md Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> * Fix TURN relaying public IP address hint * lint eturnal installation commands * Adjust synapse setup to link to existing documentation ..avoid redundant information. * remove redundant text * include alpine linux package link * Create 14293.doc * Update 14293.doc add missing dot * Update docs/setup/turn/eturnal.md Co-authored-by: reivilibre * Update docs/setup/turn/eturnal.md Co-authored-by: reivilibre * Update docs/setup/turn/coturn.md Co-authored-by: Moritz Dietz * Update docs/setup/turn/coturn.md Co-authored-by: Moritz Dietz * Update docs/setup/turn/coturn.md Co-authored-by: Moritz Dietz * Update docs/setup/turn/eturnal.md Co-authored-by: reivilibre * Update docs/setup/turn/coturn.md Co-authored-by: Moritz Dietz * Update docs/setup/turn/coturn.md Co-authored-by: Moritz Dietz * Update eturnal.md to link to official documentation ... and to simplify some aspects * Adjust coturn to link to default prefix * Mention eturnalctl location * Update docs/turn-howto.md Co-authored-by: Saarko Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Co-authored-by: reivilibre Co-authored-by: Moritz Dietz --- changelog.d/14293.doc | 1 + docs/SUMMARY.md | 2 + docs/setup/turn/coturn.md | 188 +++++++++++++++++++++++++++++++++++ docs/setup/turn/eturnal.md | 170 ++++++++++++++++++++++++++++++++ docs/turn-howto.md | 240 ++++++--------------------------------------- 5 files changed, 390 insertions(+), 211 deletions(-) create mode 100644 changelog.d/14293.doc create mode 100644 docs/setup/turn/coturn.md create mode 100644 docs/setup/turn/eturnal.md (limited to 'docs/setup') diff --git a/changelog.d/14293.doc b/changelog.d/14293.doc new file mode 100644 index 0000000000..d6410421e7 --- /dev/null +++ b/changelog.d/14293.doc @@ -0,0 +1 @@ +Add addtional TURN server configuration example based on [eturnal](https://github.com/processone/eturnal) and adjust general TURN server doc structure. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 16720bceb5..8d68719958 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -9,6 +9,8 @@ - [Configuring a Reverse Proxy](reverse_proxy.md) - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md) - [Configuring a Turn Server](turn-howto.md) + - [coturn TURN server](setup/turn/coturn.md) + - [eturnal TURN server](setup/turn/eturnal.md) - [Delegation](delegate.md) # Upgrading diff --git a/docs/setup/turn/coturn.md b/docs/setup/turn/coturn.md new file mode 100644 index 0000000000..a1bb1e934c --- /dev/null +++ b/docs/setup/turn/coturn.md @@ -0,0 +1,188 @@ +# coturn TURN server + +The following sections describe how to install [coturn]() (which implements the TURN REST API). + +## `coturn` setup + +### Initial installation + +The TURN daemon `coturn` is available from a variety of sources such as native package managers, or installation from source. + +#### Debian and Ubuntu based distributions + +Just install the debian package: + +```sh +sudo apt install coturn +``` + +This will install and start a systemd service called `coturn`. + +#### Source installation + +1. Download the [latest release](https://github.com/coturn/coturn/releases/latest) from github. Unpack it and `cd` into the directory. + +1. Configure it: + + ```sh + ./configure + ``` + + You may need to install `libevent2`: if so, you should do so in + the way recommended by your operating system. You can ignore + warnings about lack of database support: a database is unnecessary + for this purpose. + +1. Build and install it: + + ```sh + make + sudo make install + ``` + +### Configuration + +1. Create or edit the config file in `/etc/turnserver.conf`. The relevant + lines, with example values, are: + + ``` + use-auth-secret + static-auth-secret=[your secret key here] + realm=turn.myserver.org + ``` + + See `turnserver.conf` for explanations of the options. One way to generate + the `static-auth-secret` is with `pwgen`: + + ```sh + pwgen -s 64 1 + ``` + + A `realm` must be specified, but its value is somewhat arbitrary. (It is + sent to clients as part of the authentication flow.) It is conventional to + set it to be your server name. + +1. You will most likely want to configure `coturn` to write logs somewhere. The + easiest way is normally to send them to the syslog: + + ```sh + syslog + ``` + + (in which case, the logs will be available via `journalctl -u coturn` on a + systemd system). Alternatively, `coturn` can be configured to write to a + logfile - check the example config file supplied with `coturn`. + +1. Consider your security settings. TURN lets users request a relay which will + connect to arbitrary IP addresses and ports. The following configuration is + suggested as a minimum starting point: + + ``` + # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay. + no-tcp-relay + + # don't let the relay ever try to connect to private IP address ranges within your network (if any) + # given the turn server is likely behind your firewall, remember to include any privileged public IPs too. + denied-peer-ip=10.0.0.0-10.255.255.255 + denied-peer-ip=192.168.0.0-192.168.255.255 + denied-peer-ip=172.16.0.0-172.31.255.255 + + # recommended additional local peers to block, to mitigate external access to internal services. + # https://www.rtcsec.com/article/slack-webrtc-turn-compromise-and-bug-bounty/#how-to-fix-an-open-turn-relay-to-address-this-vulnerability + no-multicast-peers + denied-peer-ip=0.0.0.0-0.255.255.255 + denied-peer-ip=100.64.0.0-100.127.255.255 + denied-peer-ip=127.0.0.0-127.255.255.255 + denied-peer-ip=169.254.0.0-169.254.255.255 + denied-peer-ip=192.0.0.0-192.0.0.255 + denied-peer-ip=192.0.2.0-192.0.2.255 + denied-peer-ip=192.88.99.0-192.88.99.255 + denied-peer-ip=198.18.0.0-198.19.255.255 + denied-peer-ip=198.51.100.0-198.51.100.255 + denied-peer-ip=203.0.113.0-203.0.113.255 + denied-peer-ip=240.0.0.0-255.255.255.255 + + # special case the turn server itself so that client->TURN->TURN->client flows work + # this should be one of the turn server's listening IPs + allowed-peer-ip=10.0.0.1 + + # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS. + user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user. + total-quota=1200 + ``` + +1. Also consider supporting TLS/DTLS. To do this, add the following settings + to `turnserver.conf`: + + ``` + # TLS certificates, including intermediate certs. + # For Let's Encrypt certificates, use `fullchain.pem` here. + cert=/path/to/fullchain.pem + + # TLS private key file + pkey=/path/to/privkey.pem + + # Ensure the configuration lines that disable TLS/DTLS are commented-out or removed + #no-tls + #no-dtls + ``` + + In this case, replace the `turn:` schemes in the `turn_uris` settings below + with `turns:`. + + We recommend that you only try to set up TLS/DTLS once you have set up a + basic installation and got it working. + + NB: If your TLS certificate was provided by Let's Encrypt, TLS/DTLS will + not work with any Matrix client that uses Chromium's WebRTC library. This + currently includes Element Android & iOS; for more details, see their + [respective](https://github.com/vector-im/element-android/issues/1533) + [issues](https://github.com/vector-im/element-ios/issues/2712) as well as the underlying + [WebRTC issue](https://bugs.chromium.org/p/webrtc/issues/detail?id=11710). + Consider using a ZeroSSL certificate for your TURN server as a working alternative. + +1. Ensure your firewall allows traffic into the TURN server on the ports + you've configured it to listen on (By default: 3478 and 5349 for TURN + traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535 + for the UDP relay.) + +1. If your TURN server is behind NAT, the NAT gateway must have an external, + publicly-reachable IP address. You must configure `coturn` to advertise that + address to connecting clients: + + ``` + external-ip=EXTERNAL_NAT_IPv4_ADDRESS + ``` + + You may optionally limit the TURN server to listen only on the local + address that is mapped by NAT to the external address: + + ``` + listening-ip=INTERNAL_TURNSERVER_IPv4_ADDRESS + ``` + + If your NAT gateway is reachable over both IPv4 and IPv6, you may + configure `coturn` to advertise each available address: + + ``` + external-ip=EXTERNAL_NAT_IPv4_ADDRESS + external-ip=EXTERNAL_NAT_IPv6_ADDRESS + ``` + + When advertising an external IPv6 address, ensure that the firewall and + network settings of the system running your TURN server are configured to + accept IPv6 traffic, and that the TURN server is listening on the local + IPv6 address that is mapped by NAT to the external IPv6 address. + +1. (Re)start the turn server: + + * If you used the Debian package (or have set up a systemd unit yourself): + ```sh + sudo systemctl restart coturn + ``` + + * If you built from source: + + ```sh + /usr/local/bin/turnserver -o + ``` diff --git a/docs/setup/turn/eturnal.md b/docs/setup/turn/eturnal.md new file mode 100644 index 0000000000..2e5a45673e --- /dev/null +++ b/docs/setup/turn/eturnal.md @@ -0,0 +1,170 @@ +# eturnal TURN server + +The following sections describe how to install [eturnal]() +(which implements the TURN REST API). + +## `eturnal` setup + +### Initial installation + +The `eturnal` TURN server implementation is available from a variety of sources +such as native package managers, binary packages, installation from source or +[container image](https://eturnal.net/documentation/code/docker.html). They are +all described [here](https://github.com/processone/eturnal#installation). + +Quick-Test instructions in a [Linux Shell](https://github.com/processone/eturnal/blob/master/QUICK-TEST.md) +or with [Docker](https://github.com/processone/eturnal/blob/master/docker-k8s/QUICK-TEST.md) +are available as well. + +### Configuration + +After installation, `eturnal` usually ships a [default configuration file](https://github.com/processone/eturnal/blob/master/config/eturnal.yml) +here: `/etc/eturnal.yml` (and, if not found there, there is a backup file here: +`/opt/eturnal/etc/eturnal.yml`). It uses the (indentation-sensitive!) [YAML](https://en.wikipedia.org/wiki/YAML) +format. The file contains further explanations. + +Here are some hints how to configure eturnal on your [host machine](https://github.com/processone/eturnal#configuration) +or when using e.g. [Docker](https://eturnal.net/documentation/code/docker.html). +You may also further deep dive into the [reference documentation](https://eturnal.net/documentation/). + +`eturnal` runs out of the box with the default configuration. To enable TURN and +to integrate it with your homeserver, some aspects in `eturnal`'s default configuration file +must be edited: + +1. Homeserver's [`turn_shared_secret`](../../usage/configuration/config_documentation.md#turn_shared_secret) + and eturnal's shared `secret` for authentication + + Both need to have the same value. Uncomment and adjust this line in `eturnal`'s + configuration file: + + ```yaml + secret: "long-and-cryptic" # Shared secret, CHANGE THIS. + ``` + + One way to generate a `secret` is with `pwgen`: + + ```sh + pwgen -s 64 1 + ``` + +1. Public IP address + + If your TURN server is behind NAT, the NAT gateway must have an external, + publicly-reachable IP address. `eturnal` tries to autodetect the public IP address, + however, it may also be configured by uncommenting and adjusting this line, so + `eturnal` advertises that address to connecting clients: + + ```yaml + relay_ipv4_addr: "203.0.113.4" # The server's public IPv4 address. + ``` + + If your NAT gateway is reachable over both IPv4 and IPv6, you may + configure `eturnal` to advertise each available address: + + ```yaml + relay_ipv4_addr: "203.0.113.4" # The server's public IPv4 address. + relay_ipv6_addr: "2001:db8::4" # The server's public IPv6 address (optional). + ``` + + When advertising an external IPv6 address, ensure that the firewall and + network settings of the system running your TURN server are configured to + accept IPv6 traffic, and that the TURN server is listening on the local + IPv6 address that is mapped by NAT to the external IPv6 address. + +1. Logging + + If `eturnal` was started by systemd, log files are written into the + `/var/log/eturnal` directory by default. In order to log to the [journal](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) + instead, the `log_dir` option can be set to `stdout` in the configuration file. + +1. Security considerations + + Consider your security settings. TURN lets users request a relay which will + connect to arbitrary IP addresses and ports. The following configuration is + suggested as a minimum starting point, [see also the official documentation](https://eturnal.net/documentation/#blacklist): + + ```yaml + ## Reject TURN relaying from/to the following addresses/networks: + blacklist: # This is the default blacklist. + - "127.0.0.0/8" # IPv4 loopback. + - "::1" # IPv6 loopback. + - recommended # Expands to a number of networks recommended to be + # blocked, but includes private networks. Those + # would have to be 'whitelist'ed if eturnal serves + # local clients/peers within such networks. + ``` + + To whitelist IP addresses or specific (private) networks, you need to **add** a + whitelist part into the configuration file, e.g.: + + ```yaml + whitelist: + - "192.168.0.0/16" + - "203.0.113.113" + - "2001:db8::/64" + ``` + + The more specific, the better. + +1. TURNS (TURN via TLS/DTLS) + + Also consider supporting TLS/DTLS. To do this, adjust the following settings + in the `eturnal.yml` configuration file (TLS parts should not be commented anymore): + + ```yaml + listen: + - ip: "::" + port: 3478 + transport: udp + - ip: "::" + port: 3478 + transport: tcp + - ip: "::" + port: 5349 + transport: tls + + ## TLS certificate/key files (must be readable by 'eturnal' user!): + tls_crt_file: /etc/eturnal/tls/crt.pem + tls_key_file: /etc/eturnal/tls/key.pem + ``` + + In this case, replace the `turn:` schemes in homeserver's `turn_uris` settings + with `turns:`. More is described [here](../../usage/configuration/config_documentation.md#turn_uris). + + We recommend that you only try to set up TLS/DTLS once you have set up a + basic installation and got it working. + + NB: If your TLS certificate was provided by Let's Encrypt, TLS/DTLS will + not work with any Matrix client that uses Chromium's WebRTC library. This + currently includes Element Android & iOS; for more details, see their + [respective](https://github.com/vector-im/element-android/issues/1533) + [issues](https://github.com/vector-im/element-ios/issues/2712) as well as the underlying + [WebRTC issue](https://bugs.chromium.org/p/webrtc/issues/detail?id=11710). + Consider using a ZeroSSL certificate for your TURN server as a working alternative. + +1. Firewall + + Ensure your firewall allows traffic into the TURN server on the ports + you've configured it to listen on (By default: 3478 and 5349 for TURN + traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535 + for the UDP relay.) + +1. Reload/ restarting `eturnal` + + Changes in the configuration file require `eturnal` to reload/ restart, this + can be achieved by: + + ```sh + eturnalctl reload + ``` + + `eturnal` performs a configuration check before actually reloading/ restarting + and provides hints, if something is not correctly configured. + +### eturnalctl opterations script + +`eturnal` offers a handy [operations script](https://eturnal.net/documentation/#Operation) +which can be called e.g. to check, whether the service is up, to restart the service, +to query how many active sessions exist, to change logging behaviour and so on. + +Hint: If `eturnalctl` is not part of your `$PATH`, consider either sym-linking it (e.g. ´ln -s /opt/eturnal/bin/eturnalctl /usr/local/bin/eturnalctl´) or call it from the default `eturnal` directory directly: e.g. `/opt/eturnal/bin/eturnalctl info` diff --git a/docs/turn-howto.md b/docs/turn-howto.md index 37a311ad9c..b466cab40c 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -9,222 +9,28 @@ allows the homeserver to generate credentials that are valid for use on the TURN server through the use of a secret shared between the homeserver and the TURN server. -The following sections describe how to install [coturn]() (which implements the TURN REST API) and integrate it with synapse. +This documentation provides two TURN server configuration examples: + +* [coturn](setup/turn/coturn.md) +* [eturnal](setup/turn/eturnal.md) ## Requirements -For TURN relaying with `coturn` to work, it must be hosted on a server/endpoint with a public IP. +For TURN relaying to work, the TURN service must be hosted on a server/endpoint with a public IP. Hosting TURN behind NAT requires port forwaring and for the NAT gateway to have a public IP. However, even with appropriate configuration, NAT is known to cause issues and to often not work. -## `coturn` setup - -### Initial installation - -The TURN daemon `coturn` is available from a variety of sources such as native package managers, or installation from source. - -#### Debian installation - -Just install the debian package: - -```sh -apt install coturn -``` - -This will install and start a systemd service called `coturn`. - -#### Source installation - -1. Download the [latest release](https://github.com/coturn/coturn/releases/latest) from github. Unpack it and `cd` into the directory. - -1. Configure it: - - ```sh - ./configure - ``` - - You may need to install `libevent2`: if so, you should do so in - the way recommended by your operating system. You can ignore - warnings about lack of database support: a database is unnecessary - for this purpose. - -1. Build and install it: - - ```sh - make - make install - ``` - -### Configuration - -1. Create or edit the config file in `/etc/turnserver.conf`. The relevant - lines, with example values, are: - - ``` - use-auth-secret - static-auth-secret=[your secret key here] - realm=turn.myserver.org - ``` - - See `turnserver.conf` for explanations of the options. One way to generate - the `static-auth-secret` is with `pwgen`: - - ```sh - pwgen -s 64 1 - ``` - - A `realm` must be specified, but its value is somewhat arbitrary. (It is - sent to clients as part of the authentication flow.) It is conventional to - set it to be your server name. - -1. You will most likely want to configure coturn to write logs somewhere. The - easiest way is normally to send them to the syslog: - - ```sh - syslog - ``` - - (in which case, the logs will be available via `journalctl -u coturn` on a - systemd system). Alternatively, coturn can be configured to write to a - logfile - check the example config file supplied with coturn. - -1. Consider your security settings. TURN lets users request a relay which will - connect to arbitrary IP addresses and ports. The following configuration is - suggested as a minimum starting point: - - ``` - # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay. - no-tcp-relay - - # don't let the relay ever try to connect to private IP address ranges within your network (if any) - # given the turn server is likely behind your firewall, remember to include any privileged public IPs too. - denied-peer-ip=10.0.0.0-10.255.255.255 - denied-peer-ip=192.168.0.0-192.168.255.255 - denied-peer-ip=172.16.0.0-172.31.255.255 - - # recommended additional local peers to block, to mitigate external access to internal services. - # https://www.rtcsec.com/article/slack-webrtc-turn-compromise-and-bug-bounty/#how-to-fix-an-open-turn-relay-to-address-this-vulnerability - no-multicast-peers - denied-peer-ip=0.0.0.0-0.255.255.255 - denied-peer-ip=100.64.0.0-100.127.255.255 - denied-peer-ip=127.0.0.0-127.255.255.255 - denied-peer-ip=169.254.0.0-169.254.255.255 - denied-peer-ip=192.0.0.0-192.0.0.255 - denied-peer-ip=192.0.2.0-192.0.2.255 - denied-peer-ip=192.88.99.0-192.88.99.255 - denied-peer-ip=198.18.0.0-198.19.255.255 - denied-peer-ip=198.51.100.0-198.51.100.255 - denied-peer-ip=203.0.113.0-203.0.113.255 - denied-peer-ip=240.0.0.0-255.255.255.255 - - # special case the turn server itself so that client->TURN->TURN->client flows work - # this should be one of the turn server's listening IPs - allowed-peer-ip=10.0.0.1 - - # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS. - user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user. - total-quota=1200 - ``` - -1. Also consider supporting TLS/DTLS. To do this, add the following settings - to `turnserver.conf`: - - ``` - # TLS certificates, including intermediate certs. - # For Let's Encrypt certificates, use `fullchain.pem` here. - cert=/path/to/fullchain.pem - - # TLS private key file - pkey=/path/to/privkey.pem - - # Ensure the configuration lines that disable TLS/DTLS are commented-out or removed - #no-tls - #no-dtls - ``` - - In this case, replace the `turn:` schemes in the `turn_uris` settings below - with `turns:`. - - We recommend that you only try to set up TLS/DTLS once you have set up a - basic installation and got it working. - - NB: If your TLS certificate was provided by Let's Encrypt, TLS/DTLS will - not work with any Matrix client that uses Chromium's WebRTC library. This - currently includes Element Android & iOS; for more details, see their - [respective](https://github.com/vector-im/element-android/issues/1533) - [issues](https://github.com/vector-im/element-ios/issues/2712) as well as the underlying - [WebRTC issue](https://bugs.chromium.org/p/webrtc/issues/detail?id=11710). - Consider using a ZeroSSL certificate for your TURN server as a working alternative. - -1. Ensure your firewall allows traffic into the TURN server on the ports - you've configured it to listen on (By default: 3478 and 5349 for TURN - traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535 - for the UDP relay.) - -1. If your TURN server is behind NAT, the NAT gateway must have an external, - publicly-reachable IP address. You must configure coturn to advertise that - address to connecting clients: - - ``` - external-ip=EXTERNAL_NAT_IPv4_ADDRESS - ``` - - You may optionally limit the TURN server to listen only on the local - address that is mapped by NAT to the external address: - - ``` - listening-ip=INTERNAL_TURNSERVER_IPv4_ADDRESS - ``` - - If your NAT gateway is reachable over both IPv4 and IPv6, you may - configure coturn to advertise each available address: - - ``` - external-ip=EXTERNAL_NAT_IPv4_ADDRESS - external-ip=EXTERNAL_NAT_IPv6_ADDRESS - ``` - - When advertising an external IPv6 address, ensure that the firewall and - network settings of the system running your TURN server are configured to - accept IPv6 traffic, and that the TURN server is listening on the local - IPv6 address that is mapped by NAT to the external IPv6 address. - -1. (Re)start the turn server: - - * If you used the Debian package (or have set up a systemd unit yourself): - ```sh - systemctl restart coturn - ``` - - * If you installed from source: - - ```sh - bin/turnserver -o - ``` +Afterwards, the homeserver needs some further configuration. ## Synapse setup Your homeserver configuration file needs the following extra keys: -1. "`turn_uris`": This needs to be a yaml list of public-facing URIs - for your TURN server to be given out to your clients. Add separate - entries for each transport your TURN server supports. -2. "`turn_shared_secret`": This is the secret shared between your - homeserver and your TURN server, so you should set it to the same - string you used in turnserver.conf. -3. "`turn_user_lifetime`": This is the amount of time credentials - generated by your homeserver are valid for (in milliseconds). - Shorter times offer less potential for abuse at the expense of - increased traffic between web clients and your homeserver to - refresh credentials. The TURN REST API specification recommends - one day (86400000). -4. "`turn_allow_guests`": Whether to allow guest users to use the - TURN server. This is enabled by default, as otherwise VoIP will - not work reliably for guests. However, it does introduce a - security risk as it lets guests connect to arbitrary endpoints - without having gone through a CAPTCHA or similar to register a - real account. +1. [`turn_uris`](usage/configuration/config_documentation.md#turn_uris) +2. [`turn_shared_secret`](usage/configuration/config_documentation.md#turn_shared_secret) +3. [`turn_user_lifetime`](usage/configuration/config_documentation.md#turn_user_lifetime) +4. [`turn_allow_guests`](usage/configuration/config_documentation.md#turn_allow_guests) As an example, here is the relevant section of the config file for `matrix.org`. The `turn_uris` are appropriate for TURN servers listening on the default ports, with no TLS. @@ -263,7 +69,7 @@ Here are a few things to try: * Check that you have opened your firewall to allow UDP traffic to the UDP relay ports (49152-65535 by default). - * Try disabling `coturn`'s TLS/DTLS listeners and enable only its (unencrypted) + * Try disabling TLS/DTLS listeners and enable only its (unencrypted) TCP/UDP listeners. (This will only leave signaling traffic unencrypted; voice & video WebRTC traffic is always encrypted.) @@ -288,12 +94,19 @@ Here are a few things to try: * ensure that your TURN server uses the NAT gateway as its default route. - * Enable more verbose logging in coturn via the `verbose` setting: + * Enable more verbose logging, in `coturn` via the `verbose` setting: ``` verbose ``` + or with `eturnal` with the shell command `eturnalctl loglevel debug` or in the configuration file (the service needs to [reload](https://eturnal.net/documentation/#Operation) for it to become effective): + + ```yaml + ## Logging configuration: + log_level: debug + ``` + ... and then see if there are any clues in its logs. * If you are using a browser-based client under Chrome, check @@ -317,7 +130,7 @@ Here are a few things to try: matrix client to your homeserver in your browser's network inspector. In the response you should see `username` and `password`. Or: - * Use the following shell commands: + * Use the following shell commands for `coturn`: ```sh secret=staticAuthSecretHere @@ -327,11 +140,16 @@ Here are a few things to try: echo -e "username: $u\npassword: $p" ``` - Or: + or for `eturnal` + + ```sh + eturnalctl credentials + ``` + - * Temporarily configure coturn to accept a static username/password. To do - this, comment out `use-auth-secret` and `static-auth-secret` and add the - following: + * Or (**coturn only**): Temporarily configure `coturn` to accept a static + username/password. To do this, comment out `use-auth-secret` and + `static-auth-secret` and add the following: ``` lt-cred-mech -- cgit 1.5.1