From 684d19a11c3b93c9dd5fb90f43d38aa7e8c6005f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 4 Aug 2021 12:07:57 -0500 Subject: Add support for MSC2716 marker events (#10498) * Make historical messages available to federated servers Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716 Follow-up to https://github.com/matrix-org/synapse/pull/9247 * Debug message not available on federation * Add base starting insertion point when no chunk ID is provided * Fix messages from multiple senders in historical chunk Follow-up to https://github.com/matrix-org/synapse/pull/9247 Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716 --- Previously, Synapse would throw a 403, `Cannot force another user to join.`, because we were trying to use `?user_id` from a single virtual user which did not match with messages from other users in the chunk. * Remove debug lines * Messing with selecting insertion event extremeties * Move db schema change to new version * Add more better comments * Make a fake requester with just what we need See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080 * Store insertion events in table * Make base insertion event float off on its own See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889 Conflicts: synapse/rest/client/v1/room.py * Validate that the app service can actually control the given user See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455 Conflicts: synapse/rest/client/v1/room.py * Add some better comments on what we're trying to check for * Continue debugging * Share validation logic * Add inserted historical messages to /backfill response * Remove debug sql queries * Some marker event implemntation trials * Clean up PR * Rename insertion_event_id to just event_id * Add some better sql comments * More accurate description * Add changelog * Make it clear what MSC the change is part of * Add more detail on which insertion event came through * Address review and improve sql queries * Only use event_id as unique constraint * Fix test case where insertion event is already in the normal DAG * Remove debug changes * Add support for MSC2716 marker events * Process markers when we receive it over federation * WIP: make hs2 backfill historical messages after marker event * hs2 to better ask for insertion event extremity But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group` error * Add insertion_event_extremities table * Switch to chunk events so we can auth via power_levels Previously, we were using `content.chunk_id` to connect one chunk to another. But these events can be from any `sender` and we can't tell who should be able to send historical events. We know we only want the application service to do it but these events have the sender of a real historical message, not the application service user ID as the sender. Other federated homeservers also have no indicator which senders are an application service on the originating homeserver. So we want to auth all of the MSC2716 events via power_levels and have them be sent by the application service with proper PL levels in the room. * Switch to chunk events for federation * Add unstable room version to support new historical PL * Messy: Fix undefined state_group for federated historical events ``` 2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill await self.backfill( File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill await self._auth_and_persist_event(dest, event, context, backfilled=True) File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event await self._run_push_actions_and_persist_event(event, context, backfilled) File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event await self.persist_events_and_notify( File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify events, max_stream_token = await self.storage.persistence.persist_events( File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner return await func(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events ret_vals = await yieldable_gather_results(enqueue, partitioned.items()) File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop ret = await self._per_item_callback( File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch await self.persist_events_store._persist_events_and_state_updates( File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates await self.db_pool.runInteraction( File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction result = await self.runWithConnection( File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection return await make_deferred_yieldable( File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext result = inContext.theWork() # type: ignore[attr-defined] File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in inContext.theWork = lambda: context.call( # type: ignore[attr-defined] File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext return func(*args, **kw) File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction return function(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func return func(db_conn, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction r = func(cursor, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped return f(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn self._store_event_state_mappings_txn(txn, events_and_contexts) File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn self.db_pool.simple_insert_many_txn( File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn txn.execute_batch(sql, vals) File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch self.executemany(sql, args) File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany self._do_execute(self.txn.executemany, sql, *args) File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute return func(sql, *args) sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group ``` * Revert "Messy: Fix undefined state_group for federated historical events" This reverts commit 187ab28611546321e02770944c86f30ee2bc742a. * Fix federated events being rejected for no state_groups Add fix from https://github.com/matrix-org/synapse/pull/10439 until it merges. * Adapting to experimental room version * Some log cleanup * Add better comments around extremity fetching code and why * Rename to be more accurate to what the function returns * Add changelog * Ignore rejected events * Use simplified upsert * Add Erik's explanation of extra event checks See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332 * Clarify that the depth is not directly correlated to the backwards extremity that we return See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404 * lock only matters for sqlite See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061 * Move new SQL changes to its own delta file * Clean up upsert docstring * Bump database schema version (62) --- scripts-dev/complement.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts-dev') diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index cba015d942..5d0ef8dd3a 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -65,4 +65,4 @@ if [[ -n "$1" ]]; then fi # Run the tests! -go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/... +go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/... -- cgit 1.5.1 From 03fb99a5c8dbe67cf300986e76ea0e8183641211 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 Aug 2021 12:15:10 +0100 Subject: check-newsfragment: pass pr number explicitly use PULL_REQUEST_NUMBER instead of BUILDKITE_PULL_REQUEST remove the other user of BUILDKITE_PULL_REQUEST, namely merge_base_branch.sh --- .buildkite/.env | 1 - .buildkite/merge_base_branch.sh | 35 ----------------------------------- .github/workflows/tests.yml | 6 ++---- scripts-dev/check-newsfragment | 2 +- 4 files changed, 3 insertions(+), 41 deletions(-) delete mode 100755 .buildkite/merge_base_branch.sh (limited to 'scripts-dev') diff --git a/.buildkite/.env b/.buildkite/.env index 85b102d07f..a2969b96a1 100644 --- a/.buildkite/.env +++ b/.buildkite/.env @@ -7,7 +7,6 @@ BUILDKITE_JOB_ID BUILDKITE_BUILD_URL BUILDKITE_PROJECT_SLUG BUILDKITE_COMMIT -BUILDKITE_PULL_REQUEST BUILDKITE_TAG CODECOV_TOKEN TRIAL_FLAGS diff --git a/.buildkite/merge_base_branch.sh b/.buildkite/merge_base_branch.sh deleted file mode 100755 index 361440fd1a..0000000000 --- a/.buildkite/merge_base_branch.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then - echo "Not merging forward, as this is a release branch" - exit 0 -fi - -if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then - echo "Not a pull request, or hasn't had a PR opened yet..." - - # It probably hasn't had a PR opened yet. Since all PRs land on develop, we - # can probably assume it's based on it and will be merged into it. - GITBASE="develop" -else - # Get the reference, using the GitHub API - GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH -fi - -echo "--- merge_base_branch $GITBASE" - -# Show what we are before -git --no-pager show -s - -# Set up username so it can do a merge -git config --global user.email bot@matrix.org -git config --global user.name "A robot" - -# Fetch and merge. If it doesn't work, it will raise due to set -e. -git fetch -u origin $GITBASE -git merge --no-edit --no-commit origin/$GITBASE - -# Show what we are after. -git --no-pager show -s diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8612d1fb3a..5349e83133 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,11 +47,9 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v2 - run: pip install tox - - name: Patch Buildkite-specific test script - run: | - sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \ - scripts-dev/check-newsfragment - run: scripts-dev/check-newsfragment + env: + PULL_REQUEST_NUMBER: ${{ github.event.number }} lint-sdist: runs-on: ubuntu-latest diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment index af6d32e332..393a548d58 100755 --- a/scripts-dev/check-newsfragment +++ b/scripts-dev/check-newsfragment @@ -11,7 +11,7 @@ set -e git remote set-branches --add origin develop git fetch -q origin develop -pr="$BUILDKITE_PULL_REQUEST" +pr="$PULL_REQUEST_NUMBER" # if there are changes in the debian directory, check that the debian changelog # has been updated -- cgit 1.5.1 From c0ebdfc77e8f3e75ea162f12b2188acdde5bf4ef Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 Aug 2021 13:28:50 +0100 Subject: Kill off the .buildkite dir completely --- .buildkite/.env | 11 ----------- .buildkite/test_db.db | Bin 19296256 -> 0 bytes MANIFEST.in | 1 - ci/test_db.db | Bin 0 -> 19296256 bytes scripts-dev/lint.sh | 2 +- tox.ini | 2 +- 6 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 .buildkite/.env delete mode 100644 .buildkite/test_db.db create mode 100644 ci/test_db.db (limited to 'scripts-dev') diff --git a/.buildkite/.env b/.buildkite/.env deleted file mode 100644 index fc3606ead2..0000000000 --- a/.buildkite/.env +++ /dev/null @@ -1,11 +0,0 @@ -CI -BUILDKITE -BUILDKITE_BUILD_NUMBER -BUILDKITE_BUILD_NUMBER -BUILDKITE_JOB_ID -BUILDKITE_BUILD_URL -BUILDKITE_PROJECT_SLUG -BUILDKITE_COMMIT -BUILDKITE_TAG -CODECOV_TOKEN -TRIAL_FLAGS diff --git a/.buildkite/test_db.db b/.buildkite/test_db.db deleted file mode 100644 index a0d9f16a75..0000000000 Binary files a/.buildkite/test_db.db and /dev/null differ diff --git a/MANIFEST.in b/MANIFEST.in index 174e1b1f47..37d61f40de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -44,7 +44,6 @@ include book.toml include pyproject.toml recursive-include changelog.d * -prune .buildkite prune .circleci prune .github prune ci diff --git a/ci/test_db.db b/ci/test_db.db new file mode 100644 index 0000000000..a0d9f16a75 Binary files /dev/null and b/ci/test_db.db differ diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 869eb2372d..2c77643cda 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -94,7 +94,7 @@ else "scripts-dev/build_debian_packages" "scripts-dev/sign_json" "scripts-dev/update_database" - "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite" + "contrib" "synctl" "setup.py" "synmark" "stubs" "ci" ) fi fi diff --git a/tox.ini b/tox.ini index da77d124fc..b695126019 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ lint_targets = contrib synctl synmark - .buildkite + ci docker # default settings for all tox environments -- cgit 1.5.1 From 314a739160effac7501201cadfc8aaa9c4f34713 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 12 Aug 2021 10:40:32 +0100 Subject: Also rename in lint.sh --- scripts-dev/lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts-dev') diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 2c77643cda..809eff166a 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -94,7 +94,7 @@ else "scripts-dev/build_debian_packages" "scripts-dev/sign_json" "scripts-dev/update_database" - "contrib" "synctl" "setup.py" "synmark" "stubs" "ci" + "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci" ) fi fi -- cgit 1.5.1 From d2ad397d3cbd7e675abbb1f48072f9972c60823d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 12 Aug 2021 16:50:18 +0100 Subject: Stop building a debian package for Groovy Gorilla (#10588) --- changelog.d/10588.removal | 1 + scripts-dev/build_debian_packages | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/10588.removal (limited to 'scripts-dev') diff --git a/changelog.d/10588.removal b/changelog.d/10588.removal new file mode 100644 index 0000000000..90c4b5cee2 --- /dev/null +++ b/changelog.d/10588.removal @@ -0,0 +1 @@ +No longer build `.dev` packages for Ubuntu 20.10 LTS Groovy Gorilla, which has now EOLed. \ No newline at end of file diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 0ed1c679fd..6153cb225f 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -25,7 +25,6 @@ DISTS = ( "debian:sid", "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) - "ubuntu:groovy", # 20.10 (EOL 2021-07-07) "ubuntu:hirsute", # 21.04 (EOL 2022-01-05) ) -- cgit 1.5.1 From c8132f4a31be2717976052424abeb1ed40e947c8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:48:59 +0100 Subject: Build debs for bookworm (#10612) --- changelog.d/10612.misc | 1 + scripts-dev/build_debian_packages | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10612.misc (limited to 'scripts-dev') diff --git a/changelog.d/10612.misc b/changelog.d/10612.misc new file mode 100644 index 0000000000..c7a9457022 --- /dev/null +++ b/changelog.d/10612.misc @@ -0,0 +1 @@ +Build Debian packages for Debian 12 (Bookworm). diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 6153cb225f..e9f89e38ef 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -20,8 +20,9 @@ from concurrent.futures import ThreadPoolExecutor from typing import Optional, Sequence DISTS = ( - "debian:buster", + "debian:buster", # oldstable: EOL 2022-08 "debian:bullseye", + "debian:bookworm", "debian:sid", "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) -- cgit 1.5.1