From 07e0992a76b33de80616570582c17edd2768150f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 14 Jul 2021 14:41:23 +0100 Subject: Make GHA config more efficient (#10383) A few things here: * Build the debs for single distro for each PR, so that we can see if it breaks. Do the same for develop. Building all the debs ties up the GHA workers for ages. * Stop building the debs for release branches. Again, it takes ages, and I don't think anyone is actually going to stop and look at them. We'll know they are working when we make an RC. * Change the configs so that if we manually cancel a workflow, it actually does something. --- .github/workflows/release-artifacts.yml | 21 +++++++++++++-------- .github/workflows/tests.yml | 14 +++++++------- 2 files changed, 20 insertions(+), 15 deletions(-) (limited to '.github') diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index f292d703ed..325c1f7d39 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -3,28 +3,33 @@ name: Build release artifacts on: + # we build on PRs and develop to (hopefully) get early warning + # of things breaking (but only build one set of debs) + pull_request: push: - # we build on develop and release branches to (hopefully) get early warning - # of things breaking - branches: ["develop", "release-*"] + branches: ["develop"] - # we also rebuild on tags, so that we can be sure of picking the artifacts - # from the right tag. + # we do the full build on tags. tags: ["v*"] permissions: contents: write jobs: - # first get the list of distros to build for. get-distros: + name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - id: set-distros run: | - echo "::set-output name=distros::$(scripts-dev/build_debian_packages --show-dists-json)" + # if we're running from a tag, get the full list of distros; otherwise just use debian:sid + dists='["debian:sid"]' + if [[ $GITHUB_REF == refs/tags/* ]]; then + dists=$(scripts-dev/build_debian_packages --show-dists-json) + fi + echo "::set-output name=distros::$dists" # map the step outputs to job outputs outputs: distros: ${{ steps.set-distros.outputs.distros }} @@ -66,7 +71,7 @@ jobs: # if it's a tag, create a release and attach the artifacts to it attach-assets: name: "Attach assets to release" - if: startsWith(github.ref, 'refs/tags/') + if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} needs: - build-debs - build-sdist diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf36ee1cdf..505bac1308 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,14 +65,14 @@ jobs: # Dummy step to gate other tests on without repeating the whole list linting-done: - if: ${{ always() }} # Run this even if prior jobs were skipped + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped needs: [lint, lint-crlf, lint-newsfile, lint-sdist] runs-on: ubuntu-latest steps: - run: "true" trial: - if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest strategy: @@ -131,7 +131,7 @@ jobs: || true trial-olddeps: - if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest steps: @@ -156,7 +156,7 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' - if: ${{ contains(github.ref, 'pypy') && !failure() }} + if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest strategy: @@ -185,7 +185,7 @@ jobs: || true sytest: - if: ${{ !failure() }} + if: ${{ !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest container: @@ -245,7 +245,7 @@ jobs: /logs/**/*.log* portdb: - if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail + if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest strategy: @@ -286,7 +286,7 @@ jobs: - run: .buildkite/scripts/test_synapse_port_db.sh complement: - if: ${{ !failure() }} + if: ${{ !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest container: -- cgit 1.5.1 From 97c8ae90f7996c3d6039ce137905e87987c1be98 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 20 Jul 2021 11:41:19 +0100 Subject: Add a github actions job recording success of other jobs. (#10430) --- .github/workflows/tests.yml | 12 ++++++++++++ changelog.d/10430.misc | 1 + 2 files changed, 13 insertions(+) create mode 100644 changelog.d/10430.misc (limited to '.github') diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 505bac1308..cef4439477 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -344,3 +344,15 @@ jobs: env: COMPLEMENT_BASE_IMAGE: complement-synapse:latest working-directory: complement + + # a job which marks all the other jobs as complete, thus allowing PRs to be merged. + tests-done: + needs: + - trial + - trial-olddeps + - sytest + - portdb + - complement + runs-on: ubuntu-latest + steps: + - run: "true" \ No newline at end of file diff --git a/changelog.d/10430.misc b/changelog.d/10430.misc new file mode 100644 index 0000000000..a017cf4ac9 --- /dev/null +++ b/changelog.d/10430.misc @@ -0,0 +1 @@ +Add a github actions job recording success of other jobs. -- cgit 1.5.1 From c6509991f362cb559efbb97e1799776cd32a43d8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 21 Jul 2021 12:33:35 +0100 Subject: Move the docker image build to Github Actions (#10416) it's flaky on circleCI, and having to manage multiple CI providers is painful. --- .circleci/config.yml | 78 -------------------------------------------- .github/workflows/docker.yml | 72 ++++++++++++++++++++++++++++++++++++++++ changelog.d/10416.misc | 1 + 3 files changed, 73 insertions(+), 78 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/docker.yml create mode 100644 changelog.d/10416.misc (limited to '.github') diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index cf1989eff9..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,78 +0,0 @@ -version: 2.1 -jobs: - dockerhubuploadrelease: - docker: - - image: docker:git - steps: - - checkout - - docker_prepare - - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - # for release builds, we want to get the amd64 image out asap, so first - # we do an amd64-only build, before following up with a multiarch build. - - docker_build: - tag: -t matrixdotorg/synapse:${CIRCLE_TAG} - platforms: linux/amd64 - - docker_build: - tag: -t matrixdotorg/synapse:${CIRCLE_TAG} - platforms: linux/amd64,linux/arm64 - - dockerhubuploadlatest: - docker: - - image: docker:git - steps: - - checkout - - docker_prepare - - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - # for `latest`, we don't want the arm images to disappear, so don't update the tag - # until all of the platforms are built. - - docker_build: - tag: -t matrixdotorg/synapse:latest - platforms: linux/amd64,linux/arm64 - -workflows: - build: - jobs: - - dockerhubuploadrelease: - filters: - tags: - only: /v[0-9].[0-9]+.[0-9]+.*/ - branches: - ignore: /.*/ - - dockerhubuploadlatest: - filters: - branches: - only: [ master, main ] - -commands: - docker_prepare: - description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images - parameters: - buildx_version: - type: string - default: "v0.4.1" - steps: - - setup_remote_docker: - # 19.03.13 was the most recent available on circleci at the time of - # writing. - version: 19.03.13 - - run: apk add --no-cache curl - - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache - - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx - - run: chmod a+x ~/.docker/cli-plugins/docker-buildx - # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job - - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # create a context named `builder` for the builds - - run: docker context create builder - # create a buildx builder using the new context, and set it as the default - - run: docker buildx create builder --use - - docker_build: - description: Builds and pushed images to dockerhub using buildx - parameters: - platforms: - type: string - default: linux/amd64 - tag: - type: string - steps: - - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain . diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..8bdefb3905 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,72 @@ +# GitHub actions workflow which builds and publishes the docker images. + +name: Build docker images + +on: + push: + tags: ["v*"] + branches: [ master, main ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Inspect builder + run: docker buildx inspect + + - name: Log in to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Calculate docker image tag + id: set-tag + run: | + case "${GITHUB_REF}" in + refs/heads/master|refs/heads/main) + tag=latest + ;; + refs/tags/*) + tag=${GITHUB_REF#refs/tags/} + ;; + *) + tag=${GITHUB_SHA} + ;; + esac + echo "::set-output name=tag::$tag" + + # for release builds, we want to get the amd64 image out asap, so first + # we do an amd64-only build, before following up with a multiarch build. + - name: Build and push amd64 + uses: docker/build-push-action@v2 + if: "${{ startsWith(github.ref, 'refs/tags/v' }}" + with: + push: true + labels: "gitsha1=${{ github.sha }}" + tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}" + file: "docker/Dockerfile" + platforms: linux/amd64 + + - name: Build and push all platforms + uses: docker/build-push-action@v2 + with: + push: true + labels: "gitsha1=${{ github.sha }}" + tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}" + file: "docker/Dockerfile" + platforms: linux/amd64,linux/arm64 diff --git a/changelog.d/10416.misc b/changelog.d/10416.misc new file mode 100644 index 0000000000..fa648372f5 --- /dev/null +++ b/changelog.d/10416.misc @@ -0,0 +1 @@ +Move docker image build to Github Actions. -- cgit 1.5.1 From 4c3fdfc808a90b4ba049695e97cbf3e6cc21873e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 22 Jul 2021 21:50:30 +0100 Subject: Fix an error in the docker workflow (#10461) --- .github/workflows/docker.yml | 2 +- changelog.d/10461.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10461.misc (limited to '.github') diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8bdefb3905..af7ed21fce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,7 +54,7 @@ jobs: # we do an amd64-only build, before following up with a multiarch build. - name: Build and push amd64 uses: docker/build-push-action@v2 - if: "${{ startsWith(github.ref, 'refs/tags/v' }}" + if: "${{ startsWith(github.ref, 'refs/tags/v') }}" with: push: true labels: "gitsha1=${{ github.sha }}" diff --git a/changelog.d/10461.misc b/changelog.d/10461.misc new file mode 100644 index 0000000000..5035e26825 --- /dev/null +++ b/changelog.d/10461.misc @@ -0,0 +1 @@ +Fix an error which prevented the Github Actions workflow to build the docker images from running. -- cgit 1.5.1