diff options
| -rw-r--r-- | .github/workflows/release-artifacts.yml | 39 | ||||
| -rw-r--r-- | changelog.d/10431.misc | 1 | ||||
| -rwxr-xr-x | scripts-dev/build_debian_packages | 38 |
3 files changed, 65 insertions, 13 deletions
diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 0beb418a07..eb294f1619 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -48,12 +48,43 @@ jobs: distro: ${{ fromJson(needs.get-distros.outputs.distros) }} steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 with: path: src - - uses: actions/setup-python@v2 - - run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}" - - uses: actions/upload-artifact@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + install: true + + - name: Set up docker layer caching + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Set up python + uses: actions/setup-python@v2 + + - name: Build the packages + # see https://github.com/docker/build-push-action/issues/252 + # for the cache magic here + run: | + ./src/scripts-dev/build_debian_packages \ + --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \ + --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \ + --docker-build-arg=--progress=plain \ + --docker-build-arg=--load \ + "${{ matrix.distro }}" + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Upload debs as artifacts + uses: actions/upload-artifact@v2 with: name: debs path: debs/* diff --git a/changelog.d/10431.misc b/changelog.d/10431.misc new file mode 100644 index 0000000000..34b9b49da6 --- /dev/null +++ b/changelog.d/10431.misc @@ -0,0 +1 @@ +Use a docker image cache for the prerequisites for the debian package build. diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index e25c5bb265..0ed1c679fd 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -17,6 +17,7 @@ import subprocess import sys import threading from concurrent.futures import ThreadPoolExecutor +from typing import Optional, Sequence DISTS = ( "debian:buster", @@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) class Builder(object): - def __init__(self, redirect_stdout=False): + def __init__( + self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None + ): self.redirect_stdout = redirect_stdout + self._docker_build_args = tuple(docker_build_args or ()) self.active_containers = set() self._lock = threading.Lock() self._failed = False @@ -79,8 +83,8 @@ class Builder(object): stdout = None # first build a docker image for the build environment - subprocess.check_call( - [ + build_args = ( + ( "docker", "build", "--tag", @@ -89,8 +93,13 @@ class Builder(object): "distro=" + dist, "-f", "docker/Dockerfile-dhvirtualenv", - "docker", - ], + ) + + self._docker_build_args + + ("docker",) + ) + + subprocess.check_call( + build_args, stdout=stdout, stderr=subprocess.STDOUT, cwd=projdir, @@ -147,9 +156,7 @@ class Builder(object): self.active_containers.remove(c) -def run_builds(dists, jobs=1, skip_tests=False): - builder = Builder(redirect_stdout=(jobs > 1)) - +def run_builds(builder, dists, jobs=1, skip_tests=False): def sig(signum, _frame): print("Caught SIGINT") builder.kill_containers() @@ -181,6 +188,11 @@ if __name__ == "__main__": help="skip running tests after building", ) parser.add_argument( + "--docker-build-arg", + action="append", + help="specify an argument to pass to docker build", + ) + parser.add_argument( "--show-dists-json", action="store_true", help="instead of building the packages, just list the dists to build for, as a json array", @@ -195,4 +207,12 @@ if __name__ == "__main__": if args.show_dists_json: print(json.dumps(DISTS)) else: - run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check) + builder = Builder( + redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg + ) + run_builds( + builder, + dists=args.dist, + jobs=args.jobs, + skip_tests=args.no_check, + ) |