diff options
Diffstat (limited to 'scripts-dev/release.py')
| -rwxr-xr-x | scripts-dev/release.py | 306 |
1 files changed, 271 insertions, 35 deletions
diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 0031ba3e4b..6603bc593b 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -18,10 +18,12 @@ """ import glob +import json import os import re import subprocess import sys +import time import urllib.request from os import path from tempfile import TemporaryDirectory @@ -32,6 +34,7 @@ import click import commonmark import git from click.exceptions import ClickException +from git import GitCommandError, Repo from github import Github from packaging import version @@ -55,9 +58,12 @@ def run_until_successful( def cli() -> None: """An interactive script to walk through the parts of creating a release. - Requires the dev dependencies be installed, which can be done via: + Requirements: + - The dev dependencies be installed, which can be done via: - pip install -e .[dev] + pip install -e .[dev] + + - A checkout of the sytest repository at ../sytest Then to use: @@ -67,16 +73,21 @@ def cli() -> None: ./scripts-dev/release.py tag - # ... wait for assets to build ... + # wait for assets to build, either manually or with: + ./scripts-dev/release.py wait-for-actions ./scripts-dev/release.py publish ./scripts-dev/release.py upload - # Optional: generate some nice links for the announcement + ./scripts-dev/release.py merge-back + # Optional: generate some nice links for the announcement ./scripts-dev/release.py announce + Alternatively, `./scripts-dev/release.py full` will do all the above + as well as guiding you through the manual steps. + If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. """ @@ -84,15 +95,21 @@ def cli() -> None: @cli.command() def prepare() -> None: + _prepare() + + +def _prepare() -> None: """Do the initial stages of creating a release, including creating release branch, updating changelog and pushing to GitHub. """ # Make sure we're in a git repo. - repo = get_repo_and_check_clean_checkout() + synapse_repo = get_repo_and_check_clean_checkout() + sytest_repo = get_repo_and_check_clean_checkout("../sytest", "sytest") - click.secho("Updating git repo...") - repo.remote().fetch() + click.secho("Updating Synapse and Sytest git repos...") + synapse_repo.remote().fetch() + sytest_repo.remote().fetch() # Get the current version and AST from root Synapse module. current_version = get_package_version() @@ -166,12 +183,12 @@ def prepare() -> None: assert not parsed_new_version.is_postrelease release_branch_name = get_release_branch_name(parsed_new_version) - release_branch = find_ref(repo, release_branch_name) + release_branch = find_ref(synapse_repo, release_branch_name) if release_branch: if release_branch.is_remote(): # If the release branch only exists on the remote we check it out # locally. - repo.git.checkout(release_branch_name) + synapse_repo.git.checkout(release_branch_name) else: # If a branch doesn't exist we create one. We ask which one branch it # should be based off, defaulting to sensible values depending on the @@ -187,25 +204,34 @@ def prepare() -> None: "Which branch should the release be based on?", default=default ) - base_branch = find_ref(repo, branch_name) - if not base_branch: - print(f"Could not find base branch {branch_name}!") - click.get_current_context().abort() + for repo_name, repo in {"synapse": synapse_repo, "sytest": sytest_repo}.items(): + base_branch = find_ref(repo, branch_name) + if not base_branch: + print(f"Could not find base branch {branch_name} for {repo_name}!") + click.get_current_context().abort() + + # Check out the base branch and ensure it's up to date + repo.head.set_reference( + base_branch, f"check out the base branch for {repo_name}" + ) + repo.head.reset(index=True, working_tree=True) + if not base_branch.is_remote(): + update_branch(repo) - # Check out the base branch and ensure it's up to date - repo.head.set_reference(base_branch, "check out the base branch") - repo.head.reset(index=True, working_tree=True) - if not base_branch.is_remote(): - update_branch(repo) + # Create the new release branch + # Type ignore will no longer be needed after GitPython 3.1.28. + # See https://github.com/gitpython-developers/GitPython/pull/1419 + repo.create_head(release_branch_name, commit=base_branch) # type: ignore[arg-type] - # Create the new release branch - # Type ignore will no longer be needed after GitPython 3.1.28. - # See https://github.com/gitpython-developers/GitPython/pull/1419 - repo.create_head(release_branch_name, commit=base_branch) # type: ignore[arg-type] + # Special-case SyTest: we don't actually prepare any files so we may + # as well push it now (and only when we create a release branch; + # not on subsequent RCs or full releases). + if click.confirm("Push new SyTest branch?", default=True): + sytest_repo.git.push("-u", sytest_repo.remote().name, release_branch_name) # Switch to the release branch and ensure it's up to date. - repo.git.checkout(release_branch_name) - update_branch(repo) + synapse_repo.git.checkout(release_branch_name) + update_branch(synapse_repo) # Update the version specified in pyproject.toml. subprocess.check_output(["poetry", "version", new_version]) @@ -230,15 +256,15 @@ def prepare() -> None: run_until_successful('dch -M -r -D stable ""', shell=True) # Show the user the changes and ask if they want to edit the change log. - repo.git.add("-u") + synapse_repo.git.add("-u") subprocess.run("git diff --cached", shell=True) if click.confirm("Edit changelog?", default=False): click.edit(filename="CHANGES.md") # Commit the changes. - repo.git.add("-u") - repo.git.commit("-m", new_version) + synapse_repo.git.add("-u") + synapse_repo.git.commit("-m", new_version) # We give the option to bail here in case the user wants to make sure things # are OK before pushing. @@ -246,23 +272,31 @@ def prepare() -> None: print("") print("Run when ready to push:") print("") - print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}") + print( + f"\tgit push -u {synapse_repo.remote().name} {synapse_repo.active_branch.name}" + ) print("") sys.exit(0) # Otherwise, push and open the changelog in the browser. - repo.git.push("-u", repo.remote().name, repo.active_branch.name) + synapse_repo.git.push( + "-u", synapse_repo.remote().name, synapse_repo.active_branch.name + ) print("Opening the changelog in your browser...") print("Please ask others to give it a check.") click.launch( - f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md" + f"https://github.com/matrix-org/synapse/blob/{synapse_repo.active_branch.name}/CHANGES.md" ) @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"]) def tag(gh_token: Optional[str]) -> None: + _tag(gh_token) + + +def _tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. @@ -353,6 +387,10 @@ def tag(gh_token: Optional[str]) -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) def publish(gh_token: str) -> None: + _publish(gh_token) + + +def _publish(gh_token: str) -> None: """Publish release on GitHub.""" # Make sure we're in a git repo. @@ -390,6 +428,10 @@ def publish(gh_token: str) -> None: @cli.command() def upload() -> None: + _upload() + + +def _upload() -> None: """Upload release to pypi.""" current_version = get_package_version() @@ -423,8 +465,152 @@ def upload() -> None: ) +def _merge_into(repo: Repo, source: str, target: str) -> None: + """ + Merges branch `source` into branch `target`. + Pulls both before merging and pushes the result. + """ + + # Update our branches and switch to the target branch + for branch in [source, target]: + click.echo(f"Switching to {branch} and pulling...") + repo.heads[branch].checkout() + # Pull so we're up to date + repo.remote().pull() + + assert repo.active_branch.name == target + + try: + # TODO This seemed easier than using GitPython directly + click.echo(f"Merging {source}...") + repo.git.merge(source) + except GitCommandError as exc: + # If a merge conflict occurs, give some context and try to + # make it easy to abort if necessary. + click.echo(exc) + if not click.confirm( + f"Likely merge conflict whilst merging ({source} → {target}). " + f"Have you resolved it?" + ): + repo.git.merge("--abort") + return + + # Push result. + click.echo("Pushing...") + repo.remote().push() + + +@cli.command() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=False) +def wait_for_actions(gh_token: Optional[str]) -> None: + _wait_for_actions(gh_token) + + +def _wait_for_actions(gh_token: Optional[str]) -> None: + # Find out the version and tag name. + current_version = get_package_version() + tag_name = f"v{current_version}" + + # Authentication is optional on this endpoint, + # but use a token if we have one to reduce the chance of being rate-limited. + url = f"https://api.github.com/repos/matrix-org/synapse/actions/runs?branch={tag_name}" + headers = {"Accept": "application/vnd.github+json"} + if gh_token is not None: + headers["authorization"] = f"token {gh_token}" + req = urllib.request.Request(url, headers=headers) + + time.sleep(10 * 60) + while True: + time.sleep(5 * 60) + response = urllib.request.urlopen(req) + resp = json.loads(response.read()) + + if len(resp["workflow_runs"]) == 0: + continue + + if all( + workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] + ): + success = ( + workflow["status"] == "completed" for workflow in resp["workflow_runs"] + ) + if success: + _notify("Workflows successful. You can now continue the release.") + else: + _notify("Workflows failed.") + click.confirm("Continue anyway?", abort=True) + + break + + +def _notify(message: str) -> None: + # Send a bell character. Most terminals will play a sound or show a notification + # for this. + click.echo(f"\a{message}") + + # Try and run notify-send, but don't raise an Exception if this fails + # (This is best-effort) + # TODO Support other platforms? + subprocess.run( + [ + "notify-send", + "--app-name", + "Synapse Release Script", + "--expire-time", + "3600000", + message, + ] + ) + + +@cli.command() +def merge_back() -> None: + _merge_back() + + +def _merge_back() -> None: + """Merge the release branch back into the appropriate branches. + All branches will be automatically pulled from the remote and the results + will be pushed to the remote.""" + + synapse_repo = get_repo_and_check_clean_checkout() + branch_name = synapse_repo.active_branch.name + + if not branch_name.startswith("release-v"): + raise RuntimeError("Not on a release branch. This does not seem sensible.") + + # Pull so we're up to date + synapse_repo.remote().pull() + + current_version = get_package_version() + + if current_version.is_prerelease: + # Release candidate + if click.confirm(f"Merge {branch_name} → develop?", default=True): + _merge_into(synapse_repo, branch_name, "develop") + else: + # Full release + sytest_repo = get_repo_and_check_clean_checkout("../sytest", "sytest") + + if click.confirm(f"Merge {branch_name} → master?", default=True): + _merge_into(synapse_repo, branch_name, "master") + + if click.confirm("Merge master → develop?", default=True): + _merge_into(synapse_repo, "master", "develop") + + if click.confirm(f"On SyTest, merge {branch_name} → master?", default=True): + _merge_into(sytest_repo, branch_name, "master") + + if click.confirm("On SyTest, merge master → develop?", default=True): + _merge_into(sytest_repo, "master", "develop") + + @cli.command() def announce() -> None: + _announce() + + +def _announce() -> None: """Generate markdown to announce the release.""" current_version = get_package_version() @@ -454,10 +640,56 @@ Announce the release in - #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic - #synapse:matrix.org (Synapse Admins), bumping the version in the topic - #synapse-dev:matrix.org -- #synapse-package-maintainers:matrix.org""" +- #synapse-package-maintainers:matrix.org + +Ask the designated people to do the blog and tweets.""" ) +@cli.command() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) +def full(gh_token: str) -> None: + click.echo("1. If this is a security release, read the security wiki page.") + click.echo("2. Check for any release blockers before proceeding.") + click.echo(" https://github.com/matrix-org/synapse/labels/X-Release-Blocker") + + click.confirm("Ready?", abort=True) + + click.echo("\n*** prepare ***") + _prepare() + + click.echo("Deploy to matrix.org and ensure that it hasn't fallen over.") + click.echo("Remember to silence the alerts to prevent alert spam.") + click.confirm("Deployed?", abort=True) + + click.echo("\n*** tag ***") + _tag(gh_token) + + click.echo("\n*** wait for actions ***") + _wait_for_actions(gh_token) + + click.echo("\n*** publish ***") + _publish(gh_token) + + click.echo("\n*** upload ***") + _upload() + + click.echo("\n*** merge back ***") + _merge_back() + + click.echo("\nUpdate the Debian repository") + click.confirm("Started updating Debian repository?", abort=True) + + click.echo("\nWait for all release methods to be ready.") + # Docker should be ready because it was done by the workflows earlier + # PyPI should be ready because we just ran upload(). + # TODO Automatically poll until the Debs have made it to packages.matrix.org + click.confirm("Debs ready?", abort=True) + + click.echo("\n*** announce ***") + _announce() + + def get_package_version() -> version.Version: version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( "utf-8" @@ -469,14 +701,18 @@ def get_release_branch_name(version_number: version.Version) -> str: return f"release-v{version_number.major}.{version_number.minor}" -def get_repo_and_check_clean_checkout() -> git.Repo: +def get_repo_and_check_clean_checkout( + path: str = ".", name: str = "synapse" +) -> git.Repo: """Get the project repo and check it's not got any uncommitted changes.""" try: - repo = git.Repo() + repo = git.Repo(path=path) except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") + raise click.ClickException( + f"{path} is not a git repository (expecting a {name} repository)." + ) if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + raise click.ClickException(f"Uncommitted changes exist in {path}.") return repo |