diff options
| -rw-r--r-- | .github/workflows/docs-pr.yaml | 11 | ||||
| -rw-r--r-- | .github/workflows/docs.yaml | 10 | ||||
| -rw-r--r-- | book.toml | 5 | ||||
| -rw-r--r-- | changelog.d/16661.doc | 1 | ||||
| -rw-r--r-- | docs/upgrade.md | 9 | ||||
| -rwxr-xr-x | scripts-dev/schema_versions.py | 181 |
6 files changed, 216 insertions, 1 deletions
diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 3704bd66e2..9cf3d340a4 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -6,6 +6,7 @@ on: - docs/** - book.toml - .github/workflows/docs-pr.yaml + - scripts-dev/schema_versions.py jobs: pages: @@ -13,12 +14,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c7cb2d78e5..31b9dbe3fe 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -51,12 +51,22 @@ jobs: - pre steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/book.toml b/book.toml index fa83d86ffc..ed3f6151e0 100644 --- a/book.toml +++ b/book.toml @@ -36,4 +36,7 @@ additional-css = [ "docs/website_files/indent-section-headers.css", ] additional-js = ["docs/website_files/table-of-contents.js"] -theme = "docs/website_files/theme" \ No newline at end of file +theme = "docs/website_files/theme" + +[preprocessor.schema_versions] +command = "./scripts-dev/schema_versions.py" diff --git a/changelog.d/16661.doc b/changelog.d/16661.doc new file mode 100644 index 0000000000..74f8fc84b8 --- /dev/null +++ b/changelog.d/16661.doc @@ -0,0 +1 @@ +Add schema rollback information to documentation. diff --git a/docs/upgrade.md b/docs/upgrade.md index ba2f7703bc..329c9c7787 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,15 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +Generally Synapse database schemas are compatible across multiple versions, once +a version of Synapse is deployed you may not be able to rollback automatically. +The following table gives the version ranges and the earliest version they can +be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled +back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to +v1.61.0. + +<!-- REPLACE_WITH_SCHEMA_VERSIONS --> + # Upgrading to v1.93.0 ## Minimum supported Rust version diff --git a/scripts-dev/schema_versions.py b/scripts-dev/schema_versions.py new file mode 100755 index 0000000000..5fd73251cd --- /dev/null +++ b/scripts-dev/schema_versions.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# Copyright 2023 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to calculate which versions of Synapse have backwards-compatible +database schemas. It creates a Markdown table of Synapse versions and the earliest +compatible version. + +It is compatible with the mdbook protocol for preprocessors (see +https://rust-lang.github.io/mdBook/for_developers/preprocessors.html#implementing-a-preprocessor-with-a-different-language): + +Exit 0 to denote support for all renderers: + + ./scripts-dev/schema_versions.py supports <mdbook renderer> + +Parse a JSON list from stdin and add the table to the proper documetnation page: + + ./scripts-dev/schema_versions.py + +Additionally, the script supports dumping the table to stdout for debugging: + + ./scripts-dev/schema_versions.py dump +""" + +import io +import json +import sys +from collections import defaultdict +from typing import Any, Dict, Iterator, Optional, Tuple + +import git +from packaging import version + +# The schema version has moved around over the years. +SCHEMA_VERSION_FILES = ( + "synapse/storage/schema/__init__.py", + "synapse/storage/prepare_database.py", + "synapse/storage/__init__.py", + "synapse/app/homeserver.py", +) + + +# Skip versions of Synapse < v1.0, they're old and essentially not +# compatible with today's federation. +OLDEST_SHOWN_VERSION = version.parse("v1.0") + + +def get_schema_versions(tag: git.Tag) -> Tuple[Optional[int], Optional[int]]: + """Get the schema and schema compat versions for a tag.""" + schema_version = None + schema_compat_version = None + + for file in SCHEMA_VERSION_FILES: + try: + schema_file = tag.commit.tree / file + except KeyError: + continue + + # We (usually) can't execute the code since it might have unknown imports. + if file != "synapse/storage/schema/__init__.py": + with io.BytesIO(schema_file.data_stream.read()) as f: + for line in f.readlines(): + if line.startswith(b"SCHEMA_VERSION"): + schema_version = int(line.split()[2]) + + # Bail early. + break + else: + # SCHEMA_COMPAT_VERSION is sometimes across multiple lines, the easist + # thing to do is exec the code. Luckily it has only ever existed in + # a file which imports nothing else from Synapse. + locals: Dict[str, Any] = {} + exec(schema_file.data_stream.read().decode("utf-8"), {}, locals) + schema_version = locals["SCHEMA_VERSION"] + schema_compat_version = locals.get("SCHEMA_COMPAT_VERSION") + + return schema_version, schema_compat_version + + +def get_tags(repo: git.Repo) -> Iterator[git.Tag]: + """Return an iterator of tags sorted by version.""" + tags = [] + for tag in repo.tags: + # All "real" Synapse tags are of the form vX.Y.Z. + if not tag.name.startswith("v"): + continue + + # There's a weird tag from the initial react UI. + if tag.name == "v0.1": + continue + + try: + tag_version = version.parse(tag.name) + except version.InvalidVersion: + # Skip invalid versions. + continue + + # Skip pre- and post-release versions. + if tag_version.is_prerelease or tag_version.is_postrelease or tag_version.local: + continue + + # Skip old versions. + if tag_version < OLDEST_SHOWN_VERSION: + continue + + tags.append((tag_version, tag)) + + # Sort based on the version number (not lexically). + return (tag for _, tag in sorted(tags, key=lambda t: t[0])) + + +def calculate_version_chart() -> str: + repo = git.Repo(path=".") + + # Map of schema version -> Synapse versions which are at that schema version. + schema_versions = defaultdict(list) + # Map of schema version -> Synapse versions which are compatible with that + # schema version. + schema_compat_versions = defaultdict(list) + + # Find ranges of versions which are compatible with a schema version. + # + # There are two modes of operation: + # + # 1. Pre-schema_compat_version (i.e. schema_compat_version of None), then + # Synapse is compatible up/downgrading to a version with + # schema_version >= its current version. + # + # 2. Post-schema_compat_version (i.e. schema_compat_version is *not* None), + # then Synapse is compatible up/downgrading to a version with + # schema version >= schema_compat_version. + # + # This is more generous and avoids versions that cannot be rolled back. + # + # See https://github.com/matrix-org/synapse/pull/9933 which was included in v1.37.0. + for tag in get_tags(repo): + schema_version, schema_compat_version = get_schema_versions(tag) + + # If a schema compat version is given, prefer that over the schema version. + schema_versions[schema_version].append(tag.name) + schema_compat_versions[schema_compat_version or schema_version].append(tag.name) + + # Generate a table which maps the latest Synapse version compatible with each + # schema version. + result = f"| {'Versions': ^19} | Compatible version |\n" + result += f"|{'-' * (19 + 2)}|{'-' * (18 + 2)}|\n" + for schema_version, synapse_versions in schema_compat_versions.items(): + result += f"| {synapse_versions[0] + ' – ' + synapse_versions[-1]: ^19} | {schema_versions[schema_version][0]: ^18} |\n" + + return result + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "supports": + # We don't care about the renderer which is being used, which is the second argument. + sys.exit(0) + elif len(sys.argv) == 2 and sys.argv[1] == "dump": + print(calculate_version_chart()) + else: + # Expect JSON data on stdin. + context, book = json.load(sys.stdin) + + for section in book["sections"]: + if "Chapter" in section and section["Chapter"]["path"] == "upgrade.md": + section["Chapter"]["content"] = section["Chapter"]["content"].replace( + "<!-- REPLACE_WITH_SCHEMA_VERSIONS -->", calculate_version_chart() + ) + + # Print the result back out to stdout. + print(json.dumps(book)) |