depot/pkgs/os-specific/bsd/freebsd/update.py

#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p git "python3.withPackages (ps: with ps; [ gitpython packaging beautifulsoup4 pandas lxml ])"

import bs4
import git
import io
import json
import os
import packaging.version
import pandas
import re
import subprocess
import sys
import tempfile
import typing
import urllib.request

_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
_RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$')
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
MIN_VERSION = packaging.version.Version("13.0.0")
MAIN_BRANCH = "main"
TAG_PATTERN = re.compile(
    f"^release/({packaging.version.VERSION_PATTERN})$", re.IGNORECASE | re.VERBOSE
)
REMOTE = "origin"
BRANCH_PATTERN = re.compile(
    f"^{REMOTE}/((stable|releng)/({packaging.version.VERSION_PATTERN}))$",
    re.IGNORECASE | re.VERBOSE,
)


def request_supported_refs() -> list[str]:
    # Looks pretty shady but I think this should work with every version of the page in the last 20 years
    r = re.compile("^h\d$", re.IGNORECASE)
    soup = bs4.BeautifulSoup(
        urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
    )
    header = soup.find(
        lambda tag: r.match(tag.name) is not None
        and tag.text.lower() == "supported freebsd releases"
    )
    table = header.find_next("table")
    df = pandas.read_html(io.StringIO(table.prettify()))[0]
    return list(df["Branch"])


def query_version(repo: git.Repo) -> dict[str, typing.Any]:
    # This only works on FreeBSD 13 and later
    text = (
        subprocess.check_output(
            ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"]
        )
        .decode("utf-8")
        .strip()
    )
    fields = dict()
    for line in text.splitlines():
        m = _QUERY_VERSION_PATTERN.match(line)
        if m is None:
            continue
        fields[m[1].lower()] = m[2]

    parsed = packaging.version.parse(fields["revision"])
    fields["major"] = parsed.major
    fields["minor"] = parsed.minor

    # Extract the patch number from `RELAESE-p<patch>`, which is used
    # e.g. in the "releng" branches.
    m = _RELEASE_PATCH_PATTERN.match(fields["branch"])
    if m is not None:
        fields["patch"] = m[1]

    return fields


def handle_commit(
    repo: git.Repo,
    rev: git.objects.commit.Commit,
    ref_name: str,
    ref_type: str,
    supported_refs: list[str],
    old_versions: dict[str, typing.Any],
) -> dict[str, typing.Any]:
    if old_versions.get(ref_name, {}).get("rev", None) == rev.hexsha:
        print(f"{ref_name}: revision still {rev.hexsha}, skipping")
        return old_versions[ref_name]

    repo.git.checkout(rev)
    print(f"{ref_name}: checked out {rev.hexsha}")

    full_hash = (
        subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir])
        .decode("utf-8")
        .strip()
    )
    print(f"{ref_name}: hash is {full_hash}")

    version = query_version(repo)
    print(f"{ref_name}: version is {version['version']}")

    return {
        "rev": rev.hexsha,
        "hash": full_hash,
        "ref": ref_name,
        "refType": ref_type,
        "supported": ref_name in supported_refs,
        "version": version,
    }


def main() -> None:
    # Normally uses /run/user/*, which is on a tmpfs and too small
    temp_dir = tempfile.TemporaryDirectory(dir="/tmp")
    print(f"Selected temporary directory {temp_dir.name}")

    if len(sys.argv) >= 2:
        orig_repo = git.Repo(sys.argv[1])
        print(f"Fetching updates on {orig_repo.git_dir}")
        orig_repo.remote("origin").fetch()
    else:
        print("Cloning source repo")
        orig_repo = git.Repo.clone_from(
            "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig")
        )

    supported_refs = request_supported_refs()
    print(f"Supported refs are: {' '.join(supported_refs)}")

    print("Doing git crimes, do not run `git worktree prune` until after script finishes!")
    workdir = os.path.join(temp_dir.name, "work")
    git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)

    # Have to create object before removing .git otherwise it will complain
    repo = git.Repo(workdir)
    repo.git.set_persistent_git_options(git_dir=repo.git_dir)
    # Remove so that nix hash doesn't see the file
    os.remove(os.path.join(workdir, ".git"))

    print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")


    try:
        with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
            old_versions = json.load(f)
    except FileNotFoundError:
        old_versions = dict()

    versions = dict()
    for tag in repo.tags:
        m = TAG_PATTERN.match(tag.name)
        if m is None:
            continue
        version = packaging.version.parse(m[1])
        if version < MIN_VERSION:
            print(f"Skipping old tag {tag.name} ({version})")
            continue

        print(f"Trying tag {tag.name} ({version})")

        result = handle_commit(
            repo, tag.commit, tag.name, "tag", supported_refs, old_versions
        )

        # Hack in the patch version from parsing the tag, if we didn't
        # get one from the "branch" field (from newvers). This is
        # probably 0.
        versionObj = result["version"]
        if "patch" not in versionObj:
            versionObj["patch"] = version.micro

        versions[tag.name] = result

    for branch in repo.remote("origin").refs:
        m = BRANCH_PATTERN.match(branch.name)
        if m is not None:
            fullname = m[1]
            version = packaging.version.parse(m[3])
            if version < MIN_VERSION:
                print(f"Skipping old branch {fullname} ({version})")
                continue
            print(f"Trying branch {fullname} ({version})")
        elif branch.name == f"{REMOTE}/{MAIN_BRANCH}":
            fullname = MAIN_BRANCH
            print(f"Trying development branch {fullname}")
        else:
            continue

        result = handle_commit(
            repo, branch.commit, fullname, "branch", supported_refs, old_versions
        )
        versions[fullname] = result


    with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
        json.dump(versions, out, sort_keys=True, indent=2)
        out.write("\n")

if __name__ == '__main__':
    main()