Merge pull request #300707 from obsidiansystems/rhelmot-src-infra

freebsd: Use @rhelmot's version infra
2024-04-01 12:22:10 -04:00 · 2024-04-01 12:22:10 -04:00 · 4d1a4fd113
commit 4d1a4fd113
parent a3eaa2051d 1bcf7a8b53
7 changed files with 412 additions and 14 deletions
--- a/pkgs/os-specific/bsd/freebsd/default.nix
+++ b/pkgs/os-specific/bsd/freebsd/default.nix
@ -7,15 +7,10 @@
 let
  inherit (buildPackages.buildPackages) rsync;

-  version = "13.1.0";
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);

-  # `BuildPackages.fetchgit` avoids some probably splicing-caused infinite
-  # recursion.
-  freebsdSrc = buildPackages.fetchgit {
-    url = "https://git.FreeBSD.org/src.git";
-    rev = "release/${version}";
-    sha256 = "14nhk0kls83xfb64d5xy14vpi6k8laswjycjg80indq9pkcr2rlv";
-  };
+  version = "13.1.0";
+  branch = "release/${version}";

 in makeScopeWithSplicing' {
  otherSplices = generateSplicesForMkScope "freebsd";
@ -23,7 +18,7 @@ in makeScopeWithSplicing' {
    callPackage = self.callPackage;
    directory = ./pkgs;
  } // {
-    inherit freebsdSrc;
+    sourceData = versions.${branch};

    ports = fetchzip {
      url = "https://cgit.freebsd.org/ports/snapshot/ports-dde3b2b456c3a4bdd217d0bf3684231cc3724a0a.tar.gz";
--- a/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix
+++ b/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix
@ -33,7 +33,7 @@ stdenv.mkDerivation {
  INSTALL_AS_USER = true;
  NO_CHECKSUM = true;
  NO_MTREE = true;
-  SRC_BASE = freebsd.freebsdSrc;
+  SRC_BASE = freebsd.source;

  preUnpack = ''
    export MAKE_JOBS_NUMBER="$NIX_BUILD_CORES"
--- a/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix
+++ b/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix
@ -2,7 +2,7 @@
 , compatIfNeeded
 , runCommand, rsync
 , freebsd-lib
-, freebsdSrc
+, source
 , bsdSetupHook, freebsdSetupHook
 , makeMinimal
 , install, tsort, lorder, mandoc, groff
@ -20,7 +20,7 @@ in stdenv'.mkDerivation (rec {
      set -x
      path="$out/$p"
      mkdir -p "$(dirname "$path")"
-      src_path="${freebsdSrc}/$p"
+      src_path="${source}/$p"
      if [[ -d "$src_path" ]]; then src_path+=/; fi
      rsync --chmod="+w" -r "$src_path" "$path"
      set +x
--- a/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix
+++ b/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix
@ -1,7 +1,7 @@
-{ mkDerivation, freebsdSrc }:
+{ mkDerivation, source }:

 mkDerivation {
  path = "usr.bin/sed";
-  TESTSRC = "${freebsdSrc}/contrib/netbsd-tests";
+  TESTSRC = "${source}/contrib/netbsd-tests";
  MK_TESTS = "no";
 }
--- a/pkgs/os-specific/bsd/freebsd/pkgs/source.nix
+++ b/pkgs/os-specific/bsd/freebsd/pkgs/source.nix
@ -0,0 +1,11 @@
+{ fetchFromGitHub, sourceData }:
+
+# Using fetchFromGitHub from their mirror because it's a lot faster than their git server
+# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead.
+# The update script still pulls directly from git.freebsd.org
+fetchFromGitHub {
+  name = "src"; # Want to rename this next rebuild
+  owner = "freebsd";
+  repo = "freebsd-src";
+  inherit (sourceData) rev hash;
+}
--- a/pkgs/os-specific/bsd/freebsd/update.py
+++ b/pkgs/os-specific/bsd/freebsd/update.py
@ -0,0 +1,182 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p git "python3.withPackages (ps: with ps; [ gitpython packaging beautifulsoup4 pandas lxml ])"
+
+import bs4
+import git
+import io
+import json
+import os
+import packaging.version
+import pandas
+import re
+import subprocess
+import sys
+import tempfile
+import typing
+import urllib.request
+
+_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+MIN_VERSION = packaging.version.Version("13.0.0")
+MAIN_BRANCH = "main"
+TAG_PATTERN = re.compile(
+    f"^release/({packaging.version.VERSION_PATTERN})$", re.IGNORECASE | re.VERBOSE
+)
+REMOTE = "origin"
+BRANCH_PATTERN = re.compile(
+    f"^{REMOTE}/((stable|releng)/({packaging.version.VERSION_PATTERN}))$",
+    re.IGNORECASE | re.VERBOSE,
+)
+
+
+def request_supported_refs() -> list[str]:
+    # Looks pretty shady but I think this should work with every version of the page in the last 20 years
+    r = re.compile("^h\d$", re.IGNORECASE)
+    soup = bs4.BeautifulSoup(
+        urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
+    )
+    header = soup.find(
+        lambda tag: r.match(tag.name) is not None
+        and tag.text.lower() == "supported freebsd releases"
+    )
+    table = header.find_next("table")
+    df = pandas.read_html(io.StringIO(table.prettify()))[0]
+    return list(df["Branch"])
+
+
+def query_version(repo: git.Repo) -> dict[str, typing.Any]:
+    # This only works on FreeBSD 13 and later
+    text = (
+        subprocess.check_output(
+            ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"]
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    fields = dict()
+    for line in text.splitlines():
+        m = _QUERY_VERSION_PATTERN.match(line)
+        if m is None:
+            continue
+        fields[m[1].lower()] = m[2]
+
+    fields["major"] = packaging.version.parse(fields["revision"]).major
+    return fields
+
+
+def handle_commit(
+    repo: git.Repo,
+    rev: git.objects.commit.Commit,
+    ref_name: str,
+    ref_type: str,
+    supported_refs: list[str],
+    old_versions: dict[str, typing.Any],
+) -> dict[str, typing.Any]:
+    if old_versions.get(ref_name, {}).get("rev", None) == rev.hexsha:
+        print(f"{ref_name}: revision still {rev.hexsha}, skipping")
+        return old_versions[ref_name]
+
+    repo.git.checkout(rev)
+    print(f"{ref_name}: checked out {rev.hexsha}")
+
+    full_hash = (
+        subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir])
+        .decode("utf-8")
+        .strip()
+    )
+    print(f"{ref_name}: hash is {full_hash}")
+
+    version = query_version(repo)
+    print(f"{ref_name}: version is {version['version']}")
+
+    return {
+        "rev": rev.hexsha,
+        "hash": full_hash,
+        "ref": ref_name,
+        "refType": ref_type,
+        "supported": ref_name in supported_refs,
+        "version": query_version(repo),
+    }
+
+
+def main() -> None:
+    # Normally uses /run/user/*, which is on a tmpfs and too small
+    temp_dir = tempfile.TemporaryDirectory(dir="/tmp")
+    print(f"Selected temporary directory {temp_dir.name}")
+
+    if len(sys.argv) >= 2:
+        orig_repo = git.Repo(sys.argv[1])
+        print(f"Fetching updates on {orig_repo.git_dir}")
+        orig_repo.remote("origin").fetch()
+    else:
+        print("Cloning source repo")
+        orig_repo = git.Repo.clone_from(
+            "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig")
+        )
+
+    supported_refs = request_supported_refs()
+    print(f"Supported refs are: {' '.join(supported_refs)}")
+
+    print("Doing git crimes, do not run `git worktree prune` until after script finishes!")
+    workdir = os.path.join(temp_dir.name, "work")
+    git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)
+
+    # Have to create object before removing .git otherwise it will complain
+    repo = git.Repo(workdir)
+    repo.git.set_persistent_git_options(git_dir=repo.git_dir)
+    # Remove so that nix hash doesn't see the file
+    os.remove(os.path.join(workdir, ".git"))
+
+    print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")
+
+
+    try:
+        with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
+            old_versions = json.load(f)
+    except FileNotFoundError:
+        old_versions = dict()
+
+    versions = dict()
+    for tag in repo.tags:
+        m = TAG_PATTERN.match(tag.name)
+        if m is None:
+            continue
+        version = packaging.version.parse(m[1])
+        if version < MIN_VERSION:
+            print(f"Skipping old tag {tag.name} ({version})")
+            continue
+
+        print(f"Trying tag {tag.name} ({version})")
+
+        result = handle_commit(
+            repo, tag.commit, tag.name, "tag", supported_refs, old_versions
+        )
+        versions[tag.name] = result
+
+    for branch in repo.remote("origin").refs:
+        m = BRANCH_PATTERN.match(branch.name)
+        if m is not None:
+            fullname = m[1]
+            version = packaging.version.parse(m[3])
+            if version < MIN_VERSION:
+                print(f"Skipping old branch {fullname} ({version})")
+                continue
+            print(f"Trying branch {fullname} ({version})")
+        elif branch.name == f"{REMOTE}/{MAIN_BRANCH}":
+            fullname = MAIN_BRANCH
+            print(f"Trying development branch {fullname}")
+        else:
+            continue
+
+        result = handle_commit(
+            repo, branch.commit, fullname, "branch", supported_refs, old_versions
+        )
+        versions[fullname] = result
+
+
+    with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
+        json.dump(versions, out, sort_keys=True, indent=2)
+        out.write("\n")
+
+if __name__ == '__main__':
+    main()
--- a/pkgs/os-specific/bsd/freebsd/versions.json
+++ b/pkgs/os-specific/bsd/freebsd/versions.json
@ -0,0 +1,210 @@
+{
+  "main": {
+    "hash": "sha256-C5ucT9BK/eK8a9HNSDDi8S1uhpPmiqV22XEooxAqbPw=",
+    "ref": "main",
+    "refType": "branch",
+    "rev": "125c4560bc70971b950d035cfcd2255b89984011",
+    "supported": false,
+    "version": {
+      "branch": "CURRENT",
+      "major": 15,
+      "reldate": "1500017",
+      "release": "15.0-CURRENT",
+      "revision": "15.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 15.0-CURRENT"
+    }
+  },
+  "release/13.0.0": {
+    "hash": "sha256-2WYk/taxWc74uh2KJf9TzWDxUPrtkvt2nhU/qUZMu+Q=",
+    "ref": "release/13.0.0",
+    "refType": "tag",
+    "rev": "ea31abc261ffc01b6ff5671bffb15cf910a07f4b",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE",
+      "major": 13,
+      "reldate": "1300139",
+      "release": "13.0-RELEASE",
+      "revision": "13.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.0-RELEASE"
+    }
+  },
+  "release/13.1.0": {
+    "hash": "sha256-m2aR2bwJNxsBepJ5ybWiaJp4Nwm+l0bMcn0gTSeY0JI=",
+    "ref": "release/13.1.0",
+    "refType": "tag",
+    "rev": "fc952ac2212b121aa6eefc273f5960ec3e0a466d",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE",
+      "major": 13,
+      "reldate": "1301000",
+      "release": "13.1-RELEASE",
+      "revision": "13.1",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.1-RELEASE"
+    }
+  },
+  "release/13.2.0": {
+    "hash": "sha256-VuktVknlKYkklST0I5CUiH7OsDn3DVTE1W9O/IhaCkE=",
+    "ref": "release/13.2.0",
+    "refType": "tag",
+    "rev": "525ecfdad597980ea4cd59238e24c8530dbcd31d",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE",
+      "major": 13,
+      "reldate": "1302001",
+      "release": "13.2-RELEASE",
+      "revision": "13.2",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.2-RELEASE"
+    }
+  },
+  "release/13.3.0": {
+    "hash": "sha256-djqHlPnGlJCi9DGtX1kTULB2EEj8YUsjGTIUDQoHzAQ=",
+    "ref": "release/13.3.0",
+    "refType": "tag",
+    "rev": "80d2b634ddf0b459910b54a04bc09f5cbc7185a7",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE",
+      "major": 13,
+      "reldate": "1303001",
+      "release": "13.3-RELEASE",
+      "revision": "13.3",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.3-RELEASE"
+    }
+  },
+  "release/14.0.0": {
+    "hash": "sha256-eBKwCYcOG9Lg7gBA2gZqxQFO/3uMMrcQGtgqi8se6zA=",
+    "ref": "release/14.0.0",
+    "refType": "tag",
+    "rev": "f9716eee8ab45ad906d9b5c5233ca20c10226ca7",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE",
+      "major": 14,
+      "reldate": "1400097",
+      "release": "14.0-RELEASE",
+      "revision": "14.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 14.0-RELEASE"
+    }
+  },
+  "releng/13.0": {
+    "hash": "sha256-7PrqTb2o21IQgQ2N+zjavlzX/ju60Rw+MXjMRICmQi0=",
+    "ref": "releng/13.0",
+    "refType": "branch",
+    "rev": "5fe9c9de03ef3191d216964bc4d8e427d5ed5720",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE-p13",
+      "major": 13,
+      "reldate": "1300139",
+      "release": "13.0-RELEASE-p13",
+      "revision": "13.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.0-RELEASE-p13"
+    }
+  },
+  "releng/13.1": {
+    "hash": "sha256-9fou2HVWlpNRKkc8XToe8/aSxwbNsIZIAKpteeSjLnc=",
+    "ref": "releng/13.1",
+    "refType": "branch",
+    "rev": "39b281c2996526288c0f2ae94abe6b164bcd5954",
+    "supported": false,
+    "version": {
+      "branch": "RELEASE-p9",
+      "major": 13,
+      "reldate": "1301000",
+      "release": "13.1-RELEASE-p9",
+      "revision": "13.1",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.1-RELEASE-p9"
+    }
+  },
+  "releng/13.2": {
+    "hash": "sha256-KN508aIe02Ue4TjlonO6TmAQ7DmiOOSOYrZfg5HP9AM=",
+    "ref": "releng/13.2",
+    "refType": "branch",
+    "rev": "f5ac4e174fdd3497749e351c27aafb34171c5730",
+    "supported": true,
+    "version": {
+      "branch": "RELEASE-p11",
+      "major": 13,
+      "reldate": "1302001",
+      "release": "13.2-RELEASE-p11",
+      "revision": "13.2",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.2-RELEASE-p11"
+    }
+  },
+  "releng/13.3": {
+    "hash": "sha256-huzUiMZHfyK/mgLD3hW+DaSGgAaTUIuM51xDp+IE3qE=",
+    "ref": "releng/13.3",
+    "refType": "branch",
+    "rev": "7a0d63c9093222938f26cd63ff742e555168de77",
+    "supported": true,
+    "version": {
+      "branch": "RELEASE-p1",
+      "major": 13,
+      "reldate": "1303001",
+      "release": "13.3-RELEASE-p1",
+      "revision": "13.3",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.3-RELEASE-p1"
+    }
+  },
+  "releng/14.0": {
+    "hash": "sha256-15B9Nglshniokju88dEKj3BIffZ6L28L+ZuhAC3UqOI=",
+    "ref": "releng/14.0",
+    "refType": "branch",
+    "rev": "d338712beb16ad7740bbd00bd93299a131a68045",
+    "supported": true,
+    "version": {
+      "branch": "RELEASE-p6",
+      "major": 14,
+      "reldate": "1400097",
+      "release": "14.0-RELEASE-p6",
+      "revision": "14.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 14.0-RELEASE-p6"
+    }
+  },
+  "stable/13": {
+    "hash": "sha256-XateLKKs2A/HCP9Lx/nBm1cybB3otrbeXQvyCL40S0M=",
+    "ref": "stable/13",
+    "refType": "branch",
+    "rev": "e0a58ef24a3baf5ed4cc09a798b9fe2d85408052",
+    "supported": true,
+    "version": {
+      "branch": "STABLE",
+      "major": 13,
+      "reldate": "1303502",
+      "release": "13.3-STABLE",
+      "revision": "13.3",
+      "type": "FreeBSD",
+      "version": "FreeBSD 13.3-STABLE"
+    }
+  },
+  "stable/14": {
+    "hash": "sha256-tIKnK/SYBDk9UnE5AfhjeDpqHnzspYbor0678ye/mrs=",
+    "ref": "stable/14",
+    "refType": "branch",
+    "rev": "ab872ab0bf195e872ed8d955aab3b2a537a230cd",
+    "supported": true,
+    "version": {
+      "branch": "STABLE",
+      "major": 14,
+      "reldate": "1400510",
+      "release": "14.0-STABLE",
+      "revision": "14.0",
+      "type": "FreeBSD",
+      "version": "FreeBSD 14.0-STABLE"
+    }
+  }
+}