feat: quarantine upstream update candidates

2026-08-07 00:53:52 +00:00 · 2026-07-14 20:36:58 +07:00 · 2026-07-14 20:36:58 +07:00 · 5b0336a493
commit 5b0336a493
parent 35817ccbf4
15 changed files with 660 additions and 230 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -19,6 +19,10 @@ jobs:
        run: find . -name '*.nix' -not -path './_*' | xargs nix run nixpkgs#nixfmt-rfc-style -- --check
      - name: Lint (statix)
        run: nix run nixpkgs#statix -- check .
+      - name: Test update impact reporter
+        run: python3 -m unittest tests/test_render_update_report.py
+      - name: Test updater helpers
+        run: bash tests/update-common.sh

  check:
    needs: lint
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@ -0,0 +1,32 @@
+name: Nightly Candidate
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Detect nightly pin changes
+        id: nightly
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- nightly.nix; then
+            echo "changed=false" >>"$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >>"$GITHUB_OUTPUT"
+          fi
+      - if: steps.nightly.outputs.changed == 'true'
+        uses: DeterminateSystems/nix-installer-action@v13
+      - name: Build nightly compatibility suite
+        if: steps.nightly.outputs.changed == 'true'
+        run: nix build .#legacyPackages.x86_64-linux.nightlyChecks.all --accept-flake-config
--- a/.github/workflows/update-nightly.yml
+++ b/.github/workflows/update-nightly.yml
@ -6,8 +6,13 @@ on:
    - cron: "0 4 * * *"
  workflow_dispatch: {}

+concurrency:
+  group: hermes-agent-nightly-candidate
+  cancel-in-progress: true
+
 permissions:
  contents: write
+  pull-requests: write

 jobs:
  update:
@ -21,10 +26,36 @@ jobs:
      - name: Install Nix
        uses: DeterminateSystems/nix-installer-action@v13

+      - name: Require PR automation token
+        env:
+          HERMES_UPDATE_TOKEN: ${{ secrets.HERMES_UPDATE_TOKEN }}
+        run: |
+          if [[ -z "$HERMES_UPDATE_TOKEN" ]]; then
+            echo "Configure HERMES_UPDATE_TOKEN so candidate PRs receive required CI checks." >&2
+            exit 1
+          fi
+
      - name: Run nightly updater
+        id: update
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HERMES_UPDATE_REPORT: ${{ runner.temp }}/hermes-nightly-update.md
        run: |
-          git config user.name "nix-hermes-agent-bot"
-          git config user.email "bot@nix-hermes-agent.local"
          scripts/update-nightly.sh
+
+      - name: Open or update candidate PR
+        if: steps.update.outputs.update_available == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HERMES_UPDATE_TOKEN }}
+          branch: automation/hermes-agent-nightly-candidate
+          delete-branch: true
+          commit-message: "chore: update Hermes Agent nightly candidate"
+          title: "chore: update Hermes Agent nightly candidate"
+          body-path: ${{ runner.temp }}/hermes-nightly-update.md
+
+      - name: Mark incompatible candidate
+        if: steps.update.outputs.update_available == 'true' && steps.update.outputs.compatible != 'true'
+        run: |
+          echo "The candidate PR was created, but nightly validation failed." >&2
+          exit 1
--- a/.github/workflows/update-pins.yml
+++ b/.github/workflows/update-pins.yml
@ -2,12 +2,17 @@ name: Update Hermes Agent Pins

 on:
  schedule:
-    # Check every 6 hours — hermes-agent doesn't release as often as openclaw
-    - cron: "15 */6 * * *"
+    # Stable releases are infrequent; daily discovery avoids needless churn.
+    - cron: "15 4 * * *"
  workflow_dispatch: {}

+concurrency:
+  group: hermes-agent-stable-candidate
+  cancel-in-progress: true
+
 permissions:
  contents: write
+  pull-requests: write

 jobs:
  update:
@ -21,10 +26,36 @@ jobs:
      - name: Install Nix
        uses: DeterminateSystems/nix-installer-action@v13

+      - name: Require PR automation token
+        env:
+          HERMES_UPDATE_TOKEN: ${{ secrets.HERMES_UPDATE_TOKEN }}
+        run: |
+          if [[ -z "$HERMES_UPDATE_TOKEN" ]]; then
+            echo "Configure HERMES_UPDATE_TOKEN so candidate PRs receive required CI checks." >&2
+            exit 1
+          fi
+
      - name: Run updater
+        id: update
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HERMES_UPDATE_REPORT: ${{ runner.temp }}/hermes-stable-update.md
        run: |
-          git config user.name "nix-hermes-agent-bot"
-          git config user.email "bot@nix-hermes-agent.local"
          scripts/update-pins.sh
+
+      - name: Open or update candidate PR
+        if: steps.update.outputs.update_available == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HERMES_UPDATE_TOKEN }}
+          branch: automation/hermes-agent-stable-candidate
+          delete-branch: true
+          commit-message: "chore: update Hermes Agent stable candidate"
+          title: "chore: update Hermes Agent stable candidate"
+          body-path: ${{ runner.temp }}/hermes-stable-update.md
+
+      - name: Mark incompatible candidate
+        if: steps.update.outputs.update_available == 'true' && steps.update.outputs.compatible != 'true'
+        run: |
+          echo "The candidate PR was created, but compatibility validation failed." >&2
+          exit 1
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,4 @@
 result
 result-*
 .direnv
+__pycache__/
--- a/README.md
+++ b/README.md
@ -4,6 +4,10 @@ Declarative Nix package and NixOS module for [Hermes Agent](https://github.com/N

 Everything is configured in Nix. Config, documents, secrets, service — one `nixos-rebuild switch` and it's live.

+Upstream pins are promoted through quarantined candidate PRs so a broken Hermes
+release cannot replace the last-known-good package. See
+[the upstream update policy](docs/UPDATE-POLICY.md).
+
 ## Quick Start

 ### 1. Add to your flake
--- a/docs/UPDATE-POLICY.md
+++ b/docs/UPDATE-POLICY.md
@ -0,0 +1,54 @@
+# Upstream update policy
+
+Hermes Agent changes quickly, so upstream updates are candidates until this
+repository's package and module contracts pass. Automation never pushes an
+upstream pin directly to `main`.
+
+## Channels
+
+- **Stable** is the last reviewed, green Hermes release. Ordinary
+  `nix flake check` validates this channel only.
+- **Nightly** follows upstream `main`. Its checks are exposed at
+  `legacyPackages.<system>.nightlyChecks.all` and run only from the nightly
+  candidate workflow.
+
+The scheduled workflows each own one replaceable branch:
+
+- `automation/hermes-agent-stable-candidate`
+- `automation/hermes-agent-nightly-candidate`
+
+Do not put manual commits on those branches; the next scheduled run may
+replace them. Promotion is a normal reviewed PR merge. A failed candidate stays
+outside `main`, leaving the known-good stable pin usable.
+
+## Candidate report
+
+Each updater compares the old and new upstream sources and adds a report to its
+PR. The report covers Python bounds, direct dependencies, optional dependency
+groups, CLI entry points, config schema version, and source paths used by the
+Nix package/module. A build failure is included in the report and marks the
+updater job red, but does not discard the candidate.
+
+## Repository token
+
+Set the Actions secret `HERMES_UPDATE_TOKEN` to a fine-grained PAT or GitHub App
+token with repository contents and pull-request write access. GitHub suppresses
+new workflow runs for PRs created with the default `GITHUB_TOKEN`; the dedicated
+token allows the candidate PR to receive normal required checks. Scheduled
+updates fail clearly when this secret is absent rather than opening an unchecked
+PR.
+
+Protect `main` with the ordinary `CI / check` status and the
+`Nightly Candidate / check` status. The latter passes without installing Nix
+when a PR does not change `nightly.nix`, and builds the isolated nightly suite
+when it does.
+
+## Manual validation
+
+```bash
+# Known-good stable channel
+nix flake check --keep-going
+
+# Mutable upstream channel (replace the system when needed)
+nix build .#legacyPackages.x86_64-linux.nightlyChecks.all
+```
--- a/flake.nix
+++ b/flake.nix
@ -17,6 +17,21 @@
      let
        pkgs = import nixpkgs { inherit system; };
        inherit (pkgs) lib;
+        nightlyPackage = self.packages.${system}.hermes-agent-nightly;
+        nightlyChecks =
+          (import ./checks.nix {
+            inherit pkgs;
+            hermes-agent = nightlyPackage;
+          })
+          // {
+            skills-coexistence = import ./tests/skills-coexistence.nix {
+              inherit self nixpkgs system;
+              hermes-agent = nightlyPackage;
+            };
+          };
+        nightlyCheckSuite = pkgs.linkFarm "hermes-agent-nightly-checks" (
+          lib.mapAttrsToList (name: path: { inherit name path; }) nightlyChecks
+        );
      in
      {
        packages = {
@ -30,18 +45,18 @@
            inherit pkgs;
            inherit (self.packages.${system}) hermes-agent;
          })
-          // (lib.mapAttrs' (name: value: lib.nameValuePair "nightly-${name}" value) (
-            import ./checks.nix {
-              inherit pkgs;
-              hermes-agent = self.packages.${system}.hermes-agent-nightly;
-            }
-          ))
          // {
            skills-coexistence = import ./tests/skills-coexistence.nix {
              inherit self nixpkgs system;
            };
          };

+        # Nightly tracks mutable upstream HEAD. Keep it available for explicit
+        # validation without making upstream breakage fail the stable channel.
+        legacyPackages.nightlyChecks = nightlyChecks // {
+          all = nightlyCheckSuite;
+        };
+
        devShells.default = pkgs.mkShell {
          packages = [ self.packages.${system}.hermes-agent ];
        };
--- a/scripts/render-update-report.py
+++ b/scripts/render-update-report.py
@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Render a small, deterministic compatibility report for an upstream update."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import tomllib
+from pathlib import Path
+
+
+CONTRACT_PATHS = (
+    "hermes_cli",
+    "hermes_cli/config.py",
+    "hermes_constants.py",
+    "skills",
+    "optional-skills",
+    "hermes-tui",
+    "mini-swe-agent",
+)
+
+
+def metadata(source: Path | None) -> dict[str, object]:
+    result: dict[str, object] = {
+        "dependencies": set(),
+        "dependency_specs": {},
+        "optional_groups": set(),
+        "scripts": {},
+        "python": "unknown",
+        "config_version": "unknown",
+        "paths": set(),
+    }
+    if source is None or not source.is_dir():
+        return result
+
+    pyproject = source / "pyproject.toml"
+    if pyproject.is_file():
+        try:
+            with pyproject.open("rb") as handle:
+                project = tomllib.load(handle).get("project", {})
+            specs = {
+                dependency_name(item): item for item in project.get("dependencies", [])
+            }
+            result["dependencies"] = set(specs)
+            result["dependency_specs"] = specs
+            result["optional_groups"] = set(project.get("optional-dependencies", {}))
+            result["scripts"] = dict(project.get("scripts", {}))
+            result["python"] = project.get("requires-python", "unknown")
+        except (OSError, tomllib.TOMLDecodeError, TypeError, AttributeError):
+            pass
+
+    for config_path in (source / "hermes_cli/config.py", source / "hermes_constants.py"):
+        if not config_path.is_file():
+            continue
+        contents = config_path.read_text(encoding="utf-8", errors="replace")
+        matches = re.findall(
+            r'(?:CURRENT_CONFIG_VERSION|["\']_config_version["\'])\s*(?::|=)\s*(\d+)',
+            contents,
+        )
+        if matches:
+            result["config_version"] = matches[-1]
+            break
+
+    result["paths"] = {path for path in CONTRACT_PATHS if (source / path).exists()}
+    return result
+
+
+def dependency_name(requirement: str) -> str:
+    return re.split(r"[<>=!~;\[ @]", requirement, maxsplit=1)[0].strip().lower()
+
+
+def changes(before: set[str], after: set[str]) -> str:
+    added = sorted(after - before)
+    removed = sorted(before - after)
+    parts = []
+    if added:
+        parts.append("added: " + ", ".join(f"`{item}`" for item in added))
+    if removed:
+        parts.append("removed: " + ", ".join(f"`{item}`" for item in removed))
+    return "; ".join(parts) if parts else "none"
+
+
+def dependency_changes(before: dict[str, str], after: dict[str, str]) -> str:
+    summary = changes(set(before), set(after))
+    changed = sorted(name for name in before.keys() & after.keys() if before[name] != after[name])
+    if changed:
+        detail = ", ".join(
+            f"`{name}` (`{before[name]}` → `{after[name]}`)" for name in changed
+        )
+        summary = (summary + "; " if summary != "none" else "") + "changed: " + detail
+    return summary
+
+
+def render(args: argparse.Namespace) -> str:
+    before = metadata(args.before_source)
+    after = metadata(args.after_source)
+    status_icon = "✅" if args.build_status == "passed" else "❌"
+    lines = [
+        f"## Hermes Agent {args.channel} candidate",
+        "",
+        "This PR is a quarantined upstream candidate. Merge it only after required checks pass.",
+        "",
+        "| Contract | Current | Candidate |",
+        "| --- | --- | --- |",
+        f"| Version | `{args.current_version}` | `{args.candidate_version}` |",
+        f"| Revision | `{args.current_rev}` | `{args.candidate_rev}` |",
+        f"| Python | `{before['python']}` | `{after['python']}` |",
+        f"| Config schema | `{before['config_version']}` | `{after['config_version']}` |",
+        f"| Build validation | — | {status_icon} {args.build_status} |",
+        "",
+        "### Detected interface changes",
+        "",
+        f"- Direct dependencies: {dependency_changes(before['dependency_specs'], after['dependency_specs'])}",
+        f"- Optional dependency groups: {changes(before['optional_groups'], after['optional_groups'])}",
+        f"- CLI entry points: {changes(set(before['scripts']), set(after['scripts']))}",
+        f"- Package/module asset paths: {changes(before['paths'], after['paths'])}",
+        "",
+    ]
+    if args.upstream_url:
+        lines.extend((f"Upstream: {args.upstream_url}", ""))
+    if args.build_log and args.build_log.is_file() and args.build_status != "passed":
+        tail = args.build_log.read_text(encoding="utf-8", errors="replace").splitlines()[-80:]
+        lines.extend(
+            (
+                "<details><summary>Validation failure (last 80 lines)</summary>",
+                "",
+                "```text",
+                "\n".join(tail).replace("```", "'''"),
+                "```",
+                "</details>",
+                "",
+            )
+        )
+    return "\n".join(lines)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--channel", required=True)
+    parser.add_argument("--current-version", required=True)
+    parser.add_argument("--candidate-version", required=True)
+    parser.add_argument("--current-rev", required=True)
+    parser.add_argument("--candidate-rev", required=True)
+    parser.add_argument("--before-source", type=Path)
+    parser.add_argument("--after-source", type=Path)
+    parser.add_argument("--build-status", choices=("passed", "failed"), required=True)
+    parser.add_argument("--build-log", type=Path)
+    parser.add_argument("--upstream-url")
+    parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(render(args), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/update-common.sh
+++ b/scripts/update-common.sh
@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set_output() {
+  local name=$1 value=$2
+  if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+    printf '%s=%s\n' "$name" "$value" >>"$GITHUB_OUTPUT"
+  fi
+}
+
+extract_sri_hash() {
+  sed -nE 's/.*got:[[:space:]]*(sha256-[A-Za-z0-9+\/=]+).*/\1/p' "$1" | head -1
+}
+
+realize_flake_source() {
+  nix build "$1.src" --no-link --print-out-paths 2>/dev/null | tail -1 || true
+}
+
+current_nix_system() {
+  nix eval --impure --raw --expr builtins.currentSystem
+}
+
+publish_summary() {
+  local report=$1
+  if [[ -n "${GITHUB_STEP_SUMMARY:-}" && -f "$report" ]]; then
+    cat "$report" >>"$GITHUB_STEP_SUMMARY"
+  fi
+}
+
+# Validation failure is data for the candidate PR, not a reason to discard it.
+run_validation() {
+  local log_file=$1
+  shift
+  if "$@" >"$log_file" 2>&1; then
+    printf 'passed\n'
+  else
+    printf 'failed\n'
+  fi
+}
--- a/scripts/update-nightly.sh
+++ b/scripts/update-nightly.sh
@ -1,102 +1,96 @@
 #!/usr/bin/env bash
 set -euo pipefail

-# Auto-update nightly.nix to track HEAD of NousResearch/hermes-agent main branch.
-# Designed to run in GitHub Actions (see .github/workflows/update-nightly.yml).
+# Generate a reviewable nightly candidate without pushing to main.

 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 nightly_file="$repo_root/nightly.nix"
+report=${HERMES_UPDATE_REPORT:-/tmp/hermes-nightly-update.md}

-log() {
-  printf '>> %s\n' "$*"
-}
+# shellcheck source=scripts/update-common.sh
+source "$repo_root/scripts/update-common.sh"

-# --- Resolve HEAD of main ---
-log "Fetching HEAD of NousResearch/hermes-agent main branch"
-head_sha=$(git ls-remote https://github.com/NousResearch/hermes-agent.git refs/heads/main | awk '{print $1}')
-if [[ -z "$head_sha" ]]; then
-  echo "Failed to resolve HEAD of main" >&2
-  exit 1
-fi
-log "HEAD SHA: $head_sha"
+log() { printf '>> %s\n' "$*"; }
+die() { printf 'error: %s\n' "$*" >&2; exit 1; }

-# --- Compare with current ---
-current_rev=$(awk -F'"' '/pinRev = /{print $2}' "$nightly_file" | head -1)
-log "Current pinned rev: $current_rev"
+command -v gh >/dev/null || die "gh is required"
+command -v python3 >/dev/null || die "python3 is required"

+head_sha=$(gh api /repos/NousResearch/hermes-agent/commits/main --jq '.sha')
+[[ -n "$head_sha" ]] || die "failed to resolve upstream main"
+current_rev=$(awk -F'"' '/pinRev = /{print $2; exit}' "$nightly_file")
+current_version=$(awk -F'"' '/pinVersion = /{print $2; exit}' "$nightly_file")
+[[ -n "$current_rev" && -n "$current_version" ]] || die "could not read current nightly pin"
 if [[ "$current_rev" == "$head_sha" ]]; then
-  log "Already up to date ($current_rev). Nothing to do."
+  log "Nightly already tracks $head_sha"
+  set_output update_available false
+  set_output compatible true
  exit 0
 fi
-log "Update available: ${current_rev:0:12} → ${head_sha:0:12}"

-# --- Determine base version from package.nix ---
-base_version=$(awk -F'"' '/pinVersion \? "/{print $2}' "$repo_root/package.nix" | head -1)
-if [[ -z "$base_version" ]]; then
-  base_version="0.0.0"
-fi
-nightly_version="${base_version}-unstable-$(date -u +%Y-%m-%d)"
-log "Nightly version: $nightly_version"
+upstream_version=$(
+  gh api -H "Accept: application/vnd.github.raw+json" \
+    "/repos/NousResearch/hermes-agent/contents/pyproject.toml?ref=${head_sha}" |
+    python3 -c 'import sys, tomllib; print(tomllib.load(sys.stdin.buffer)["project"]["version"])'
+)
+python3 - "$upstream_version" <<'PY' || die "upstream pyproject contains an invalid version"
+import re, sys
+raise SystemExit(0 if re.fullmatch(r"\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?", sys.argv[1]) else 1)
+PY
+nightly_version="${upstream_version}-unstable-$(date -u +%Y-%m-%d).${head_sha:0:8}"
+set_output update_available true

-# --- Update nightly.nix with new rev and empty hash ---
+before_source=$(realize_flake_source ".#hermes-agent-nightly")
+backup=$(mktemp)
+build_log=$(mktemp)
+cp "$nightly_file" "$backup"
+candidate_ready=false
+cleanup() {
+  [[ "$candidate_ready" == true ]] || cp "$backup" "$nightly_file"
+  rm -f "$backup" "$build_log"
+}
+trap cleanup EXIT
+
+log "Preparing nightly candidate ${current_rev:0:12} -> ${head_sha:0:12}"
+[[ $(grep -c 'pinVersion = "' "$nightly_file") == 1 ]] || die "unexpected nightly version layout"
+[[ $(grep -c 'pinRev = "' "$nightly_file") == 1 ]] || die "unexpected nightly revision layout"
+[[ $(grep -c 'pinHash = "' "$nightly_file") == 1 ]] || die "unexpected nightly hash layout"
 perl -0pi -e "s|pinVersion = \"[^\"]+\";|pinVersion = \"${nightly_version}\";|" "$nightly_file"
 perl -0pi -e "s|pinRev = \"[^\"]+\";|pinRev = \"${head_sha}\";|" "$nightly_file"
-perl -0pi -e 's|pinHash = "[^"]*";|pinHash = "";|' "$nightly_file"
+perl -0pi -e 's|pinHash = "[^"]*";|pinHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";|' "$nightly_file"
+grep -Fq "pinRev = \"${head_sha}\";" "$nightly_file" || die "failed to update nightly revision"

-# --- Prefetch to get correct hash ---
-build_log=$(mktemp)
-log "Running nix build to compute source hash..."
-if nix build .#hermes-agent-nightly --accept-flake-config >"$build_log" 2>&1; then
-  log "Build succeeded with empty hash?! Unexpected, but OK."
-  source_hash=""
-else
-  source_hash=$(grep -oP 'got: *\Ksha256-[A-Za-z0-9+/=]+' "$build_log" | head -1 || true)
-  if [[ -z "$source_hash" ]]; then
-    log "Build failed but couldn't extract hash. Build log:"
-    tail -50 "$build_log" >&2
-    # Restore original
-    git checkout -- "$nightly_file"
-    rm -f "$build_log"
-    exit 1
-  fi
+if nix build '.#hermes-agent-nightly.src' --no-link --accept-flake-config >"$build_log" 2>&1; then
+  cp "$backup" "$nightly_file"
+  die "the fake source hash unexpectedly succeeded"
 fi
-rm -f "$build_log"
-log "Source hash: $source_hash"
-
-# Update with the correct hash
-if [[ -n "$source_hash" ]]; then
-  perl -0pi -e "s|pinHash = \"[^\"]*\";|pinHash = \"${source_hash}\";|" "$nightly_file"
+source_hash=$(extract_sri_hash "$build_log")
+if [[ -z "$source_hash" ]]; then
+  cp "$backup" "$nightly_file"
+  tail -80 "$build_log" >&2
+  die "could not extract the candidate source hash"
 fi
+perl -0pi -e "s|pinHash = \"[^\"]*\";|pinHash = \"${source_hash}\";|" "$nightly_file"

-# --- Validate build ---
-build_log=$(mktemp)
-log "Validating full nightly build..."
-if ! nix build .#hermes-agent-nightly --accept-flake-config >"$build_log" 2>&1; then
-  log "Build validation FAILED. This likely means dependencies changed upstream."
-  log "Build log (last 100 lines):"
-  tail -100 "$build_log" >&2
-  git checkout -- "$nightly_file"
-  rm -f "$build_log"
-  exit 1
-fi
-rm -f "$build_log"
-log "Build validation PASSED ✅"
+after_source=$(realize_flake_source ".#hermes-agent-nightly")
+system=$(current_nix_system)
+build_status=$(
+  run_validation "$build_log" nix build ".#legacyPackages.${system}.nightlyChecks.all" --accept-flake-config
+)

-# --- Commit and push ---
-if git diff --quiet "$nightly_file"; then
-  log "No changes to commit (shouldn't happen)"
-  exit 0
-fi
+source_args=()
+[[ -n "$before_source" ]] && source_args+=(--before-source "$before_source")
+[[ -n "$after_source" ]] && source_args+=(--after-source "$after_source")
+python3 "$repo_root/scripts/render-update-report.py" \
+  --channel nightly \
+  --current-version "$current_version" --candidate-version "$nightly_version" \
+  --current-rev "$current_rev" --candidate-rev "$head_sha" \
+  "${source_args[@]}" \
+  --build-status "$build_status" --build-log "$build_log" \
+  --upstream-url "https://github.com/NousResearch/hermes-agent/commit/${head_sha}" \
+  --output "$report"

-log "Committing nightly update"
-git add "$nightly_file"
-git commit -m "🤖 bump hermes-agent-nightly to ${head_sha:0:12} (${nightly_version})" \
-  -m "Upstream HEAD: https://github.com/NousResearch/hermes-agent/commit/${head_sha}" \
-  -m "Tests: nix build .#hermes-agent-nightly (passed)"
-
-log "Pushing to main"
-git fetch origin main
-git rebase origin/main
-git push origin HEAD:main
-
-log "Done! Updated hermes-agent-nightly to ${head_sha:0:12} (${nightly_version})"
+publish_summary "$report"
+set_output compatible "$([[ "$build_status" == passed ]] && echo true || echo false)"
+candidate_ready=true
+log "Candidate prepared; validation: $build_status. The workflow will open or update its PR."
--- a/scripts/update-pins.sh
+++ b/scripts/update-pins.sh
@ -1,157 +1,120 @@
 #!/usr/bin/env bash
 set -euo pipefail

-# Auto-update nix-hermes-agent to track latest stable release from NousResearch/hermes-agent.
-# Designed to run in GitHub Actions (see .github/workflows/update-pins.yml).
-# Similar to nix-openclaw's update-pins.sh but tracks releases instead of HEAD.
+# Generate a reviewable stable candidate. The workflow owns commits and PRs;
+# this script intentionally never mutates main or any remote branch.

 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 package_file="$repo_root/package.nix"
+report=${HERMES_UPDATE_REPORT:-/tmp/hermes-stable-update.md}

-log() {
-  printf '>> %s\n' "$*"
-}
+# shellcheck source=scripts/update-common.sh
+source "$repo_root/scripts/update-common.sh"

-if ! command -v jq >/dev/null 2>&1; then
-  echo "jq is required but not installed." >&2
-  exit 1
-fi
+log() { printf '>> %s\n' "$*"; }
+die() { printf 'error: %s\n' "$*" >&2; exit 1; }

-# --- Resolve latest stable release ---
-log "Fetching latest release from NousResearch/hermes-agent"
-# Use the releases endpoint to find the latest non-prerelease
-release_json=$(gh api /repos/NousResearch/hermes-agent/releases/latest 2>/dev/null || true)
-if [[ -z "$release_json" ]]; then
-  echo "Failed to fetch latest release" >&2
-  exit 1
-fi
+command -v gh >/dev/null || die "gh is required"
+command -v jq >/dev/null || die "jq is required"
+command -v python3 >/dev/null || die "python3 is required"

+release_json=$(gh api /repos/NousResearch/hermes-agent/releases/latest)
 release_tag=$(printf '%s' "$release_json" | jq -r '.tag_name // empty')
-release_name=$(printf '%s' "$release_json" | jq -r '.name // empty')
-if [[ -z "$release_tag" ]]; then
-  echo "No release tag found" >&2
-  exit 1
-fi
-log "Latest release: $release_tag ($release_name)"
+[[ -n "$release_tag" ]] || die "latest release has no tag"

-# Extract version from release name or tag (e.g. "Hermes Agent v0.3.0 (v2026.3.17)" → "0.3.0")
-upstream_version=$(printf '%s' "$release_name" | grep -oP 'v\K[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
-if [[ -z "$upstream_version" ]]; then
-  # Fallback: try tag itself
-  upstream_version=$(printf '%s' "$release_tag" | grep -oP 'v?\K[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
-fi
-if [[ -z "$upstream_version" ]]; then
-  echo "Could not parse version from release tag=$release_tag name=$release_name" >&2
-  exit 1
-fi
-log "Parsed version: $upstream_version"
+log "Resolving immutable commit for $release_tag"
+tag_object=$(gh api "/repos/NousResearch/hermes-agent/git/ref/tags/${release_tag}")
+tag_sha=$(printf '%s' "$tag_object" | jq -r '.object.sha // empty')
+tag_type=$(printf '%s' "$tag_object" | jq -r '.object.type // empty')
+for _depth in 1 2 3 4 5; do
+  [[ "$tag_type" == "tag" ]] || break
+  tag_object=$(gh api "/repos/NousResearch/hermes-agent/git/tags/${tag_sha}")
+  tag_sha=$(printf '%s' "$tag_object" | jq -r '.object.sha // empty')
+  tag_type=$(printf '%s' "$tag_object" | jq -r '.object.type // empty')
+done
+[[ "$tag_type" == "commit" && -n "$tag_sha" ]] || die "release tag does not resolve to a commit"

-# --- Compare with current ---
-current_version=$(awk -F'"' '/pinVersion \? "/{print $2}' "$package_file" | head -1)
-log "Current pinned version: $current_version"
+upstream_version=$(
+  gh api -H "Accept: application/vnd.github.raw+json" \
+    "/repos/NousResearch/hermes-agent/contents/pyproject.toml?ref=${tag_sha}" |
+    python3 -c 'import sys, tomllib; print(tomllib.load(sys.stdin.buffer)["project"]["version"])'
+)
+current_version=$(awk -F'"' '/pinVersion \? "/{print $2; exit}' "$package_file")
+current_rev=$(awk -F'"' '/pinRev \? "/{print $2; exit}' "$package_file")
+[[ -n "$current_version" && -n "$current_rev" ]] || die "could not read current stable pin"
+
+if [[ "$current_rev" == "$tag_sha" ]]; then
+  log "Already tracking $release_tag at $tag_sha"
+  set_output update_available false
+  set_output compatible true
+  exit 0
+fi

 if [[ "$current_version" == "$upstream_version" ]]; then
-  log "Already up to date ($current_version). Nothing to do."
-  exit 0
-fi
-log "Update available: $current_version → $upstream_version"
-
-# --- Resolve the commit SHA for the release tag ---
-# Properly handle dereferencing annotated tags
-tag_sha=$(gh api "/repos/NousResearch/hermes-agent/git/ref/tags/${release_tag}" --jq '.object.sha' 2>/dev/null || true)
-if [[ -n "$tag_sha" ]]; then
-    tag_object_type=$(gh api "/repos/NousResearch/hermes-agent/git/ref/tags/${release_tag}" --jq '.object.type' 2>/dev/null || true)
-    if [[ "$tag_object_type" == "tag" ]]; then
-        log "Tag is annotated, dereferencing to commit SHA..."
-        tag_sha=$(gh api "/repos/NousResearch/hermes-agent/git/tags/${tag_sha}" --jq '.object.sha' 2>/dev/null || true)
-    fi
+  die "release version $upstream_version now points at a different commit; review the retag manually"
 fi

-if [[ -z "$tag_sha" ]]; then
-  # Last resort: ls-remote
-  tag_sha=$(git ls-remote https://github.com/NousResearch/hermes-agent.git "refs/tags/${release_tag}^{}" | awk '{print $1}' || true)
-fi
+python3 - "$current_version" "$upstream_version" <<'PY' || die "refusing an automatic version downgrade"
+import re, sys
+def version(value):
+    match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:[-+].*)?", value)
+    if not match:
+        raise SystemExit(2)
+    return tuple(map(int, match.groups()))
+raise SystemExit(0 if version(sys.argv[2]) >= version(sys.argv[1]) else 1)
+PY

-if [[ -z "$tag_sha" ]]; then
-  echo "Failed to resolve commit SHA for tag $release_tag" >&2
-  exit 1
-fi
-log "Release commit SHA: $tag_sha"
-
-# --- Prefetch source ---
-source_url="https://github.com/NousResearch/hermes-agent/archive/${tag_sha}.tar.gz"
-log "Prefetching source tarball (with submodules via fetchFromGitHub)..."
-
-# Strategy: use nix to evaluate the hash by building with a fake hash
-log "Computing fetchFromGitHub hash (with submodules)..."
-
-# Save original
-cp "$package_file" "$package_file.bak"
-
-# Update pinVersion, pinRev, and set pinHash to empty
-perl -0pi -e "s|pinVersion \\? \"[^\"]+\"|pinVersion \\? \"${upstream_version}\"|" "$package_file"
-perl -0pi -e "s|pinRev \\? \"[^\"]+\"|pinRev \\? \"${tag_sha}\"|" "$package_file"
-perl -0pi -e 's|pinHash \? "sha256-[^"]+"|pinHash \? ""|' "$package_file"
-
-# Build and capture the correct hash from the error
+set_output update_available true
+before_source=$(realize_flake_source ".#hermes-agent")
+backup=$(mktemp)
 build_log=$(mktemp)
-log "Running nix build to get correct hash..."
-if nix build .#hermes-agent --accept-flake-config >"$build_log" 2>&1; then
-  log "Build succeeded with empty hash?! Unexpected, but OK."
-  source_hash=""
-else
-  # Nix 2.4+ output format
-  source_hash=$(grep -oP 'got: *\Ksha256-[A-Za-z0-9+/=]+' "$build_log" | head -1 || true)
-  if [[ -z "$source_hash" ]]; then
-    # Fallback to older Nix formats or different diagnostic styles
-    source_hash=$(grep -oP 'specified: .*, got: *\Ksha256-[A-Za-z0-9+/=]+' "$build_log" | head -1 || true)
-  fi
+cp "$package_file" "$backup"
+candidate_ready=false
+cleanup() {
+  [[ "$candidate_ready" == true ]] || cp "$backup" "$package_file"
+  rm -f "$backup" "$build_log"
+}
+trap cleanup EXIT

-  if [[ -z "$source_hash" ]]; then
-    log "Build failed but couldn't extract hash. Build log:"
-    tail -50 "$build_log" >&2
-    cp "$package_file.bak" "$package_file"
-    rm -f "$build_log" "$package_file.bak"
-    exit 1
-  fi
+log "Preparing stable candidate $current_version -> $upstream_version"
+[[ $(grep -c 'pinVersion ? "' "$package_file") == 1 ]] || die "unexpected pinVersion layout"
+[[ $(grep -c 'pinRev ? "' "$package_file") == 1 ]] || die "unexpected pinRev layout"
+[[ $(grep -c 'pinHash ? "' "$package_file") == 1 ]] || die "unexpected pinHash layout"
+perl -0pi -e "s|pinVersion \\? \"[^\"]+\"|pinVersion ? \"${upstream_version}\"|" "$package_file"
+perl -0pi -e "s|pinRev \\? \"[^\"]+\"|pinRev ? \"${tag_sha}\"|" "$package_file"
+perl -0pi -e 's|pinHash \? "[^"]*"|pinHash ? "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="|' "$package_file"
+grep -Fq "pinRev ? \"${tag_sha}\"" "$package_file" || die "failed to update stable revision"
+
+if nix build '.#hermes-agent.src' --no-link --accept-flake-config >"$build_log" 2>&1; then
+  cp "$backup" "$package_file"
+  die "the fake source hash unexpectedly succeeded"
 fi
-rm -f "$build_log"
-log "Source hash: $source_hash"
-
-# Update with the correct hash
-if [[ -n "$source_hash" ]]; then
-  perl -0pi -e "s|pinHash \\? \"[^\"]*\"|pinHash \\? \"${source_hash}\"|" "$package_file"
+source_hash=$(extract_sri_hash "$build_log")
+if [[ -z "$source_hash" ]]; then
+  cp "$backup" "$package_file"
+  tail -80 "$build_log" >&2
+  die "could not extract the candidate source hash"
 fi
+perl -0pi -e "s|pinHash \\? \"[^\"]*\"|pinHash ? \"${source_hash}\"|" "$package_file"

-# --- Validate build ---
-build_log=$(mktemp)
-log "Validating full build..."
-if ! nix build .#hermes-agent --accept-flake-config >"$build_log" 2>&1; then
-  log "Build validation FAILED. This likely means dependencies changed upstream."
-  log "Build log (last 100 lines):"
-  tail -100 "$build_log" >&2
-  cp "$package_file.bak" "$package_file"
-  rm -f "$build_log" "$package_file.bak"
-  exit 1
-fi
-rm -f "$build_log" "$package_file.bak"
-log "Build validation PASSED ✅"
+after_source=$(realize_flake_source ".#hermes-agent")
+build_status=$(
+  run_validation "$build_log" nix flake check --keep-going --no-write-lock-file --accept-flake-config
+)

-# --- Commit and push ---
-if git diff --quiet "$package_file"; then
-  log "No changes to commit (shouldn't happen)"
-  exit 0
-fi
+source_args=()
+[[ -n "$before_source" ]] && source_args+=(--before-source "$before_source")
+[[ -n "$after_source" ]] && source_args+=(--after-source "$after_source")
+python3 "$repo_root/scripts/render-update-report.py" \
+  --channel stable \
+  --current-version "$current_version" --candidate-version "$upstream_version" \
+  --current-rev "$current_rev" --candidate-rev "$tag_sha" \
+  "${source_args[@]}" \
+  --build-status "$build_status" --build-log "$build_log" \
+  --upstream-url "https://github.com/NousResearch/hermes-agent/releases/tag/${release_tag}" \
+  --output "$report"

-log "Committing update"
-git add "$package_file"
-git commit -m "🤖 bump hermes-agent ${current_version} → ${upstream_version} (${release_tag})" \
-  -m "Upstream: https://github.com/NousResearch/hermes-agent/releases/tag/${release_tag}" \
-  -m "Tests: nix build .#hermes-agent (passed)"
-
-log "Pushing to main"
-git fetch origin main
-git rebase origin/main
-git push origin HEAD:main
-
-log "Done! Updated hermes-agent to ${upstream_version} (${release_tag})"
+publish_summary "$report"
+set_output compatible "$([[ "$build_status" == passed ]] && echo true || echo false)"
+candidate_ready=true
+log "Candidate prepared; validation: $build_status. The workflow will open or update its PR."
--- a/tests/skills-coexistence.nix
+++ b/tests/skills-coexistence.nix
@ -2,6 +2,7 @@
  self,
  nixpkgs,
  system,
+  hermes-agent ? self.packages.${system}.hermes-agent,
 }:

 let
@ -21,7 +22,7 @@ pkgs.testers.runNixOSTest {

      services.hermes-agent = {
        enable = true;
-        package = self.packages.${system}.hermes-agent;
+        package = hermes-agent;
        skills = {
          bundled.enable = false;
          custom.repo-watch = {
--- a/tests/test_render_update_report.py
+++ b/tests/test_render_update_report.py
@ -0,0 +1,76 @@
+import argparse
+import importlib.util
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[1] / "scripts" / "render-update-report.py"
+SPEC = importlib.util.spec_from_file_location("render_update_report", SCRIPT)
+REPORT = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+SPEC.loader.exec_module(REPORT)
+
+
+class RenderUpdateReportTest(unittest.TestCase):
+    def write_source(self, root: Path, dependencies: list[str], config_version: int) -> None:
+        (root / "hermes_cli").mkdir(parents=True)
+        deps = ", ".join(f'"{item}"' for item in dependencies)
+        (root / "pyproject.toml").write_text(
+            "[project]\n"
+            'requires-python = ">=3.11"\n'
+            f"dependencies = [{deps}]\n"
+            "[project.scripts]\n"
+            'hermes = "hermes_cli.main:main"\n',
+            encoding="utf-8",
+        )
+        (root / "hermes_cli" / "config.py").write_text(
+            f'CURRENT_CONFIG_VERSION = {config_version}\n', encoding="utf-8"
+        )
+
+    def test_reports_dependency_and_schema_changes(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            before, after = root / "before", root / "after"
+            self.write_source(before, ["openai>=1", "pydantic"], 3)
+            self.write_source(after, ["openai>=2", "pillow", "cryptography"], 4)
+            args = argparse.Namespace(
+                channel="stable",
+                current_version="0.16.0",
+                candidate_version="0.18.2",
+                current_rev="old",
+                candidate_rev="new",
+                before_source=before,
+                after_source=after,
+                build_status="failed",
+                build_log=None,
+                upstream_url="https://example.invalid/release",
+            )
+
+            rendered = REPORT.render(args)
+
+            self.assertIn("added: `cryptography`, `pillow`; removed: `pydantic`", rendered)
+            self.assertIn("changed: `openai` (`openai>=1` → `openai>=2`)", rendered)
+            self.assertIn("| Config schema | `3` | `4` |", rendered)
+            self.assertIn("❌ failed", rendered)
+
+    def test_missing_sources_are_reported_without_crashing(self) -> None:
+        args = argparse.Namespace(
+            channel="nightly",
+            current_version="old",
+            candidate_version="new",
+            current_rev="old-rev",
+            candidate_rev="new-rev",
+            before_source=None,
+            after_source=None,
+            build_status="passed",
+            build_log=None,
+            upstream_url=None,
+        )
+        rendered = REPORT.render(args)
+        self.assertIn("| Python | `unknown` | `unknown` |", rendered)
+        self.assertIn("✅ passed", rendered)
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/tests/update-common.sh
+++ b/tests/update-common.sh
@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+# shellcheck source=scripts/update-common.sh
+source "$repo_root/scripts/update-common.sh"
+
+log=$(mktemp)
+trap 'rm -f "$log"' EXIT
+
+printf 'error: hash mismatch\n  got:    sha256-AbCd0123+/=\n' >"$log"
+[[ "$(extract_sri_hash "$log")" == "sha256-AbCd0123+/=" ]]
+
+printf 'specified: sha256-old\n       got: sha256-newValue=\n' >"$log"
+[[ "$(extract_sri_hash "$log")" == "sha256-newValue=" ]]
+
+printf 'an unrelated dependency failure\n' >"$log"
+[[ -z "$(extract_sri_hash "$log")" ]]
+
+status=$(run_validation "$log" bash -c 'echo dependency-broke; exit 42')
+[[ "$status" == "failed" ]]
+grep -q dependency-broke "$log"
+
+status=$(run_validation "$log" bash -c 'echo all-good')
+[[ "$status" == "passed" ]]
+grep -q all-good "$log"