fix(devtools): package Codex daemon and use upstream Pi
This commit is contained in:
parent
89bebe5610
commit
037e817b29
9 changed files with 151 additions and 121 deletions
40
packages/codex-daemon-test.py
Normal file
40
packages/codex-daemon-test.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
"""Run manually outside the Nix sandbox: python3 packages/codex-daemon-test.py /path/to/codex."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
binary = str(Path(sys.argv[1]).resolve())
|
||||
with tempfile.TemporaryDirectory(prefix="codex-daemon-test-") as directory:
|
||||
home = Path(directory)
|
||||
settings = home / "app-server-daemon/settings.json"
|
||||
settings.parent.mkdir()
|
||||
settings.write_text(json.dumps({"updater": {"autoUpdateEnabled": False}}))
|
||||
env = {**os.environ, "CODEX_HOME": directory}
|
||||
|
||||
def run(*arguments, check=True):
|
||||
return subprocess.run(
|
||||
[binary, "app-server", "daemon", *arguments],
|
||||
env=env, check=check, timeout=60, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
try:
|
||||
started = run("start", check=False)
|
||||
print(started.stdout)
|
||||
started.check_returncode()
|
||||
release = home / "packages/app-server-daemon/current"
|
||||
assert (release / "codex-package.json").is_file()
|
||||
assert (release / "bin/codex").is_file()
|
||||
version = run("version")
|
||||
print(version.stdout)
|
||||
status = json.loads(version.stdout)
|
||||
assert status["status"] == "running", status
|
||||
assert status["appServerVersion"] == status["cliVersion"], status
|
||||
print("Codex daemon startup passed")
|
||||
finally:
|
||||
stopped = run("stop", check=False)
|
||||
print(stopped.stdout)
|
||||
stopped.check_returncode()
|
||||
25
packages/codex-package-test.py
Normal file
25
packages/codex-package-test.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
"""Check the layout required by Codex daemon provisioning."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
root = Path(sys.argv[1]).resolve()
|
||||
version, target = sys.argv[2:]
|
||||
manifest = json.loads((root / "codex-package.json").read_text())
|
||||
assert manifest == {
|
||||
"layoutVersion": 1,
|
||||
"version": version,
|
||||
"target": target,
|
||||
"entrypoint": "bin/codex",
|
||||
}
|
||||
required = ["bin/codex", "bin/codex-code-mode-host", "codex-path/rg"]
|
||||
if "linux" in target:
|
||||
required.append("codex-resources/bwrap")
|
||||
for name in required:
|
||||
path = root / name
|
||||
assert path.is_file() and os.access(path, os.X_OK), name
|
||||
for path in root.rglob("*"):
|
||||
assert path.resolve().is_relative_to(root), f"Escaping link: {path}"
|
||||
assert not (path.is_symlink() and path.is_dir()), f"Directory link: {path}"
|
||||
print("Codex package layout checks passed")
|
||||
39
packages/codex.md
Normal file
39
packages/codex.md
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# Local Codex package adapter
|
||||
|
||||
`codex.nix` adds the complete package layout required by Codex daemon startup.
|
||||
The pinned Numtide package supplies binaries but lacks the manifest and bundled
|
||||
ripgrep. The adapter copies the binaries without rebuilding Rust.
|
||||
|
||||
Remove this adapter and select `llmPkgs.codex` when Numtide supplies the complete
|
||||
layout. Also remove the custom package export and adjust the flake check.
|
||||
|
||||
## Validation
|
||||
|
||||
Build the package and run its layout checks:
|
||||
|
||||
```sh
|
||||
nix build .#codex
|
||||
```
|
||||
|
||||
Test daemon provisioning outside the Nix sandbox:
|
||||
|
||||
```sh
|
||||
python3 packages/codex-daemon-test.py ./result/bin/codex
|
||||
```
|
||||
|
||||
The daemon test uses a temporary `CODEX_HOME`, disables automatic updates there,
|
||||
and stops the test daemon. It does not use your normal Codex sessions or settings.
|
||||
|
||||
## Runtime limits
|
||||
|
||||
Codex copies the package into `~/.codex/packages/app-server-daemon`.
|
||||
The copied Nix binaries still reference dependencies in `/nix/store`.
|
||||
Keep the corresponding Nix system generation while using that daemon version.
|
||||
After a Nix upgrade, `codex app-server daemon update --help` describes how to
|
||||
select the current CLI package with `--from-cli`.
|
||||
|
||||
The adapter does not change your daemon update settings. Codex can automatically
|
||||
update its private daemon independently of the CLI installed by Nix.
|
||||
|
||||
Only aarch64-darwin daemon startup has been tested locally. The derivation also
|
||||
provides the required bubblewrap executable on Linux.
|
||||
38
packages/codex.nix
Normal file
38
packages/codex.nix
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
{ pkgs, llm-agents }:
|
||||
let
|
||||
upstream = llm-agents.packages.${pkgs.stdenv.hostPlatform.system}.codex;
|
||||
manifest = pkgs.writeText "codex-package.json" (
|
||||
builtins.toJSON {
|
||||
layoutVersion = 1;
|
||||
version = upstream.version;
|
||||
target = pkgs.stdenv.hostPlatform.rust.rustcTarget;
|
||||
entrypoint = "bin/codex";
|
||||
}
|
||||
);
|
||||
in
|
||||
# Temporary adapter for Numtide's source build. Remove when upstream supplies
|
||||
# the complete daemon package layout. Reuse binaries without rebuilding Rust.
|
||||
pkgs.runCommand "codex-${upstream.version}"
|
||||
{
|
||||
inherit (upstream) version meta;
|
||||
nativeBuildInputs = [ pkgs.python3 ];
|
||||
passthru = { inherit upstream; };
|
||||
}
|
||||
''
|
||||
mkdir -p "$out/bin" "$out/codex-path"
|
||||
source=${upstream}/bin
|
||||
if [ -d ${upstream}/libexec/codex/bin ]; then
|
||||
source=${upstream}/libexec/codex/bin
|
||||
fi
|
||||
# Copy real executables: daemon provisioning rejects escaping symlinks.
|
||||
cp -L "$source/codex" "$source/codex-code-mode-host" "$source/logs_client" "$out/bin/"
|
||||
cp -L ${pkgs.ripgrep}/bin/rg "$out/codex-path/rg"
|
||||
cp ${manifest} "$out/codex-package.json"
|
||||
${pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isLinux ''
|
||||
mkdir -p "$out/codex-resources"
|
||||
cp -L ${pkgs.bubblewrap}/bin/bwrap "$out/codex-resources/bwrap"
|
||||
''}
|
||||
cp -RL ${upstream}/share "$out/share"
|
||||
python ${./codex-package-test.py} "$out" '${upstream.version}' '${pkgs.stdenv.hostPlatform.rust.rustcTarget}'
|
||||
"$out/bin/codex" --version
|
||||
''
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{ pkgs, llm-agents, ... }:
|
||||
{
|
||||
cf = pkgs.callPackage ./cf.nix { };
|
||||
pi = pkgs.callPackage ./pi.nix { inherit llm-agents; };
|
||||
codex = pkgs.callPackage ./codex.nix { inherit llm-agents; };
|
||||
pikpak-cli = pkgs.callPackage ./pikpak-cli.nix { };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { Type } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
createCodemodeExtension,
|
||||
type ExtensionAPI,
|
||||
type ExtensionToolContext,
|
||||
type ToolDefinition,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
|
||||
// Run the embedded sandbox without a model request or provider credentials.
|
||||
export default function (pi: ExtensionAPI) {
|
||||
let codemode: ToolDefinition | undefined;
|
||||
createCodemodeExtension({ models: false })({
|
||||
...pi,
|
||||
registerTool: (tool) => {
|
||||
codemode = tool;
|
||||
},
|
||||
});
|
||||
|
||||
pi.on("session_start", async (_event, ctx) => {
|
||||
try {
|
||||
assert.ok(codemode);
|
||||
let nestedCalls = 0;
|
||||
const toolContext: ExtensionToolContext = {
|
||||
...ctx,
|
||||
tools: [
|
||||
{
|
||||
name: "echo",
|
||||
description: "Return the input value.",
|
||||
parameters: Type.Object({ value: Type.Number() }),
|
||||
outputSchema: Type.Object({ value: Type.Number() }),
|
||||
},
|
||||
],
|
||||
executeTool: async (name, args) => {
|
||||
assert.equal(name, "echo");
|
||||
nestedCalls += 1;
|
||||
return {
|
||||
toolCall: { type: "toolCall", id: `test/${nestedCalls}`, name, arguments: args },
|
||||
isError: false,
|
||||
result: {
|
||||
content: [{ type: "text", text: JSON.stringify(args) }],
|
||||
structuredContent: args,
|
||||
details: undefined,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const run = (code: string) =>
|
||||
codemode!.execute("codemode-test", { code }, new AbortController().signal, undefined, toolContext);
|
||||
const output = (result: Awaited<ReturnType<typeof run>>) =>
|
||||
result.content.filter((item) => item.type === "text").map((item) => item.text).join("\n");
|
||||
|
||||
const arithmetic = await run("return 6 * 7;");
|
||||
assert.match(output(arithmetic), /Script completed/);
|
||||
assert.match(output(arithmetic), /42/);
|
||||
|
||||
const nested = await run(`
|
||||
const values = await Promise.all([
|
||||
tools.echo({ value: 20 }),
|
||||
tools.echo({ value: 22 }),
|
||||
]);
|
||||
return values.reduce((sum, item) => sum + item.value, 0);
|
||||
`);
|
||||
assert.match(output(nested), /Script completed/);
|
||||
assert.match(output(nested), /42/);
|
||||
assert.equal(nestedCalls, 2);
|
||||
|
||||
const failure = await run('throw new Error("expected-test-error");');
|
||||
assert.match(output(failure), /Script failed/);
|
||||
assert.match(output(failure), /expected-test-error/);
|
||||
console.error("codemode regression test passed");
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
ctx.shutdown();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
{ pkgs, llm-agents }:
|
||||
let
|
||||
upstream = llm-agents.packages.${pkgs.stdenv.hostPlatform.system}.pi;
|
||||
compileCommand = "bun build --compile ./dist/bun/cli.js ./src/utils/image-resize-worker.ts --outfile dist/pi";
|
||||
in
|
||||
upstream.overrideAttrs (old: {
|
||||
# The standalone binary resolves this exact source path at runtime.
|
||||
# The npm source only ships dist/, so create and embed a worker entry.
|
||||
# Fail evaluation if the upstream build changes instead of dropping the fix.
|
||||
preInstall =
|
||||
assert pkgs.lib.hasInfix compileCommand old.preInstall;
|
||||
pkgs.lib.replaceStrings
|
||||
[ compileCommand ]
|
||||
[
|
||||
''
|
||||
mkdir -p src/extensions/codemode
|
||||
echo 'import "../../../dist/extensions/codemode/worker.js";' > src/extensions/codemode/worker.ts
|
||||
bun build --compile ./dist/bun/cli.js ./src/utils/image-resize-worker.ts ./src/extensions/codemode/worker.ts --outfile dist/pi
|
||||
''
|
||||
]
|
||||
old.preInstall;
|
||||
|
||||
postInstallCheck = (old.postInstallCheck or "") + ''
|
||||
test_home=$(mktemp -d)
|
||||
HOME="$test_home" PI_CODING_AGENT_DIR="$test_home/agent" \
|
||||
"$out/bin/pi" --offline --no-session --no-extensions --no-skills \
|
||||
--no-prompt-templates --no-themes --no-context-files --no-approve \
|
||||
--extension ${./pi-codemode-test.ts} --mode rpc < /dev/null > codemode-test.log 2>&1
|
||||
tail -n 60 codemode-test.log
|
||||
# RPC shutdown can return zero after an extension failure.
|
||||
grep -Fx 'codemode regression test passed' codemode-test.log
|
||||
'';
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue