{ "run_id": "TayHCode--agent-bench-claude-haiku-5-5-1792641802162398530", "date": "2026-10-22T04:10:55.624589+00:00", "repo": "TayHCode/agent-bench", "kind": "repo", "source": "https://github.com/TayHCode/agent-bench", "commit": "ad01b757a0565c35fa590fe531964019363e70d5", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "test_version": "readme-6f8cbfa597c6d59f7ece0e29599b48d61f780e82e853b6ab300c3a8265684642", "pass": true, "outcome": "pass", "time_to_working_s": 418.00403682899196, "total_s": 453.4621648788452, "steps": [ "cd && pwd && ls -la && id -u && uname -a && (git status 2>&1 | head -3) && cat README.md 2>/dev/null | head -200", "ls -la 2>&1; ls /inputs 2>&1; /opt/homebrew/bin/python3 --version 2>&1; which node npm uv 2>&1; cat /etc/os-release | head -2", "cd && which git && git --version && git init -q && git remote add origin https://github.com/TayHCode/agent-bench && git fetch --depth 1 origin ad01b757a0565c35fa590fe531964019363e70d5 2>&1 | tail -3 && git reset --mixed FETCH_HEAD && git log --oneline -1 && git status --short | head", "cd && /opt/homebrew/bin/python3 -m venv && -m pip install -q --upgrade pip 2>&1 | tail -3; -m pip install . 2>&1 | tail -5; ls | grep agentbench", "mkdir -p && cd && export AGENTBENCH_HOME= && doctor 2>&1; echo \"rc=$?\"; demo --no-open 2>&1; echo \"rc=$?\"", "cd && export AGENTBENCH_HOME= && demo 2>&1; echo \"rc=$?\"; ls -la ", "mkdir -p && cd && export AGENTBENCH_HOME=\"$PWD/bench-home\" && export PATH= && -m agentbench.toy toy 2>&1; echo \"rc=$?\"; FIX_COMMIT=$(git -C toy rev-parse HEAD) && echo \"fix=$FIX_COMMIT\" && agentbench task from-commit ./toy \"$FIX_COMMIT\" --id toy-null-total --check 'python3 -m unittest discover -s tests' 2>&1; echo \"rc=$?\"; -c \"from pathlib import Path; Path('bench-home/tasks/toy-null-total/prompt.md').write_text('Fix total(None) so it returns zero.\\n')\" && agentbench plan --tasks toy-null-total --contestants stub-solver --ledger-cap 1 2>&1; echo \"rc=$?\"; agentbench run --task toy-null-total --contestant stub-solver --profile byo --out runs 2>&1; echo \"rc=$?\"", "cd && cat CONTRIBUTING.md; ls .github/*; cat .github/*/*.yml 2>/dev/null | head -80; echo ---; grep -rniE \"cron|schedul|--once|--now|webhook|while True|loop\" src | head -20; echo ---; ls src/agentbench | head -50", "cd && PYTHONDONTWRITEBYTECODE=1 -m unittest discover -s tests -q 2>&1 | tail -6; echo \"rc=${PIPESTATUS[0]}\"; cd && git status --short | head", "cd && ls -la .coldpass && grep -nF '`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.' README.md | cut -c1-40; grep -nF '`passed=True` and $0; the stand-in copies gold, so this checks the pipeline only.' README.md | cut -c1-40; grep -nF 'Run `python -m unittest discover -s tests -q` in one process' CONTRIBUTING.md | cut -c1-30", "cd && AB= && cat > probe.json <<'EOF'\n{\"kind\":\"cli\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"stub-solver pass\"}\nEOF\ncat > claims.json <<'EOF'\n[\n{\"kind\":\"claim\",\"claim\":\"demo builds a toy repo and runs two built-in stand-in agents\",\"source\":\"`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.\",\"input\":\"agentbench demo\",\"expected\":\"stub-solver passes and stub-lazy fails\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"stub-solver pass\"},\n{\"kind\":\"claim\",\"claim\":\"from-commit rehearsal run reports passed=True at zero cost\",\"source\":\"`passed=True` and $0; the stand-in copies gold, so this checks the pipeline only.\",\"input\":\"agentbench run --task toy-null-total --contestant stub-solver on the toy from-commit task\",\"expected\":\"passed=True\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"run\",\"--task\",\"toy-null-total\",\"--contestant\",\"stub-solver\",\"--profile\",\"byo\",\"--out\",\"runs\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"passed=True\"}\n]\nEOF\ncat > examples.json <<'EOF'\n[\n{\"kind\":\"example\",\"claim\":\"Free quickstart demo runs the two stand-in agents\",\"source\":\"README.md Quickstart: agentbench demo\",\"input\":\"agentbench demo\",\"expected\":\"stub-solver pass, stub-lazy FAIL\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"stub-solver pass\"},\n{\"kind\":\"example\",\"claim\":\"Repository offline test suite passes\",\"source\":\"CONTRIBUTING.md\",\"input\":\"python -m unittest discover -s tests -q\",\"expected\":\"all tests run and pass (1 skipped)\",\"argv\":[\"\",\"-m\",\"unittest\",\"discover\",\"-s\",\"tests\",\"-q\"],\"cwd\":\"\",\"env\":{\"PYTHONDONTWRITEBYTECODE\":\"1\"},\"expected_text\":\"Privacy mode: external private-term list\"}\n]\nEOF\ncat > cycles.json <<'EOF'\n{\"detected\":false,\"reason\":\"Inspected README.md and src/agentbench (grep for cron, schedule, --once, --now, webhook, loop, while True found nothing). agentbench runs one command per invocation (doctor, demo, start, run, matrix, report); there is no product scheduler or loop.\",\"checks\":[]}\nEOF\nfor f in probe.json claims.json examples.json cycles.json; do -m json.tool $f >/dev/null && echo \"$f ok\"; done", "rm -rf && cd && COLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay 2>&1 | tail -60; echo \"rc=${PIPESTATUS[0]}\"", "cd && grep -n '\"env\"\\|env=\\|\\.get(\"env\\|environ' check.py | head -40", "cd && grep -nF '`doctor` tells you exactly what to run if not.' README.md | cut -c1-20 && cd .coldpass && cat > claims.json <<'EOF'\n[\n{\"kind\":\"claim\",\"claim\":\"demo builds a toy repo and runs two built-in stand-in agents\",\"source\":\"`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.\",\"input\":\"agentbench demo\",\"expected\":\"stub-solver passes and stub-lazy fails\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"expected_text\":\"stub-solver pass\"},\n{\"kind\":\"claim\",\"claim\":\"doctor tells you exactly what to run if the real agent CLIs are missing\",\"source\":\"`doctor` tells you exactly what to run if not.\",\"input\":\"agentbench doctor on a guest without claude installed\",\"expected\":\"a fix line naming the npm install command for claude-code\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"doctor\"],\"cwd\":\"\",\"expected_text\":\"npm install -g @anthropic-ai/claude-code\"}\n]\nEOF\ncat > examples.json <<'EOF'\n[\n{\"kind\":\"example\",\"claim\":\"Free quickstart demo runs the two stand-in agents\",\"source\":\"README.md Quickstart: agentbench demo\",\"input\":\"agentbench demo\",\"expected\":\"stub-solver pass, stub-lazy FAIL\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"stub-solver pass\"},\n{\"kind\":\"example\",\"claim\":\"Free from-commit rehearsal: build toy task and run stub agent with passed=True\",\"source\":\"README.md Free from-commit rehearsal\",\"input\":\"agentbench run --task toy-null-total --contestant stub-solver --profile byo --out runs-example (toy task built with agentbench task from-commit)\",\"expected\":\"passed=True at $0\",\"argv\":[\"\",\"run\",\"--task\",\"toy-null-total\",\"--contestant\",\"stub-solver\",\"--profile\",\"byo\",\"--out\",\"runs-example\"],\"cwd\":\"\",\"env\":{\"AGENTBENCH_HOME\":\"\"},\"expected_text\":\"passed=True\"},\n{\"kind\":\"example\",\"claim\":\"Repository offline test suite passes\",\"source\":\"CONTRIBUTING.md\",\"input\":\"python -m unittest discover -s tests -q\",\"expected\":\"all tests run and pass (1 skipped)\",\"argv\":[\"\",\"-m\",\"unittest\",\"discover\",\"-s\",\"tests\",\"-q\"],\"cwd\":\"\",\"env\":{\"PYTHONDONTWRITEBYTECODE\":\"1\"},\"expected_text\":\"OK (skipped=1)\"}\n]\nEOF\n -m json.tool claims.json >/dev/null && -m json.tool examples.json >/dev/null && echo ok; rm -rf ; cd && COLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay > 2>&1; echo \"rc=$?\"; /opt/homebrew/bin/python3 -I -c \"import json;d=json.load(open(''));print(d.get('valid'),d.get('problems'));r=d.get('replay',{});print(r.get('status'));[print(c['command'][:90],c['verdict'],c['outcome'][:90]) for c in r['details'].get('claims',[])];[print(c['command'][:90],c['outcome'][:60]) for c in r['details'].get('example_checks',[])]\" 2>&1 | head -30", "cd && cat > claims.json <<'EOF'\n[\n{\"kind\":\"claim\",\"claim\":\"demo builds a toy repo and runs two built-in stand-in agents\",\"source\":\"`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.\",\"input\":\"agentbench demo\",\"expected\":\"stub-solver passes and stub-lazy fails\",\"probe_kind\":\"cli\",\"argv\":[\"\",\"demo\"],\"cwd\":\"\",\"expected_text\":\"stub-solver pass\"}\n]\nEOF\n -m json.tool claims.json >/dev/null && echo ok; rm -rf ; cd && COLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay > 2>&1; echo \"rc=$?\"; /opt/homebrew/bin/python3 -I -c \"import json;d=json.load(open(''));print(d.get('valid'),d.get('problems'));r=d.get('replay',{});print(r.get('status'));[print(c['command'][:100],c['verdict'],c['outcome'][:90]) for c in r['details'].get('claims',[])];[print(c['command'][:100],c['outcome'][:60]) for c in r['details'].get('example_checks',[])]\" 2>&1 | head -30", "/opt/homebrew/bin/python3 -I -c \"\nimport json\nd=json.load(open(''))\nr=d['replay']\nprint(json.dumps(r.get('details',{}).get('example_checks',r),default=str)[:1500])\n\" ; grep -o '\"verdict\": \"[a-z]*\"' | sort | uniq -c; grep -o '\"passed=True\"\\|passed=True' | head -2; ps aux | grep -i agentbench | grep -v grep | wc -l", "cd && cat > report.json <<'EOF'\n{\n \"outcome\": \"working\",\n \"fault_owner\": null,\n \"operation\": \"Install agent-bench (agentbench CLI) from the pinned checkout and run the free offline demo, which builds a toy repo and scores two stand-in agents; also run the documented from-commit rehearsal (toy task build, then stub-solver run reporting passed=True).\",\n \"operation_scope\": \"Offline, no model login, $0. Covers the free stand-in agents only. Real agents (claude-code, codex) were NOT installed or logged in, so real model runs were not exercised. Platform: macOS (Darwin arm64) guest, Python 3.14.8 in a venv at , git 2.39.5. Not verified: uv-based tasks, built-in code task preparation (needs GitHub network).\",\n \"steps\": [\n \"git init; git remote add origin https://github.com/TayHCode/agent-bench; git fetch --depth 1 origin ad01b757a0565c35fa590fe531964019363e70d5; git reset --mixed FETCH_HEAD\",\n \"/opt/homebrew/bin/python3 -m venv ; -m pip install --upgrade pip\",\n \" -m pip install . (README Quickstart: python -m pip install .) -> agentbench-0.0.1 installed\",\n \"AGENTBENCH_HOME= doctor (README: agentbench doctor)\",\n \"AGENTBENCH_HOME= demo (README: agentbench demo) -> stub-solver pass, stub-lazy FAIL\",\n \"AGENTBENCH_HOME= -m agentbench.toy toy (README Free from-commit rehearsal)\",\n \"agentbench task from-commit ./toy --id toy-null-total --check 'python3 -m unittest discover -s tests' -> verified: unfixed fails, fixed passes\",\n \"write bench-home/tasks/toy-null-total/prompt.md; agentbench plan --tasks toy-null-total --contestants stub-solver --ledger-cap 1\",\n \"agentbench run --task toy-null-total --contestant stub-solver --profile byo --out runs -> ok, passed=True, $0.0000\",\n \" -m unittest discover -s tests -q (CONTRIBUTING.md) -> Ran 209 tests, OK (skipped=1)\",\n \"COLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay -> valid, no problems\"\n ],\n \"deviations\": [\n \"Workspace was not a git repo; ran the git init / fetch --depth 1 / reset --mixed sequence at the pinned commit as instructed.\",\n \"Used a venv () for pip install . instead of the system Python, per the task's absolute-venv rule. Same command otherwise.\",\n \"Ran `agentbench demo --no-open` first: failed with 'unrecognized arguments: --no-open' (that flag belongs to `start`). Reran `agentbench demo` without it; it passed.\",\n \"claude and codex are not installed in the guest, and uv is not installed. Real agent runs were not attempted (no login, and the task forbids credentials). Only the free stand-in agents were run.\",\n \"First replay of a claim using the from-commit rehearsal failed: the checker does not pass a claim's env, so AGENTBENCH_HOME was not set and the built-in task lookup failed. Fixed by moving the rehearsal into examples.json (which honors env) and keeping only the demo claim.\",\n \"A claim using `agentbench doctor` was rejected as setup-only evidence and was removed.\",\n \"The checker path /inputs/coldpass-preflight.py from the task text does not exist; used as the coldpass instructions direct.\",\n \"unittest output contains ResourceWarning noise and a 'refusing to plan' message printed by a test; these are from the repo's tests, not failures.\"\n ],\n \"stuck_step\": null,\n \"main_blocker\": null,\n \"plain_failure\": null,\n \"suggested_fix\": \"None required for the free demo path. For real agent runs, install and log in to claude or codex in the guest (npm install -g @anthropic-ai/claude-code), then rerun agentbench doctor and a real run.\"\n}\nEOF\n -m json.tool report.json >/dev/null && echo report-ok; ls -la ; ls | head -3", "rm -rf ; cd && COLDPASS_WORKSPACE= /opt/homebrew/bin/python3 -I --validate --replay > 2>&1; echo \"rc=$?\"; /opt/homebrew/bin/python3 -I -c \"import json;d=json.load(open(''));print('valid=',d.get('valid'),'problems=',d.get('problems'),'replay=',d.get('replay',{}).get('status'))\"" ], "stuck_step": null, "error_excerpt": null, "what_it_did": "Install agent-bench (agentbench CLI) from the pinned checkout and run the free offline demo, which builds a toy repo and scores two stand-in agents; also run the documented from-commit rehearsal (toy task build, then stub-solver run reporting passed=True).", "suggested_fix": null, "tokens_in": 402901, "tokens_out": 17613, "cost_usd": 0.019631069999999997, "usage_complete": true, "checks": [ { "check": "operation", "status": "PASS", "details": { "working": true, "argv": [ "", "demo" ], "kind": "cli", "returncode": 0, "stdout": "1/3 Building a toy task: a small repo, a bug, and a hidden test that proves it is fixed ...\n2/3 Running two free stand-in agents on it (no model, no network, $0) ...\n3/3 Results:\n\nContestant Result Time Tokens $ equiv\n----------- ------ ---- ------ -------\nstub-solver pass 0.0s 1,060 $0.0000\nstub-lazy FAIL 0.0s 1,060 $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start", "stderr": "", "needs_credentials": false, "needs_browser": false, "scope": "product operation" } } ], "safety": { "version": "z2x-1", "rules_version": "z2x-1", "summary": "No problems traced to this repository code in these checks.", "findings": [], "context": [], "limitations": [ "A short run cannot establish that code is safe.", "Connections show IP addresses; proxies can hide the final hostname.", "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing.", "macos behavior observation needs an OS-native file/network/process collector; only the static scan ran.", "No Linux syscall trace was collected." ], "static": { "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check." }, "evidence_url": "/safety/?run=TayHCode--agent-bench-claude-haiku-5-5-1792641802162398530", "evidence_file": "/data/safety/TayHCode--agent-bench-claude-haiku-5-5-1792641802162398530.json.gz" }, "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "platform": "macos", "reasoning": "low", "service_tier": null, "resources": { "cpus": null, "memory_mib": 4096, "disk_mib": 32768 }, "native_tool_calls": 18, "native_bundle": null, "architecture": null, "backend": "tart", "platform_details": { "backend": "tart", "os": "macos" }, "platform_conflicts": null, "sandbox_source": null, "source_pin": { "commit": "ad01b757a0565c35fa590fe531964019363e70d5", "url": "https://github.com/TayHCode/agent-bench" }, "purpose": "initial", "profile": { "model": "claude-haiku-5-5", "reasoning": "low" }, "fault": { "owner": null, "status": "resolved", "reason": null }, "fix_id": null, "verifies_run_id": null, "hosting": null, "operation_scope": "product operation", "tool_calls": 18, "guest_setup": null, "runtime": null, "runtime_version": null, "runtime_revision": null, "source_revision": null, "evidence_version": null, "failure_origin": null, "evidence_strength": "readme_checked", "works": "yes", "install_s": null, "first_use_s": null, "mcp": "none", "needs": [], "needs_which": null, "agent_interface": [ "cli" ], "license": "MIT", "repo_url": "https://github.com/TayHCode/agent-bench", "cost_to_try": "free", "hardware": null, "what_failed": null, "local_or_hosted": "local", "cycle_checks": [ { "kind": "cycle", "claim": "Scheduled or looping main behavior", "source": "repository entry points", "input": "source inspection", "expected": "identify the product trigger", "verdict": "yes", "outcome": "Inspected README.md and src/agentbench (grep for cron, schedule, --once, --now, webhook, loop, while True found nothing). agentbench runs one command per invocation (doctor, demo, start, run, matrix, report); there is no product scheduler or loop.", "evidence": "Inspected README.md and src/agentbench (grep for cron, schedule, --once, --now, webhook, loop, while True found nothing). agentbench runs one command per invocation (doctor, demo, start, run, matrix, report); there is no product scheduler or loop.", "command": "" } ], "claims": [ { "kind": "claim", "claim": "demo builds a toy repo and runs two built-in stand-in agents", "source": "`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.", "input": "agentbench demo", "expected": "stub-solver passes and stub-lazy fails", "command": " demo", "verdict": "yes", "outcome": "Real input produced the expected result: stub-solver pass", "evidence": "stub-solver pass 0.0s 1,060 $0.0000\nstub-lazy FAIL 0.0s 1,060 $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start\n", "returncode": 0, "source_checked": true, "source_file": "README.md" } ], "claims_note": null, "example_checks": [ { "kind": "example", "claim": "Free quickstart demo runs the two stand-in agents", "source": "README.md Quickstart: agentbench demo", "input": "agentbench demo", "command": " demo", "evidence": "1/3 Building a toy task: a small repo, a bug, and a hidden test that proves it is fixed ...\n2/3 Running two free stand-in agents on it (no model, no network, $0) ...\n3/3 Results:\n\nContestant Result Time Tokens $ equiv\n----------- ------ ---- ------ -------\nstub-solver pass 0.0s 1,060 $0.0000\nstub-lazy FAIL 0.1s 1,060 $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start", "deferred": false, "expected": "stub-solver pass, stub-lazy FAIL", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" }, { "kind": "example", "claim": "Free from-commit rehearsal: build toy task and run stub agent with passed=True", "source": "README.md Free from-commit rehearsal", "input": "agentbench run --task toy-null-total --contestant stub-solver --profile byo --out runs-example (toy task built with agentbench task from-commit)", "command": " run --task toy-null-total --contestant stub-solver --profile byo --out runs-example-checker-2d1e1f973089488dbee6e7d876367c53", "evidence": "toy-null-total / stub-solver: ok, passed=True, 0.0s, 1,060 tokens, $0.0000", "deferred": false, "expected": "passed=True at $0", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" }, { "kind": "example", "claim": "Repository offline test suite passes", "source": "CONTRIBUTING.md", "input": "python -m unittest discover -s tests -q", "command": " -m unittest discover -s tests -q", "evidence": "coldpass/workspace/tests/test_agentbench.py:409: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpnx3r7yxx/w/matched.csv' mode='r' encoding='utf-8'>\n rows = list(csv.reader(open(work / \"matched.csv\", newline=\"\", encoding=\"utf-8\")))\nResourceWarning: Enable tracemalloc to get the object allocation traceback\nerror: refusing to plan without --ledger-cap USD: the ledger cap is the most you are willing to spend, and the matrix stops when recorded spend gets within 10% of it\n----------------------------------------------------------------------\nRan 209 tests in 47.956s\n\nOK (skipped=1)", "deferred": false, "expected": "all tests run and pass (1 skipped)", "returncode": 0, "verdict": "yes", "outcome": "command completed successfully" } ], "does_what_it_says": "yes", "maintenance": { "archived": false, "commits_90d": 5, "last_commit_at": "2026-10-15T17:13:33Z", "last_release_at": null, "median_reply_hours": null, "merged_prs_90d": 1, "note": "actively maintained: last commit 6 days ago; 5 commits and 1 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases", "open_issues": 0, "open_prs": 0, "reply_share": null, "reply_threads": 0, "verdict": "maintained" }, "maintenance_verdict": "maintained", "maintenance_note": "actively maintained: last commit 6 days ago; 5 commits and 1 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases", "maintained": "active", "maintained_note": "last commit 6 days ago", "last_commit_at": "2026-10-15T17:23:57Z", "last_release_at": null, "open_issues": null, "answered_issues": null, "platforms_claimed": [ "linux", "mac", "windows" ], "topic": "agent-harness", "agent_interfaces": [ { "interface": "cli", "evidence": "independent checker", "observed": true, "operation_checked": true } ], "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.", "mcp_context": null, "tested_at": "2026-10-22T04:10:55.624589+00:00", "tested_commit": "ad01b757a0565c35fa590fe531964019363e70d5", "tested_with": { "agent": "claude-code", "agent_version": "2.1.300 (Claude Code)", "model": "claude-haiku-5-5", "model_route": "claude-haiku-5-5", "reasoning": "low", "os": "macos", "backend": "tart", "transport": "mcp-assigned-vm" }, "setup_identity": { "repo": "TayHCode/agent-bench", "source": "https://github.com/TayHCode/agent-bench", "commit": "ad01b757a0565c35fa590fe531964019363e70d5", "source_revision": null, "os": "mac", "architecture": null, "backend": "tart", "model": "claude-haiku-5-5", "model_version": "claude-haiku-5-5", "agent_execution": "native_agent", "agent_runtime": "claude-code", "agent_runtime_version": "2.1.300 (Claude Code)", "tool_transport": "mcp-assigned-vm", "runtime": null, "runtime_version": null, "runtime_revision": null, "sandbox_source": null, "guest_setup": null, "source_pin": { "commit": "ad01b757a0565c35fa590fe531964019363e70d5", "url": "https://github.com/TayHCode/agent-bench" }, "platform_details": { "os": "macos", "backend": "tart" }, "platform_conflicts": null, "reasoning": "low", "service_tier": null, "resources": { "cpus": null, "memory_mib": 4096, "disk_mib": 32768 }, "operation": "Install agent-bench (agentbench CLI) from the pinned checkout and run the free offline demo, which builds a toy repo and scores two stand-in agents; also run the documented from-commit rehearsal (toy task build, then stub-solver run reporting passed=True).", "operation_scope": "product operation", "hosting": null, "kind": "repo", "test_version": "readme-6f8cbfa597c6d59f7ece0e29599b48d61f780e82e853b6ab300c3a8265684642", "evidence_version": null }, "other_model": null, "other_run_id": null, "models_agree": null, "furthest_model": null, "transcript_url": "/transcripts/TayHCode--agent-bench-claude-haiku-5-5-1792641802162398530.txt", "report_url": "/reports/TayHCode--agent-bench-claude-haiku-5-5-1792641802162398530.json" }