{
  "run_id": "TayHCode--agent-bench-gpt-6-luna-1792682175058527795",
  "date": "2026-10-22T15:18:59.743475+00:00",
  "repo": "TayHCode/agent-bench",
  "kind": "repo",
  "source": "https://github.com/TayHCode/agent-bench",
  "commit": "ad01b757a0565c35fa590fe531964019363e70d5",
  "model": "gpt-6-luna",
  "model_version": "gpt-6-luna",
  "test_version": "readme-1002bc876f0002f622e14f9c5b99f9f66721fccc0356f4d53ed24f2f79a3ac5d",
  "pass": true,
  "outcome": "pass",
  "time_to_working_s": 139.22086574696004,
  "total_s": 164.6849241256714,
  "steps": [
    "pwd; ls -la; rg --files -g 'README*' -g 'AGENTS.md' -g 'pyproject.toml' -g 'package.json' -g 'Makefile' -g '.github/workflows/**' | head -100; vault-search 'agent-bench README MCP operation' || true",
    "cat README.md; cat pyproject.toml; python3 --version; python3 -m pip --version; find . -name AGENTS.md -o -name '*.py' | head -100",
    "id -u; git status --short; python3 -m venv .venv; .venv/bin/python -m pip --version; .venv/bin/python -m pip install .",
    "sudo -n apt-get update && sudo -n apt-get install -y python3.12-venv git; python3 -m venv .venv; .venv/bin/python -m pip install .",
    "mkdir -p .coldpass; .venv/bin/agentbench demo; .venv/bin/agentbench doctor; cat .gitignore; git init; git remote add origin https://github.com/TayHCode/agent-bench; git fetch --depth 1 origin ad01b757a0565c35fa590fe531964019363e70d; git reset --mixed FETCH_HEAD",
    "python3 - <<'PY'\nimport json, pathlib\np=pathlib.Path('.coldpass')\np.mkdir(exist_ok=True)\nprobe={\"kind\":\"cli\",\"argv\":[\"<guest-path>\",\"demo\"],\"cwd\":\"<guest-path>\",\"expected_text\":\"stub-solver  pass\"}\nclaims=[{\"kind\":\"claim\",\"claim\":\"The demo runs two free stand-in agents without a model or network.\",\"source\":\"They cost nothing and use no model, so they test the plumbing, not any agent.\",\"input\":\"documented agentbench demo\",\"expected\":\"stub-solver passes and stub-lazy fails\",\"probe_kind\":\"cli\",\"argv\":[\"<guest-path>\",\"demo\"],\"expected_text\":\"stub-solver  pass\",\"cwd\":\"<guest-path>\"}]\nexamples=[{\"kind\":\"example\",\"claim\":\"README quickstart demo\",\"source\":\"README.md#Quickstart: three commands, no login, no cost\",\"input\":\"demo\",\"expected\":\"stub-solver passes and stub-lazy fails\",\"argv\":[\"<guest-path>\",\"demo\"],\"cwd\":\"<guest-path>\",\"expected_text\":\"stub-solver  pass\"}]\ncycles={\"detected\":False,\"reason\":\"README describes one-shot CLI commands and no product scheduler, loop or webhook; demo is a single invocation.\",\"checks\":[]}\nfor name,data in [('probe.json',probe),('claims.json',claims),('examples.json',examples),('cycles.json',cycles)]: (p/name).write_text(json.dumps(data,indent=2)+'\\n')\nPY\nCOLDPASS_WORKSPACE=<guest-path> python3 -I <guest-path> --validate --replay",
    "python3 - <<'PY'\nimport json\np='.coldpass/report.json'\nd={\"outcome\":\"working\",\"fault_owner\":None,\"operation\":\"agentbench demo\",\"operation_scope\":\"Installed from the checkout in a Python 3.12.3 virtual environment; free stand-in pipeline without model login.\",\"steps\":[\"Created .venv after installing Ubuntu python3.12-venv and git.\",\"Ran the README install command: .venv/bin/python -m pip install .\",\"Ran .venv/bin/agentbench demo; stub-solver passed and stub-lazy failed as expected.\",\"Ran .venv/bin/agentbench doctor; core prerequisites are ready and real agent CLIs are optional for demo.\",\"Validated and replayed .coldpass/probe.json, claims, cycles and examples successfully.\"],\"deviations\":[\"Initial venv creation failed because ensurepip was unavailable; installed python3.12-venv with sudo and retried successfully.\",\"Requested pinned git fetch failed: fatal: couldn't find remote ref ad01b757a0565c35fa590fe531964019363e70d.\"],\"stuck_step\":None,\"main_blocker\":None,\"plain_failure\":None,\"suggested_fix\":None}\nopen(p,'w').write(json.dumps(d,indent=2)+'\\n')\nPY\nls -la .coldpass; git status --short"
  ],
  "stuck_step": null,
  "error_excerpt": null,
  "what_it_did": "agentbench demo",
  "suggested_fix": null,
  "tokens_in": 133778,
  "tokens_out": 1446,
  "cost_usd": null,
  "usage_complete": true,
  "checks": [
    {
      "check": "operation",
      "status": "PASS",
      "details": {
        "working": true,
        "argv": [
          "<guest-path>",
          "demo"
        ],
        "kind": "cli",
        "returncode": 0,
        "stdout": "1/3  Building a toy task: a small repo, a bug, and a hidden test that proves it is fixed ...\n2/3  Running two free stand-in agents on it (no model, no network, $0) ...\n3/3  Results:\n\nContestant   Result  Time  Tokens  $ equiv\n-----------  ------  ----  ------  -------\nstub-solver  pass    0.1s   1,060  $0.0000\nstub-lazy    FAIL    0.1s   1,060  $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start",
        "stderr": "",
        "needs_credentials": false,
        "needs_browser": false,
        "scope": "product operation"
      }
    }
  ],
  "safety": {
    "version": "z2x-1",
    "rules_version": "z2x-1",
    "summary": "No problems traced to this repository code in these checks.",
    "findings": [],
    "context": [],
    "limitations": [
      "A short run cannot establish that code is safe.",
      "Connections show IP addresses; proxies can hide the final hostname.",
      "Tracing covers dispatched commands and their children, not the whole OS; guest root can interfere with tracing."
    ],
    "static": {
      "dependency_note": "OSV scanned supported lockfiles; unpinned dependencies and unsupported formats are outside this check."
    },
    "evidence_url": "/safety/?run=TayHCode--agent-bench-gpt-6-luna-1792682175058527795",
    "evidence_file": "/data/safety/TayHCode--agent-bench-gpt-6-luna-1792682175058527795.json.gz"
  },
  "agent_execution": "native_agent",
  "agent_runtime": "codex",
  "agent_runtime_version": "codex-cli 0.163.0",
  "tool_transport": "mcp-assigned-vm",
  "platform": "linux",
  "reasoning": "low",
  "service_tier": "standard",
  "resources": {
    "cpus": 2,
    "memory_mib": 4096,
    "disk_mib": 32768
  },
  "native_tool_calls": 7,
  "native_bundle": null,
  "architecture": null,
  "backend": "hyperv",
  "platform_details": {
    "backend": "hyperv",
    "os": "linux"
  },
  "platform_conflicts": null,
  "sandbox_source": null,
  "source_pin": {
    "commit": "ad01b757a0565c35fa590fe531964019363e70d5",
    "url": "https://github.com/TayHCode/agent-bench"
  },
  "purpose": "initial",
  "profile": {
    "model": "gpt-6-luna",
    "reasoning": "low"
  },
  "fault": {
    "owner": null,
    "status": "resolved",
    "reason": null
  },
  "fix_id": null,
  "verifies_run_id": null,
  "hosting": null,
  "operation_scope": "product operation",
  "tool_calls": 7,
  "guest_setup": null,
  "runtime": null,
  "runtime_version": null,
  "runtime_revision": null,
  "source_revision": null,
  "evidence_version": null,
  "failure_origin": null,
  "evidence_strength": "readme_checked",
  "works": "yes",
  "install_s": null,
  "first_use_s": null,
  "mcp": "none",
  "needs": [],
  "needs_which": null,
  "agent_interface": [
    "cli"
  ],
  "license": "MIT",
  "repo_url": "https://github.com/TayHCode/agent-bench",
  "cost_to_try": "free",
  "hardware": null,
  "what_failed": null,
  "local_or_hosted": "local",
  "cycle_checks": [
    {
      "kind": "cycle",
      "claim": "Scheduled or looping main behavior",
      "source": "repository entry points",
      "input": "source inspection",
      "expected": "identify the product trigger",
      "verdict": "yes",
      "outcome": "README describes one-shot CLI commands and no product scheduler, loop or webhook; demo is a single invocation.",
      "evidence": "README describes one-shot CLI commands and no product scheduler, loop or webhook; demo is a single invocation.",
      "command": ""
    }
  ],
  "claims": [
    {
      "kind": "claim",
      "claim": "The demo runs two free stand-in agents without a model or network.",
      "source": "They cost nothing and use no model, so they test the plumbing, not any agent.",
      "input": "documented agentbench demo",
      "expected": "stub-solver passes and stub-lazy fails",
      "command": "<guest-path> demo",
      "verdict": "yes",
      "outcome": "Real input produced the expected result: stub-solver  pass",
      "evidence": "stub-solver  pass    0.1s   1,060  $0.0000\nstub-lazy    FAIL    0.1s   1,060  $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start\n",
      "returncode": 0,
      "source_checked": true,
      "source_file": "README.md"
    }
  ],
  "claims_note": null,
  "example_checks": [
    {
      "kind": "example",
      "claim": "README quickstart demo",
      "source": "README.md#Quickstart: three commands, no login, no cost",
      "input": "demo",
      "command": "<guest-path> demo",
      "evidence": "1/3  Building a toy task: a small repo, a bug, and a hidden test that proves it is fixed ...\n2/3  Running two free stand-in agents on it (no model, no network, $0) ...\n3/3  Results:\n\nContestant   Result  Time  Tokens  $ equiv\n-----------  ------  ----  ------  -------\nstub-solver  pass    0.1s   1,060  $0.0000\nstub-lazy    FAIL    0.1s   1,060  $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start",
      "deferred": false,
      "expected": "stub-solver passes and stub-lazy fails",
      "returncode": 0,
      "verdict": "yes",
      "outcome": "command completed successfully"
    }
  ],
  "does_what_it_says": "yes",
  "maintenance": {
    "archived": false,
    "commits_90d": 5,
    "last_commit_at": "2026-10-15T17:13:33Z",
    "last_release_at": null,
    "median_reply_hours": null,
    "merged_prs_90d": 1,
    "note": "actively maintained: last commit 6 days ago; 5 commits and 1 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases",
    "open_issues": 0,
    "open_prs": 0,
    "reply_share": null,
    "reply_threads": 0,
    "verdict": "maintained"
  },
  "maintenance_verdict": "maintained",
  "maintenance_note": "actively maintained: last commit 6 days ago; 5 commits and 1 merged PRs in 90 days; 0 open issues, 0 open PRs; no releases",
  "maintained": "active",
  "maintained_note": "last commit 6 days ago",
  "last_commit_at": "2026-10-15T17:23:57Z",
  "last_release_at": null,
  "open_issues": null,
  "answered_issues": null,
  "platforms_claimed": [
    "linux",
    "mac",
    "windows"
  ],
  "topic": "agent-harness",
  "agent_interfaces": [
    {
      "interface": "cli",
      "evidence": "independent checker",
      "observed": true,
      "operation_checked": true
    }
  ],
  "agent_interfaces_note": "README metadata is a discovery hint, not pinned documentation proof. Observed interfaces were exercised in this run. An empty list means unknown.",
  "mcp_context": null,
  "tested_at": "2026-10-22T15:18:59.743475+00:00",
  "tested_commit": "ad01b757a0565c35fa590fe531964019363e70d5",
  "tested_with": {
    "agent": "codex",
    "agent_version": "codex-cli 0.163.0",
    "model": "gpt-6-luna",
    "model_route": "gpt-6-luna",
    "reasoning": "low",
    "os": "linux",
    "backend": "hyperv",
    "transport": "mcp-assigned-vm"
  },
  "setup_identity": {
    "repo": "TayHCode/agent-bench",
    "source": "https://github.com/TayHCode/agent-bench",
    "commit": "ad01b757a0565c35fa590fe531964019363e70d5",
    "source_revision": null,
    "os": "linux",
    "architecture": null,
    "backend": "hyperv",
    "model": "gpt-6-luna",
    "model_version": "gpt-6-luna",
    "agent_execution": "native_agent",
    "agent_runtime": "codex",
    "agent_runtime_version": "codex-cli 0.163.0",
    "tool_transport": "mcp-assigned-vm",
    "runtime": null,
    "runtime_version": null,
    "runtime_revision": null,
    "sandbox_source": null,
    "guest_setup": null,
    "source_pin": {
      "commit": "ad01b757a0565c35fa590fe531964019363e70d5",
      "url": "https://github.com/TayHCode/agent-bench"
    },
    "platform_details": {
      "os": "linux",
      "backend": "hyperv"
    },
    "platform_conflicts": null,
    "reasoning": "low",
    "service_tier": "standard",
    "resources": {
      "cpus": 2,
      "memory_mib": 4096,
      "disk_mib": 32768
    },
    "operation": "agentbench demo",
    "operation_scope": "product operation",
    "hosting": null,
    "kind": "repo",
    "test_version": "readme-1002bc876f0002f622e14f9c5b99f9f66721fccc0356f4d53ed24f2f79a3ac5d",
    "evidence_version": null
  },
  "other_model": null,
  "other_run_id": null,
  "models_agree": null,
  "furthest_model": null,
  "transcript_url": "/transcripts/TayHCode--agent-bench-gpt-6-luna-1792682175058527795.txt",
  "report_url": "/reports/TayHCode--agent-bench-gpt-6-luna-1792682175058527795.json"
}