{
  "run_id": "TayHCode--agent-bench-claude-haiku-5-5-1792640189152867590",
  "checker": {
    "claims": [
      {
        "claim": "demo builds a toy repo and runs two free stand-in agents, one passing and one failing",
        "command": "<guest-path> demo",
        "evidence": "stub-solver  pass    0.1s   1,060  $0.0000\nstub-lazy    FAIL    0.1s   1,060  $0.0000\n\nFull report: results.html\nstub-solver copies the known fix and stub-lazy changes nothing: they test the pipeline, not any model. Run `agentbench start` to compare real agents on your own code.\nNext: agentbench doctor, then agentbench start\n",
        "expected": "stub-solver passes and stub-lazy fails",
        "input": "agentbench demo",
        "kind": "claim",
        "outcome": "Real input produced the expected result: stub-solver  pass",
        "returncode": 0,
        "source": "`demo` builds a toy repo, hides the test that proves a bug fixed, and runs two built-in stand-in agents: one copies the known fix, one does nothing.",
        "source_checked": true,
        "source_file": "README.md",
        "verdict": "yes"
      }
    ],
    "cycle_checks": [
      {
        "claim": "Scheduled or looping main behavior",
        "command": "",
        "evidence": "Inspected src/agentbench for cron, schedule, loop, webhook, --once and --now entry points: none found. The CLI (doctor, demo, plan, run, task from-commit, report) is one-shot per invocation; agentbench start is interactive and not a recurring cycle. No product cycle to trigger.",
        "expected": "identify the product trigger",
        "input": "source inspection",
        "kind": "cycle",
        "outcome": "Inspected src/agentbench for cron, schedule, loop, webhook, --once and --now entry points: none found. The CLI (doctor, demo, plan, run, task from-commit, report) is one-shot per invocation; agentbench start is interactive and not a recurring cycle. No product cycle to trigger.",
        "source": "repository entry points",
        "verdict": "yes"
      }
    ],
    "details": {
      "argv": [
        "<guest-path>",
        "run",
        "--task",
        "toy-null-total",
        "--contestant",
        "stub-solver",
        "--profile",
        "byo",
        "--out",
        "/tmp/ab-rehearsal/probe-runs-checker-50f530ca59714697a5b5ad89be7f8b30"
      ],
      "kind": "cli",
      "needs_browser": false,
      "needs_credentials": false,
      "returncode": 0,
      "scope": "product operation",
      "stderr": "",
      "stdout": "toy-null-total / stub-solver: ok, passed=True, 0.1s, 1,060 tokens, $0.0000",
      "working": true
    },
    "evidence_strength": "readme_checked",
    "example_checks": [
      {
        "claim": "Run the documented test suite",
        "command": "<guest-path> -m unittest discover -s tests -q",
        "deferred": false,
        "evidence": "oldpass/workspace/tests/test_agentbench.py:409: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp6vt842o7/w/matched.csv' mode='r' encoding='utf-8'>\n  rows = list(csv.reader(open(work / \"matched.csv\", newline=\"\", encoding=\"utf-8\")))\nResourceWarning: Enable tracemalloc to get the object allocation traceback\nerror: refusing to plan without --ledger-cap USD: the ledger cap is the most you are willing to spend, and the matrix stops when recorded spend gets within 10% of it\n----------------------------------------------------------------------\nRan 209 tests in 139.590s\n\nOK (skipped=1)",
        "expected": "tests pass",
        "input": "python -m unittest discover -s tests -q",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "README.md ## Development",
        "verdict": "yes"
      },
      {
        "claim": "Run the free from-commit rehearsal stand-in on the built toy task",
        "command": "<guest-path> run --task toy-null-total --contestant stub-solver --profile byo --out /tmp/ab-rehearsal/example-runs-checker-8d600b8b49c54bed89df4ab8bbf6771a",
        "deferred": false,
        "evidence": "toy-null-total / stub-solver: ok, passed=True, 0.1s, 1,060 tokens, $0.0000",
        "expected": "passed=True at zero dollar cost",
        "input": "agentbench run --task toy-null-total --contestant stub-solver --profile byo --out /tmp/ab-rehearsal/example-runs",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "README.md ### Free from-commit rehearsal",
        "verdict": "yes"
      }
    ],
    "status": "PASS"
  },
  "diagnosis": null
}