{
  "checker": {
    "claims": [
      {
        "claim": "README claims",
        "command": "",
        "evidence": "expected_file must match an argv output destination",
        "expected": "",
        "input": "",
        "kind": "claim",
        "outcome": "Claim file could not be read: expected_file must match an argv output destination",
        "source": "",
        "verdict": "blocked"
      }
    ],
    "cycle_checks": [
      {
        "claim": "Scheduled or looping main behavior",
        "command": "",
        "evidence": "Inspected README, scripts, and benchmark entry point: this is a manual inference/benchmark kit with no product scheduler or recurring cycle. Run the documented benchmark command for each measurement.",
        "expected": "identify the product trigger",
        "input": "source inspection",
        "kind": "cycle",
        "outcome": "Inspected README, scripts, and benchmark entry point: this is a manual inference/benchmark kit with no product scheduler or recurring cycle. Run the documented benchmark command for each measurement.",
        "source": "repository entry points",
        "verdict": "yes"
      }
    ],
    "details": {
      "argv": [
        ".tools/uv",
        "run",
        "--no-project",
        "--python",
        ".venv",
        "python",
        "benchmarks/bench_laya_cpu.py",
        "--model",
        ".cache/model/typed-decisions",
        "--threads",
        "4",
        "--warmup",
        "1",
        "--repeats",
        "1",
        "--counts",
        "1,3",
        "--output",
        ".coldpass/replayed-benchmark.json-checker-73bc33ade4144276bc340c195fd662fa"
      ],
      "expected_file": ".coldpass/replayed-benchmark.json-checker-73bc33ade4144276bc340c195fd662fa",
      "file_checked": true,
      "file_excerpt": "\"device\": \"cpu\",\n    \"model_path\": \"<local typed-decisions checkpoint>\",\n    \"model_config\": {\n      \"encoder\": \"answerdotai/ModernBERT-large\",\n      \"head_layers\": 2,\n      \"max_len\": 1024,\n      \"head_max_len\": 256,\n      \"max_prefixes\": 6,\n      \"act_costs\": {\n        \"escalate\": 0.5\n      },\n      \"cost_wrong_act\": 3.0,\n      \"amp_dtype\": \"bf16\",\n      \"model_name\": \"laya-typed-decisions\",\n      \"training\": {\n        \"updates\": 7313,\n        \"epochs_completed\": 1,\n        \"hours\": 1.96,\n        \"world_size\": 1,\n        \"fine_tuned_from_checkpoint\": true\n      },\n      \"gradient_checkpointing\": true,\n      \"max_tokens_per_batch\": 4096,\n      \"fine_tuned\": true\n    },\n    \"calibration_source\": \"pinned upstream rl_agent_config.json\",\n    \"cold_load_ms\": 44833.099,\n    \"rss_before_mib\": 189.11,\n    \"rss_after_load_mib\": 2286.02\n  },\n  \"workloads\": {\n    \"1\": {\n      \"p50_ms\": 358.415,\n      \"p95_ms\": 358.415,\n      \"min_ms\": 358.415,\n      \"max_ms\": 358.415,\n      \"mean_ms\": 358.415,\n      \"questions\": 1,\n      \"questions_per_second_p50\": 2.79,\n      \"rss_peak_mib\": 2291.5,\n      \"cpu_percent_mean_sampled\": 334.4,\n      \"output_sha256_first\": \"615045d6d0cc8bcc75c059903638b8198377b74f90ab179d89f8f006b21cecdf\",\n      \"output_sha256_last\": \"615045d6d0cc8bcc75c059903638b8198377b74f90ab179d89f8f006b21cecdf\",\n      \"deterministic_repeated_output\": true,\n      \"sample_ms\": [\n        358.415\n      ]\n    },\n    \"3\": {\n      \"p50_ms\": 1138.692,\n      \"p95_ms\": 1138.692,\n      \"min_ms\": 1138.692,\n      \"max_ms\": 1138.692,\n      \"mean_ms\": 1138.692,\n      \"questions\": 3,\n      \"questions_per_second_p50\": 2.635,\n      \"rss_peak_mib\": 2292.7,\n      \"cpu_percent_mean_sampled\": 346.7,\n      \"output_sha256_first\": \"62e8e2041019b6894716c30d3050e0beec1e70ea2b9c0e0405e80fb01a4e3aa4\",\n      \"output_sha256_last\": \"62e8e2041019b6894716c30d3050e0beec1e70ea2b9c0e0405e80fb01a4e3aa4\",\n      \"deterministic_repeated_output\": true,\n      \"sample_ms\": [\n        1138.692\n      ]\n    }\n  },\n  \"batch_equivalence\": {\n    \"same_public_answers\": true,\n    \"batched_digest\": \"9b003fbe3501b995d7d9255fe97b2044dec65b44825b40302b949c40eb2ed62d\",\n    \"singles_digest\": \"9b003fbe3501b995d7d9255fe97b2044dec65b44825b40302b949c40eb2ed62d\",\n    \"batched_answers\": {\n      \"department\": {\n        \"type\": \"choice\",\n        \"choice\": \"billing\",\n        \"probabilities\": {\n          \"billing\": 0.6989,\n          \"technical\": 0.0814,\n          \"sales\": 0.1338,\n          \"other\": 0.086\n        },\n        \"confidence\": 0.3258,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"urgency\": {\n        \"type\": \"score\",\n        \"score\": 1.6111,\n        \"legend\": {\n          \"0\": \"not urgent\",\n          \"1\": \"soon\",\n          \"2\": \"critical or blocking\"\n        },\n        \"probabilities\": {\n          \"0\": 0.0408,\n          \"1\": 0.3073,\n          \"2\": 0.6519\n        },\n        \"confidence\": 0.2972,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"refund\": {\n        \"type\": \"noul\",\n        \"noul\": 0.68,\n        \"confidence\": 0.68,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      }\n    },\n    \"single_answers\": {\n      \"department\": {\n        \"type\": \"choice\",\n        \"choice\": \"billing\",\n        \"probabilities\": {\n          \"billing\": 0.6989,\n          \"technical\": 0.0814,\n          \"sales\": 0.1338,\n          \"other\": 0.086\n        },\n        \"confidence\": 0.3258,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"urgency\": {\n        \"type\": \"score\",\n        \"score\": 1.6111,\n        \"legend\": {\n          \"0\": \"not urgent\",\n          \"1\": \"soon\",\n          \"2\": \"critical or blocking\"\n        },\n        \"probabilities\": {\n          \"0\": 0.0408,\n          \"1\": 0.3073,\n          \"2\": 0.6519\n        },\n        \"confidence\": 0.2972,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"refund\": {\n        \"type\": \"noul\",\n        \"noul\": 0.68,\n        \"confidence\": 0.68,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      }\n    }\n  },\n  \"fixture\": {\n    \"state\": {\n      \"ticket\": {\n        \"subject\": \"Duplicate charge\",\n        \"body\": \"The customer was billed twice for invoice 4411 and asks for a refund today. They say the duplicate charge is blocking month-end close.\"\n      }\n    },\n    \"questions\": {\n      \"department\": {\n        \"type\": \"choice\",\n        \"instructions\": \"Which team should handle this ticket?\",\n        \"criteria\": {\n          \"billing\": \"invoices, payments, refunds\",\n          \"technical\": \"bugs, outages, integrations\",\n          \"sales\": \"pricing, demos, new purchases\",\n          \"other\": \"everything else\"\n        }\n      },\n      \"urgency\": {\n        \"type\": \"score\",\n        \"instructions\": \"How urgent is this ticket?\",\n        \"criteria\": [\n          \"not urgent\",\n          \"soon\",\n          \"critical or blocking\"\n        ]\n      },\n      \"refund\": {\n        \"type\": \"noul\",\n        \"instructions\": \"Does the customer ask for money back?\"\n      }\n    },\n    \"answers\": {\n      \"department\": {\n        \"type\": \"choice\",\n        \"choice\": \"billing\",\n        \"probabilities\": {\n          \"billing\": 0.6989,\n          \"technical\": 0.0814,\n          \"sales\": 0.1338,\n          \"other\": 0.086\n        },\n        \"confidence\": 0.3258,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"urgency\": {\n        \"type\": \"score\",\n        \"score\": 1.6111,\n        \"legend\": {\n          \"0\": \"not urgent\",\n          \"1\": \"soon\",\n          \"2\": \"critical or blocking\"\n        },\n        \"probabilities\": {\n          \"0\": 0.0408,\n          \"1\": 0.3073,\n          \"2\": 0.6519\n        },\n        \"confidence\": 0.2972,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      },\n      \"refund\": {\n        \"type\": \"noul\",\n        \"noul\": 0.68,\n        \"confidence\": 0.68,\n        \"action\": {\n          \"act_probability\": 1.0\n        }\n      }\n    }\n  }\n}\n",
      "kind": "cli",
      "needs_browser": false,
      "needs_credentials": false,
      "returncode": 0,
      "scope": "product operation",
      "stderr": "",
      "stdout": "timing 1 questions ...\n  p50=358.4 ms p95=358.4 ms 2.79 q/s RSS=2291.5 MiB CPU=334.4%\ntiming 3 questions ...\n  p50=1138.7 ms p95=1138.7 ms 2.63 q/s RSS=2292.7 MiB CPU=346.7%\nwrote .coldpass/replayed-benchmark.json-checker-73bc33ade4144276bc340c195fd662fa",
      "working": false
    },
    "evidence_strength": "partly_checked",
    "example_checks": [
      {
        "claim": "Run README PyTorch FP32 quickstart benchmark",
        "command": ".tools/uv run --no-project --python .venv python benchmarks/bench_laya_cpu.py --model .cache/model/typed-decisions --threads 4 --warmup 1 --repeats 1 --counts 1,3 --output .coldpass/example-benchmark.json",
        "deferred": false,
        "evidence": "timing 1 questions ...\n  p50=682.0 ms p95=682.0 ms 1.47 q/s RSS=2284.6 MiB CPU=299.4%\ntiming 3 questions ...\n  p50=1156.6 ms p95=1156.6 ms 2.59 q/s RSS=2286.0 MiB CPU=315.5%\nwrote .coldpass/example-benchmark.json",
        "expected": "CPU benchmark result file",
        "input": "--threads 4 --warmup 1 --repeats 1 --counts 1,3",
        "kind": "example",
        "outcome": "command completed successfully",
        "returncode": 0,
        "source": "README.md / Quickstart: PyTorch FP32",
        "verdict": "yes"
      }
    ],
    "status": "PASS"
  },
  "deviations": [
    "The preinstalled Python was 3.14; uv provisioned required Python 3.12. The benchmark invocation uses the README's documented options with a fresh .coldpass output path for replay."
  ],
  "fault_owner": null,
  "main_blocker": "The saved checks do not fully confirm documented product behavior.",
  "operation": "Run the README PyTorch FP32 CPU inference benchmark",
  "operation_scope": "macOS arm64, Python 3.12, pinned model and documented dependencies",
  "outcome": "unverified",
  "plain_failure": null,
  "steps": [
    "Installed uv using the README setup installer",
    "Created Python 3.12 venv and installed pinned wheels",
    "Downloaded and verified typed-decisions checkpoint",
    "Ran documented benchmark for synthetic workloads of 1 and 3 questions"
  ],
  "stuck_step": ".tools/uv run --no-project --python .venv python benchmarks/bench_laya_cpu.py --model .cache/model/typed-decisions --threads 4 --warmup 1 --repeats 1 --counts 1,3 --output .coldpass/replayed-benchmark.json-checker-73bc33ade4144276bc340c195fd662fa",
  "suggested_fix": "Keep the partial evidence and check the documented behavior with real input and expected output."
}