{ "run_id": "TayHCode--sysone-cpu-gpt-6-luna-1792683877827961506", "checker": { "claims": [ { "claim": "Benchmark runs the synthetic workload and records inference results.", "command": ".tools/uv run --no-project --python .venv python benchmarks/bench_laya_cpu.py --model .cache/model/typed-decisions --threads 4 --warmup 1 --repeats 1 --counts 1,3 --output .cache/claim-run.json-checker-bcf933f28d0848b8ba6a144eb5554a2c", "evidence": "\"device\": \"cpu\",\n \"model_path\": \"\",\n \"model_config\": {\n \"encoder\": \"answerdotai/ModernBERT-large\",\n \"head_layers\": 2,\n \"max_len\": 1024,\n \"head_max_len\": 256,\n \"max_prefixes\": 6,\n \"act_costs\": {\n \"escalate\": 0.5\n },\n \"cost_wrong_act\": 3.0,\n \"amp_dtype\": \"bf16\",\n \"model_name\": \"laya-typed-decisions\",\n \"training\": {\n \"updates\": 7313,\n \"epochs_completed\": 1,\n \"hours\": 1.96,\n \"world_size\": 1,\n \"fine_tuned_from_checkpoint\": true\n },\n \"gradient_checkpointi", "expected": "CPU metadata and equivalent batched answers are present in the JSON result", "input": "Synthetic README quickstart ticket questions at counts 1 and 3", "kind": "claim", "outcome": "Real input produced the expected result: \"device\": \"cpu\"", "returncode": 0, "source": "Expect `meta.device` to be `\"cpu\"`", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Inspected README entry points, scripts, and benchmark harnesses. The product downloads a checkpoint and runs an explicitly invoked benchmark; no product scheduler, recurring loop, or webhook cycle exists.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Inspected README entry points, scripts, and benchmark harnesses. The product downloads a checkpoint and runs an explicitly invoked benchmark; no product scheduler, recurring loop, or webhook cycle exists.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ ".tools/uv", "run", "--no-project", "--python", ".venv", "python", "benchmarks/bench_laya_cpu.py", "--model", ".cache/model/typed-decisions", "--threads", "4", "--warmup", "1", "--repeats", "1", "--counts", "1,3", "--output", ".cache/replay-run.json-checker-816b4bc112054695ac5a0bff5a114001" ], "expected_file": ".cache/replay-run.json-checker-816b4bc112054695ac5a0bff5a114001", "file_checked": true, "file_excerpt": "\"device\": \"cpu\",\n \"model_path\": \"\",\n \"model_config\": {\n \"encoder\": \"answerdotai/ModernBERT-large\",\n \"head_layers\": 2,\n \"max_len\": 1024,\n \"head_max_len\": 256,\n \"max_prefixes\": 6,\n \"act_costs\": {\n \"escalate\": 0.5\n },\n \"cost_wrong_act\": 3.0,\n \"amp_dtype\": \"bf16\",\n \"model_name\": \"laya-typed-decisions\",\n \"training\": {\n \"updates\": 7313,\n \"epochs_completed\": 1,\n \"hours\": 1.96,\n \"world_size\": 1,\n \"fine_tuned_from_checkpoint\": true\n },\n \"gradient_checkpointing\": true,\n \"max_tokens_per_batch\": 4096,\n \"fine_tuned\": true\n },\n \"calibration_source\": \"pinned upstream rl_agent_config.json\",\n \"cold_load_ms\": 24687.351,\n \"rss_before_mib\": 222.05,\n \"rss_after_load_mib\": 2036.53\n },\n \"workloads\": {\n \"1\": {\n \"p50_ms\": 677.864,\n \"p95_ms\": 677.864,\n \"min_ms\": 677.864,\n \"max_ms\": 677.864,\n \"mean_ms\": 677.864,\n \"questions\": 1,\n \"questions_per_second_p50\": 1.475,\n \"rss_peak_mib\": 2052.75,\n \"cpu_percent_mean_sampled\": 109.0,\n \"output_sha256_first\": \"615045d6d0cc8bcc75c059903638b8198377b74f90ab179d89f8f006b21cecdf\",\n \"output_sha256_last\": \"615045d6d0cc8bcc75c059903638b8198377b74f90ab179d89f8f006b21cecdf\",\n \"deterministic_repeated_output\": true,\n \"sample_ms\": [\n 677.864\n ]\n },\n \"3\": {\n \"p50_ms\": 1584.688,\n \"p95_ms\": 1584.688,\n \"min_ms\": 1584.688,\n \"max_ms\": 1584.688,\n \"mean_ms\": 1584.688,\n \"questions\": 3,\n \"questions_per_second_p50\": 1.893,\n \"rss_peak_mib\": 2056.77,\n \"cpu_percent_mean_sampled\": 128.0,\n \"output_sha256_first\": \"62e8e2041019b6894716c30d3050e0beec1e70ea2b9c0e0405e80fb01a4e3aa4\",\n \"output_sha256_last\": \"62e8e2041019b6894716c30d3050e0beec1e70ea2b9c0e0405e80fb01a4e3aa4\",\n \"deterministic_repeated_output\": true,\n \"sample_ms\": [\n 1584.688\n ]\n }\n },\n \"batch_equivalence\": {\n \"same_public_answers\": true,\n \"batched_digest\": \"9b003fbe3501b995d7d9255fe97b2044dec65b44825b40302b949c40eb2ed62d\",\n \"singles_digest\": \"9b003fbe3501b995d7d9255fe97b2044dec65b44825b40302b949c40eb2ed62d\",\n \"batched_answers\": {\n \"department\": {\n \"type\": \"choice\",\n \"choice\": \"billing\",\n \"probabilities\": {\n \"billing\": 0.6989,\n \"technical\": 0.0814,\n \"sales\": 0.1338,\n \"other\": 0.086\n },\n \"confidence\": 0.3258,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"score\": 1.6111,\n \"legend\": {\n \"0\": \"not urgent\",\n \"1\": \"soon\",\n \"2\": \"critical or blocking\"\n },\n \"probabilities\": {\n \"0\": 0.0408,\n \"1\": 0.3073,\n \"2\": 0.6519\n },\n \"confidence\": 0.2972,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"refund\": {\n \"type\": \"noul\",\n \"noul\": 0.68,\n \"confidence\": 0.68,\n \"action\": {\n \"act_probability\": 1.0\n }\n }\n },\n \"single_answers\": {\n \"department\": {\n \"type\": \"choice\",\n \"choice\": \"billing\",\n \"probabilities\": {\n \"billing\": 0.6989,\n \"technical\": 0.0814,\n \"sales\": 0.1338,\n \"other\": 0.086\n },\n \"confidence\": 0.3258,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"score\": 1.6111,\n \"legend\": {\n \"0\": \"not urgent\",\n \"1\": \"soon\",\n \"2\": \"critical or blocking\"\n },\n \"probabilities\": {\n \"0\": 0.0408,\n \"1\": 0.3073,\n \"2\": 0.6519\n },\n \"confidence\": 0.2972,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"refund\": {\n \"type\": \"noul\",\n \"noul\": 0.68,\n \"confidence\": 0.68,\n \"action\": {\n \"act_probability\": 1.0\n }\n }\n }\n },\n \"fixture\": {\n \"state\": {\n \"ticket\": {\n \"subject\": \"Duplicate charge\",\n \"body\": \"The customer was billed twice for invoice 4411 and asks for a refund today. They say the duplicate charge is blocking month-end close.\"\n }\n },\n \"questions\": {\n \"department\": {\n \"type\": \"choice\",\n \"instructions\": \"Which team should handle this ticket?\",\n \"criteria\": {\n \"billing\": \"invoices, payments, refunds\",\n \"technical\": \"bugs, outages, integrations\",\n \"sales\": \"pricing, demos, new purchases\",\n \"other\": \"everything else\"\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"instructions\": \"How urgent is this ticket?\",\n \"criteria\": [\n \"not urgent\",\n \"soon\",\n \"critical or blocking\"\n ]\n },\n \"refund\": {\n \"type\": \"noul\",\n \"instructions\": \"Does the customer ask for money back?\"\n }\n },\n \"answers\": {\n \"department\": {\n \"type\": \"choice\",\n \"choice\": \"billing\",\n \"probabilities\": {\n \"billing\": 0.6989,\n \"technical\": 0.0814,\n \"sales\": 0.1338,\n \"other\": 0.086\n },\n \"confidence\": 0.3258,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"urgency\": {\n \"type\": \"score\",\n \"score\": 1.6111,\n \"legend\": {\n \"0\": \"not urgent\",\n \"1\": \"soon\",\n \"2\": \"critical or blocking\"\n },\n \"probabilities\": {\n \"0\": 0.0408,\n \"1\": 0.3073,\n \"2\": 0.6519\n },\n \"confidence\": 0.2972,\n \"action\": {\n \"act_probability\": 1.0\n }\n },\n \"refund\": {\n \"type\": \"noul\",\n \"noul\": 0.68,\n \"confidence\": 0.68,\n \"action\": {\n \"act_probability\": 1.0\n }\n }\n }\n }\n}\n", "kind": "cli", "needs_browser": false, "needs_credentials": false, "returncode": 0, "scope": "product operation", "stderr": "", "stdout": "timing 1 questions ...\n p50=677.9 ms p95=677.9 ms 1.48 q/s RSS=2052.8 MiB CPU=109.0%\ntiming 3 questions ...\n p50=1584.7 ms p95=1584.7 ms 1.89 q/s RSS=2056.8 MiB CPU=128.0%\nwrote .cache/replay-run.json-checker-816b4bc112054695ac5a0bff5a114001", "working": true }, "evidence_strength": "readme_checked", "example_checks": [ { "claim": "Run README PyTorch FP32 quickstart benchmark", "command": ".tools/uv run --no-project --python .venv python benchmarks/bench_laya_cpu.py --model .cache/model/typed-decisions --threads 4 --warmup 1 --repeats 1 --counts 1,3 --output .cache/example-run.json", "deferred": false, "evidence": "timing 1 questions ...\n p50=685.3 ms p95=685.3 ms 1.46 q/s RSS=2051.2 MiB CPU=107.8%\ntiming 3 questions ...\n p50=1595.0 ms p95=1595.0 ms 1.88 q/s RSS=2055.2 MiB CPU=126.6%\nwrote .cache/example-run.json", "expected": "CPU inference results are written to JSON with batch equivalence", "input": "--threads 4 --warmup 1 --repeats 1 --counts 1,3", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md, Quickstart: PyTorch FP32", "verdict": "yes" }, { "claim": "Run local standard library checks documented in README", "command": "python3 -m unittest discover -s tests -q", "deferred": false, "evidence": "----------------------------------------------------------------------\nRan 19 tests in 0.020s\n\nOK", "expected": "documented checks complete", "input": "python3 -m unittest discover -s tests -q", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md, Local checks", "verdict": "yes" }, { "claim": "Run documented syntax check", "command": "python3 scripts/check_syntax.py", "deferred": false, "evidence": "Python syntax checked.", "expected": "syntax check completes", "input": "python3 scripts/check_syntax.py", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md, Local checks", "verdict": "yes" } ], "status": "PASS" }, "diagnosis": null }