{ "checker": { "claims": [ { "claim": "Search existing public GitHub projects for an idea and return a ranked shortlist.", "command": " --python 3.12 --from search-first tools/call {\"tool\": \"search_existing\", \"arguments\": {\"idea\": \"a Python library for rate limiting outgoing HTTP requests\", \"queries\": [\"python rate limiter\", \"http client rate limit\"], \"limit\": 2}}", "evidence": "alisaifee/flask-limiter\\\",\\n \\\"url\\\": \\\"https://github.com/alisaifee/flask-limiter\\\"\\n },\\n \\\"plausible_candidates\\\": 15,\\n \\\"adjacent_candidates\\\": 40,\\n \\\"recommendation\\\": \\\"use as is\\\",\\n \\\"reasoning\\\": \\\"Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.\\\",\\n \\\"provisional\\\": true,\\n \\\"search_complete\\\": true\\n },\\n \\\"warnings\\\": [],\\n \\\"candidates\\\": [\\n {\\n \\\"name\\\": \\\"alisaifee/flask-limiter\\\",\\n \\\"url\\\": \\\"https://github.com/alisaifee/flask-limiter\\\",\\n ", "expected": "Ranked candidates and coverage metadata", "input": "{\"tool\": \"search_existing\", \"arguments\": {\"idea\": \"a Python library for rate limiting outgoing HTTP requests\", \"queries\": [\"python rate limiter\", \"http client rate limit\"], \"limit\": 2}}", "kind": "claim", "outcome": "Real input produced the expected result: alisaifee/flask-limiter", "source": "The default response returns the top **8** candidates", "source_checked": true, "source_file": "README.md", "verdict": "yes" } ], "cycle_checks": [ { "claim": "Scheduled or looping main behavior", "command": "", "evidence": "Inspected README and CLI entry point: the product serves MCP requests on demand over stdio or Streamable HTTP and defines no scheduler, recurring task, webhook, or cycle trigger.", "expected": "identify the product trigger", "input": "source inspection", "kind": "cycle", "outcome": "Inspected README and CLI entry point: the product serves MCP requests on demand over stdio or Streamable HTTP and defines no scheduler, recurring task, webhook, or cycle trigger.", "source": "repository entry points", "verdict": "yes" } ], "details": { "argv": [ "", "--python", "3.12", "--from", "", "search-first" ], "kind": "mcp", "scope": "product operation", "serverInfo": { "name": "search-first", "version": "0.1.0" }, "tool_result": { "content": [ { "text": "{\n \"verdict\": {\n \"closest_match\": {\n \"name\": \"alisaifee/flask-limiter\",\n \"url\": \"https://github.com/alisaifee/flask-limiter\"\n },\n \"plausible_candidates\": 15,\n \"adjacent_candidates\": 40,\n \"recommendation\": \"use as is\",\n \"reasoning\": \"Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.\",\n \"provisional\": true,\n \"search_complete\": true\n },\n \"warnings\": [],\n \"candidates\": [\n {\n \"name\": \"alisaifee/flask-limiter\",\n \"url\": \"https://github.com/alisaifee/flask-limiter\",\n \"stars\": 1207,\n \"last_push\": \"2026-05-10T23:07:49Z\",\n \"license\": \"MIT\",\n \"relevance_score\": 0.937,\n \"idea_term_coverage\": 1.0,\n \"missing_key_terms\": [],\n \"missing_constraints\": [],\n \"why_it_matched\": \"Strong core match; Core terms: 2/2 (limit, rate); 1207 stars, push 163 days ago; returned by 1 query variant(s), best GitHub rank 1.\"\n },\n {\n \"name\": \"laurentS/slowapi\",\n \"url\": \"https://github.com/laurentS/slowapi\",\n \"stars\": 2070,\n \"last_push\": \"2026-10-21T19:50:53Z\",\n \"license\": \"MIT\",\n \"relevance_score\": 0.912,\n \"idea_term_coverage\": 1.0,\n \"missing_key_terms\": [],\n \"missing_constraints\": [],\n \"why_it_matched\": \"Strong core match; Core terms: 2/2 (limit, rate); 2070 stars, push 0 days ago; returned by 1 query variant(s), best GitHub rank 3.\"\n }\n ],\n \"unique_candidates_evaluated\": 60,\n \"errors\": [],\n \"message\": null,\n \"retry_after_seconds\": null,\n \"coverage\": {\n \"bounded\": true,\n \"truncated\": true,\n \"repositories_per_query\": 30,\n \"queries_completed\": 2,\n \"queries_failed\": 0,\n \"queries_skipped\": 1,\n \"query_budget\": 2,\n \"bounded_plan_complete\": true,\n \"results_skipped\": 58,\n \"unretrieved_results\": null,\n \"note\": \"Counts refer to deduplicated retrieved projects, not all of GitHub. Metadata-only lexical ranking; no candidate has been run or tested.\"\n },\n \"notes\": [\n \"Unauthenticated mode: at most 2 search variants per call; set GITHUB_TOKEN for complete searches. Shared-IP quota still applies.\"\n ]\n}", "type": "text" } ], "isError": false, "structuredContent": { "candidates": [ { "idea_term_coverage": 1.0, "last_push": "2026-05-10T23:07:49Z", "license": "MIT", "missing_constraints": [], "missing_key_terms": [], "name": "alisaifee/flask-limiter", "relevance_score": 0.937, "stars": 1207, "url": "https://github.com/alisaifee/flask-limiter", "why_it_matched": "Strong core match; Core terms: 2/2 (limit, rate); 1207 stars, push 163 days ago; returned by 1 query variant(s), best GitHub rank 1." }, { "idea_term_coverage": 1.0, "last_push": "2026-10-21T19:50:53Z", "license": "MIT", "missing_constraints": [], "missing_key_terms": [], "name": "laurentS/slowapi", "relevance_score": 0.912, "stars": 2070, "url": "https://github.com/laurentS/slowapi", "why_it_matched": "Strong core match; Core terms: 2/2 (limit, rate); 2070 stars, push 0 days ago; returned by 1 query variant(s), best GitHub rank 3." } ], "coverage": { "bounded": true, "bounded_plan_complete": true, "note": "Counts refer to deduplicated retrieved projects, not all of GitHub. Metadata-only lexical ranking; no candidate has been run or tested.", "queries_completed": 2, "queries_failed": 0, "queries_skipped": 1, "query_budget": 2, "repositories_per_query": 30, "results_skipped": 58, "truncated": true, "unretrieved_results": null }, "errors": [], "message": null, "notes": [ "Unauthenticated mode: at most 2 search variants per call; set GITHUB_TOKEN for complete searches. Shared-IP quota still applies." ], "retry_after_seconds": null, "unique_candidates_evaluated": 60, "verdict": { "adjacent_candidates": 40, "closest_match": { "name": "alisaifee/flask-limiter", "url": "https://github.com/alisaifee/flask-limiter" }, "plausible_candidates": 15, "provisional": true, "reasoning": "Strong metadata overlap and a recognized license. Inspect README and test the required behavior before deciding that it works unchanged.", "recommendation": "use as is", "search_complete": true }, "warnings": [] } }, "tool_result_checked": true, "tool_schema_bytes": 4129, "tools": [ "search_existing", "inspect_repo", "compare_candidates" ], "working": true }, "evidence_strength": "readme_checked", "example_checks": [ { "claim": "Install and run documented local CLI help", "command": " --python 3.12 --from search-first --help", "deferred": false, "evidence": " SEARCH_FIRST_AUTH_TOKEN (only behind a trusted\n gateway)\n --rank-weights RANK_WEIGHTS\n Ranking weights as name=value,... over the defaults\n (or SEARCH_FIRST_RANK_WEIGHTS); names: idea_match,\n query_match, github_rank, variant_agreement,\n freshness, stars\n --overall-weights OVERALL_WEIGHTS\n Compare blend weights as name=value,... (or\n SEARCH_FIRST_OVERALL_WEIGHTS); names: fit,\n maintenance, stars", "expected": "CLI usage and documented transport options", "input": "uvx --python 3.12 --from . search-first --help", "kind": "example", "outcome": "command completed successfully", "returncode": 0, "source": "README.md \u2014 Install and run", "verdict": "yes" } ], "status": "PASS" }, "deviations": [], "fault_owner": null, "main_blocker": null, "operation": "Installed the documented local uvx package entrypoint and called search_existing through an MCP stdio client.", "operation_scope": "Unauthenticated public GitHub repository search, limited to the first two query variants by product design.", "outcome": "working", "plain_failure": null, "steps": [ "Initialized Git metadata and checked out pinned commit 2b7c5fd54b96bdd2a736eb2ae25db35d2cb71d7d.", "Ran scripts/check_python.py with Python 3.14.8; it passed the Python 3.11+ requirement.", "Installed uv with the documented macOS installer.", "Ran uvx --python 3.12 --from . search-first --help.", "Called search_existing with a real idea and two variants; the result had 2 completed queries, 0 failed, and returned ranked candidates including alisaifee/flask-limiter.", "Replayed probe, README claim, example and cycle evidence using coldpass checker.", "README development checks and package build were not run; they are merge/development checks, not necessary for this operation.", "vault-search was unavailable in this VM (command not found)." ], "stuck_step": null, "suggested_fix": null }