{"id":"ccb8cf9c-594e-45b5-a6cd-3b6db7d2a6d9","entityType":"agent","slug":"clawhub-zning1994-brainharness-autoresearch","name":"BrainHarness Autoresearch","canonicalUrl":"https://www.xpersona.co/agent/clawhub-zning1994-brainharness-autoresearch","canonicalPath":"/agent/clawhub-zning1994-brainharness-autoresearch","generatedAt":"2026-10-13T00:05:43.279Z","source":"CLAWHUB","claimStatus":"UNCLAIMED","verificationTier":"NONE","summary":{"evidence":{"source":"editorial-content","verified":true,"confidence":"high","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":null},"description":"Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy autoresearch pattern. Skill: BrainHarness Autoresearch Owner: zning1994 Summary: Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy","descriptionLabel":"Technical summary","evidenceSummary":"Capability contract not published. No trust telemetry is available yet. 956 downloads reported by the source. Last updated 10/12/2026.","installCommand":"clawhub skill install s173qj6menrea848d3v13ma9qh83gb7k:brainharness-autoresearch","sourceUrl":"https://clawhub.ai/zning1994/brainharness-autoresearch","homepage":"https://clawhub.ai/zning1994/skills/brainharness-autoresearch","primaryLinks":[{"label":"View on ClawHub","url":"https://clawhub.ai/zning1994/brainharness-autoresearch","kind":"source"},{"label":"Homepage","url":"https://clawhub.ai/zning1994/skills/brainharness-autoresearch","kind":"homepage"}],"safetyScore":84,"overallRank":62,"popularityScore":60,"trustScore":null,"claimedByName":null,"isOwner":false,"seoDescription":"Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eva"},"coverage":{"evidence":{"source":"public-profile","verified":false,"confidence":"medium","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":null},"protocols":[{"protocol":"OPENCLEW","label":"OpenClaw","status":"self-declared","notes":"Declared in the public agent profile."}],"capabilities":[],"verifiedCount":0,"selfDeclaredCount":1,"capabilityMatrix":{"rows":[{"key":"OPENCLEW","type":"protocol","support":"unknown","confidenceSource":"profile","notes":"Listed on profile"}],"flattenedTokens":"protocol:OPENCLEW|unknown|profile"}},"adoption":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":null},"stars":null,"forks":null,"downloads":956,"likes":null,"task":null,"library":null,"packageName":null,"latestVersion":"0.3.0","tractionLabel":"956 downloads"},"release":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-12T02:13:06.054Z","emptyReason":null},"lastUpdatedAt":"2026-10-12T02:13:06.135Z","lastCrawledAt":"2026-10-12T02:13:06.054Z","lastIndexedAt":null,"nextCrawlAt":"2026-10-13T02:13:06.054Z","lastVerifiedAt":null,"highlights":[{"version":"0.3.0","createdAt":"2026-10-03T20:28:18.885Z","changelog":"统一 BrainHarness 名称、安装入口和源仓库链接，优化器行为保持不变。","fileCount":11,"zipByteSize":34810},{"version":"0.2.5","createdAt":"2026-04-23T02:31:47.121Z","changelog":"Renamed skill to brainforge-autoresearch. Old slug openclaw-autoresearch will be merged as redirect.","fileCount":10,"zipByteSize":34253}]},"execution":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No published capability contract is available yet."},"installCommand":"clawhub skill install s173qj6menrea848d3v13ma9qh83gb7k:brainharness-autoresearch","setupComplexity":"low","setupSteps":["Setup complexity is LOW. This package is likely designed for quick installation with minimal external side-effects.","Final validation: Expose the agent to a mock request payload inside a sandbox and trace the network egress before allowing access to real customer data."],"contract":{"contractStatus":"missing","authModes":[],"requires":[],"forbidden":[],"supportsMcp":false,"supportsA2a":false,"supportsStreaming":false,"inputSchemaRef":null,"outputSchemaRef":null,"dataRegion":null,"contractUpdatedAt":null,"sourceUpdatedAt":null,"freshnessSeconds":null},"invocationGuide":{"preferredApi":{"snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/trust"},"curlExamples":["curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/snapshot\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/contract\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/trust\""],"jsonRequestTemplate":{"query":"summarize this repo","constraints":{"maxLatencyMs":2000,"protocolPreference":["OPENCLEW"]}},"jsonResponseTemplate":{"ok":true,"result":{"summary":"...","confidence":0.9},"meta":{"source":"CLAWHUB","generatedAt":"2026-10-13T00:05:43.278Z"}},"retryPolicy":{"maxAttempts":3,"backoffMs":[500,1500,3500],"retryableConditions":["HTTP_429","HTTP_503","NETWORK_TIMEOUT"]}},"endpoints":{"dossierUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/dossier","snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-zning1994-brainharness-autoresearch/trust"}},"reliability":{"evidence":{"source":"runtime-metrics","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No trust, reliability, or runtime telemetry is available."},"trust":{"status":"unavailable","handshakeStatus":"UNKNOWN","verificationFreshnessHours":null,"reputationScore":null,"p95LatencyMs":null,"successRate30d":null,"fallbackRate":null,"attempts30d":null,"trustUpdatedAt":null,"trustConfidence":"unknown","sourceUpdatedAt":null,"freshnessSeconds":null},"decisionGuardrails":{"doNotUseIf":["Contract metadata is missing or unavailable for deterministic execution."],"safeUseWhen":[],"riskFlags":["missing_or_unavailable_contract","trust_data_unavailable","schema_references_missing"],"operationalConfidence":"low"},"executionMetrics":{"observedLatencyMsP50":null,"observedLatencyMsP95":null,"estimatedCostUsd":null,"uptime30d":null,"rateLimitRpm":null,"rateLimitBurst":null,"lastVerifiedAt":null,"verificationSource":null},"runtimeMetrics":{"successRate":null,"avgLatencyMs":null,"avgCostUsd":null,"hallucinationRate":null,"retryRate":null,"disputeRate":null,"p50Latency":null,"p95Latency":null,"lastUpdated":null}},"benchmarks":{"evidence":{"source":"no-benchmark-data","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No benchmark suites or observed failure patterns are available."},"suites":[],"failurePatterns":[]},"artifacts":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"high","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":null},"readme":"Skill: BrainHarness Autoresearch\n\nOwner: zning1994\n\nSummary: Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy autoresearch pattern.\n\nTags: latest:0.3.0\n\nVersion history:\n\nv0.3.0 | 2026-10-03T20:28:18.885Z | user\n\n统一 BrainHarness 名称、安装入口和源仓库链接，优化器行为保持不变。\n\nv0.2.5 | 2026-04-23T02:31:47.121Z | user\n\nRenamed skill to brainforge-autoresearch. Old slug openclaw-autoresearch will be merged as redirect.\n\nArchive index:\n\nArchive v0.3.0: 11 files, 34810 bytes\n\nFiles: autoresearch.py (47237b), CHANGELOG.md (6397b), eval-guide.md (9191b), examples/search-skill.json (2957b), examples/weekly-report.json (5209b), LICENSE (1066b), README.md (5188b), self-eval.json (5906b), skill-card.md (1980b), SKILL.md (8714b), _meta.json (144b)\n\nFile v0.3.0:SKILL.md\n\n---\nname: brainharness-autoresearch\ndescription: >-\n  Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt.\n  Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\",\n  \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\",\n  \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\".\n  Runs automated prompt experiments using the Karpathy autoresearch pattern.\nversion: 0.3.0\nmetadata:\n  author: zning1994\n  openclaw:\n    requires:\n      anyEnv:\n        - MINIMAX_API_KEY\n        - OPENAI_API_KEY\n        - ANTHROPIC_API_KEY\n      anyBins:\n        - python3\n        - python\n    primaryEnv: OPENAI_API_KEY\n    optionalEnv:\n      - OPENAI_BASE_URL\n      - OPENAI_API_BASE\n    homepage: https://github.com/zning1994/brainharness-autoresearch\n    os:\n      - macos\n      - linux\n---\n\n# brainharness-autoresearch\n\n> Previously published as `brainforge-autoresearch`, with earlier names `autoresearch` / `openclaw-autoresearch`. Renamed for the BrainHarness marketplace; functionality unchanged.\n\nAutonomous prompt optimization for AI agent skills. Runs controlled experiments to find better prompt variants using the [Karpathy autoresearch pattern](https://github.com/karpathy/autoresearch): generate hypothesis, mutate prompt, evaluate, repeat.\n\n## When to use\n\n- 用户说\"优化一下这个 skill\" / User says \"optimize this skill's prompt\"\n- 用户要对比不同 prompt 版本的效果 / User wants to benchmark prompt variants\n- 用户说\"run autoresearch on X\" / \"eval skill X\" / \"improve skill X\"\n- 用户对 skill 输出质量不满，想系统性改进 / User is unhappy with skill output quality and wants systematic improvement\n\n**Do not use:**\n- 一次性的小改动（直接改 prompt 即可） / One-off prompt tweaks — just edit the prompt directly\n- 调试某个特定失败 case / Debugging a specific failure — investigate the root cause instead\n- Skill 脚本本身有 bug（代码逻辑问题不是 prompt 问题） / Skill script has a bug — fix the code, not the prompt\n\n## Requirements\n\n- Python 3.10+\n- `autoresearch.py` script in the skill directory\n- LLM API access (MiniMax, OpenAI, or Anthropic)\n- Target skill must have a prompt file (SKILL.md, SYSTEM.md, or similar)\n\n## Procedure\n\nAlways follow these steps in order: (1) Create eval.json, (2) Run autoresearch command, (3) Review results and apply best prompt.\n\n### Step 1: Gather context\n\nBefore running, you need:\n\n| Parameter | Description | Example |\n|-----------|-------------|---------|\n| `--target` | Path to the skill directory or prompt file to optimize | `../workspace/skills/brain-search/SKILL.md` |\n| `--evals` | Path to eval definition JSON file | `eval.json` |\n| `--provider` | LLM provider for running experiments | `minimax` (default), `openai`, `anthropic` |\n| `--runs` | Number of runs per experiment (statistical significance) | `5` (default) |\n| `--max-experiments` | Maximum experiments before stopping | `30` (default) |\n| `--dashboard` | Open live results dashboard in browser | flag, no value |\n\n### Step 2: Create eval.json\n\nDefine test inputs and evaluation criteria. Each eval is a binary pass/fail check.\n\n```json\n{\n  \"test_inputs\": [\n    \"search for latest AI agent frameworks\",\n    \"find news about LLM inference optimization\",\n    \"搜一下 transformer 架构的最新进展\"\n  ],\n  \"evals\": [\n    {\n      \"name\": \"has_sources\",\n      \"type\": \"rule\",\n      \"rule\": \"regex\",\n      \"pattern\": \"(https?://|Source:|来源:)\"\n    },\n    {\n      \"name\": \"no_hallucinated_urls\",\n      \"type\": \"rule\",\n      \"rule\": \"banned_phrases\",\n      \"phrases\": [\"example.com\", \"placeholder.url\"]\n    },\n    {\n      \"name\": \"sufficient_detail\",\n      \"type\": \"rule\",\n      \"rule\": \"word_count\",\n      \"min\": 50,\n      \"max\": 500\n    },\n    {\n      \"name\": \"contains_summary\",\n      \"type\": \"rule\",\n      \"rule\": \"contains\",\n      \"values\": [\"summary\", \"key findings\", \"结论\"]\n    },\n    {\n      \"name\": \"no_apology_prefix\",\n      \"type\": \"rule\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"I apologize\", \"I'm sorry, but\"]\n    },\n    {\n      \"name\": \"actionable_output\",\n      \"type\": \"llm\",\n      \"question\": \"Does the response provide actionable information the user can immediately use (links, specific facts, concrete next steps)?\",\n      \"pass_description\": \"The response contains specific actionable items like URLs, concrete facts, or clear next steps\",\n      \"fail_description\": \"The response is vague, generic, or lacks specific actionable information\"\n    }\n  ]\n}\n```\n\n**Rule types:**\n\n| Rule | Parameters | Description |\n|------|-----------|-------------|\n| `regex` | `pattern` | Pass if regex matches output |\n| `banned_phrases` | `phrases` (list) | Pass if NONE of the phrases appear |\n| `word_count` | `min`, `max` (optional) | Pass if word count is within range |\n| `contains` | `values` (list), optional `match`: `\"any\"` (default) or `\"all\"` | Pass if any/all values appear in output (case-insensitive) |\n| `not_contains` | `values` (list) | Pass if NONE of the values appear in output (case-insensitive) |\n\n**LLM eval type:**\n\n| Field | Description |\n|-------|-------------|\n| `type` | Must be `\"llm\"` |\n| `name` | Unique name for this eval |\n| `question` | What to ask the judge LLM about the output |\n| `pass_description` | Description of what a passing output looks like |\n| `fail_description` | Description of what a failing output looks like |\n\nSee `eval-guide.md` for detailed guidance on writing effective evals.\n\n### Step 3: Run autoresearch\n\n```bash\npython autoresearch.py \\\n  --target ../workspace/skills/brain-search/SKILL.md \\\n  --evals eval.json \\\n  --provider minimax \\\n  --runs 5 \\\n  --max-experiments 30 \\\n  --dashboard\n```\n\n### Step 4: Review results and apply changes\n\nThe script writes results to `results.tsv` in the working directory. Each row is one experiment:\n\n```\nexperiment_id  parent_id  mutation_description  avg_score  pass_rate  evals_detail  prompt_diff\n```\n\nFind the best performing variant:\n```bash\ncat results.tsv | sort -k4 -nr | head -5\n```\n\nApply the winning prompt to your skill by copying the optimized prompt text to replace the original.\n\n## Example: optimizing brain-search\n\n```\nUser: brain-search 的搜索结果经常缺少来源链接，帮我优化一下\n\n完整流程:\n\n1. 创建 eval.json:\n   {\n     \"test_inputs\": [\n       \"search for latest news on OpenAI\",\n       \"搜一下最新的 AI 芯片进展\",\n       \"find recent papers on RAG optimization\",\n       \"what happened with Anthropic this week\",\n       \"查查 GPU 价格趋势\"\n     ],\n     \"evals\": [\n       {\n         \"name\": \"has_urls\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"min_2_sources\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+.*https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"structured_output\",\n         \"type\": \"llm\",\n         \"question\": \"Is the output well-structured with clear sections?\",\n         \"pass_description\": \"Output uses clear structure like bullets or headers\",\n         \"fail_description\": \"Output is a wall of text without clear structure\"\n       }\n     ]\n   }\n\n2. 运行命令:\n   python autoresearch.py \\\n     --target ../workspace/skills/brain-search/SKILL.md \\\n     --evals eval.json \\\n     --runs 5 \\\n     --max-experiments 20\n\n3. 查看并应用结果:\n   - 检查 results.tsv 找最高分变体\n   - 查看 mutation_description 了解关键改动\n   - 将最佳 prompt 应用到原始 SKILL.md\n```\n\n## Failure handling\n\n| Issue | Action |\n|-------|--------|\n| LLM API rate limit | Script auto-retries with backoff; if persistent, reduce `--runs` |\n| Target file not found | Check path, must be readable prompt/skill file |\n| All experiments score 0 | Evals may be too strict — review eval definitions, loosen criteria |\n| Script crashes mid-run | Results already written to `results.tsv` are preserved; re-run continues |\n\n## Gotchas\n\n- 每次实验会调用 LLM 多次（runs x test_inputs x llm_evals），注意 API 用量 / Each experiment makes multiple LLM calls — watch API usage\n- LLM eval 本身有噪声，`--runs` 设高一点（5+）才有统计意义 / LLM evals are noisy, use 5+ runs for statistical significance\n- Rule evals 比 LLM evals 更稳定、更便宜，优先用 rule / Rule evals are more stable and cheaper — prefer them\n- Baseline 分数太低（< 20%）说明 eval 定义可能有问题，先修 eval / If baseline score is very low, fix evals first\n- 优化 prompt 不能解决架构问题（比如搜索 API 本身返回差结果） / Prompt optimization cannot fix architectural issues\n\nFile v0.3.0:README.md\n\n# brainharness-autoresearch\n\n> Previously named `brainforge-autoresearch`; earlier names include `openclaw-autoresearch` and `autoresearch`. Use the current installation commands below.\n\nAutonomous skill prompt optimizer based on Karpathy's autoresearch methodology.\n\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)\n[![ClawHub](https://img.shields.io/badge/ClawHub-brainharness--autoresearch-blue)](https://clawhub.ai/zning1994/brainharness-autoresearch)\n[![brainharness](https://img.shields.io/badge/brainharness-marketplace-purple)](https://github.com/zning1994/brainharness)\n\n## What it does\n\nDefine what \"good\" means as binary evals, then let an agent loop to optimize your skill prompt automatically. The optimizer reads your current prompt, runs it against eval cases, analyzes failures, mutates the prompt, tests the mutation, and keeps or discards it -- repeating until evals pass or the experiment budget is exhausted.\n\nBased on [Andrej Karpathy's autoresearch](https://github.com/karpathy/autoresearch) and [Ole Lehmann's Claude Code adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill). Works with any AI coding agent: OpenClaw, Claude Code, Cursor, Cline.\n\n## Quick Start\n\n```bash\n# Claude Code (via brainharness marketplace)\n/plugin marketplace add zning1994/brainharness\n/plugin install brainharness-autoresearch@brainharness\n\n# Universal (npx skills)\nnpx skills add zning1994/brainharness-autoresearch\n\n# OpenClaw ClawHub\nnpx clawhub@0.23.3 install brainharness-autoresearch\n\n# Standalone\ngit clone https://github.com/zning1994/brainharness-autoresearch\ncd brainharness-autoresearch\npython autoresearch.py --target ./my-skill/SKILL.md --evals eval.json\n```\n\n## How it works\n\n```\n                        +------------------+\n                        |  Read SKILL.md   |\n                        +--------+---------+\n                                 |\n                        +--------v---------+\n                   +--->|  Run baseline    |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    | Analyze failures |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |  Mutate prompt   |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |   Test mutation  |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   +----+ Keep or discard  |\n                        +------------------+\n```\n\n## Creating eval.json\n\nMinimal example with one rule eval and one LLM eval:\n\n```json\n[\n  {\n    \"name\": \"includes_greeting\",\n    \"input\": \"Say hello to the user\",\n    \"type\": \"rule\",\n    \"rule\": \"contains\",\n    \"expected\": \"hello\"\n  },\n  {\n    \"name\": \"tone_is_professional\",\n    \"input\": \"Draft a project update email\",\n    \"type\": \"llm\",\n    \"criteria\": \"The output uses a professional tone with no slang or emojis\"\n  }\n]\n```\n\nSee [eval-guide.md](eval-guide.md) for details.\n\n## Eval types\n\n| Type | Rule | Description |\n|------|------|-------------|\n| `rule` | `regex` | Match output against a regex pattern |\n| `rule` | `contains` | Output must contain the expected string |\n| `rule` | `not_contains` | Output must not contain the string |\n| `rule` | `banned_phrases` | Output must not contain any listed phrase |\n| `rule` | `word_count` | Output word count within min/max range |\n| `llm` | -- | LLM judges output against freeform criteria |\n\n## CLI Reference\n\n| Flag | Default | Description |\n|------|---------|-------------|\n| `--target` | `./SKILL.md` | Path to the skill prompt file to optimize |\n| `--evals` | `./eval.json` | Path to the eval definitions file |\n| `--provider` | auto-detect | LLM provider: `minimax`, `openai`, `anthropic` |\n| `--runs` | `3` | Number of runs per eval case per experiment |\n| `--max-experiments` | `10` | Maximum optimization iterations |\n| `--dashboard` | off | Generate an HTML dashboard of results |\n| `--output-dir` | `./results` | Directory for output artifacts |\n| `--verbose` | off | Print detailed logs to stderr |\n\n## LLM Providers\n\nAuto-detection order:\n\n1. `MINIMAX_API_KEY`\n2. `OPENAI_API_KEY`\n3. `ANTHROPIC_API_KEY`\n\nCustom endpoint support via `OPENAI_BASE_URL` (works with any OpenAI-compatible API).\n\n## Output\n\n| File | Description |\n|------|-------------|\n| `results.tsv` | Tab-separated eval scores per experiment |\n| `changelog.md` | Human-readable log of each mutation and its effect |\n| `results.json` | Full structured results for programmatic use |\n| `dashboard.html` | Visual dashboard (when `--dashboard` is set) |\n| `SKILL.md.baseline` | Backup of the original prompt before optimization |\n\n## Credits\n\n- [Andrej Karpathy's autoresearch](https://github.com/karpathy/autoresearch) -- the original methodology\n- [Ole Lehmann's Claude Code adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill) -- adapted for prompt optimization with coding agents\n\n## License\n\n[MIT](LICENSE)\n\nFile v0.3.0:_meta.json\n\n{\n  \"ownerId\": \"kn7emz9j02vyab03rzn6fx4q7582xae6\",\n  \"slug\": \"brainharness-autoresearch\",\n  \"version\": \"0.3.0\",\n  \"publishedAt\": 1791059298885\n}\n\nFile v0.3.0:CHANGELOG.md\n\n# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nFormat based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).\n\n## [0.3.0] - 2026-10-04\n\n### Changed\n\n- Rename `brainforge-autoresearch` to `brainharness-autoresearch` and align repository links and installation names with BrainHarness.\n- Update distribution metadata and documentation; preserve skill behavior and historical release notes.\n\n## [0.2.5] - 2026-04-23\n\n### Changed\n\n- **Renamed repo and skill**: `openclaw-autoresearch` → `brainforge-autoresearch`\n- SKILL.md `name` field: `autoresearch` → `brainforge-autoresearch`\n- Updated README, plugin.json, and SKILL.md metadata to point at the new repo URL\n- Added to the [brainforge](https://github.com/zning1994/brainforge) Claude Code plugin marketplace\n- Published to ClawHub as `brainforge-autoresearch`; old slug `openclaw-autoresearch` merged in as redirect\n\n### Compatibility\n\n- GitHub: old URL `https://github.com/zning1994/openclaw-autoresearch` auto-redirects\n- ClawHub: `openclaw skills install openclaw-autoresearch` continues to work via ClawHub slug merge/redirect\n- No behavior changes to the optimizer itself\n\n## [0.2.4] - 2026-03-29\n\n### Fixed\n\n- SKILL.md: `pass`/`fail` → `pass_description`/`fail_description` in LLM eval docs and examples (field names didn't match actual API)\n- SKILL.md: `contains`/`not_contains` parameter documented as `value` (string) → `values` (list) to match actual usage\n- SKILL.md: LLM eval table now includes required `type` and `name` fields\n\n### Changed\n\n- SKILL.md self-optimized via autoresearch (2 rounds: MiniMax 48.6%→68.1%, Claude 80.6%→91.7%)\n- Procedure rewritten: added 3-step summary upfront, renamed steps for clarity, added actionable result-review guidance\n- Example section restructured: removed \"confirm target\" preamble, simplified to \"create eval → run → review\" flow\n- Added `self-eval.json` for self-optimization (dogfooding eval definition)\n\n### Tested\n\n- Self-optimization: Round 1 (MiniMax M2.7, 15 exp, 3 keep) and Round 2 (Claude, 15 exp, 1 keep) independently discovered same doc bugs\n\n## [0.2.3] - 2026-03-25\n\n### Added\n\n- `--timeout` CLI flag to control HTTP timeout per LLM call (default: 180s)\n  - Large prompts (e.g., 14KB style guides) need longer timeouts for mutation calls\n  - Previously hardcoded, now user-configurable\n\n### Changed\n\n- Default HTTP timeout increased from 60s to 180s (was causing mutation failures on Anthropic API with large prompts)\n- Timeout is now stored on the provider instance, passed through `detect_provider()`\n\n## [0.2.2] - 2026-03-25\n\n### Fixed\n\n- `.gitignore`: removed incorrect entries (`.git`, `.github`), added `autoresearch-*/` and `*.baseline`\n- README: added ClawHub badge and link, fixed install slug to `openclaw-autoresearch`\n\n## [0.2.1] - 2026-03-25\n\n### Fixed\n\n- ClawHub security flag: changed `requires.env` (all required) to `requires.anyEnv` (any one suffices)\n- Declared optional env vars `OPENAI_BASE_URL` / `OPENAI_API_BASE` in metadata\n\n## [0.2.0] - 2026-03-25\n\n### Fixed\n\n- MiniMax extended thinking support — models with thinking blocks (M2.7, Claude) no longer crash the parser\n- Fallback to thinking content when text block is missing (thinking exhausts `max_tokens`)\n- LLM judge `max_tokens` increased from 16 to 256 (thinking models need headroom)\n- Rule type field: accept both `\"rule\"` and `\"check\"` keys for backwards compatibility\n- `contains`/`not_contains`: accept both `\"values\"` (list) and `\"value\"` (string)\n- LLM eval fields: accept both `\"pass_description\"`/`\"fail_description\"` and `\"pass\"`/`\"fail\"`\n- `test_inputs`: accept both plain strings and `{\"name\": ..., \"input\": ...}` objects\n- `contains` rule now supports `\"match\": \"all\"` mode (require all values present)\n- Convergence counter double-increment bug fixed\n- `regex` rule now uses `re.MULTILINE` for correct `^`/`$` matching\n- CJK-aware word counting in `word_count` rule (counts Chinese/Japanese/Korean characters individually)\n\n### Added\n\n- `--model` CLI flag to override the default model per provider\n- JSON fallback: `runs_per_experiment` and `max_experiments` from eval.json used when CLI defaults unchanged\n- Shared `_extract_text()` helper for Anthropic-style response parsing across all providers\n\n### Tested\n\n- Successfully optimized `brain-search` skill: 37.5% → 54.2% pass rate in 5 experiments with MiniMax M2.7\n\n## [0.1.0] - 2026-03-24\n\n### Added\n\n- Core `autoresearch.py` script — autonomous skill prompt optimizer\n  - Zero external dependencies (Python 3.9+ stdlib only)\n  - LLM providers: MiniMax, OpenAI, Anthropic (auto-detect from env vars)\n  - Custom endpoint support via `OPENAI_BASE_URL`\n  - `--model` flag to override default model per provider\n- Hybrid eval system\n  - Rule-based evals: `regex`, `banned_phrases`, `word_count`, `contains`, `not_contains`\n  - `contains` supports `match: \"all\"` and `match: \"any\"` modes\n  - LLM-as-judge evals with binary YES/NO scoring\n  - CJK-aware word counting for Chinese/Japanese/Korean text\n- Experiment loop\n  - Baseline measurement before any changes\n  - One mutation per experiment (targeted, not bulk rewrites)\n  - Automatic keep/discard based on score comparison\n  - Convergence exit: 95%+ pass rate for 3 consecutive experiments\n  - Budget cap via `--max-experiments`\n- Output artifacts\n  - `results.tsv` — tab-separated score log\n  - `changelog.md` — detailed mutation history with reasoning\n  - `results.json` — structured data for tooling\n  - `dashboard.html` — self-contained Chart.js dashboard (opt-in via `--dashboard`)\n  - `SKILL.md.baseline` — original skill backup\n- `SKILL.md` — agent instructions for OpenClaw / Claude Code / Cursor / Cline\n- `eval-guide.md` — practical guide for writing binary eval criteria\n- Example eval configs: `weekly-report.json`, `search-skill.json`\n- Compatible with `npx skills add` (Vercel Labs skills ecosystem)\n\n[0.2.4]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.4\n[0.2.3]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.3\n[0.2.2]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.2\n[0.2.1]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.1\n[0.2.0]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.0\n[0.1.0]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.1.0\n\nFile v0.3.0:eval-guide.md\n\n# Eval Guide: Writing Effective Eval Criteria\n\nHow to write eval criteria that actually improve your skill's output. This guide covers the hybrid rule+LLM eval system used by brainharness-autoresearch (previously `openclaw-autoresearch`).\n\n## The Golden Rule\n\n**Binary yes/no only. No scales.**\n\nEvery eval must produce a pass or fail. Not \"7/10\", not \"mostly good\", not \"needs improvement\". Pass or fail.\n\nWhy: Scales are subjective (your 7 is my 5), hard to optimize against, and create noise. Binary forces you to define exactly what \"good\" means.\n\nBad: \"Rate the tone from 1-5\"\nGood: \"Does the output use a professional tone without slang or emojis?\"\n\n## Rule-Based Evals\n\nRule-based evals cost zero tokens, run instantly, and are 100% consistent. Use them whenever possible.\n\n### `regex`\n\nPattern matching against the output text.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_date_format\",\n  \"rule\": \"regex\",\n  \"pattern\": \"\\\\d{4}-\\\\d{2}-\\\\d{2}\",\n  \"description\": \"Output contains a date in YYYY-MM-DD format\"\n}\n```\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_markdown_headers\",\n  \"rule\": \"regex\",\n  \"pattern\": \"^#{1,3} .+\",\n  \"description\": \"Output uses markdown headers\"\n}\n```\n\nUseful for: format validation, structure checks, required patterns.\n\n### `banned_phrases`\n\nExact phrases that must not appear in the output.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_filler\",\n  \"rule\": \"banned_phrases\",\n  \"phrases\": [\"as an AI\", \"I cannot\", \"I'm sorry but\", \"it depends\", \"各方面\", \"总的来说\"],\n  \"description\": \"No AI disclaimers or vague filler phrases\"\n}\n```\n\nUseful for: removing AI-speak, enforcing brand voice, blocking lazy language.\n\n### `word_count`\n\nMin and/or max word limits. For Chinese text, counts characters.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"length_check\",\n  \"rule\": \"word_count\",\n  \"min\": 100,\n  \"max\": 500,\n  \"description\": \"Output is between 100-500 words\"\n}\n```\n\nUseful for: preventing one-line answers, keeping outputs concise, matching format requirements.\n\n### `contains`\n\nRequired keywords or phrases that must appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_source\",\n  \"rule\": \"contains\",\n  \"values\": [\"Source:\", \"Reference:\", \"来源:\"],\n  \"match\": \"any\",\n  \"description\": \"Output cites at least one source\"\n}\n```\n\n`match` can be `\"any\"` (at least one) or `\"all\"` (every value must appear).\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_all_sections\",\n  \"rule\": \"contains\",\n  \"values\": [\"## Summary\", \"## Action Items\", \"## Risks\"],\n  \"match\": \"all\",\n  \"description\": \"Output has all required sections\"\n}\n```\n\nUseful for: enforcing structure, required sections, key terminology.\n\n### `not_contains`\n\nPhrases or patterns that must NOT appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_placeholder\",\n  \"rule\": \"not_contains\",\n  \"values\": [\"TODO\", \"TBD\", \"FIXME\", \"[insert\", \"[placeholder\"],\n  \"description\": \"No placeholder text left in output\"\n}\n```\n\nUseful for: catching incomplete outputs, blocking specific content.\n\n## LLM-Based Evals\n\nLLM evals handle semantic judgments that rules cannot express. They cost tokens and have slight variance, so use them only when necessary.\n\n### When to Use LLM Evals\n\nUse LLM when you need to judge:\n- Accuracy against source material (hallucination detection)\n- Logical coherence or reasoning quality\n- Whether advice is actionable vs. generic\n- Tone, style, or audience-appropriateness\n- Semantic completeness (did it cover all key points?)\n\nDo NOT use LLM for things rules can handle:\n- Word count, format checks, required sections (use rules)\n- Presence of specific keywords (use contains)\n- Forbidden phrases (use banned_phrases)\n\n### Writing Good LLM Eval Questions\n\n```json\n{\n  \"type\": \"llm\",\n  \"name\": \"no_hallucination\",\n  \"question\": \"Compare the output against the provided input data. Does every claim in the output have a corresponding fact in the input? Ignore formatting and phrasing differences.\",\n  \"pass_description\": \"Every factual claim maps to input data. No invented statistics, dates, or details.\",\n  \"fail_description\": \"Contains at least one claim not supported by the input data.\"\n}\n```\n\nKey principles:\n1. **Be specific about what to check.** Not \"is this good?\" but \"does every claim have a source?\"\n2. **Tell the LLM what to compare against.** \"Compare against the input data\" or \"based on the search results provided.\"\n3. **Define pass and fail concretely.** The LLM judge needs clear criteria, not vibes.\n4. **One judgment per eval.** Don't ask \"is it accurate AND well-structured?\" -- split into two evals.\n\n### Bad vs Good Examples\n\nBad:\n```json\n{\n  \"question\": \"Is this a good summary?\",\n  \"pass_description\": \"It's good\",\n  \"fail_description\": \"It's bad\"\n}\n```\n\nGood:\n```json\n{\n  \"question\": \"Does the summary capture the 3 most important findings from the search results, without adding information not present in the results?\",\n  \"pass_description\": \"All 3 key findings are mentioned. No information is fabricated beyond what the search results contain.\",\n  \"fail_description\": \"Misses one or more key findings, or includes claims not supported by the search results.\"\n}\n```\n\n## Examples by Skill Type\n\n### Text/Copy Skills (newsletters, reports, summaries)\n\nTypical eval mix:\n- `banned_phrases`: remove filler (\"总的来说\", \"as mentioned above\", \"综上所述\")\n- `word_count`: enforce length constraints\n- `contains`: required sections or keywords\n- LLM: accuracy check against source material\n- LLM: actionability or specificity check\n\n### Search Skills (web search, knowledge search)\n\nTypical eval mix:\n- `contains`: must cite URLs or sources\n- `not_contains`: no hallucination markers (\"[Source needed]\", made-up URLs)\n- LLM: does summary reflect actual search results\n- LLM: is output well-structured with clear sections\n\n### Code Generation Skills\n\nTypical eval mix:\n- `regex`: output contains code blocks (`` ```\\w+ ``)\n- `not_contains`: no placeholder code (\"pass\", \"# TODO\")\n- `contains`: includes error handling patterns (\"try\", \"except\" or \"catch\")\n- LLM: does the code solve the stated problem\n\n### Translation Skills\n\nTypical eval mix:\n- `banned_phrases`: no untranslated source language fragments\n- `word_count`: reasonable length ratio to source\n- LLM: semantic equivalence to source text\n- LLM: natural fluency in target language (no translationese)\n\n### Data Extraction Skills\n\nTypical eval mix:\n- `regex`: output matches expected data format (JSON, CSV, etc.)\n- `contains`: all required fields present\n- LLM: extracted values are accurate against source\n- LLM: edge cases handled (missing data, ambiguous values)\n\n## Common Mistakes\n\n### Too many evals (> 6)\n\nMore evals means more constraints. Past 6, the optimizer starts gaming individual evals at the expense of overall quality. The prompt becomes a checklist-satisfier instead of genuinely good.\n\n**Fix**: Pick the 4-5 that matter most. If you can't decide, you don't understand what \"good\" means for this skill yet.\n\n### Overlapping evals\n\nIf two evals test similar things, a failure on one is always a failure on the other. You're just adding noise.\n\nExample of overlap:\n- Eval A: \"No filler language\"\n- Eval B: \"Every sentence adds new information\"\n\nThese catch the same problem. Pick one.\n\n### Too vague\n\n\"Is the output helpful?\" -- helpful to whom, for what? The LLM judge can't read your mind.\n\n**Fix**: Replace with \"Does the output provide at least 2 specific, actionable next steps the reader can take this week?\"\n\n### Too narrow\n\n\"Does the output mention React hooks?\" -- this only works for one specific input. It will fail or be irrelevant for other test cases.\n\n**Fix**: Evals should apply to ALL test inputs. If it's input-specific, it's not an eval -- it's a test assertion.\n\n### Unmeasurable by LLM\n\n\"Will this output generate more revenue?\" -- no LLM can judge this. Stick to things observable in the text.\n\n**Fix**: Replace with proxies: \"Does the output include a clear call-to-action?\" or \"Are the benefits described in concrete terms with numbers?\"\n\n## The 3-Question Test\n\nBefore adding any eval, ask:\n\n1. **Can I explain pass/fail to a stranger in one sentence?**\n   If not, the eval is too vague.\n\n2. **Will this eval produce the same result on every test input?**\n   If it only applies to some inputs, it's too narrow.\n\n3. **Can a rule handle this instead?**\n   If yes, use a rule. Save LLM evals for semantic judgments.\n\n## Template\n\n```json\n{\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"short_snake_case_name\",\n      \"rule\": \"contains|banned_phrases|word_count|regex|not_contains\",\n      \"description\": \"One sentence: what this checks\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"short_snake_case_name\",\n      \"question\": \"Specific question for the LLM judge to answer about the output.\",\n      \"pass_description\": \"Concrete description of what a passing output looks like.\",\n      \"fail_description\": \"Concrete description of what a failing output looks like.\"\n    }\n  ],\n  \"test_inputs\": [\n    {\n      \"name\": \"descriptive_scenario_name\",\n      \"input\": \"The actual input that will be sent to the skill.\"\n    }\n  ]\n}\n```\n\nStart with 2-3 test inputs and 3-4 evals. Add more only after running a few optimization rounds and seeing where the gaps are.\n\nFile v0.3.0:skill-card.md\n\n## Description:\n\nRuns iterative prompt experiments to evaluate and improve AI agent skills against user-defined criteria.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[zning1994](https://clawhub.ai/user/zning1994)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nDevelopers and skill authors use it to benchmark prompt variants against defined evaluations and improve an existing skill's instructions.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: Experiments can automatically rewrite the target skill prompt.\n\nMitigation: Run on a copy or version-controlled skill and review the diff before retaining changes.\n\nRisk: Prompt and evaluation content is sent to third-party LLM APIs.\n\nMitigation: Do not use confidential prompts or evaluation data without approval for the chosen provider.\n\nRisk: Installation commands may fetch a changing release.\n\nMitigation: Pin the installed version where possible.\n\n## Reference(s):\n\n- [ClawHub skill listing](https://clawhub.ai/zning1994/skills/brainharness-autoresearch)\n- [Project homepage (listed in skill metadata)](https://github.com/zning1994/brainharness-autoresearch)\n- [Evaluation guide](eval-guide.md)\n- [Autoresearch methodology](https://github.com/karpathy/autoresearch)\n\n## Skill Output:\n\n**Output Type(s):** [Guidance, Shell commands, Configuration, Files]\n\n**Output Format:** [Markdown guidance and JSON evaluation definitions; experiment results in TSV and JSON]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [May write revised prompt files, experiment logs, and an original-prompt backup.]\n\n## Skill Version(s):\n\n0.3.0 (source: frontmatter and ClawHub release metadata)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.\n\nFile v0.3.0:examples/search-skill.json\n\n{\n  \"skill_description\": \"Perform a web search on a given query and return a structured summary of findings with source citations. The output should be accurate to search results, well-organized, and immediately useful.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"technical_english\",\n      \"input\": \"What are the main differences between SQLite WAL mode and journal mode? When should I use each?\"\n    },\n    {\n      \"name\": \"current_events_chinese\",\n      \"input\": \"2026年3月 OpenAI 和 Anthropic 最新发布了什么模型？各自有什么特点？\"\n    },\n    {\n      \"name\": \"comparison_query\",\n      \"input\": \"Compare Cloudflare Workers vs AWS Lambda for edge computing: cold start times, pricing, and language support\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"has_source_citation\",\n      \"rule\": \"regex\",\n      \"pattern\": \"https?://[^\\\\s)\\\\]]+\",\n      \"description\": \"Output contains at least one URL as a source citation\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_hallucination_markers\",\n      \"rule\": \"not_contains\",\n      \"values\": [\n        \"[Source needed]\",\n        \"[citation needed]\",\n        \"I couldn't find\",\n        \"I don't have access\",\n        \"As of my knowledge cutoff\",\n        \"I'm not able to browse\"\n      ],\n      \"description\": \"No placeholder citations or disclaimers about inability to search\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"captures_key_findings\",\n      \"question\": \"Based on the search query, does the output identify and summarize the most important findings? Check that: (1) the core question is directly answered, not deflected, (2) key distinctions or comparisons requested are addressed, and (3) the information appears factually consistent (no self-contradictions).\",\n      \"pass_description\": \"The output directly answers the query with specific, substantive information. Key points are covered (e.g., for a comparison query, both sides are discussed with concrete details like numbers or features). No self-contradictions.\",\n      \"fail_description\": \"The output is vague, generic, or fails to address the core question. For comparison queries, one side may be missing or described only in generalities. Contains self-contradicting statements.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"clear_structure\",\n      \"question\": \"Is the output organized with clear visual structure? Look for: distinct sections or categories (using headers, bullet points, or numbered lists), a logical flow from overview to details, and separation between different sub-topics.\",\n      \"pass_description\": \"Output uses headers, bullets, or numbered lists to separate information into logical sections. A reader can scan the output and find specific information without reading everything.\",\n      \"fail_description\": \"Output is a wall of text, or uses formatting inconsistently. Information about different sub-topics is mixed together without clear separation.\"\n    }\n  ]\n}\n\nFile v0.3.0:examples/weekly-report.json\n\n{\n  \"skill_description\": \"Generate a weekly project status report from raw project updates. Output should be structured, accurate, and highlight risks clearly.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"multi_project_normal\",\n      \"input\": \"本周项目进展：\\n1. 用户系统重构：完成了数据库迁移脚本编写，已在 staging 环境测试通过。下周计划上线生产环境。风险：生产数据量是 staging 的 50 倍，迁移预计需要 4 小时停机。\\n2. 支付网关集成：对接了 Stripe webhook，完成了 3 种支付方式（信用卡、Apple Pay、Google Pay）的联调。剩余：退款流程还没做，需要 2 天。\\n3. 移动端 App：iOS 已提交 App Store 审核，Android 在做性能优化。Android 冷启动时间从 3.2 秒降到 1.8 秒。风险：iOS 审核可能因为隐私政策描述被拒。\"\n    },\n    {\n      \"name\": \"single_project_delayed\",\n      \"input\": \"本周项目进展：\\nAPI Gateway 重写项目：原计划本周完成所有路由迁移，实际只完成了 60%（18/30 个路由）。原因：发现旧系统有 5 个未文档化的中间件，需要逆向工程理解逻辑后才能迁移。团队两人本周请假，人力不足。下周需要从其他组借调 1 名后端开发。风险：如果下周无法补上进度，整体上线时间将从 4 月 15 日推迟到 4 月 30 日。已与产品经理沟通过延期可能性。\"\n    },\n    {\n      \"name\": \"infra_project_chinese\",\n      \"input\": \"本周项目进展：\\n1. K8s 集群升级：从 1.27 升级到 1.29，已完成 dev 和 staging 集群。生产集群计划下周二滚动升级。已验证所有 Helm chart 兼容性，发现 ingress-nginx 需要从 4.7 升到 4.9。风险：升级期间可能有 2-3 分钟的 ingress 流量中断。\\n2. 监控告警优化：重新调整了 CPU/内存告警阈值，误报率从每天 15 次降到 2 次。新增了 SLO 面板，覆盖 3 个核心服务的 P99 延迟和可用性。\\n3. 成本优化：将 3 台 m5.xlarge 节点替换为 m6g.xlarge（ARM），预计月度成本降低 $420。已运行 1 周无异常。\"\n    },\n    {\n      \"name\": \"mixed_good_and_bad\",\n      \"input\": \"本周项目进展：\\n1. 搜索服务：Elasticsearch 索引重建完成，搜索延迟从 P95 800ms 降到 200ms。新增了同义词词典和拼音搜索支持。本周无风险。\\n2. 数据管道：Spark 作业频繁 OOM，本周花了 3 天排查。根因是上游数据量突增 3 倍（从 200GB/天到 600GB/天）但没有通知我们。临时方案：增加了 executor 内存到 16GB。长期方案：需要重新设计分区策略，预计 2 周。风险：如果上游数据量继续增长，当前临时方案也会撑不住。\\n3. CI/CD：Jenkins 迁移到 GitHub Actions 完成了 80%（16/20 个 pipeline）。剩余 4 个涉及自建 runner 的 pipeline 需要额外配置。\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"has_status_keywords\",\n      \"rule\": \"contains\",\n      \"values\": [\"进度\", \"风险\", \"下周\"],\n      \"match\": \"all\",\n      \"description\": \"Report must contain status tracking keywords: progress, risks, and next week plan\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_vague_language\",\n      \"rule\": \"banned_phrases\",\n      \"phrases\": [\n        \"基本完成\",\n        \"差不多\",\n        \"还行\",\n        \"大概\",\n        \"应该没问题\",\n        \"可能会好\",\n        \"总体来说还可以\",\n        \"各方面都不错\",\n        \"正在推进中\"\n      ],\n      \"description\": \"No vague or non-committal status language\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"report_length\",\n      \"rule\": \"word_count\",\n      \"min\": 200,\n      \"max\": 800,\n      \"description\": \"Report is between 200-800 characters (concise but complete)\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"no_hallucination\",\n      \"question\": \"Compare every factual claim in the weekly report output (dates, numbers, percentages, project names, technical details) against the input data. Is there any information in the output that was not present in or directly derivable from the input?\",\n      \"pass_description\": \"Every fact in the report (numbers, dates, project names, status descriptions) can be traced back to the input. No invented details, no embellished metrics, no assumed timelines.\",\n      \"fail_description\": \"The report contains at least one claim that cannot be found in or reasonably derived from the input data -- for example, an invented completion percentage, a made-up deadline, or a fabricated metric.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"actionable_risks\",\n      \"question\": \"Look at each risk mentioned in the output. Does every risk description include: (1) what specifically could go wrong, (2) what the impact would be, and (3) what mitigation or next step is suggested?\",\n      \"pass_description\": \"Each risk is specific (not generic like 'might have issues'), states the potential impact (delay, cost, outage, etc.), and suggests or implies a concrete next step.\",\n      \"fail_description\": \"At least one risk is vague (e.g., '可能有风险' without specifics), missing impact assessment, or has no suggested mitigation.\"\n    }\n  ]\n}\n\nFile v0.3.0:self-eval.json\n\n{\n  \"skill_description\": \"Guide an AI agent on how to use autoresearch to optimize skill prompts. The agent should correctly explain the workflow, create proper eval definitions, formulate correct CLI commands, interpret results, and know when NOT to use autoresearch.\",\n  \"system_context\": \"You are an AI assistant with access to the autoresearch tool. When the user asks about optimizing skills or prompts, follow the instructions in your skill definition to help them.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"optimize_skill_chinese\",\n      \"input\": \"我的 brain-search skill 搜索结果总是缺少来源链接，帮我优化一下这个 skill 的 prompt\"\n    },\n    {\n      \"name\": \"write_eval_criteria\",\n      \"input\": \"I want to optimize my weekly-report skill. The reports are often too long and miss action items. How do I write the eval.json for this?\"\n    },\n    {\n      \"name\": \"interpret_results\",\n      \"input\": \"autoresearch 跑完了，结果：baseline 45%, experiment 3 到了 72%, experiment 8 到了 91% 然后就停了。这个结果怎么解读？我要怎么做？\"\n    },\n    {\n      \"name\": \"should_not_use\",\n      \"input\": \"My ingest skill crashes when I upload a PDF file larger than 10MB. Can you run autoresearch to fix it?\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_hallucinated_flags\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"--optimize\", \"--auto-fix\", \"--improve\", \"--tune\", \"--benchmark\", \"--score\"],\n      \"description\": \"Must not invent CLI flags that don't exist\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_wrong_provider_names\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"--provider google\", \"--provider gemini\", \"--provider claude\", \"--provider gpt\"],\n      \"description\": \"Provider names are minimax/openai/anthropic only — not model names\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"cli_command_accuracy\",\n      \"question\": \"Evaluate the CLI command usage in the response: (1) If the user asks to optimize a skill or run autoresearch, the response MUST include a CLI command with both '--target' and '--evals' flags pointing to specific file paths. (2) If the user asks about interpreting results or asks about a code bug, the response should NOT necessarily include a full CLI command — that's fine. (3) If a CLI command IS present, verify it uses only valid flags: --target, --evals, --provider, --model, --runs, --max-experiments, --dashboard, --output-dir, --timeout, --verbose.\",\n      \"pass_description\": \"Optimization requests include correct CLI with --target and --evals. Result interpretation and code bug questions are not penalized for lacking CLI commands. Any CLI commands shown use only valid flags.\",\n      \"fail_description\": \"Optimization request is missing CLI command or missing --target/--evals. Or a CLI command uses invalid/hallucinated flags.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"appropriate_response\",\n      \"question\": \"Does the response appropriately handle the user's request? (1) For optimization requests: outlines the workflow (create eval.json → run autoresearch → review results), mentions eval definitions. (2) For eval writing questions: shows correct eval JSON structure with 'type' and 'name' fields, uses valid rule types (regex, banned_phrases, word_count, contains, not_contains) or llm type with 'question'/'pass_description'/'fail_description'. (3) For result interpretation: explains what the scores and convergence mean, suggests concrete next steps (apply the prompt, review the mutations, adjust evals). (4) For code bug reports: clearly states autoresearch is NOT the right tool and suggests fixing the code/debugging instead.\",\n      \"pass_description\": \"Response matches the request type with specific, actionable guidance. Each scenario gets the right kind of help.\",\n      \"fail_description\": \"Response mismatches: suggests autoresearch for a code bug, gives vague advice, omits eval workflow for optimization requests, or fails to interpret scores when asked.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"eval_json_correctness\",\n      \"question\": \"If the response includes eval JSON examples: (1) Does each eval have both 'type' and 'name' fields? (2) Do rule evals use one of: regex, banned_phrases, word_count, contains, not_contains? (3) Do LLM evals have 'question', 'pass_description', and 'fail_description' fields (NOT 'pass'/'fail')? (4) Are the evals tailored to the user's specific problem, not just generic examples? If no eval JSON is shown (e.g., for result interpretation or code bug questions), this eval passes automatically.\",\n      \"pass_description\": \"Eval JSON is well-formed with correct field names, OR no eval JSON is needed for this type of question. Evals are relevant to the user's stated problem.\",\n      \"fail_description\": \"Eval JSON uses wrong field names (e.g., 'pass' instead of 'pass_description'), invalid rule types, missing required fields, or is clearly generic copy-paste unrelated to the user's problem.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"workflow_completeness\",\n      \"question\": \"For optimization and eval-writing requests: does the response cover the full autoresearch workflow? Check for: (1) Creating eval.json with test_inputs and evals, (2) Running the CLI command, (3) Reviewing/interpreting results. For result interpretation: does it explain scores AND suggest next steps? For code bugs: does it redirect away from autoresearch clearly? A complete response covers all relevant steps, not just one.\",\n      \"pass_description\": \"Response covers all relevant workflow steps for the given request type. Nothing critical is missing.\",\n      \"fail_description\": \"Response covers only part of the workflow (e.g., shows eval JSON but forgets the run command, or shows the command but skips eval creation).\"\n    }\n  ],\n  \"runs_per_experiment\": 3,\n  \"max_experiments\": 15\n}\n\nFile v0.3.0:LICENSE\n\nMIT License\n\nCopyright (c) 2026 zning1994\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\nArchive v0.2.5: 10 files, 34253 bytes\n\nFiles: autoresearch.py (47237b), CHANGELOG.md (6119b), eval-guide.md (9189b), examples/search-skill.json (2957b), examples/weekly-report.json (5209b), README.md (5259b), self-eval.json (5906b), skill-card.md (2564b), SKILL.md (8671b), _meta.json (144b)\n\nFile v0.2.5:SKILL.md\n\n---\nname: brainforge-autoresearch\ndescription: >-\n  Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt.\n  Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\",\n  \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\",\n  \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\".\n  Runs automated prompt experiments using the Karpathy autoresearch pattern.\nversion: 0.2.5\nmetadata:\n  author: zning1994\n  openclaw:\n    requires:\n      anyEnv:\n        - MINIMAX_API_KEY\n        - OPENAI_API_KEY\n        - ANTHROPIC_API_KEY\n      anyBins:\n        - python3\n        - python\n    primaryEnv: OPENAI_API_KEY\n    optionalEnv:\n      - OPENAI_BASE_URL\n      - OPENAI_API_BASE\n    homepage: https://github.com/zning1994/brainforge-autoresearch\n    os:\n      - macos\n      - linux\n---\n\n# brainforge-autoresearch\n\n> Previously published as `autoresearch` / `openclaw-autoresearch`. Renamed for the brainforge marketplace rollout — functionality unchanged.\n\nAutonomous prompt optimization for AI agent skills. Runs controlled experiments to find better prompt variants using the [Karpathy autoresearch pattern](https://github.com/karpathy/autoresearch): generate hypothesis, mutate prompt, evaluate, repeat.\n\n## When to use\n\n- 用户说\"优化一下这个 skill\" / User says \"optimize this skill's prompt\"\n- 用户要对比不同 prompt 版本的效果 / User wants to benchmark prompt variants\n- 用户说\"run autoresearch on X\" / \"eval skill X\" / \"improve skill X\"\n- 用户对 skill 输出质量不满，想系统性改进 / User is unhappy with skill output quality and wants systematic improvement\n\n**Do not use:**\n- 一次性的小改动（直接改 prompt 即可） / One-off prompt tweaks — just edit the prompt directly\n- 调试某个特定失败 case / Debugging a specific failure — investigate the root cause instead\n- Skill 脚本本身有 bug（代码逻辑问题不是 prompt 问题） / Skill script has a bug — fix the code, not the prompt\n\n## Requirements\n\n- Python 3.10+\n- `autoresearch.py` script in the skill directory\n- LLM API access (MiniMax, OpenAI, or Anthropic)\n- Target skill must have a prompt file (SKILL.md, SYSTEM.md, or similar)\n\n## Procedure\n\nAlways follow these steps in order: (1) Create eval.json, (2) Run autoresearch command, (3) Review results and apply best prompt.\n\n### Step 1: Gather context\n\nBefore running, you need:\n\n| Parameter | Description | Example |\n|-----------|-------------|---------|\n| `--target` | Path to the skill directory or prompt file to optimize | `../workspace/skills/brain-search/SKILL.md` |\n| `--evals` | Path to eval definition JSON file | `eval.json` |\n| `--provider` | LLM provider for running experiments | `minimax` (default), `openai`, `anthropic` |\n| `--runs` | Number of runs per experiment (statistical significance) | `5` (default) |\n| `--max-experiments` | Maximum experiments before stopping | `30` (default) |\n| `--dashboard` | Open live results dashboard in browser | flag, no value |\n\n### Step 2: Create eval.json\n\nDefine test inputs and evaluation criteria. Each eval is a binary pass/fail check.\n\n```json\n{\n  \"test_inputs\": [\n    \"search for latest AI agent frameworks\",\n    \"find news about LLM inference optimization\",\n    \"搜一下 transformer 架构的最新进展\"\n  ],\n  \"evals\": [\n    {\n      \"name\": \"has_sources\",\n      \"type\": \"rule\",\n      \"rule\": \"regex\",\n      \"pattern\": \"(https?://|Source:|来源:)\"\n    },\n    {\n      \"name\": \"no_hallucinated_urls\",\n      \"type\": \"rule\",\n      \"rule\": \"banned_phrases\",\n      \"phrases\": [\"example.com\", \"placeholder.url\"]\n    },\n    {\n      \"name\": \"sufficient_detail\",\n      \"type\": \"rule\",\n      \"rule\": \"word_count\",\n      \"min\": 50,\n      \"max\": 500\n    },\n    {\n      \"name\": \"contains_summary\",\n      \"type\": \"rule\",\n      \"rule\": \"contains\",\n      \"values\": [\"summary\", \"key findings\", \"结论\"]\n    },\n    {\n      \"name\": \"no_apology_prefix\",\n      \"type\": \"rule\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"I apologize\", \"I'm sorry, but\"]\n    },\n    {\n      \"name\": \"actionable_output\",\n      \"type\": \"llm\",\n      \"question\": \"Does the response provide actionable information the user can immediately use (links, specific facts, concrete next steps)?\",\n      \"pass_description\": \"The response contains specific actionable items like URLs, concrete facts, or clear next steps\",\n      \"fail_description\": \"The response is vague, generic, or lacks specific actionable information\"\n    }\n  ]\n}\n```\n\n**Rule types:**\n\n| Rule | Parameters | Description |\n|------|-----------|-------------|\n| `regex` | `pattern` | Pass if regex matches output |\n| `banned_phrases` | `phrases` (list) | Pass if NONE of the phrases appear |\n| `word_count` | `min`, `max` (optional) | Pass if word count is within range |\n| `contains` | `values` (list), optional `match`: `\"any\"` (default) or `\"all\"` | Pass if any/all values appear in output (case-insensitive) |\n| `not_contains` | `values` (list) | Pass if NONE of the values appear in output (case-insensitive) |\n\n**LLM eval type:**\n\n| Field | Description |\n|-------|-------------|\n| `type` | Must be `\"llm\"` |\n| `name` | Unique name for this eval |\n| `question` | What to ask the judge LLM about the output |\n| `pass_description` | Description of what a passing output looks like |\n| `fail_description` | Description of what a failing output looks like |\n\nSee `eval-guide.md` for detailed guidance on writing effective evals.\n\n### Step 3: Run autoresearch\n\n```bash\npython autoresearch.py \\\n  --target ../workspace/skills/brain-search/SKILL.md \\\n  --evals eval.json \\\n  --provider minimax \\\n  --runs 5 \\\n  --max-experiments 30 \\\n  --dashboard\n```\n\n### Step 4: Review results and apply changes\n\nThe script writes results to `results.tsv` in the working directory. Each row is one experiment:\n\n```\nexperiment_id  parent_id  mutation_description  avg_score  pass_rate  evals_detail  prompt_diff\n```\n\nFind the best performing variant:\n```bash\ncat results.tsv | sort -k4 -nr | head -5\n```\n\nApply the winning prompt to your skill by copying the optimized prompt text to replace the original.\n\n## Example: optimizing brain-search\n\n```\nUser: brain-search 的搜索结果经常缺少来源链接，帮我优化一下\n\n完整流程:\n\n1. 创建 eval.json:\n   {\n     \"test_inputs\": [\n       \"search for latest news on OpenAI\",\n       \"搜一下最新的 AI 芯片进展\",\n       \"find recent papers on RAG optimization\",\n       \"what happened with Anthropic this week\",\n       \"查查 GPU 价格趋势\"\n     ],\n     \"evals\": [\n       {\n         \"name\": \"has_urls\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"min_2_sources\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+.*https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"structured_output\",\n         \"type\": \"llm\",\n         \"question\": \"Is the output well-structured with clear sections?\",\n         \"pass_description\": \"Output uses clear structure like bullets or headers\",\n         \"fail_description\": \"Output is a wall of text without clear structure\"\n       }\n     ]\n   }\n\n2. 运行命令:\n   python autoresearch.py \\\n     --target ../workspace/skills/brain-search/SKILL.md \\\n     --evals eval.json \\\n     --runs 5 \\\n     --max-experiments 20\n\n3. 查看并应用结果:\n   - 检查 results.tsv 找最高分变体\n   - 查看 mutation_description 了解关键改动\n   - 将最佳 prompt 应用到原始 SKILL.md\n```\n\n## Failure handling\n\n| Issue | Action |\n|-------|--------|\n| LLM API rate limit | Script auto-retries with backoff; if persistent, reduce `--runs` |\n| Target file not found | Check path, must be readable prompt/skill file |\n| All experiments score 0 | Evals may be too strict — review eval definitions, loosen criteria |\n| Script crashes mid-run | Results already written to `results.tsv` are preserved; re-run continues |\n\n## Gotchas\n\n- 每次实验会调用 LLM 多次（runs x test_inputs x llm_evals），注意 API 用量 / Each experiment makes multiple LLM calls — watch API usage\n- LLM eval 本身有噪声，`--runs` 设高一点（5+）才有统计意义 / LLM evals are noisy, use 5+ runs for statistical significance\n- Rule evals 比 LLM evals 更稳定、更便宜，优先用 rule / Rule evals are more stable and cheaper — prefer them\n- Baseline 分数太低（< 20%）说明 eval 定义可能有问题，先修 eval / If baseline score is very low, fix evals first\n- 优化 prompt 不能解决架构问题（比如搜索 API 本身返回差结果） / Prompt optimization cannot fix architectural issues\n\nFile v0.2.5:README.md\n\n# brainforge-autoresearch\n\n> **Renamed.** Previously published as `openclaw-autoresearch` on GitHub and `autoresearch` on ClawHub. The old GitHub URL redirects to this repo automatically; the ClawHub skill name is still `autoresearch` for now (rename pending).\n\nAutonomous skill prompt optimizer based on Karpathy's autoresearch methodology.\n\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)\n[![ClawHub](https://img.shields.io/badge/ClawHub-autoresearch-blue)](https://clawhub.ai/zning1994/openclaw-autoresearch)\n[![brainforge](https://img.shields.io/badge/brainforge-marketplace-purple)](https://github.com/zning1994/brainforge)\n\n## What it does\n\nDefine what \"good\" means as binary evals, then let an agent loop to optimize your skill prompt automatically. The optimizer reads your current prompt, runs it against eval cases, analyzes failures, mutates the prompt, tests the mutation, and keeps or discards it -- repeating until evals pass or the experiment budget is exhausted.\n\nBased on [Andrej Karpathy's autoresearch](https://github.com/karpathy/autoresearch) and [Ole Lehmann's Claude Code adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill). Works with any AI coding agent: OpenClaw, Claude Code, Cursor, Cline.\n\n## Quick Start\n\n```bash\n# Claude Code (via brainforge marketplace)\n/plugin marketplace add zning1994/brainforge\n/plugin install brainforge-autoresearch@brainforge\n\n# Universal (npx skills)\nnpx skills add zning1994/brainforge-autoresearch\n\n# OpenClaw ClawHub (skill name is still \"autoresearch\" until rename)\nopenclaw skills install autoresearch\n\n# Standalone\ngit clone https://github.com/zning1994/brainforge-autoresearch\ncd brainforge-autoresearch\npython autoresearch.py --target ./my-skill/SKILL.md --evals eval.json\n```\n\n## How it works\n\n```\n                        +------------------+\n                        |  Read SKILL.md   |\n                        +--------+---------+\n                                 |\n                        +--------v---------+\n                   +--->|  Run baseline    |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    | Analyze failures |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |  Mutate prompt   |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |   Test mutation  |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   +----+ Keep or discard  |\n                        +------------------+\n```\n\n## Creating eval.json\n\nMinimal example with one rule eval and one LLM eval:\n\n```json\n[\n  {\n    \"name\": \"includes_greeting\",\n    \"input\": \"Say hello to the user\",\n    \"type\": \"rule\",\n    \"rule\": \"contains\",\n    \"expected\": \"hello\"\n  },\n  {\n    \"name\": \"tone_is_professional\",\n    \"input\": \"Draft a project update email\",\n    \"type\": \"llm\",\n    \"criteria\": \"The output uses a professional tone with no slang or emojis\"\n  }\n]\n```\n\nSee [eval-guide.md](eval-guide.md) for details.\n\n## Eval types\n\n| Type | Rule | Description |\n|------|------|-------------|\n| `rule` | `regex` | Match output against a regex pattern |\n| `rule` | `contains` | Output must contain the expected string |\n| `rule` | `not_contains` | Output must not contain the string |\n| `rule` | `banned_phrases` | Output must not contain any listed phrase |\n| `rule` | `word_count` | Output word count within min/max range |\n| `llm` | -- | LLM judges output against freeform criteria |\n\n## CLI Reference\n\n| Flag | Default | Description |\n|------|---------|-------------|\n| `--target` | `./SKILL.md` | Path to the skill prompt file to optimize |\n| `--evals` | `./eval.json` | Path to the eval definitions file |\n| `--provider` | auto-detect | LLM provider: `minimax`, `openai`, `anthropic` |\n| `--runs` | `3` | Number of runs per eval case per experiment |\n| `--max-experiments` | `10` | Maximum optimization iterations |\n| `--dashboard` | off | Generate an HTML dashboard of results |\n| `--output-dir` | `./results` | Directory for output artifacts |\n| `--verbose` | off | Print detailed logs to stderr |\n\n## LLM Providers\n\nAuto-detection order:\n\n1. `MINIMAX_API_KEY`\n2. `OPENAI_API_KEY`\n3. `ANTHROPIC_API_KEY`\n\nCustom endpoint support via `OPENAI_BASE_URL` (works with any OpenAI-compatible API).\n\n## Output\n\n| File | Description |\n|------|-------------|\n| `results.tsv` | Tab-separated eval scores per experiment |\n| `changelog.md` | Human-readable log of each mutation and its effect |\n| `results.json` | Full structured results for programmatic use |\n| `dashboard.html` | Visual dashboard (when `--dashboard` is set) |\n| `SKILL.md.baseline` | Backup of the original prompt before optimization |\n\n## Credits\n\n- [Andrej Karpathy's autoresearch](https://github.com/karpathy/autoresearch) -- the original methodology\n- [Ole Lehmann's Claude Code adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill) -- adapted for prompt optimization with coding agents\n\n## License\n\n[MIT](LICENSE)\n\nFile v0.2.5:_meta.json\n\n{\n  \"ownerId\": \"kn7emz9j02vyab03rzn6fx4q7582xae6\",\n  \"slug\": \"brainharness-autoresearch\",\n  \"version\": \"0.2.5\",\n  \"publishedAt\": 1776911507121\n}\n\nFile v0.2.5:CHANGELOG.md\n\n# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nFormat based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).\n\n## [0.2.5] - 2026-04-23\n\n### Changed\n\n- **Renamed repo and skill**: `openclaw-autoresearch` → `brainforge-autoresearch`\n- SKILL.md `name` field: `autoresearch` → `brainforge-autoresearch`\n- Updated README, plugin.json, and SKILL.md metadata to point at the new repo URL\n- Added to the [brainforge](https://github.com/zning1994/brainforge) Claude Code plugin marketplace\n- Published to ClawHub as `brainforge-autoresearch`; old slug `openclaw-autoresearch` merged in as redirect\n\n### Compatibility\n\n- GitHub: old URL `https://github.com/zning1994/openclaw-autoresearch` auto-redirects\n- ClawHub: `openclaw skills install openclaw-autoresearch` continues to work via ClawHub slug merge/redirect\n- No behavior changes to the optimizer itself\n\n## [0.2.4] - 2026-03-29\n\n### Fixed\n\n- SKILL.md: `pass`/`fail` → `pass_description`/`fail_description` in LLM eval docs and examples (field names didn't match actual API)\n- SKILL.md: `contains`/`not_contains` parameter documented as `value` (string) → `values` (list) to match actual usage\n- SKILL.md: LLM eval table now includes required `type` and `name` fields\n\n### Changed\n\n- SKILL.md self-optimized via autoresearch (2 rounds: MiniMax 48.6%→68.1%, Claude 80.6%→91.7%)\n- Procedure rewritten: added 3-step summary upfront, renamed steps for clarity, added actionable result-review guidance\n- Example section restructured: removed \"confirm target\" preamble, simplified to \"create eval → run → review\" flow\n- Added `self-eval.json` for self-optimization (dogfooding eval definition)\n\n### Tested\n\n- Self-optimization: Round 1 (MiniMax M2.7, 15 exp, 3 keep) and Round 2 (Claude, 15 exp, 1 keep) independently discovered same doc bugs\n\n## [0.2.3] - 2026-03-25\n\n### Added\n\n- `--timeout` CLI flag to control HTTP timeout per LLM call (default: 180s)\n  - Large prompts (e.g., 14KB style guides) need longer timeouts for mutation calls\n  - Previously hardcoded, now user-configurable\n\n### Changed\n\n- Default HTTP timeout increased from 60s to 180s (was causing mutation failures on Anthropic API with large prompts)\n- Timeout is now stored on the provider instance, passed through `detect_provider()`\n\n## [0.2.2] - 2026-03-25\n\n### Fixed\n\n- `.gitignore`: removed incorrect entries (`.git`, `.github`), added `autoresearch-*/` and `*.baseline`\n- README: added ClawHub badge and link, fixed install slug to `openclaw-autoresearch`\n\n## [0.2.1] - 2026-03-25\n\n### Fixed\n\n- ClawHub security flag: changed `requires.env` (all required) to `requires.anyEnv` (any one suffices)\n- Declared optional env vars `OPENAI_BASE_URL` / `OPENAI_API_BASE` in metadata\n\n## [0.2.0] - 2026-03-25\n\n### Fixed\n\n- MiniMax extended thinking support — models with thinking blocks (M2.7, Claude) no longer crash the parser\n- Fallback to thinking content when text block is missing (thinking exhausts `max_tokens`)\n- LLM judge `max_tokens` increased from 16 to 256 (thinking models need headroom)\n- Rule type field: accept both `\"rule\"` and `\"check\"` keys for backwards compatibility\n- `contains`/`not_contains`: accept both `\"values\"` (list) and `\"value\"` (string)\n- LLM eval fields: accept both `\"pass_description\"`/`\"fail_description\"` and `\"pass\"`/`\"fail\"`\n- `test_inputs`: accept both plain strings and `{\"name\": ..., \"input\": ...}` objects\n- `contains` rule now supports `\"match\": \"all\"` mode (require all values present)\n- Convergence counter double-increment bug fixed\n- `regex` rule now uses `re.MULTILINE` for correct `^`/`$` matching\n- CJK-aware word counting in `word_count` rule (counts Chinese/Japanese/Korean characters individually)\n\n### Added\n\n- `--model` CLI flag to override the default model per provider\n- JSON fallback: `runs_per_experiment` and `max_experiments` from eval.json used when CLI defaults unchanged\n- Shared `_extract_text()` helper for Anthropic-style response parsing across all providers\n\n### Tested\n\n- Successfully optimized `brain-search` skill: 37.5% → 54.2% pass rate in 5 experiments with MiniMax M2.7\n\n## [0.1.0] - 2026-03-24\n\n### Added\n\n- Core `autoresearch.py` script — autonomous skill prompt optimizer\n  - Zero external dependencies (Python 3.9+ stdlib only)\n  - LLM providers: MiniMax, OpenAI, Anthropic (auto-detect from env vars)\n  - Custom endpoint support via `OPENAI_BASE_URL`\n  - `--model` flag to override default model per provider\n- Hybrid eval system\n  - Rule-based evals: `regex`, `banned_phrases`, `word_count`, `contains`, `not_contains`\n  - `contains` supports `match: \"all\"` and `match: \"any\"` modes\n  - LLM-as-judge evals with binary YES/NO scoring\n  - CJK-aware word counting for Chinese/Japanese/Korean text\n- Experiment loop\n  - Baseline measurement before any changes\n  - One mutation per experiment (targeted, not bulk rewrites)\n  - Automatic keep/discard based on score comparison\n  - Convergence exit: 95%+ pass rate for 3 consecutive experiments\n  - Budget cap via `--max-experiments`\n- Output artifacts\n  - `results.tsv` — tab-separated score log\n  - `changelog.md` — detailed mutation history with reasoning\n  - `results.json` — structured data for tooling\n  - `dashboard.html` — self-contained Chart.js dashboard (opt-in via `--dashboard`)\n  - `SKILL.md.baseline` — original skill backup\n- `SKILL.md` — agent instructions for OpenClaw / Claude Code / Cursor / Cline\n- `eval-guide.md` — practical guide for writing binary eval criteria\n- Example eval configs: `weekly-report.json`, `search-skill.json`\n- Compatible with `npx skills add` (Vercel Labs skills ecosystem)\n\n[0.2.4]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.4\n[0.2.3]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.3\n[0.2.2]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.2\n[0.2.1]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.1\n[0.2.0]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.2.0\n[0.1.0]: https://github.com/zning1994/openclaw-autoresearch/releases/tag/v0.1.0\n\nFile v0.2.5:eval-guide.md\n\n# Eval Guide: Writing Effective Eval Criteria\n\nHow to write eval criteria that actually improve your skill's output. This guide covers the hybrid rule+LLM eval system used by brainforge-autoresearch (previously `openclaw-autoresearch`).\n\n## The Golden Rule\n\n**Binary yes/no only. No scales.**\n\nEvery eval must produce a pass or fail. Not \"7/10\", not \"mostly good\", not \"needs improvement\". Pass or fail.\n\nWhy: Scales are subjective (your 7 is my 5), hard to optimize against, and create noise. Binary forces you to define exactly what \"good\" means.\n\nBad: \"Rate the tone from 1-5\"\nGood: \"Does the output use a professional tone without slang or emojis?\"\n\n## Rule-Based Evals\n\nRule-based evals cost zero tokens, run instantly, and are 100% consistent. Use them whenever possible.\n\n### `regex`\n\nPattern matching against the output text.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_date_format\",\n  \"rule\": \"regex\",\n  \"pattern\": \"\\\\d{4}-\\\\d{2}-\\\\d{2}\",\n  \"description\": \"Output contains a date in YYYY-MM-DD format\"\n}\n```\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_markdown_headers\",\n  \"rule\": \"regex\",\n  \"pattern\": \"^#{1,3} .+\",\n  \"description\": \"Output uses markdown headers\"\n}\n```\n\nUseful for: format validation, structure checks, required patterns.\n\n### `banned_phrases`\n\nExact phrases that must not appear in the output.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_filler\",\n  \"rule\": \"banned_phrases\",\n  \"phrases\": [\"as an AI\", \"I cannot\", \"I'm sorry but\", \"it depends\", \"各方面\", \"总的来说\"],\n  \"description\": \"No AI disclaimers or vague filler phrases\"\n}\n```\n\nUseful for: removing AI-speak, enforcing brand voice, blocking lazy language.\n\n### `word_count`\n\nMin and/or max word limits. For Chinese text, counts characters.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"length_check\",\n  \"rule\": \"word_count\",\n  \"min\": 100,\n  \"max\": 500,\n  \"description\": \"Output is between 100-500 words\"\n}\n```\n\nUseful for: preventing one-line answers, keeping outputs concise, matching format requirements.\n\n### `contains`\n\nRequired keywords or phrases that must appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_source\",\n  \"rule\": \"contains\",\n  \"values\": [\"Source:\", \"Reference:\", \"来源:\"],\n  \"match\": \"any\",\n  \"description\": \"Output cites at least one source\"\n}\n```\n\n`match` can be `\"any\"` (at least one) or `\"all\"` (every value must appear).\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_all_sections\",\n  \"rule\": \"contains\",\n  \"values\": [\"## Summary\", \"## Action Items\", \"## Risks\"],\n  \"match\": \"all\",\n  \"description\": \"Output has all required sections\"\n}\n```\n\nUseful for: enforcing structure, required sections, key terminology.\n\n### `not_contains`\n\nPhrases or patterns that must NOT appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_placeholder\",\n  \"rule\": \"not_contains\",\n  \"values\": [\"TODO\", \"TBD\", \"FIXME\", \"[insert\", \"[placeholder\"],\n  \"description\": \"No placeholder text left in output\"\n}\n```\n\nUseful for: catching incomplete outputs, blocking specific content.\n\n## LLM-Based Evals\n\nLLM evals handle semantic judgments that rules cannot express. They cost tokens and have slight variance, so use them only when necessary.\n\n### When to Use LLM Evals\n\nUse LLM when you need to judge:\n- Accuracy against source material (hallucination detection)\n- Logical coherence or reasoning quality\n- Whether advice is actionable vs. generic\n- Tone, style, or audience-appropriateness\n- Semantic completeness (did it cover all key points?)\n\nDo NOT use LLM for things rules can handle:\n- Word count, format checks, required sections (use rules)\n- Presence of specific keywords (use contains)\n- Forbidden phrases (use banned_phrases)\n\n### Writing Good LLM Eval Questions\n\n```json\n{\n  \"type\": \"llm\",\n  \"name\": \"no_hallucination\",\n  \"question\": \"Compare the output against the provided input data. Does every claim in the output have a corresponding fact in the input? Ignore formatting and phrasing differences.\",\n  \"pass_description\": \"Every factual claim maps to input data. No invented statistics, dates, or details.\",\n  \"fail_description\": \"Contains at least one claim not supported by the input data.\"\n}\n```\n\nKey principles:\n1. **Be specific about what to check.** Not \"is this good?\" but \"does every claim have a source?\"\n2. **Tell the LLM what to compare against.** \"Compare against the input data\" or \"based on the search results provided.\"\n3. **Define pass and fail concretely.** The LLM judge needs clear criteria, not vibes.\n4. **One judgment per eval.** Don't ask \"is it accurate AND well-structured?\" -- split into two evals.\n\n### Bad vs Good Examples\n\nBad:\n```json\n{\n  \"question\": \"Is this a good summary?\",\n  \"pass_description\": \"It's good\",\n  \"fail_description\": \"It's bad\"\n}\n```\n\nGood:\n```json\n{\n  \"question\": \"Does the summary capture the 3 most important findings from the search results, without adding information not present in the results?\",\n  \"pass_description\": \"All 3 key findings are mentioned. No information is fabricated beyond what the search results contain.\",\n  \"fail_description\": \"Misses one or more key findings, or includes claims not supported by the search results.\"\n}\n```\n\n## Examples by Skill Type\n\n### Text/Copy Skills (newsletters, reports, summaries)\n\nTypical eval mix:\n- `banned_phrases`: remove filler (\"总的来说\", \"as mentioned above\", \"综上所述\")\n- `word_count`: enforce length constraints\n- `contains`: required sections or keywords\n- LLM: accuracy check against source material\n- LLM: actionability or specificity check\n\n### Search Skills (web search, knowledge search)\n\nTypical eval mix:\n- `contains`: must cite URLs or sources\n- `not_contains`: no hallucination markers (\"[Source needed]\", made-up URLs)\n- LLM: does summary reflect actual search results\n- LLM: is output well-structured with clear sections\n\n### Code Generation Skills\n\nTypical eval mix:\n- `regex`: output contains code blocks (`` ```\\w+ ``)\n- `not_contains`: no placeholder code (\"pass\", \"# TODO\")\n- `contains`: includes error handling patterns (\"try\", \"except\" or \"catch\")\n- LLM: does the code solve the stated problem\n\n### Translation Skills\n\nTypical eval mix:\n- `banned_phrases`: no untranslated source language fragments\n- `word_count`: reasonable length ratio to source\n- LLM: semantic equivalence to source text\n- LLM: natural fluency in target language (no translationese)\n\n### Data Extraction Skills\n\nTypical eval mix:\n- `regex`: output matches expected data format (JSON, CSV, etc.)\n- `contains`: all required fields present\n- LLM: extracted values are accurate against source\n- LLM: edge cases handled (missing data, ambiguous values)\n\n## Common Mistakes\n\n### Too many evals (> 6)\n\nMore evals means more constraints. Past 6, the optimizer starts gaming individual evals at the expense of overall quality. The prompt becomes a checklist-satisfier instead of genuinely good.\n\n**Fix**: Pick the 4-5 that matter most. If you can't decide, you don't understand what \"good\" means for this skill yet.\n\n### Overlapping evals\n\nIf two evals test similar things, a failure on one is always a failure on the other. You're just adding noise.\n\nExample of overlap:\n- Eval A: \"No filler language\"\n- Eval B: \"Every sentence adds new information\"\n\nThese catch the same problem. Pick one.\n\n### Too vague\n\n\"Is the output helpful?\" -- helpful to whom, for what? The LLM judge can't read your mind.\n\n**Fix**: Replace with \"Does the output provide at least 2 specific, actionable next steps the reader can take this week?\"\n\n### Too narrow\n\n\"Does the output mention React hooks?\" -- this only works for one specific input. It will fail or be irrelevant for other test cases.\n\n**Fix**: Evals should apply to ALL test inputs. If it's input-specific, it's not an eval -- it's a test assertion.\n\n### Unmeasurable by LLM\n\n\"Will this output generate more revenue?\" -- no LLM can judge this. Stick to things observable in the text.\n\n**Fix**: Replace with proxies: \"Does the output include a clear call-to-action?\" or \"Are the benefits described in concrete terms with numbers?\"\n\n## The 3-Question Test\n\nBefore adding any eval, ask:\n\n1. **Can I explain pass/fail to a stranger in one sentence?**\n   If not, the eval is too vague.\n\n2. **Will this eval produce the same result on every test input?**\n   If it only applies to some inputs, it's too narrow.\n\n3. **Can a rule handle this instead?**\n   If yes, use a rule. Save LLM evals for semantic judgments.\n\n## Template\n\n```json\n{\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"short_snake_case_name\",\n      \"rule\": \"contains|banned_phrases|word_count|regex|not_contains\",\n      \"description\": \"One sentence: what this checks\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"short_snake_case_name\",\n      \"question\": \"Specific question for the LLM judge to answer about the output.\",\n      \"pass_description\": \"Concrete description of what a passing output looks like.\",\n      \"fail_description\": \"Concrete description of what a failing output looks like.\"\n    }\n  ],\n  \"test_inputs\": [\n    {\n      \"name\": \"descriptive_scenario_name\",\n      \"input\": \"The actual input that will be sent to the skill.\"\n    }\n  ]\n}\n```\n\nStart with 2-3 test inputs and 3-4 evals. Add more only after running a few optimization rounds and seeing where the gaps are.\n\nFile v0.2.5:skill-card.md\n\n## Description:\n\nBrainforge Autoresearch guides an agent through automated prompt optimization for skill prompts using eval definitions, iterative experiments, and result review.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[zning1994](https://clawhub.ai/user/zning1994)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nDevelopers and agent builders use this skill to improve skill prompts by defining binary evals, running controlled prompt experiments, and reviewing the best-performing mutation before applying it. It is suited for systematic prompt quality work, not for fixing code defects in a skill implementation.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: The optimizer can rewrite the target prompt file with model-generated content.\n\nMitigation: Run on a disposable branch or copy, then inspect diffs and result artifacts before keeping changes.\n\nRisk: Prompt and eval data may be sent to the selected external LLM provider.\n\nMitigation: Avoid sensitive prompts unless the provider and endpoint are approved for that data.\n\nRisk: The optional dashboard uses a remote Chart.js dependency.\n\nMitigation: Avoid the dashboard or review the remote dependency when offline or self-contained reporting matters.\n\n## Reference(s):\n\n- [ClawHub skill page](https://clawhub.ai/zning1994/skills/brainforge-autoresearch)\n- [Project homepage](https://github.com/zning1994/brainforge-autoresearch)\n- [Karpathy autoresearch pattern](https://github.com/karpathy/autoresearch)\n- [Claude Code autoresearch adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill)\n- [Eval guide](artifact/eval-guide.md)\n- [Changelog](artifact/CHANGELOG.md)\n\n## Skill Output:\n\n**Output Type(s):** [Guidance, Markdown, Code, Shell commands, Configuration, Files]\n\n**Output Format:** [Markdown guidance with JSON and bash code blocks; runtime artifacts may include TSV, JSON, Markdown, HTML, and prompt backup files.]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [Can create eval definitions, command invocations, prompt changes, results.tsv, results.json, changelog.md, dashboard.html, and SKILL.md.baseline.]\n\n## Skill Version(s):\n\n0.2.5 (source: server release metadata, SKILL.md frontmatter, CHANGELOG dated 2026-04-23)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.\n\nFile v0.2.5:examples/search-skill.json\n\n{\n  \"skill_description\": \"Perform a web search on a given query and return a structured summary of findings with source citations. The output should be accurate to search results, well-organized, and immediately useful.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"technical_english\",\n      \"input\": \"What are the main differences between SQLite WAL mode and journal mode? When should I use each?\"\n    },\n    {\n      \"name\": \"current_events_chinese\",\n      \"input\": \"2026年3月 OpenAI 和 Anthropic 最新发布了什么模型？各自有什么特点？\"\n    },\n    {\n      \"name\": \"comparison_query\",\n      \"input\": \"Compare Cloudflare Workers vs AWS Lambda for edge computing: cold start times, pricing, and language support\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"has_source_citation\",\n      \"rule\": \"regex\",\n      \"pattern\": \"https?://[^\\\\s)\\\\]]+\",\n      \"description\": \"Output contains at least one URL as a source citation\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_hallucination_markers\",\n      \"rule\": \"not_contains\",\n      \"values\": [\n        \"[Source needed]\",\n        \"[citation needed]\",\n        \"I couldn't find\",\n        \"I don't have access\",\n        \"As of my knowledge cutoff\",\n        \"I'm not able to browse\"\n      ],\n      \"description\": \"No placeholder citations or disclaimers about inability to search\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"captures_key_findings\",\n      \"question\": \"Based on the search query, does the output identify and summarize the most important findings? Check that: (1) the core question is directly answered, not deflected, (2) key distinctions or comparisons requested are addressed, and (3) the information appears factually consistent (no self-contradictions).\",\n      \"pass_description\": \"The output directly answers the query with specific, substantive information. Key points are covered (e.g., for a comparison query, both sides are discussed with concrete details like numbers or features). No self-contradictions.\",\n      \"fail_description\": \"The output is vague, generic, or fails to address the core question. For comparison queries, one side may be missing or described only in generalities. Contains self-contradicting statements.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"clear_structure\",\n      \"question\": \"Is the output organized with clear visual structure? Look for: distinct sections or categories (using headers, bullet points, or numbered lists), a logical flow from overview to details, and separation between different sub-topics.\",\n      \"pass_description\": \"Output uses headers, bullets, or numbered lists to separate information into logical sections. A reader can scan the output and find specific information without reading everything.\",\n      \"fail_description\": \"Output is a wall of text, or uses formatting inconsistently. Information about different sub-topics is mixed together without clear separation.\"\n    }\n  ]\n}\n\nFile v0.2.5:examples/weekly-report.json\n\n{\n  \"skill_description\": \"Generate a weekly project status report from raw project updates. Output should be structured, accurate, and highlight risks clearly.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"multi_project_normal\",\n      \"input\": \"本周项目进展：\\n1. 用户系统重构：完成了数据库迁移脚本编写，已在 staging 环境测试通过。下周计划上线生产环境。风险：生产数据量是 staging 的 50 倍，迁移预计需要 4 小时停机。\\n2. 支付网关集成：对接了 Stripe webhook，完成了 3 种支付方式（信用卡、Apple Pay、Google Pay）的联调。剩余：退款流程还没做，需要 2 天。\\n3. 移动端 App：iOS 已提交 App Store 审核，Android 在做性能优化。Android 冷启动时间从 3.2 秒降到 1.8 秒。风险：iOS 审核可能因为隐私政策描述被拒。\"\n    },\n    {\n      \"name\": \"single_project_delayed\",\n      \"input\": \"本周项目进展：\\nAPI Gateway 重写项目：原计划本周完成所有路由迁移，实际只完成了 60%（18/30 个路由）。原因：发现旧系统有 5 个未文档化的中间件，需要逆向工程理解逻辑后才能迁移。团队两人本周请假，人力不足。下周需要从其他组借调 1 名后端开发。风险：如果下周无法补上进度，整体上线时间将从 4 月 15 日推迟到 4 月 30 日。已与产品经理沟通过延期可能性。\"\n    },\n    {\n      \"name\": \"infra_project_chinese\",\n      \"input\": \"本周项目进展：\\n1. K8s 集群升级：从 1.27 升级到 1.29，已完成 dev 和 staging 集群。生产集群计划下周二滚动升级。已验证所有 Helm chart 兼容性，发现 ingress-nginx 需要从 4.7 升到 4.9。风险：升级期间可能有 2-3 分钟的 ingress 流量中断。\\n2. 监控告警优化：重新调整了 CPU/内存告警阈值，误报率从每天 15 次降到 2 次。新增了 SLO 面板，覆盖 3 个核心服务的 P99 延迟和可用性。\\n3. 成本优化：将 3 台 m5.xlarge 节点替换为 m6g.xlarge（ARM），预计月度成本降低 $420。已运行 1 周无异常。\"\n    },\n    {\n      \"name\": \"mixed_good_and_bad\",\n      \"input\": \"本周项目进展：\\n1. 搜索服务：Elasticsearch 索引重建完成，搜索延迟从 P95 800ms 降到 200ms。新增了同义词词典和拼音搜索支持。本周无风险。\\n2. 数据管道：Spark 作业频繁 OOM，本周花了 3 天排查。根因是上游数据量突增 3 倍（从 200GB/天到 600GB/天）但没有通知我们。临时方案：增加了 executor 内存到 16GB。长期方案：需要重新设计分区策略，预计 2 周。风险：如果上游数据量继续增长，当前临时方案也会撑不住。\\n3. CI/CD：Jenkins 迁移到 GitHub Actions 完成了 80%（16/20 个 pipeline）。剩余 4 个涉及自建 runner 的 pipeline 需要额外配置。\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"has_status_keywords\",\n      \"rule\": \"contains\",\n      \"values\": [\"进度\", \"风险\", \"下周\"],\n      \"match\": \"all\",\n      \"description\": \"Report must contain status tracking keywords: progress, risks, and next week plan\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_vague_language\",\n      \"rule\": \"banned_phrases\",\n      \"phrases\": [\n        \"基本完成\",\n        \"差不多\",\n        \"还行\",\n        \"大概\",\n        \"应该没问题\",\n        \"可能会好\",\n        \"总体来说还可以\",\n        \"各方面都不错\",\n        \"正在推进中\"\n      ],\n      \"description\": \"No vague or non-committal status language\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"report_length\",\n      \"rule\": \"word_count\",\n      \"min\": 200,\n      \"max\": 800,\n      \"description\": \"Report is between 200-800 characters (concise but complete)\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"no_hallucination\",\n      \"question\": \"Compare every factual claim in the weekly report output (dates, numbers, percentages, project names, technical details) against the input data. Is there any information in the output that was not present in or directly derivable from the input?\",\n      \"pass_description\": \"Every fact in the report (numbers, dates, project names, status descriptions) can be traced back to the input. No invented details, no embellished metrics, no assumed timelines.\",\n      \"fail_description\": \"The report contains at least one claim that cannot be found in or reasonably derived from the input data -- for example, an invented completion percentage, a made-up deadline, or a fabricated metric.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"actionable_risks\",\n      \"question\": \"Look at each risk mentioned in the output. Does every risk description include: (1) what specifically could go wrong, (2) what the impact would be, and (3) what mitigation or next step is suggested?\",\n      \"pass_description\": \"Each risk is specific (not generic like 'might have issues'), states the potential impact (delay, cost, outage, etc.), and suggests or implies a concrete next step.\",\n      \"fail_description\": \"At least one risk is vague (e.g., '可能有风险' without specifics), missing impact assessment, or has no suggested mitigation.\"\n    }\n  ]\n}\n\nFile v0.2.5:self-eval.json\n\n{\n  \"skill_description\": \"Guide an AI agent on how to use autoresearch to optimize skill prompts. The agent should correctly explain the workflow, create proper eval definitions, formulate correct CLI commands, interpret results, and know when NOT to use autoresearch.\",\n  \"system_context\": \"You are an AI assistant with access to the autoresearch tool. When the user asks about optimizing skills or prompts, follow the instructions in your skill definition to help them.\",\n  \"test_inputs\": [\n    {\n      \"name\": \"optimize_skill_chinese\",\n      \"input\": \"我的 brain-search skill 搜索结果总是缺少来源链接，帮我优化一下这个 skill 的 prompt\"\n    },\n    {\n      \"name\": \"write_eval_criteria\",\n      \"input\": \"I want to optimize my weekly-report skill. The reports are often too long and miss action items. How do I write the eval.json for this?\"\n    },\n    {\n      \"name\": \"interpret_results\",\n      \"input\": \"autoresearch 跑完了，结果：baseline 45%, experiment 3 到了 72%, experiment 8 到了 91% 然后就停了。这个结果怎么解读？我要怎么做？\"\n    },\n    {\n      \"name\": \"should_not_use\",\n      \"input\": \"My ingest skill crashes when I upload a PDF file larger than 10MB. Can you run autoresearch to fix it?\"\n    }\n  ],\n  \"evals\": [\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_hallucinated_flags\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"--optimize\", \"--auto-fix\", \"--improve\", \"--tune\", \"--benchmark\", \"--score\"],\n      \"description\": \"Must not invent CLI flags that don't exist\"\n    },\n    {\n      \"type\": \"rule\",\n      \"name\": \"no_wrong_provider_names\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"--provider google\", \"--provider gemini\", \"--provider claude\", \"--provider gpt\"],\n      \"description\": \"Provider names are minimax/openai/anthropic only — not model names\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"cli_command_accuracy\",\n      \"question\": \"Evaluate the CLI command usage in the response: (1) If the user asks to optimize a skill or run autoresearch, the response MUST include a CLI command with both '--target' and '--evals' flags pointing to specific file paths. (2) If the user asks about interpreting results or asks about a code bug, the response should NOT necessarily include a full CLI command — that's fine. (3) If a CLI command IS present, verify it uses only valid flags: --target, --evals, --provider, --model, --runs, --max-experiments, --dashboard, --output-dir, --timeout, --verbose.\",\n      \"pass_description\": \"Optimization requests include correct CLI with --target and --evals. Result interpretation and code bug questions are not penalized for lacking CLI commands. Any CLI commands shown use only valid flags.\",\n      \"fail_description\": \"Optimization request is missing CLI command or missing --target/--evals. Or a CLI command uses invalid/hallucinated flags.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"appropriate_response\",\n      \"question\": \"Does the response appropriately handle the user's request? (1) For optimization requests: outlines the workflow (create eval.json → run autoresearch → review results), mentions eval definitions. (2) For eval writing questions: shows correct eval JSON structure with 'type' and 'name' fields, uses valid rule types (regex, banned_phrases, word_count, contains, not_contains) or llm type with 'question'/'pass_description'/'fail_description'. (3) For result interpretation: explains what the scores and convergence mean, suggests concrete next steps (apply the prompt, review the mutations, adjust evals). (4) For code bug reports: clearly states autoresearch is NOT the right tool and suggests fixing the code/debugging instead.\",\n      \"pass_description\": \"Response matches the request type with specific, actionable guidance. Each scenario gets the right kind of help.\",\n      \"fail_description\": \"Response mismatches: suggests autoresearch for a code bug, gives vague advice, omits eval workflow for optimization requests, or fails to interpret scores when asked.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"eval_json_correctness\",\n      \"question\": \"If the response includes eval JSON examples: (1) Does each eval have both 'type' and 'name' fields? (2) Do rule evals use one of: regex, banned_phrases, word_count, contains, not_contains? (3) Do LLM evals have 'question', 'pass_description', and 'fail_description' fields (NOT 'pass'/'fail')? (4) Are the evals tailored to the user's specific problem, not just generic examples? If no eval JSON is shown (e.g., for result interpretation or code bug questions), this eval passes automatically.\",\n      \"pass_description\": \"Eval JSON is well-formed with correct field names, OR no eval JSON is needed for this type of question. Evals are relevant to the user's stated problem.\",\n      \"fail_description\": \"Eval JSON uses wrong field names (e.g., 'pass' instead of 'pass_description'), invalid rule types, missing required fields, or is clearly generic copy-paste unrelated to the user's problem.\"\n    },\n    {\n      \"type\": \"llm\",\n      \"name\": \"workflow_completeness\",\n      \"question\": \"For optimization and eval-writing requests: does the response cover the full autoresearch workflow? Check for: (1) Creating eval.json with test_inputs and evals, (2) Running the CLI command, (3) Reviewing/interpreting results. For result interpretation: does it explain scores AND suggest next steps? For code bugs: does it redirect away from autoresearch clearly? A complete response covers all relevant steps, not just one.\",\n      \"pass_description\": \"Response covers all relevant workflow steps for the given request type. Nothing critical is missing.\",\n      \"fail_description\": \"Response covers only part of the workflow (e.g., shows eval JSON but forgets the run command, or shows the command but skips eval creation).\"\n    }\n  ],\n  \"runs_per_experiment\": 3,\n  \"max_experiments\": 15\n}","readmeExcerpt":"Skill: BrainHarness Autoresearch Owner: zning1994 Summary: Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy","codeSnippets":[],"executableExamples":[{"language":"json","snippet":"{\n  \"test_inputs\": [\n    \"search for latest AI agent frameworks\",\n    \"find news about LLM inference optimization\",\n    \"搜一下 transformer 架构的最新进展\"\n  ],\n  \"evals\": [\n    {\n      \"name\": \"has_sources\",\n      \"type\": \"rule\",\n      \"rule\": \"regex\",\n      \"pattern\": \"(https?://|Source:|来源:)\"\n    },\n    {\n      \"name\": \"no_hallucinated_urls\",\n      \"type\": \"rule\",\n      \"rule\": \"banned_phrases\",\n      \"phrases\": [\"example.com\", \"placeholder.url\"]\n    },\n    {\n      \"name\": \"sufficient_detail\",\n      \"type\": \"rule\",\n      \"rule\": \"word_count\",\n      \"min\": 50,\n      \"max\": 500\n    },\n    {\n      \"name\": \"contains_summary\",\n      \"type\": \"rule\",\n      \"rule\": \"contains\",\n      \"values\": [\"summary\", \"key findings\", \"结论\"]\n    },\n    {\n      \"name\": \"no_apology_prefix\",\n      \"type\": \"rule\",\n      \"rule\": \"not_contains\",\n      \"values\": [\"I apologize\", \"I'm sorry, but\"]\n    },\n    {\n      \"name\": \"actionable_output\",\n      \"type\": \"llm\",\n      \"question\": \"Does the response provide actionable information the user can immediately use (links, specific facts, concrete next steps)?\",\n      \"pass_description\": \"The response contains specific actionable items like URLs, concrete facts, or clear next steps\",\n      \"fail_description\": \"The response is vague, generic, or lacks specific actionable information\"\n    }\n  ]\n}"},{"language":"bash","snippet":"python autoresearch.py \\\n  --target ../workspace/skills/brain-search/SKILL.md \\\n  --evals eval.json \\\n  --provider minimax \\\n  --runs 5 \\\n  --max-experiments 30 \\\n  --dashboard"},{"language":"text","snippet":"experiment_id  parent_id  mutation_description  avg_score  pass_rate  evals_detail  prompt_diff"},{"language":"bash","snippet":"cat results.tsv | sort -k4 -nr | head -5"},{"language":"text","snippet":"User: brain-search 的搜索结果经常缺少来源链接，帮我优化一下\n\n完整流程:\n\n1. 创建 eval.json:\n   {\n     \"test_inputs\": [\n       \"search for latest news on OpenAI\",\n       \"搜一下最新的 AI 芯片进展\",\n       \"find recent papers on RAG optimization\",\n       \"what happened with Anthropic this week\",\n       \"查查 GPU 价格趋势\"\n     ],\n     \"evals\": [\n       {\n         \"name\": \"has_urls\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"min_2_sources\",\n         \"type\": \"rule\",\n         \"rule\": \"regex\",\n         \"pattern\": \"https?://[^\\\\s]+.*https?://[^\\\\s]+\"\n       },\n       {\n         \"name\": \"structured_output\",\n         \"type\": \"llm\",\n         \"question\": \"Is the output well-structured with clear sections?\",\n         \"pass_description\": \"Output uses clear structure like bullets or headers\",\n         \"fail_description\": \"Output is a wall of text without clear structure\"\n       }\n     ]\n   }\n\n2. 运行命令:\n   python autoresearch.py \\\n     --target ../workspace/skills/brain-search/SKILL.md \\\n     --evals eval.json \\\n     --runs 5 \\\n     --max-experiments 20\n\n3. 查看并应用结果:\n   - 检查 results.tsv 找最高分变体\n   - 查看 mutation_description 了解关键改动\n   - 将最佳 prompt 应用到原始 SKILL.md"},{"language":"bash","snippet":"# Claude Code (via brainharness marketplace)\n/plugin marketplace add zning1994/brainharness\n/plugin install brainharness-autoresearch@brainharness\n\n# Universal (npx skills)\nnpx skills add zning1994/brainharness-autoresearch\n\n# OpenClaw ClawHub\nnpx clawhub@0.23.3 install brainharness-autoresearch\n\n# Standalone\ngit clone https://github.com/zning1994/brainharness-autoresearch\ncd brainharness-autoresearch\npython autoresearch.py --target ./my-skill/SKILL.md --evals eval.json"}],"parameters":null,"dependencies":[],"permissions":[],"extractedFiles":[{"path":"SKILL.md","content":"---\nname: brainharness-autoresearch\ndescription: >-\n  Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt.\n  Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\",\n  \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\",\n  \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\".\n  Runs automated prompt experiments using the Karpathy autoresearch pattern.\nversion: 0.3.0\nmetadata:\n  author: zning1994\n  openclaw:\n    requires:\n      anyEnv:\n        - MINIMAX_API_KEY\n        - OPENAI_API_KEY\n        - ANTHROPIC_API_KEY\n      anyBins:\n        - python3\n        - python\n    primaryEnv: OPENAI_API_KEY\n    optionalEnv:\n      - OPENAI_BASE_URL\n      - OPENAI_API_BASE\n    homepage: https://github.com/zning1994/brainharness-autoresearch\n    os:\n      - macos\n      - linux\n---\n\n# brainharness-autoresearch\n\n> Previously published as `brainforge-autoresearch`, with earlier names `autoresearch` / `openclaw-autoresearch`. Renamed for the BrainHarness marketplace; functionality unchanged.\n\nAutonomous prompt optimization for AI agent skills. Runs controlled experiments to find better prompt variants using the [Karpathy autoresearch pattern](https://github.com/karpathy/autoresearch): generate hypothesis, mutate prompt, evaluate, repeat.\n\n## When to use\n\n- 用户说\"优化一下这个 skill\" / User says \"optimize this skill's prompt\"\n- 用户要对比不同 prompt 版本的效果 / User wants to benchmark prompt variants\n- 用户说\"run autoresearch on X\" / \"eval skill X\" / \"improve skill X\"\n- 用户对 skill 输出质量不满，想系统性改进 / User is unhappy with skill output quality and wants systematic improvement\n\n**Do not use:**\n- 一次性的小改动（直接改 prompt 即可） / One-off prompt tweaks — just edit the prompt directly\n- 调试某个特定失败 case / Debugging a specific failure — investigate the root cause instead\n- Skill 脚本本身有 bug（代码逻辑问题不是 prompt 问题） / Skill script has a bug — fix the code, not the prompt\n\n## Requirements\n\n- Python 3.10+\n- `autoresearch.py` script in the skill directory\n- LLM API access (MiniMax, OpenAI, or Anthropic)\n- Target skill must have a prompt file (SKILL.md, SYSTEM.md, or similar)\n\n## Procedure\n\nAlways follow these steps in order: (1) Create eval.json, (2) Run autoresearch command, (3) Review results and apply best prompt.\n\n### Step 1: Gather context\n\nBefore running, you need:\n\n| Parameter | Description | Example |\n|-----------|-------------|---------|\n| `--target` | Path to the skill directory or prompt file to optimize | `../workspace/skills/brain-search/SKILL.md` |\n| `--evals` | Path to eval definition JSON file | `eval.json` |\n| `--provider` | LLM provider for running experiments | `minimax` (default), `openai`, `anthropic` |\n| `--runs` | Number of runs per experiment (statistical significance) | `5` (default) |\n| `--max-experiments` | Maximum experiments before stopping | `30` (default) |\n| `--dashboard` | Open live results dashboard in browser | flag, no value |\n\n### Step 2: Create eval.json\n\nDefine test inputs and evaluation criteria. "},{"path":"README.md","content":"# brainharness-autoresearch\n\n> Previously named `brainforge-autoresearch`; earlier names include `openclaw-autoresearch` and `autoresearch`. Use the current installation commands below.\n\nAutonomous skill prompt optimizer based on Karpathy's autoresearch methodology.\n\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)\n[![ClawHub](https://img.shields.io/badge/ClawHub-brainharness--autoresearch-blue)](https://clawhub.ai/zning1994/brainharness-autoresearch)\n[![brainharness](https://img.shields.io/badge/brainharness-marketplace-purple)](https://github.com/zning1994/brainharness)\n\n## What it does\n\nDefine what \"good\" means as binary evals, then let an agent loop to optimize your skill prompt automatically. The optimizer reads your current prompt, runs it against eval cases, analyzes failures, mutates the prompt, tests the mutation, and keeps or discards it -- repeating until evals pass or the experiment budget is exhausted.\n\nBased on [Andrej Karpathy's autoresearch](https://github.com/karpathy/autoresearch) and [Ole Lehmann's Claude Code adaptation](https://github.com/olelehmann100kMRR/autoresearch-skill). Works with any AI coding agent: OpenClaw, Claude Code, Cursor, Cline.\n\n## Quick Start\n\n```bash\n# Claude Code (via brainharness marketplace)\n/plugin marketplace add zning1994/brainharness\n/plugin install brainharness-autoresearch@brainharness\n\n# Universal (npx skills)\nnpx skills add zning1994/brainharness-autoresearch\n\n# OpenClaw ClawHub\nnpx clawhub@0.23.3 install brainharness-autoresearch\n\n# Standalone\ngit clone https://github.com/zning1994/brainharness-autoresearch\ncd brainharness-autoresearch\npython autoresearch.py --target ./my-skill/SKILL.md --evals eval.json\n```\n\n## How it works\n\n```\n                        +------------------+\n                        |  Read SKILL.md   |\n                        +--------+---------+\n                                 |\n                        +--------v---------+\n                   +--->|  Run baseline    |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    | Analyze failures |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |  Mutate prompt   |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   |    |   Test mutation  |\n                   |    +--------+---------+\n                   |             |\n                   |    +--------v---------+\n                   +----+ Keep or discard  |\n                        +------------------+\n```\n\n## Creating eval.json\n\nMinimal example with one rule eval and one LLM eval:\n\n```json\n[\n  {\n    \"name\": \"includes_greeting\",\n    \"input\": \"Say hello to the user\",\n    \"type\": \"rule\",\n    \"rule\": \"contains\",\n    \"expected\": \"hello\"\n  },\n  {\n    \"name\": \"tone_is_professi"},{"path":"_meta.json","content":"{\n  \"ownerId\": \"kn7emz9j02vyab03rzn6fx4q7582xae6\",\n  \"slug\": \"brainharness-autoresearch\",\n  \"version\": \"0.3.0\",\n  \"publishedAt\": 1791059298885\n}"},{"path":"CHANGELOG.md","content":"# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nFormat based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).\n\n## [0.3.0] - 2026-10-04\n\n### Changed\n\n- Rename `brainforge-autoresearch` to `brainharness-autoresearch` and align repository links and installation names with BrainHarness.\n- Update distribution metadata and documentation; preserve skill behavior and historical release notes.\n\n## [0.2.5] - 2026-04-23\n\n### Changed\n\n- **Renamed repo and skill**: `openclaw-autoresearch` → `brainforge-autoresearch`\n- SKILL.md `name` field: `autoresearch` → `brainforge-autoresearch`\n- Updated README, plugin.json, and SKILL.md metadata to point at the new repo URL\n- Added to the [brainforge](https://github.com/zning1994/brainforge) Claude Code plugin marketplace\n- Published to ClawHub as `brainforge-autoresearch`; old slug `openclaw-autoresearch` merged in as redirect\n\n### Compatibility\n\n- GitHub: old URL `https://github.com/zning1994/openclaw-autoresearch` auto-redirects\n- ClawHub: `openclaw skills install openclaw-autoresearch` continues to work via ClawHub slug merge/redirect\n- No behavior changes to the optimizer itself\n\n## [0.2.4] - 2026-03-29\n\n### Fixed\n\n- SKILL.md: `pass`/`fail` → `pass_description`/`fail_description` in LLM eval docs and examples (field names didn't match actual API)\n- SKILL.md: `contains`/`not_contains` parameter documented as `value` (string) → `values` (list) to match actual usage\n- SKILL.md: LLM eval table now includes required `type` and `name` fields\n\n### Changed\n\n- SKILL.md self-optimized via autoresearch (2 rounds: MiniMax 48.6%→68.1%, Claude 80.6%→91.7%)\n- Procedure rewritten: added 3-step summary upfront, renamed steps for clarity, added actionable result-review guidance\n- Example section restructured: removed \"confirm target\" preamble, simplified to \"create eval → run → review\" flow\n- Added `self-eval.json` for self-optimization (dogfooding eval definition)\n\n### Tested\n\n- Self-optimization: Round 1 (MiniMax M2.7, 15 exp, 3 keep) and Round 2 (Claude, 15 exp, 1 keep) independently discovered same doc bugs\n\n## [0.2.3] - 2026-03-25\n\n### Added\n\n- `--timeout` CLI flag to control HTTP timeout per LLM call (default: 180s)\n  - Large prompts (e.g., 14KB style guides) need longer timeouts for mutation calls\n  - Previously hardcoded, now user-configurable\n\n### Changed\n\n- Default HTTP timeout increased from 60s to 180s (was causing mutation failures on Anthropic API with large prompts)\n- Timeout is now stored on the provider instance, passed through `detect_provider()`\n\n## [0.2.2] - 2026-03-25\n\n### Fixed\n\n- `.gitignore`: removed incorrect entries (`.git`, `.github`), added `autoresearch-*/` and `*.baseline`\n- README: added ClawHub badge and link, fixed install slug to `openclaw-autoresearch`\n\n## [0.2.1] - 2026-03-25\n\n### Fixed\n\n- ClawHub security flag: changed `requires.env` (all required) to `requires.anyEnv` (any one suffices)\n- Declared optional env vars `OPENAI_BASE_URL` / `OPENAI_API_BA"},{"path":"eval-guide.md","content":"# Eval Guide: Writing Effective Eval Criteria\n\nHow to write eval criteria that actually improve your skill's output. This guide covers the hybrid rule+LLM eval system used by brainharness-autoresearch (previously `openclaw-autoresearch`).\n\n## The Golden Rule\n\n**Binary yes/no only. No scales.**\n\nEvery eval must produce a pass or fail. Not \"7/10\", not \"mostly good\", not \"needs improvement\". Pass or fail.\n\nWhy: Scales are subjective (your 7 is my 5), hard to optimize against, and create noise. Binary forces you to define exactly what \"good\" means.\n\nBad: \"Rate the tone from 1-5\"\nGood: \"Does the output use a professional tone without slang or emojis?\"\n\n## Rule-Based Evals\n\nRule-based evals cost zero tokens, run instantly, and are 100% consistent. Use them whenever possible.\n\n### `regex`\n\nPattern matching against the output text.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_date_format\",\n  \"rule\": \"regex\",\n  \"pattern\": \"\\\\d{4}-\\\\d{2}-\\\\d{2}\",\n  \"description\": \"Output contains a date in YYYY-MM-DD format\"\n}\n```\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_markdown_headers\",\n  \"rule\": \"regex\",\n  \"pattern\": \"^#{1,3} .+\",\n  \"description\": \"Output uses markdown headers\"\n}\n```\n\nUseful for: format validation, structure checks, required patterns.\n\n### `banned_phrases`\n\nExact phrases that must not appear in the output.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_filler\",\n  \"rule\": \"banned_phrases\",\n  \"phrases\": [\"as an AI\", \"I cannot\", \"I'm sorry but\", \"it depends\", \"各方面\", \"总的来说\"],\n  \"description\": \"No AI disclaimers or vague filler phrases\"\n}\n```\n\nUseful for: removing AI-speak, enforcing brand voice, blocking lazy language.\n\n### `word_count`\n\nMin and/or max word limits. For Chinese text, counts characters.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"length_check\",\n  \"rule\": \"word_count\",\n  \"min\": 100,\n  \"max\": 500,\n  \"description\": \"Output is between 100-500 words\"\n}\n```\n\nUseful for: preventing one-line answers, keeping outputs concise, matching format requirements.\n\n### `contains`\n\nRequired keywords or phrases that must appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_source\",\n  \"rule\": \"contains\",\n  \"values\": [\"Source:\", \"Reference:\", \"来源:\"],\n  \"match\": \"any\",\n  \"description\": \"Output cites at least one source\"\n}\n```\n\n`match` can be `\"any\"` (at least one) or `\"all\"` (every value must appear).\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"has_all_sections\",\n  \"rule\": \"contains\",\n  \"values\": [\"## Summary\", \"## Action Items\", \"## Risks\"],\n  \"match\": \"all\",\n  \"description\": \"Output has all required sections\"\n}\n```\n\nUseful for: enforcing structure, required sections, key terminology.\n\n### `not_contains`\n\nPhrases or patterns that must NOT appear.\n\n```json\n{\n  \"type\": \"rule\",\n  \"name\": \"no_placeholder\",\n  \"rule\": \"not_contains\",\n  \"values\": [\"TODO\", \"TBD\", \"FIXME\", \"[insert\", \"[placeholder\"],\n  \"description\": \"No placeholder text left in output\"\n}\n```\n\nUseful for: catching incomplete outputs, blocking specific content.\n\n## LLM-Based Evals\n\nLLM evals handle semantic judgments"}],"languages":[],"docsSourceLabel":"CLAWHUB","editorialOverview":"Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy autoresearch pattern. Skill: BrainHarness Autoresearch Owner: zning1994 Summary: Use when user wants to optimize, improve, benchmark, or evaluate a skill's prompt. Triggers on \"optimize skill\", \"improve skill prompt\", \"benchmark skill\", \"eval skill\", \"run autoresearch\", \"tune prompt\", \"prompt optimization\", \"skill evaluation\", \"A/B test prompt\", \"find best prompt\", \"auto-improve skill\". Runs automated prompt experiments using the Karpathy","editorialQuality":{"score":100,"threshold":65,"status":"ready","wordCount":1042,"uniquenessScore":49,"reasons":[]}},"media":{"evidence":{"source":"no-media","verified":false,"confidence":"low","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":"No screenshots, media assets, or demo links are available."},"primaryImageUrl":null,"mediaAssetCount":0,"assets":[],"demoUrl":null},"ownerResources":{"evidence":{"source":"unclaimed","verified":false,"confidence":"low","updatedAt":"2026-10-12T02:13:06.135Z","emptyReason":"This page has not been claimed by the agent owner."},"hasCustomPage":false,"customPageUpdatedAt":null,"customLinks":[],"structuredLinks":{"docsUrl":null,"demoUrl":null,"supportUrl":null,"pricingUrl":null,"statusUrl":null},"customPage":null},"relatedAgents":{"evidence":{"source":"protocol-neighbors","verified":false,"confidence":"medium","updatedAt":"2026-10-13T00:05:43.279Z","emptyReason":null},"items":[{"id":"8ebccd8e-3863-4187-8355-c3f14e1f9edf","entityType":"agent","canonicalPath":"/agent/iofficeai-aionui","slug":"iofficeai-aionui","name":"AionUi","description":"Free, local, open-source 24/7 Cowork app and OpenClaw for Gemini CLI, Claude Code, Codex, OpenCode, Qwen Code, Goose CLI, Auggie, and more | 🌟 Star if you like it!","url":"https://github.com/iOfficeAI/AionUi","homepage":"https://www.aionui.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-10-09T19:11:12.944Z","createdAt":"2026-02-25T03:38:16.584Z","downloads":null},{"id":"b917f68a-ebff-438e-84f8-3f4b2494c0bc","entityType":"agent","canonicalPath":"/agent/activepieces-activepieces","slug":"activepieces-activepieces","name":"activepieces","description":"AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents","url":"https://github.com/activepieces/activepieces","homepage":"https://www.activepieces.com","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-15T02:22:12.426Z","createdAt":"2026-02-25T03:38:12.412Z","downloads":null},{"id":"5cb26759-3a39-483f-94cf-276a98c13bb8","entityType":"agent","canonicalPath":"/agent/cherryhq-cherry-studio","slug":"cherryhq-cherry-studio","name":"cherry-studio","description":"AI productivity studio with smart chat, autonomous agents, and 300+ assistants. Unified access to frontier LLMs","url":"https://github.com/CherryHQ/cherry-studio","homepage":"https://cherry-ai.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-11T14:38:40.986Z","createdAt":"2026-02-25T03:38:19.379Z","downloads":null},{"id":"6f6582d0-5d76-4f0f-b81d-86520247950b","entityType":"agent","canonicalPath":"/agent/copilotkit-copilotkit","slug":"copilotkit-copilotkit","name":"CopilotKit","description":"The Frontend for Agents & Generative UI. React + Angular","url":"https://github.com/CopilotKit/CopilotKit","homepage":"https://docs.copilotkit.ai","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-03-25T09:50:57.846Z","createdAt":"2026-02-25T03:39:14.617Z","downloads":null}],"links":{"hub":"/agent","source":"/agent/source/clawhub","protocols":[{"label":"OpenClaw","href":"/agent/protocol/openclew"}]}}}