{"id":"48350384-30f8-4203-a672-bc0c78a16ca7","entityType":"agent","slug":"clawhub-openlark-crawl4ai-web-crawler","name":"Crawl4AI Web Crawler","canonicalUrl":"https://www.xpersona.co/agent/clawhub-openlark-crawl4ai-web-crawler","canonicalPath":"/agent/clawhub-openlark-crawl4ai-web-crawler","generatedAt":"2026-10-10T21:46:30.926Z","source":"CLAWHUB","claimStatus":"UNCLAIMED","verificationTier":"NONE","summary":{"evidence":{"source":"editorial-content","verified":true,"confidence":"high","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":null},"description":"Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf... Skill: Crawl4AI Web Crawler Owner: openlark Summary: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf... Tags: latest:1.0.1 Version history: v1.0.1 | 2026-05-12T01:15:47.164Z | user Version 1.0.1 - Renamed and rebranded the skill to \"crawl4ai-web-crawler\" with updated purpose and trigger words. - Replaced the","descriptionLabel":"Technical summary","evidenceSummary":"Capability contract not published. No trust telemetry is available yet. 1.3K downloads reported by the source. Last updated 10/10/2026.","installCommand":"clawhub skill install s1727wv2g20pc729snzcm4nf8183hy72:crawl4ai-web-crawler","sourceUrl":"https://clawhub.ai/openlark/crawl4ai-web-crawler","homepage":"https://clawhub.ai/openlark/skills/crawl4ai-web-crawler","primaryLinks":[{"label":"View on ClawHub","url":"https://clawhub.ai/openlark/crawl4ai-web-crawler","kind":"source"},{"label":"Homepage","url":"https://clawhub.ai/openlark/skills/crawl4ai-web-crawler","kind":"homepage"}],"safetyScore":84,"overallRank":62,"popularityScore":62,"trustScore":null,"claimedByName":null,"isOwner":false,"seoDescription":"Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf..."},"coverage":{"evidence":{"source":"public-profile","verified":false,"confidence":"medium","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":null},"protocols":[{"protocol":"OPENCLEW","label":"OpenClaw","status":"self-declared","notes":"Declared in the public agent profile."}],"capabilities":[],"verifiedCount":0,"selfDeclaredCount":1,"capabilityMatrix":{"rows":[{"key":"OPENCLEW","type":"protocol","support":"unknown","confidenceSource":"profile","notes":"Listed on profile"}],"flattenedTokens":"protocol:OPENCLEW|unknown|profile"}},"adoption":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":null},"stars":null,"forks":null,"downloads":1298,"packageName":null,"latestVersion":"1.0.1","tractionLabel":"1.3K downloads"},"release":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":null},"lastUpdatedAt":"2026-10-10T18:23:08.768Z","lastCrawledAt":"2026-10-10T18:23:08.768Z","lastIndexedAt":null,"nextCrawlAt":"2026-10-11T18:23:08.768Z","lastVerifiedAt":null,"highlights":[{"version":"1.0.1","createdAt":"2026-05-12T01:15:47.164Z","changelog":"Version 1.0.1 - Renamed and rebranded the skill to \"crawl4ai-web-crawler\" with updated purpose and trigger words. - Replaced the prior RAGFlow documentation with comprehensive Crawl4AI usage guide, including installation, core classes, and real-world examples. - Added API reference pointer: references/api-reference.md. - Removed prior references: architecture.md, cli-reference.md, and deployment.md, focusing documentation on Crawl4AI features and usage.","fileCount":4,"zipByteSize":9337},{"version":"1.0.0","createdAt":"2026-05-12T01:09:22.272Z","changelog":"- Initial release of the crawl4ai-web-crawler skill under the name \"ragflow\". - Provides guidance for deploying, configuring, managing, and troubleshooting the RAGFlow open-source Retrieval-Augmented Generation (RAG) engine. - Includes detailed Docker deployment and quick-start instructions, system prerequisites, and configuration file references. - Covers CLI usage for managing datasets, documents, agents, and chats. - Offers troubleshooting tips, architecture overview, and links to documentation, support channels, and source code.","fileCount":5,"zipByteSize":10385}]},"execution":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No published capability contract is available yet."},"installCommand":"clawhub skill install s1727wv2g20pc729snzcm4nf8183hy72:crawl4ai-web-crawler","setupComplexity":"low","setupSteps":["Setup complexity is LOW. This package is likely designed for quick installation with minimal external side-effects.","Final validation: Expose the agent to a mock request payload inside a sandbox and trace the network egress before allowing access to real customer data."],"contract":{"contractStatus":"missing","authModes":[],"requires":[],"forbidden":[],"supportsMcp":false,"supportsA2a":false,"supportsStreaming":false,"inputSchemaRef":null,"outputSchemaRef":null,"dataRegion":null,"contractUpdatedAt":null,"sourceUpdatedAt":null,"freshnessSeconds":null},"invocationGuide":{"preferredApi":{"snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/trust"},"curlExamples":["curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/snapshot\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/contract\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/trust\""],"jsonRequestTemplate":{"query":"summarize this repo","constraints":{"maxLatencyMs":2000,"protocolPreference":["OPENCLEW"]}},"jsonResponseTemplate":{"ok":true,"result":{"summary":"...","confidence":0.9},"meta":{"source":"CLAWHUB","generatedAt":"2026-10-10T21:46:30.925Z"}},"retryPolicy":{"maxAttempts":3,"backoffMs":[500,1500,3500],"retryableConditions":["HTTP_429","HTTP_503","NETWORK_TIMEOUT"]}},"endpoints":{"dossierUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/dossier","snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-openlark-crawl4ai-web-crawler/trust"}},"reliability":{"evidence":{"source":"runtime-metrics","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No trust, reliability, or runtime telemetry is available."},"trust":{"status":"unavailable","handshakeStatus":"UNKNOWN","verificationFreshnessHours":null,"reputationScore":null,"p95LatencyMs":null,"successRate30d":null,"fallbackRate":null,"attempts30d":null,"trustUpdatedAt":null,"trustConfidence":"unknown","sourceUpdatedAt":null,"freshnessSeconds":null},"decisionGuardrails":{"doNotUseIf":["Contract metadata is missing or unavailable for deterministic execution."],"safeUseWhen":[],"riskFlags":["missing_or_unavailable_contract","trust_data_unavailable","schema_references_missing"],"operationalConfidence":"low"},"executionMetrics":{"observedLatencyMsP50":null,"observedLatencyMsP95":null,"estimatedCostUsd":null,"uptime30d":null,"rateLimitRpm":null,"rateLimitBurst":null,"lastVerifiedAt":null,"verificationSource":null},"runtimeMetrics":{"successRate":null,"avgLatencyMs":null,"avgCostUsd":null,"hallucinationRate":null,"retryRate":null,"disputeRate":null,"p50Latency":null,"p95Latency":null,"lastUpdated":null}},"benchmarks":{"evidence":{"source":"no-benchmark-data","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No benchmark suites or observed failure patterns are available."},"suites":[],"failurePatterns":[]},"artifacts":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"high","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":null},"readme":"Skill: Crawl4AI Web Crawler\n\nOwner: openlark\n\nSummary: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf...\n\nTags: latest:1.0.1\n\nVersion history:\n\nv1.0.1 | 2026-05-12T01:15:47.164Z | user\n\nVersion 1.0.1\n\n- Renamed and rebranded the skill to \"crawl4ai-web-crawler\" with updated purpose and trigger words.\n- Replaced the prior RAGFlow documentation with comprehensive Crawl4AI usage guide, including installation, core classes, and real-world examples.\n- Added API reference pointer: references/api-reference.md.\n- Removed prior references: architecture.md, cli-reference.md, and deployment.md, focusing documentation on Crawl4AI features and usage.\n\nv1.0.0 | 2026-05-12T01:09:22.272Z | user\n\n- Initial release of the crawl4ai-web-crawler skill under the name \"ragflow\".\n- Provides guidance for deploying, configuring, managing, and troubleshooting the RAGFlow open-source Retrieval-Augmented Generation (RAG) engine.\n- Includes detailed Docker deployment and quick-start instructions, system prerequisites, and configuration file references.\n- Covers CLI usage for managing datasets, documents, agents, and chats.\n- Offers troubleshooting tips, architecture overview, and links to documentation, support channels, and source code.\n\nArchive index:\n\nArchive v1.0.1: 4 files, 9337 bytes\n\nFiles: references/api-reference.md (10165b), skill-card.md (2270b), SKILL.md (10414b), _meta.json (139b)\n\nFile v1.0.1:SKILL.md\n\n---\nname: crawl4ai-web-crawler\ndescription: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perform batch crawling, or use AI-driven web data collection.\n---\n\n# Crawl4AI Web Crawler\n\n[Crawl4AI](https://github.com/unclecode/crawl4ai) is an open-source, LLM-friendly web crawler on GitHub that converts web pages into clean Markdown or structured JSON, ideal for RAG, AI Agents, and data pipelines.\n\nFor detailed API parameters, see [references/api-reference.md](references/api-reference.md).\n\n## Trigger Words\n\n\"scrape,\" \"crawl,\" \"crawl,\" \"extract webpage,\" \"convert webpage to markdown,\" \"structured extraction,\" etc.\n\n## Installation\n\n```bash\npip install -U crawl4ai\ncrawl4ai-setup          # Automatically installs the Playwright browser\ncrawl4ai-doctor         # Verifies the installation\n```\n\nIf the browser installation fails, run manually:\n```bash\npython -m playwright install --with-deps chromium\n```\n\n## Core Architecture\n\nThree core classes:\n\n| Class | Purpose |\n|-------|---------|\n| `AsyncWebCrawler` | Main async crawler class, manages the browser lifecycle |\n| `BrowserConfig` | Browser settings (headless, UA, proxy, viewport, etc.) |\n| `CrawlerRunConfig` | Per-crawl settings (cache, extraction strategy, JS, screenshots, etc.) |\n\n## Basic Usage\n\n### Simplest Crawl\n\n```python\nimport asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n    async with AsyncWebCrawler() as crawler:\n        result = await crawler.arun(url=\"https://example.com\")\n        print(result.markdown)  # LLM-ready Markdown\n\nasyncio.run(main())\n```\n\n### Crawl with Configuration\n\n```python\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n\nbrowser_cfg = BrowserConfig(headless=True, verbose=True)\nrun_cfg = CrawlerRunConfig(\n    cache_mode=CacheMode.BYPASS,     # BYPASS=no cache, ENABLED=enable, WRITE_ONLY, READ_ONLY\n    css_selector=\"main.article\",     # Only extract the specified area\n    word_count_threshold=10,         # Filter out short text blocks\n    screenshot=True,                 # Take a screenshot\n)\n\nasync with AsyncWebCrawler(config=browser_cfg) as crawler:\n    result = await crawler.arun(url=\"https://example.com\", config=run_cfg)\n    print(result.markdown)\n    if result.screenshot:\n        print(f\"Screenshot: {len(result.screenshot)} bytes base64\")\n```\n\n### Command Line Tool\n\n```bash\n# Basic crawl\ncrwl https://example.com -o markdown\n\n# Deep crawl (BFS, up to 10 pages)\ncrwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10\n\n# LLM extraction\ncrwl https://example.com/products -q \"Extract all product prices\"\n```\n\n## Markdown Generation\n\n### Using Content Filters\n\nRaw Markdown is generated by default. Use `DefaultMarkdownGenerator` + content filters to get cleaner output:\n\n```python\nfrom crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter\nfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator\n\n# Method 1: PruningContentFilter — density-based pruning\nmd_gen = DefaultMarkdownGenerator(\n    content_filter=PruningContentFilter(\n        threshold=0.48,           # 0-1; the lower the value, the more is pruned\n        threshold_type=\"fixed\",   # \"fixed\" or \"dynamic\"\n        min_word_threshold=0\n    )\n)\n\n# Method 2: BM25ContentFilter — query relevance-based filtering\nmd_gen = DefaultMarkdownGenerator(\n    content_filter=BM25ContentFilter(\n        user_query=\"machine learning\",  # Keywords to focus on\n        bm25_threshold=1.0\n    )\n)\n\nrun_cfg = CrawlerRunConfig(markdown_generator=md_gen)\n\nasync with AsyncWebCrawler() as crawler:\n    result = await crawler.arun(url=\"...\", config=run_cfg)\n    print(len(result.markdown.raw_markdown))   # Raw MD\n    print(len(result.markdown.fit_markdown))   # Filtered MD\n```\n\n## Structured Data Extraction\n\n### CSS/XPath Extraction (No LLM Required, Fast and Free)\n\n```python\nfrom crawl4ai import JsonCssExtractionStrategy\nimport json\n\nschema = {\n    \"name\": \"Articles\",\n    \"baseSelector\": \"article.post\",     # Container for repeating elements\n    \"fields\": [\n        {\"name\": \"title\", \"selector\": \"h2\", \"type\": \"text\"},\n        {\"name\": \"url\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"},\n        {\"name\": \"image\", \"selector\": \"img\", \"type\": \"attribute\", \"attribute\": \"src\"},\n    ]\n}\n\nrun_cfg = CrawlerRunConfig(\n    extraction_strategy=JsonCssExtractionStrategy(schema)\n)\n\nasync with AsyncWebCrawler() as crawler:\n    result = await crawler.arun(url=\"https://example.com/blog\", config=run_cfg)\n    data = json.loads(result.extracted_content)\n    print(data)  # [{\"title\": \"...\", \"url\": \"...\", \"image\": \"...\"}, ...]\n```\n\n**Auto-Generate Schema** (one-time LLM cost, then reuse for free):\n```python\nfrom crawl4ai import LLMConfig\n\nschema = JsonCssExtractionStrategy.generate_schema(\n    html=\"<div class='product'>...\",\n    llm_config=LLMConfig(provider=\"openai/gpt-4o\", api_token=\"your-key\")\n    # Or use a local model: LLMConfig(provider=\"ollama/llama3.3\", api_token=None)\n)\n```\n\n### LLM Extraction (Suitable for Unstructured Content)\n\n```python\nfrom pydantic import BaseModel, Field\nfrom crawl4ai import LLMExtractionStrategy, LLMConfig\n\nclass Product(BaseModel):\n    name: str = Field(..., description=\"Product name\")\n    price: str = Field(..., description=\"Price as string\")\n    description: str = Field(..., description=\"Short description\")\n\nllm_strategy = LLMExtractionStrategy(\n    llm_config=LLMConfig(\n        provider=\"openai/gpt-4o-mini\",     # Also supports ollama/llama3, anthropic/claude-3, etc.\n        api_token=\"your-api-key\"\n    ),\n    schema=Product.model_json_schema(),\n    extraction_type=\"schema\",              # \"schema\" or \"block\"\n    instruction=\"Extract all product objects with name, price, and description.\",\n    chunk_token_threshold=1000,            # Auto-chunk when exceeding this token count\n    overlap_rate=0.1,                      # 10% overlap between chunks\n    apply_chunking=True,\n    input_format=\"markdown\",               # \"markdown\" | \"html\" | \"fit_markdown\"\n    extra_args={\"temperature\": 0.0, \"max_tokens\": 800}\n)\n\nrun_cfg = CrawlerRunConfig(extraction_strategy=llm_strategy)\n\nasync with AsyncWebCrawler() as crawler:\n    result = await crawler.arun(url=\"https://example.com/products\", config=run_cfg)\n    data = json.loads(result.extracted_content)\n    llm_strategy.show_usage()  # Print token usage statistics\n```\n\n### Extraction Strategy Selection Guide\n\n| Scenario | Strategy |\n|----------|----------|\n| Repeating lists (products, articles, search results) | `JsonCssExtractionStrategy` |\n| Unstructured text requiring AI understanding | `LLMExtractionStrategy` |\n| High-frequency crawling of the same site | Generate Schema with LLM first, then extract via CSS |\n\n## Dynamic Page Handling\n\n```python\nrun_cfg = CrawlerRunConfig(\n    js_code=[                          # JS executed on the page\n        \"window.scrollTo(0, document.body.scrollHeight)\",\n        \"await new Promise(r => setTimeout(r, 2000))\",\n    ],\n    wait_for=\"css:.content-loaded\",     # Wait for a specific element to appear\n    delay_before_return_html=2.0,       # Additional wait in seconds before returning\n)\n```\n\n## Batch Crawling\n\n```python\nurls = [\"https://example.com/page1\", \"https://example.com/page2\", ...]\n\nasync with AsyncWebCrawler() as crawler:\n    results = await crawler.arun_many(urls=urls, config=run_cfg)\n    for result in results:\n        if result.success:\n            print(result.markdown[:200])\n```\n\n`arun_many()` automatically handles rate limiting, memory monitoring, and concurrency control.\n\n## Browser Management\n\n```python\nbrowser_cfg = BrowserConfig(\n    browser_type=\"chromium\",       # \"chromium\" | \"firefox\" | \"webkit\"\n    headless=True,\n    viewport_width=1920,\n    viewport_height=1080,\n    user_agent=\"Mozilla/5.0 ...\",\n    proxy=\"http://user:pass@proxy:8080\",\n    use_managed_browser=True,      # Use an existing browser instance\n    user_data_dir=\"/path/to/profile\",  # Persistent profile (to retain login state)\n)\n```\n\n## Deep Crawl (Site-Level Crawling)\n\n```python\nfrom crawl4ai import DeepCrawlStrategy, BFSDeepCrawlStrategy\n\ndeep_crawl = BFSDeepCrawlStrategy(\n    max_depth=3,                    # Maximum depth\n    max_pages=50,                   # Maximum number of pages\n    include_paths=[\"/docs/*\"],      # Only crawl specified paths\n    exclude_paths=[\"/blog/*\"],      # Exclude specified paths\n)\n\nrun_cfg = CrawlerRunConfig(deep_crawl_strategy=deep_crawl)\n\nasync with AsyncWebCrawler() as crawler:\n    results = await crawler.arun(url=\"https://example.com\", config=run_cfg)\n    for r in results:\n        print(f\"{r.url} → {len(r.markdown)} chars\")\n```\n\n## Docker Deployment\n\n```bash\ndocker pull unclecode/crawl4ai:latest\ndocker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest\n\n# Dashboard: http://localhost:11235/dashboard\n# Playground: http://localhost:11235/playground\n```\n\nPython Client:\n```python\nimport requests\n\nresp = requests.post(\"http://localhost:11235/crawl\",\n    json={\"urls\": [\"https://example.com\"], \"priority\": 10})\n\ntask_id = resp.json()[\"task_id\"]\nresult = requests.get(f\"http://localhost:11235/task/{task_id}\")\nprint(result.json())\n```\n\n## CrawlResult Key Fields\n\n```python\nresult.url              # Final URL (after any redirects)\nresult.html             # Raw HTML\nresult.cleaned_html     # Cleaned HTML\nresult.markdown         # Markdown formatted output (contains raw_markdown and fit_markdown)\nresult.extracted_content # JSON string returned by the extraction strategy\nresult.screenshot       # Base64 screenshot\nresult.media            # Image/video information\nresult.links            # Internal and external link information\nresult.success          # Whether the crawl was successful\nresult.error_message    # Error message\n```\n\n## FAQ\n\n**Playwright browser not installed:**\n```bash\npython -m playwright install --with-deps chromium\n```\n\n**Cache issues causing stale data to be returned:**\nSet `cache_mode=CacheMode.BYPASS` to skip the cache.\n\n**Dynamic content not loading:**\nUse `wait_for=\"css:selector\"` to wait for the target element, or `js_code` to execute scrolling.\n\n**Out of memory (batch crawling):**\nReduce the concurrency level; `arun_many()` automatically monitors memory and adapts.\n\n**Anti-bot / detection:**\nEnable `use_managed_browser=True` in `BrowserConfig` or configure a proxy.\n\nFile v1.0.1:_meta.json\n\n{\n  \"ownerId\": \"kn75qrtb885pznwsjwsh18dvf1813bv0\",\n  \"slug\": \"crawl4ai-web-crawler\",\n  \"version\": \"1.0.1\",\n  \"publishedAt\": 1778548547164\n}\n\nFile v1.0.1:references/api-reference.md\n\n# Crawl4AI API Reference\n\nDetailed class and parameter reference. Read the main SKILL.md first before using this reference.\n\n## BrowserConfig\n\n```python\nBrowserConfig(\n    browser_type: str = \"chromium\",        # \"chromium\" | \"firefox\" | \"webkit\"\n    headless: bool = True,\n    viewport_width: int = 1080,\n    viewport_height: int = 600,\n    user_agent: Optional[str] = None,      # Custom User-Agent\n    proxy: Optional[str] = None,            # Proxy URL\n    proxy_config: Optional[dict] = None,    # Advanced proxy configuration\n    use_managed_browser: bool = False,      # Use an existing browser (anti-detection)\n    user_data_dir: Optional[str] = None,    # Browser profile path\n    channel: Optional[str] = None,          # Playwright channel\n    ignore_https_errors: bool = True,\n    java_script_enabled: bool = True,\n    cookies: list = [],                     # Pre-set cookies\n    headers: dict = {},                     # Additional HTTP headers\n    accept_downloads: bool = False,\n    downloads_path: Optional[str] = None,\n    storage_state: Optional[str] = None,    # Authentication state file path\n    text_mode: bool = False,                # Disable image loading (faster)\n    light_mode: bool = False,               # Lightweight mode\n    verbose: bool = True,\n    extra_args: Optional[list] = None,      # Additional browser launch arguments\n    cdp_url: Optional[str] = None,          # Connect to remote Chrome DevTools\n)\n```\n\n## CrawlerRunConfig\n\n```python\nCrawlerRunConfig(\n    # Cache\n    cache_mode: CacheMode = CacheMode.BYPASS,   # BYPASS | ENABLED | WRITE_ONLY | READ_ONLY\n    \n    # Content Extraction\n    css_selector: Optional[str] = None,          # Only extract matching CSS regions\n    word_count_threshold: int = 10,              # Filter short text (discard if below this value)\n    excluded_tags: list = [],                     # Excluded HTML tags\n    excluded_selector: Optional[str] = None,      # Excluded CSS selector\n    keep_data_attributes: bool = False,           # Preserve data-* attributes\n    remove_forms: bool = False,\n    remove_overlay_elements: bool = False,        # Remove pop-ups/masks\n    \n    # Markdown Generation\n    markdown_generator: Optional[DefaultMarkdownGenerator] = None,\n    \n    # Extraction Strategy\n    extraction_strategy: Optional = None,         # JsonCssExtractionStrategy | LLMExtractionStrategy\n    \n    # Dynamic Pages\n    js_code: Optional[list] = None,               # List of JS code snippets to execute\n    js_only: bool = False,                        # Use JS only (do not load HTML)\n    wait_for: Optional[str] = None,               # CSS selector to wait for\n    wait_for_images: bool = False,                # Wait for images to load\n    delay_before_return_html: float = 0.0,        # Additional wait in seconds before returning\n    page_timeout: int = 60000,                    # Page load timeout (ms)\n    \n    # Media\n    screenshot: bool = False,\n    screenshot_wait_for: Optional[float] = None,  # Wait before taking screenshot\n    pdf: bool = False,\n    \n    # Session\n    session_id: Optional[str] = None,             # Reuse browser session\n    magic: bool = False,                          # Automatic anti-detection\n    \n    # Hooks\n    on_before_goto: Optional = None,              # Callback before navigation\n    on_after_goto: Optional = None,               # Callback after navigation\n    \n    # Links\n    extract_links: bool = True,\n    check_robots_txt: bool = False,\n    \n    # LLM Content Filtering\n    llm_filter: Optional[str] = None,             # LLM instruction to filter content\n    \n    # Deep Crawl\n    deep_crawl_strategy: Optional = None,          # BFSDeepCrawlStrategy | BestFirstCrawlStrategy\n    \n    # Adaptive Crawl\n    adaptive_config: Optional[AdaptiveConfig] = None,\n    \n    # Other\n    scan_full_page: bool = False,                 # Full page scroll\n    process_iframes: bool = False,\n    remove_overlay_elements: bool = False,\n    mean_delay: float = 0.1,                      # Average delay between requests\n    max_range: float = 0.3,                       # Random range for delay\n    verbose: bool = True,\n)\n```\n\n## CacheMode\n\n```python\nclass CacheMode:\n    BYPASS = \"bypass\"           # Do not use cache; re-crawl every time\n    ENABLED = \"enabled\"         # Read from and write to cache\n    WRITE_ONLY = \"write_only\"   # Write only, do not read\n    READ_ONLY = \"read_only\"     # Read only, do not write (return cached results only)\n```\n\n## LLMConfig\n\n```python\nLLMConfig(\n    provider: str,              # e.g., \"openai/gpt-4o\", \"ollama/llama3.3\", \"anthropic/claude-3\"\n    api_token: Optional[str] = None,   # API key (can be left blank for local models)\n    base_url: Optional[str] = None,    # Custom API endpoint\n    temperature: float = 0.0,\n    max_tokens: int = 2000,\n    top_p: float = 1.0,\n    frequency_penalty: float = 0.0,\n    presence_penalty: float = 0.0,\n)\n```\n\nSupported provider prefixes (via LiteLLM):\n- `openai/` — OpenAI models (gpt-4o, gpt-4o-mini, gpt-4, ...)\n- `ollama/` — Local Ollama models (llama3.3, qwen2.5, ...)\n- `anthropic/` — Claude models (claude-3-5-sonnet, ...)\n- `groq/` — Groq models\n- `deepseek/` — DeepSeek models\n- `gemini/` — Google Gemini\n- And all other providers supported by LiteLLM\n\n## LLMExtractionStrategy\n\n```python\nLLMExtractionStrategy(\n    llm_config: LLMConfig,\n    schema: Optional[dict] = None,               # Pydantic model_json_schema()\n    extraction_type: str = \"schema\",             # \"schema\" | \"block\"\n    instruction: Optional[str] = None,           # Prompt for the LLM\n    chunk_token_threshold: int = 4000,           # Chunking threshold\n    overlap_rate: float = 0.1,                   # Chunk overlap rate\n    apply_chunking: bool = True,\n    input_format: str = \"markdown\",              # \"markdown\" | \"html\" | \"fit_markdown\"\n    extra_args: Optional[dict] = None,           # temperature, max_tokens, etc.\n    verbose: bool = False,\n)\n```\n\nMethods:\n- `show_usage()` — Print token usage and cost\n- `generate_schema(html, llm_config)` — Class method, auto-generates a CSS extraction schema\n\n## JsonCssExtractionStrategy\n\n```python\nJsonCssExtractionStrategy(\n    schema: dict,       # Schema definition\n    verbose: bool = False\n)\n```\n\nSchema format:\n```python\n{\n    \"name\": \"SchemaName\",\n    \"baseSelector\": \"div.item\",           # CSS selector for repeating elements (optional)\n    \"fields\": [\n        {\n            \"name\": \"field_name\",          # Key in the output JSON\n            \"selector\": \"h2.title\",        # CSS selector\n            \"type\": \"text\",                # \"text\" | \"attribute\" | \"html\" | \"regex\"\n            \"attribute\": \"href\",           # Attribute name when type=\"attribute\"\n            \"regex\": r\"pattern\",           # Extraction pattern when type=\"regex\"\n            \"transform\": lambda x: ...,    # Optional transformation function\n            \"default\": \"N/A\"               # Default value\n        }\n    ]\n}\n```\n\nClass method:\n- `generate_schema(html, llm_config, target_elements_description=\"...\")` — Auto-generate schema via LLM\n\n## JsonXPathExtractionStrategy\n\nSimilar to `JsonCssExtractionStrategy`, but uses XPath expressions in the `selector` field for each field.\n\n## BFSDeepCrawlStrategy\n\n```python\nBFSDeepCrawlStrategy(\n    max_depth: int = 3,\n    max_pages: int = 50,\n    include_paths: Optional[list] = None,      # Whitelist paths, e.g., [\"/docs/*\"]\n    exclude_paths: Optional[list] = None,      # Blacklist paths, e.g., [\"/blog/*\"]\n    filter_patterns: Optional[list] = None,\n    url_filter: Optional[callable] = None,     # Custom URL filter\n    on_result: Optional[callable] = None,       # Per-page result callback\n)\n```\n\n## Content Filters\n\n### PruningContentFilter\n\n```python\nPruningContentFilter(\n    threshold: float = 0.5,               # 0-1; the lower the score, the more is pruned\n    threshold_type: str = \"dynamic\",      # \"fixed\" | \"dynamic\"\n    min_word_threshold: int = 0,          # Minimum word count threshold\n)\n```\n\n### BM25ContentFilter\n\n```python\nBM25ContentFilter(\n    user_query: Optional[str] = None,     # Search query, used for relevance scoring\n    bm25_threshold: float = 1.0,          # BM25 score threshold\n)\n```\n\n## CrawlResult\n\n```python\nresult.url: str                          # Final URL\nresult.html: str                         # Raw HTML\nresult.cleaned_html: str                 # Cleaned HTML\nresult.markdown: MarkdownGenerationResult # Markdown result\nresult.markdown.raw_markdown: str        # Raw Markdown\nresult.markdown.fit_markdown: str        # Filtered Markdown\nresult.markdown.references_markdown: str # Reference list Markdown\nresult.extracted_content: str            # Extracted JSON string\nresult.screenshot: str                   # Base64 screenshot\nresult.pdf: str                          # Base64 PDF\nresult.media: dict                       # {\"images\": [...], \"videos\": [...]}\nresult.links: dict                       # {\"internal\": [...], \"external\": [...]}\nresult.metadata: dict                    # Page metadata\nresult.success: bool                     # Whether the crawl was successful\nresult.error_message: str                # Error message\nresult.status_code: int                  # HTTP status code\nresult.response_headers: dict            # Response headers\nresult.downloaded_files: list            # Paths of downloaded files\n```\n\n## Complete CLI Reference\n\n```bash\ncrwl <url> [options]\n\nOptions:\n  -o, --output FORMAT     Output format: markdown, html, cleaned_html, screenshot, all\n  -q, --question TEXT     Ask the LLM a question (requires API key configuration)\n  --css-selector TEXT     CSS selector to limit the crawl scope\n  --deep-crawl STRATEGY   Deep crawl strategy: bfs\n  --max-pages N           Maximum number of pages for deep crawl\n  --max-depth N           Maximum depth for deep crawl\n  --screenshot            Take a page screenshot\n  --json                  Output in JSON format\n  --headless / --no-headless  Whether to run in headless mode\n  --verbose               Verbose output\n```\n\nEnvironment Variables:\n- `OPENAI_API_KEY` — OpenAI API key for LLM extraction\n- `CRAWL4AI_CACHE_DIR` — Cache directory\n\nFile v1.0.1:skill-card.md\n\n## Description:\n\nUse Crawl4AI for web scraping and content extraction, including structured data extraction, Markdown conversion, batch crawling, and AI-driven web data collection.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[openlark](https://clawhub.ai/user/openlark)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nDevelopers and engineers use this skill to configure and run Crawl4AI for web scraping, content-to-Markdown conversion, structured JSON extraction, batch crawling, deep crawling, and optional LLM-assisted extraction.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: Crawler and browser tooling can access broad network targets, persistent browser profiles, downloads, and cached page content.\n\nMitigation: Use an isolated environment, avoid private or authenticated sites unless explicitly intended, avoid normal browser profiles, and keep downloads and caches in disposable directories.\n\nRisk: Unpinned package and Docker image installation can change behavior across releases.\n\nMitigation: Pin the Crawl4AI package version and Docker image digest before use.\n\nRisk: Remote LLM extraction can disclose crawled page content to the configured provider.\n\nMitigation: Use remote LLM extraction only for content approved for that provider, or use a local model when page content should remain local.\n\n## Reference(s):\n\n- [Crawl4AI API Reference](references/api-reference.md)\n- [Crawl4AI GitHub Repository](https://github.com/unclecode/crawl4ai)\n- [Crawl4AI Documentation](https://docs.crawl4ai.com)\n\n## Skill Output:\n\n**Output Type(s):** [text, markdown, code, shell commands, configuration, guidance]\n\n**Output Format:** [Markdown guidance with Python and bash code blocks]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [May include Crawl4AI installation commands, crawler configuration, API usage examples, and safety scoping guidance.]\n\n## Skill Version(s):\n\n1.0.1 (source: server release evidence)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.\n\nArchive v1.0.0: 5 files, 10385 bytes\n\nFiles: references/architecture.md (6354b), references/cli-reference.md (5031b), references/deployment.md (5885b), SKILL.md (6412b), _meta.json (139b)\n\nFile v1.0.0:SKILL.md\n\n---\r\nname: ragflow\r\ndescription: RAGFlow open-source Retrieval-Augmented Generation (RAG) engine — deployment, configuration, management, and troubleshooting.\r\n---\r\n\r\n# RAGFlow\r\n\r\nOpen-source RAG engine fusing RAG with Agent capabilities. Full-stack: Python backend (Flask), React/TypeScript frontend, Docker-deployed microservices.\r\n\r\n## Use Cases\r\n\r\n- Deploying or self-hosting RAGFlow via Docker Compose or from source\r\n- Configuring RAGFlow (LLM providers, API keys, document engines, ports)\r\n- Managing knowledge bases, datasets, documents, agents, and chats via RAGFlow CLI\r\n- Understanding RAGFlow architecture (DeepDoc, Agent system, RAG pipeline)\r\n- Integrating with RAGFlow REST API\r\n- Troubleshooting RAGFlow deployment or runtime issues.\r\n\r\n\r\n## Quick Reference\r\n\r\n- **Website**: https://ragflow.io\r\n- **Cloud**: https://cloud.ragflow.io\r\n- **Docs**: https://ragflow.io/docs/dev/\r\n- **Repo**: https://github.com/infiniflow/ragflow\r\n- **Discord**: https://discord.gg/NjYzJD3GM3\r\n- **Docker Hub**: `infiniflow/ragflow`\r\n- **License**: Apache 2.0\r\n\r\n## When to Use Which Reference\r\n\r\n- **Deploying / troubleshooting deployment** → [references/deployment.md](references/deployment.md)\r\n- **Understanding architecture / components / data flow** → [references/architecture.md](references/architecture.md)\r\n- **Using CLI to manage datasets, agents, models** → [references/cli-reference.md](references/cli-reference.md)\r\n\r\n## Prerequisites\r\n\r\n- CPU >= 4 cores, RAM >= 16 GB, Disk >= 50 GB\r\n- Docker >= 24.0.0 & Docker Compose >= v2.26.1\r\n- `vm.max_map_count` >= 262144 (Linux, for Elasticsearch)\r\n- gVisor: optional, only needed for the code executor (sandbox) feature\r\n\r\n## Docker Deployment (Quick Start)\r\n\r\n```bash\r\ngit clone https://github.com/infiniflow/ragflow.git\r\ncd ragflow/docker\r\ndocker compose -f docker-compose.yml up -d\r\ndocker logs -f docker-ragflow-cpu-1  # wait for the banner, then login\r\n# Open http://YOUR_SERVER_IP in browser\r\n```\r\n\r\nConfigure LLM API keys in `docker/service_conf.yaml.template` under `user_default_llm`, then restart:\r\n\r\n```bash\r\ndocker compose -f docker-compose.yml up -d\r\n```\r\n\r\n### Chinese Mirror Images\r\n\r\nIf Docker Hub is slow:\r\n- Huawei Cloud: `swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow`\r\n- Alibaba Cloud: `registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow`\r\n\r\nSet HuggingFace mirror if needed: `HF_ENDPOINT=https://hf-mirror.com` in `docker/.env`.\r\n\r\n## Key Configuration Files\r\n\r\n| File | Scope |\r\n|------|-------|\r\n| `docker/.env` | Environment variables: `SVR_HTTP_PORT`, `MYSQL_PASSWORD`, `MINIO_PASSWORD`, `DOC_ENGINE`, `RAGFLOW_IMAGE`, `HF_ENDPOINT` |\r\n| `docker/service_conf.yaml.template` | Backend services: LLM factory, API keys, embedding/rerank/ASR/TTS models |\r\n| `docker/docker-compose.yml` | Full stack orchestration |\r\n| `docker/docker-compose-base.yml` | Infrastructure services only (dev mode) |\r\n\r\n## Post-Deployment LLM Setup\r\n\r\n1. Log in via browser\r\n2. Edit `docker/service_conf.yaml.template`:\r\n   ```yaml\r\n   user_default_llm:\r\n     factory: \"OpenAI\"   # or \"DeepSeek\", \"Gemini\", etc.\r\n     api_key: \"sk-...\"\r\n     base_url: \"https://api.openai.com/v1/\"\r\n   ```\r\n3. Run `docker compose -f docker-compose.yml up -d` to apply.\r\n\r\n## CLI Quick Reference\r\n\r\nAll CLI commands end with `;`. Full reference: [references/cli-reference.md](references/cli-reference.md).\r\n\r\n```bash\r\n# Datasets\r\nLIST DATASETS;\r\nCREATE DATASET 'my_kb' WITH EMBEDDING 'text-embedding-ada-002' PARSER 'pdf';\r\nDROP DATASET 'my_kb';\r\nLIST FILES OF DATASET 'my_kb';\r\n\r\n# Documents\r\nIMPORT '/path/to/doc.pdf' INTO DATASET 'my_kb';\r\nPARSE DATASET 'my_kb' SYNC;\r\nPARSE DATASET 'my_kb' ASYNC;\r\n\r\n# Search\r\nSEARCH 'What is RAG?' ON DATASETS 'my_kb';\r\n\r\n# Models\r\nCREATE MODEL PROVIDER 'openai' 'sk-...';\r\nSET DEFAULT LLM 'gpt-4';\r\nLIST MODEL PROVIDERS;\r\nLIST DEFAULT MODELS;\r\n\r\n# Agents & Chats\r\nLIST AGENTS;\r\nLIST CHATS;\r\nCREATE CHAT 'my_session';\r\nDROP CHAT 'my_session';\r\n\r\n# Connection\r\nPING;\r\nSHOW CURRENT USER;\r\n```\r\n\r\n## Switching Doc Engine (Elasticsearch → Infinity)\r\n\r\n```bash\r\ndocker compose -f docker/docker-compose.yml down -v  # WARNING: clears data\r\n# Edit docker/.env: set DOC_ENGINE=infinity\r\ndocker compose -f docker-compose.yml up -d\r\n```\r\n\r\nInfinity is lighter weight but Linux/arm64 is not officially supported.\r\n\r\n## Architecture at a Glance\r\n\r\n```\r\nWeb UI (React+TS+vitejs+shadcn) → Flask API (/api/) → RAG Core (/rag/) + Agent (/agent/)\r\n                                                          ↓\r\nInfrastructure: MySQL + Elasticsearch/Infinity + Redis + MinIO\r\n```\r\n\r\n- **Backend** (`/api/`): Flask blueprints — kb, dialog, document, canvas, file, user\r\n- **RAG Core** (`/rag/`): DeepDoc parsing, LLM/embedding/rerank abstractions, chunking, GraphRAG\r\n- **Agent** (`/agent/`): Canvas-based workflow builder with components (LLM, Retrieval, Code Executor, MCP, Search, SQL)\r\n- **Frontend** (`/web/`): React 18 + TypeScript + Vite\r\n\r\nSee [references/architecture.md](references/architecture.md) for detailed component breakdown.\r\n\r\n## Development from Source\r\n\r\n```bash\r\ngit clone https://github.com/infiniflow/ragflow.git && cd ragflow\r\nuv sync --python 3.12 && uv run python3 download_deps.py\r\ndocker compose -f docker/docker-compose-base.yml up -d\r\n# Add to /etc/hosts: 127.0.0.1 es01 infinity mysql minio redis sandbox-executor-manager\r\nsource .venv/bin/activate && export PYTHONPATH=$(pwd)\r\nbash docker/launch_backend_service.sh\r\n# Separate terminal:\r\ncd web && npm install && npm run dev\r\n```\r\n\r\n## Troubleshooting Quick Reference\r\n\r\n| Problem | Fix |\r\n|---------|-----|\r\n| `network abnormal` browser error | Wait for Docker logs to show the RAGFlow banner — server is initializing |\r\n| Docker pull timeout in China | Use `RAGFLOW_IMAGE` mirrors (Huawei Cloud / Alibaba Cloud) |\r\n| HuggingFace unreachable | `export HF_ENDPOINT=https://hf-mirror.com` |\r\n| ARM64 platform | Build Docker image from source (no official ARM64 image) |\r\n| Port conflict | Change `80:80` to `<PORT>:80` in `docker-compose.yml` |\r\n| Elasticsearch exits with 137 | Increase Docker memory allocation |\r\n| `vm.max_map_count` too low | `sudo sysctl -w vm.max_map_count=262144` |\r\n\r\n## API & SDK\r\n\r\n- REST API: `http://SERVER_IP/api/` — Swagger docs at `/api/docs`\r\n- Python SDK: available in `sdk/python/`\r\n- CLI client: `python admin/client/ragflow_cli.py <command>`\r\n- Guide: [https://ragflow.io/docs/dev/](https://ragflow.io/docs/dev/)\n\nFile v1.0.0:_meta.json\n\n{\n  \"ownerId\": \"kn75qrtb885pznwsjwsh18dvf1813bv0\",\n  \"slug\": \"crawl4ai-web-crawler\",\n  \"version\": \"1.0.0\",\n  \"publishedAt\": 1778548162272\n}\n\nFile v1.0.0:references/architecture.md\n\n# RAGFlow Architecture Reference\r\n\r\n## System Overview\r\n\r\nRAGFlow is a full-stack microservices application with the following layers:\r\n\r\n```\r\n┌─────────────────────────────────────────────────┐\r\n│                  Web UI (React/TS)               │\r\n│               vitejs + shadcn/ui                 │\r\n├─────────────────────────────────────────────────┤\r\n│              Flask API Server (/api/)            │\r\n│    kb_app | dialog_app | document_app | ...      │\r\n├──────────────────┬──────────────────────────────┤\r\n│   RAG Core (/rag/)   │   Agent System (/agent/)  │\r\n│   DeepDoc | LLM      │   Canvas | Components     │\r\n│   Chunking | GraphRAG│   Templates | Tools       │\r\n├──────────────────┴──────────────────────────────┤\r\n│              Infrastructure                      │\r\n│   MySQL | ES/Infinity | Redis | MinIO           │\r\n└─────────────────────────────────────────────────┘\r\n```\r\n\r\n## Backend (`/api/`)\r\n\r\nFlask-based REST API server. Entry point: `api/ragflow_server.py`\r\n\r\n### Blueprint Apps (`api/apps/`)\r\n\r\n| App | File | Purpose |\r\n|-----|------|---------|\r\n| Knowledge Base | `kb_app.py` | Create, list, delete knowledge bases/datasets |\r\n| Dialog | `dialog_app.py` | Chat/conversation sessions, message handling |\r\n| Document | `document_app.py` | Upload, parse, manage documents |\r\n| Canvas | `canvas_app.py` | Agent workflow canvas CRUD |\r\n| File | `file_app.py` | File upload, management, chunking |\r\n| User | `user_app.py` | User management, authentication |\r\n\r\n### Data Layer (`api/db/`)\r\n\r\n- `db_models.py` — SQLAlchemy ORM models\r\n- `services/` — Business logic layer for all entities\r\n\r\n## RAG Core (`/rag/`)\r\n\r\n### DeepDoc (`deepdoc/`)\r\n\r\nDeep document understanding engine:\r\n- PDF parsing with layout analysis\r\n- OCR for scanned documents\r\n- Table extraction\r\n- Multi-modal model support for images in PDF/DOCX\r\n\r\n### LLM Integration (`rag/llm/`)\r\n\r\nModel abstraction layer supporting:\r\n- **Chat models**: OpenAI, DeepSeek, Gemini, GPT-5, local models\r\n- **Embedding models**: Various text embedding providers\r\n- **Reranking models**: Cross-encoder rerankers (BGE, etc.)\r\n- **ASR models**: Whisper and others\r\n- **TTS models**: Text-to-speech\r\n\r\n### RAG Pipeline (`rag/flow/`)\r\n\r\n- Chunking strategies (template-based)\r\n- Parser selection\r\n- Tokenization\r\n- Pipeline orchestration\r\n\r\n### GraphRAG (`rag/graphrag/`)\r\n\r\n- Knowledge graph construction from documents\r\n- Graph-based querying and retrieval\r\n\r\n## Agent System (`/agent/`)\r\n\r\n### Canvas (`agent/canvas.py`)\r\n\r\nVisual workflow builder for AI agents. Users can drag-and-drop components to create agent pipelines.\r\n\r\n### Components (`agent/component/`)\r\n\r\nModular workflow components:\r\n\r\n| Component | Purpose |\r\n|-----------|---------|\r\n| LLM | Chat completion, reasoning |\r\n| Retrieval | Knowledge base query |\r\n| Categorize | Classify input into categories |\r\n| Code Executor | Run Python/JavaScript in sandbox |\r\n| MCP | Model Context Protocol integration |\r\n| Web Search | External search (Tavily, Wikipedia) |\r\n| SQL Executor | Database query execution |\r\n| Begin/End | Workflow start and end nodes |\r\n\r\n### Templates (`agent/templates/`)\r\n\r\nPre-built agent workflows for common scenarios:\r\n- Q&A bots\r\n- Customer support\r\n- Data analysis\r\n- RAG pipelines\r\n\r\n### Tools (`agent/tools/`)\r\n\r\nExternal integrations: Tavily search, Wikipedia, SQL execution, API calls, etc.\r\n\r\n## Frontend (`/web/`)\r\n\r\n- **Framework**: React 18 + TypeScript\r\n- **Build**: Vite\r\n- **UI**: shadcn/ui components\r\n- **State**: Zustand\r\n- **CSS**: Tailwind CSS\r\n- **Testing**: Jest + React Testing Library\r\n\r\n## Infrastructure Services\r\n\r\n| Service | Port | Purpose |\r\n|---------|------|---------|\r\n| MySQL | 3306 | Primary database (users, KBs, configs) |\r\n| Elasticsearch | 9200 | Full-text search + vector storage (default) |\r\n| Infinity | 23817 | Alternative doc engine (lighter weight) |\r\n| Redis | 6379 | Caching, task queues |\r\n| MinIO | 9000 | Object storage for documents/files |\r\n| Sandbox Executor | — | gVisor-based code execution isolation |\r\n\r\n## Document Engines\r\n\r\n### Elasticsearch (Default)\r\n- Full-text search + dense vector search\r\n- Higher resource usage\r\n- Production-proven\r\n\r\n### Infinity\r\n- Lightweight alternative by infiniflow\r\n- Lower resource footprint\r\n- Switch via `DOC_ENGINE=infinity` in `docker/.env`\r\n\r\n## Data Flow\r\n\r\n1. **Ingestion**: User uploads documents → MinIO storage\r\n2. **Parsing**: DeepDoc extracts text, tables, images from documents\r\n3. **Chunking**: Documents split into chunks via template-based strategies\r\n4. **Embedding**: Chunks vectorized via configured embedding model\r\n5. **Indexing**: Vectors + text stored in Elasticsearch/Infinity\r\n6. **Retrieval**: User query → embedding → vector search → rerank → LLM generation\r\n7. **Agent Pipeline**: Optional agent workflow orchestrates multi-step reasoning\r\n\r\n## Python Dependencies\r\n\r\n- Package manager: `uv`\r\n- Python version: 3.10-3.12\r\n- Key dependencies: Flask, SQLAlchemy, pyPDF, transformers (optional), onnxruntime (optional)\r\n- Test framework: pytest (p1/p2/p3 priority levels)\r\n\r\n## CLI (`admin/client/`)\r\n\r\nPython-based CLI client with SQL-like syntax:\r\n- `ragflow_cli.py` — Entry point\r\n- `ragflow_client.py` — HTTP client (REST API wrapper)\r\n- `parser.py` — Command parser\r\n- `http_client.py` — HTTP transport layer\r\n- `user.py` — User management\r\n\r\n## Configuration Files\r\n\r\n| File | Scope |\r\n|------|-------|\r\n| `docker/.env` | Environment variables for Docker deployment |\r\n| `docker/service_conf.yaml.template` | Backend service configuration |\r\n| `docker/docker-compose.yml` | Full stack service orchestration |\r\n| `docker/docker-compose-base.yml` | Infra services only (dev mode) |\r\n| `pyproject.toml` | Python project metadata and deps |\r\n| `web/package.json` | Frontend dependencies |\n\nFile v1.0.0:references/cli-reference.md\n\n# RAGFlow CLI Reference\r\n\r\nAll CLI commands end with a semicolon (`;`). Prompt: `ragflow>`.\r\n\r\n## Server & User Commands\r\n\r\n### PING — Test Connection\r\n\r\n```\r\nPING;\r\n```\r\n\r\nTests connectivity to the RAGFlow server.\r\n\r\n### SHOW CURRENT USER — Display User Info\r\n\r\n```\r\nSHOW CURRENT USER;\r\n```\r\n\r\nShows currently logged-in user information.\r\n\r\n---\r\n\r\n## Model Provider Management\r\n\r\n### CREATE MODEL PROVIDER — Add Provider\r\n\r\n```\r\nCREATE MODEL PROVIDER <provider_name> <provider_key>;\r\n```\r\n\r\n**Parameters:**\r\n- `provider_name`: Provider name (quoted string), e.g., `'openai'`, `'deepseek'`\r\n- `provider_key`: API key (quoted string)\r\n\r\n**Example:**\r\n```\r\nCREATE MODEL PROVIDER 'openai' 'sk-xxxxxxxx';\r\n```\r\n\r\n### DROP MODEL PROVIDER — Remove Provider\r\n\r\n```\r\nDROP MODEL PROVIDER <provider_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nDROP MODEL PROVIDER 'openai';\r\n```\r\n\r\n### LIST MODEL PROVIDERS — Show All Providers\r\n\r\n```\r\nLIST MODEL PROVIDERS;\r\n```\r\n\r\n---\r\n\r\n## Default Model Configuration\r\n\r\n### SET DEFAULT LLM / VLM / EMBEDDING / RERANKER / ASR / TTS\r\n\r\n```\r\nSET DEFAULT LLM <llm_id>;\r\nSET DEFAULT VLM <vlm_id>;\r\nSET DEFAULT EMBEDDING <embedding_id>;\r\nSET DEFAULT RERANKER <reranker_id>;\r\nSET DEFAULT ASR <asr_id>;\r\nSET DEFAULT TTS <tts_id>;\r\n```\r\n\r\n**Examples:**\r\n```\r\nSET DEFAULT LLM 'gpt-4';\r\nSET DEFAULT VLM 'clip-vit-large';\r\nSET DEFAULT EMBEDDING 'text-embedding-ada-002';\r\nSET DEFAULT RERANKER 'bge-reranker-large';\r\nSET DEFAULT ASR 'whisper-large';\r\nSET DEFAULT TTS 'tts-1';\r\n```\r\n\r\n### RESET DEFAULT Models\r\n\r\n```\r\nRESET DEFAULT LLM;\r\nRESET DEFAULT VLM;\r\nRESET DEFAULT EMBEDDING;\r\nRESET DEFAULT RERANKER;\r\nRESET DEFAULT ASR;\r\nRESET DEFAULT TTS;\r\n```\r\n\r\nResets to system default values.\r\n\r\n### LIST DEFAULT MODELS — Show Current Defaults\r\n\r\n```\r\nLIST DEFAULT MODELS;\r\n```\r\n\r\n---\r\n\r\n## Dataset Management\r\n\r\n### CREATE DATASET — With Parser\r\n\r\n```\r\nCREATE DATASET <dataset_name> WITH EMBEDDING <embedding> PARSER <parser_type>;\r\n```\r\n\r\n**Parameters:**\r\n- `dataset_name`: Dataset name (quoted)\r\n- `embedding`: Embedding model name (quoted)\r\n- `parser_type`: Parser type, e.g., `'pdf'`, `'docx'`, `'txt'`\r\n\r\n**Example:**\r\n```\r\nCREATE DATASET 'my_kb' WITH EMBEDDING 'text-embedding-ada-002' PARSER 'pdf';\r\n```\r\n\r\n### CREATE DATASET — With Pipeline\r\n\r\n```\r\nCREATE DATASET <dataset_name> WITH EMBEDDING <embedding> PIPELINE <pipeline>;\r\n```\r\n\r\n**Example:**\r\n```\r\nCREATE DATASET 'my_kb' WITH EMBEDDING 'text-embedding-ada-002' PIPELINE 'standard';\r\n```\r\n\r\n### DROP DATASET — Delete Dataset\r\n\r\n```\r\nDROP DATASET <dataset_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nDROP DATASET 'my_kb';\r\n```\r\n\r\n### LIST DATASETS — Show All Datasets\r\n\r\n```\r\nLIST DATASETS;\r\n```\r\n\r\n### LIST FILES OF DATASET — Show Documents\r\n\r\n```\r\nLIST FILES OF DATASET <dataset_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nLIST FILES OF DATASET 'my_kb';\r\n```\r\n\r\n---\r\n\r\n## Document Operations\r\n\r\n### IMPORT — Add Documents to Dataset\r\n\r\n```\r\nIMPORT <document_list> INTO DATASET <dataset_name>;\r\n```\r\n\r\n**Parameters:**\r\n- `document_list`: Comma-separated file paths (quoted)\r\n- `dataset_name`: Target dataset name (quoted)\r\n\r\n**Example:**\r\n```\r\nIMPORT '/path/to/doc1.pdf,/path/to/doc2.pdf' INTO DATASET 'my_kb';\r\n```\r\n\r\n### PARSE Documents — Specific Files\r\n\r\n```\r\nPARSE <document_names> OF DATASET <dataset_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nPARSE 'doc1.pdf,doc2.pdf' OF DATASET 'my_kb';\r\n```\r\n\r\n### PARSE DATASET — Entire Dataset\r\n\r\n```\r\nPARSE DATASET <dataset_name> SYNC;\r\nPARSE DATASET <dataset_name> ASYNC;\r\n```\r\n\r\n- `SYNC`: Wait for parsing to complete\r\n- `ASYNC`: Return immediately, parse in background\r\n\r\n**Example:**\r\n```\r\nPARSE DATASET 'my_kb' SYNC;\r\n```\r\n\r\n---\r\n\r\n## Search\r\n\r\n### SEARCH — Query Datasets\r\n\r\n```\r\nSEARCH <question> ON DATASETS <dataset_list>;\r\n```\r\n\r\n**Parameters:**\r\n- `question`: Search query (quoted)\r\n- `dataset_list`: Comma-separated dataset names (quoted)\r\n\r\n**Example:**\r\n```\r\nSEARCH 'What is RAG?' ON DATASETS 'kb1,kb2';\r\n```\r\n\r\n---\r\n\r\n## Agent Management\r\n\r\n### LIST AGENTS — Show All Agents\r\n\r\n```\r\nLIST AGENTS;\r\n```\r\n\r\n---\r\n\r\n## Chat Session Management\r\n\r\n### CREATE CHAT — New Session\r\n\r\n```\r\nCREATE CHAT <chat_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nCREATE CHAT 'support_session';\r\n```\r\n\r\n### DROP CHAT — Delete Session\r\n\r\n```\r\nDROP CHAT <chat_name>;\r\n```\r\n\r\n**Example:**\r\n```\r\nDROP CHAT 'support_session';\r\n```\r\n\r\n### LIST CHATS — Show All Sessions\r\n\r\n```\r\nLIST CHATS;\r\n```\r\n\r\n---\r\n\r\n## Performance Testing\r\n\r\n### BENCHMARK — Test Command Performance\r\n\r\n```\r\nBENCHMARK <concurrency> <iterations> <user_command>;\r\n```\r\n\r\n**Parameters:**\r\n- `concurrency`: Number of concurrent requests\r\n- `iterations`: Number of iterations per request\r\n- `user_command`: Any valid command (including semicolon)\r\n\r\n**Example:**\r\n```\r\nBENCHMARK 5 10 PING;\r\n```\r\n\r\n---\r\n\r\n## Notes\r\n\r\n- All string parameters must be quoted with single (`'`) or double (`\"`) quotes\r\n- Commands must end with `;`\r\n- CLI client: `python admin/client/ragflow_cli.py <command>`\r\n- Requires authentication (login first or configure credentials in environment)\n\nFile v1.0.0:references/deployment.md\n\n# RAGFlow Deployment Reference\r\n\r\n## Self-Hosting with Docker (Recommended)\r\n\r\n### Prerequisites\r\n\r\n- CPU >= 4 cores\r\n- RAM >= 16 GB\r\n- Disk >= 50 GB\r\n- Docker >= 24.0.0 & Docker Compose >= v2.26.1\r\n- `vm.max_map_count` >= 262144 (Linux only)\r\n\r\n### Quick Start\r\n\r\n```bash\r\n# 1. Clone the repo\r\ngit clone https://github.com/infiniflow/ragflow.git\r\ncd ragflow/docker\r\n\r\n# 2. (Optional) Checkout a stable release\r\n# git checkout v0.25.2\r\n\r\n# 3. Start services (CPU mode)\r\ndocker compose -f docker-compose.yml up -d\r\n\r\n# For GPU acceleration:\r\n# sed -i '1i DEVICE=gpu' .env\r\n# docker compose -f docker-compose.yml up -d\r\n```\r\n\r\n### Verify Startup\r\n\r\n```bash\r\ndocker logs -f docker-ragflow-cpu-1\r\n```\r\n\r\nSuccessful launch shows the RAGFlow ASCII banner and `Running on all addresses (0.0.0.0)`.\r\n\r\n### Access\r\n\r\nOpen `http://YOUR_SERVER_IP` in browser. Default port is 80.\r\n\r\n### Post-Deployment Configuration\r\n\r\n1. Log in to the web UI\r\n2. Edit `docker/service_conf.yaml.template`:\r\n   - Set `user_default_llm.factory` to your LLM provider\r\n   - Set `user_default_llm.api_key` with your API key\r\n3. Restart to apply:\r\n   ```bash\r\n   docker compose -f docker-compose.yml up -d\r\n   ```\r\n\r\n## Configuration Files Reference\r\n\r\n### `docker/.env` — Environment Variables\r\n\r\n| Variable | Description | Default |\r\n|----------|-------------|---------|\r\n| `SVR_HTTP_PORT` | HTTP serving port | `80` |\r\n| `MYSQL_PASSWORD` | MySQL root password | `infini_rag_flow` |\r\n| `MINIO_PASSWORD` | MinIO access password | `infini_rag_flow` |\r\n| `DOC_ENGINE` | Document engine (`elasticsearch` or `infinity`) | `elasticsearch` |\r\n| `RAGFLOW_IMAGE` | Docker image to pull | `infiniflow/ragflow:v0.25.2` |\r\n| `HF_ENDPOINT` | HuggingFace mirror | `https://huggingface.co` |\r\n| `DEVICE` | `cpu` or `gpu` for DeepDoc | `cpu` |\r\n\r\n### Chinese Mirror Images\r\n\r\nIf Docker Hub is slow in China:\r\n```\r\n# Huawei Cloud\r\nRAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.25.2\r\n# Alibaba Cloud  \r\nRAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.25.2\r\n```\r\n\r\nHF mirror:\r\n```\r\nHF_ENDPOINT=https://hf-mirror.com\r\n```\r\n\r\n### `docker/service_conf.yaml.template` — Backend Config\r\n\r\nKey sections:\r\n- `user_default_llm`: Default LLM factory and API key\r\n- `embedding_model`: Default embedding model\r\n- `rerank_model`: Default reranking model\r\n- `asr_model`: Speech recognition model\r\n- `tts_model`: Text-to-speech model\r\n\r\nExample for OpenAI:\r\n```yaml\r\nuser_default_llm:\r\n  factory: \"OpenAI\"\r\n  api_key: \"sk-...\"\r\n  base_url: \"https://api.openai.com/v1/\"\r\n```\r\n\r\nExample for DeepSeek:\r\n```yaml\r\nuser_default_llm:\r\n  factory: \"DeepSeek\"\r\n  api_key: \"sk-...\"\r\n  base_url: \"https://api.deepseek.com/v1\"\r\n```\r\n\r\n## Port Configuration\r\n\r\nTo change the HTTP port (default 80), edit `docker/docker-compose.yml`:\r\n```yaml\r\nports:\r\n  - \"YOUR_PORT:80\"\r\n```\r\n\r\nThen restart: `docker compose -f docker-compose.yml up -d`\r\n\r\n## Switching Document Engine\r\n\r\n### Elasticsearch → Infinity\r\n\r\n```bash\r\n# 1. Stop and clean volumes (WARNING: data loss)\r\ndocker compose -f docker/docker-compose.yml down -v\r\n\r\n# 2. Edit docker/.env\r\n# Set: DOC_ENGINE=infinity\r\n\r\n# 3. Restart\r\ndocker compose -f docker-compose.yml up -d\r\n```\r\n\r\n> Infinity on Linux/arm64 is not officially supported.\r\n\r\n## Development from Source\r\n\r\n### Prerequisites\r\n- Python 3.10-3.12\r\n- Node.js >= 18.20.4\r\n- `uv` package manager\r\n- Docker & Docker Compose\r\n- `jemalloc` (optional, recommended)\r\n\r\n### Setup\r\n\r\n```bash\r\n# 1. Clone and install Python deps\r\ngit clone https://github.com/infiniflow/ragflow.git\r\ncd ragflow/\r\nuv sync --python 3.12\r\nuv run python3 download_deps.py\r\npre-commit install\r\n\r\n# 2. Start infrastructure services\r\ndocker compose -f docker/docker-compose-base.yml up -d\r\n\r\n# 3. Add to /etc/hosts\r\n# 127.0.0.1  es01 infinity mysql minio redis sandbox-executor-manager\r\n\r\n# 4. (If in China) Set HF mirror\r\n# export HF_ENDPOINT=https://hf-mirror.com\r\n\r\n# 5. Launch backend\r\nsource .venv/bin/activate\r\nexport PYTHONPATH=$(pwd)\r\nbash docker/launch_backend_service.sh\r\n\r\n# 6. Launch frontend (separate terminal)\r\ncd web\r\nnpm install\r\nnpm run dev\r\n```\r\n\r\n### Stop Development Services\r\n\r\n```bash\r\npkill -f \"ragflow_server.py|task_executor.py\"\r\n```\r\n\r\n## Building Docker Image\r\n\r\n```bash\r\ngit clone https://github.com/infiniflow/ragflow.git\r\ncd ragflow/\r\ndocker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .\r\n```\r\n\r\nWith proxy:\r\n```bash\r\ndocker build --platform linux/amd64 \\\r\n  --build-arg http_proxy=http://PROXY:PORT \\\r\n  --build-arg https_proxy=http://PROXY:PORT \\\r\n  -f Dockerfile -t infiniflow/ragflow:nightly .\r\n```\r\n\r\n## ARM64 Platforms\r\n\r\nNo official ARM64 Docker images. Must build from source:\r\n```bash\r\ndocker build -f Dockerfile -t infiniflow/ragflow:custom-arm64 .\r\n```\r\nThen update `RAGFLOW_IMAGE` in `docker/.env` to use your custom image.\r\n\r\n## Linux vm.max_map_count\r\n\r\nElasticsearch requires at least 262144. Check:\r\n```bash\r\nsysctl vm.max_map_count\r\n```\r\n\r\nTemporary fix:\r\n```bash\r\nsudo sysctl -w vm.max_map_count=262144\r\n```\r\n\r\nPermanent fix (add to /etc/sysctl.conf):\r\n```\r\nvm.max_map_count=262144\r\n```\r\n\r\n## Troubleshooting\r\n\r\n| Issue | Solution |\r\n|-------|----------|\r\n| `network abnormal` on login | Wait until Docker logs show the RAGFlow ASCII banner — server may still be initializing |\r\n| Docker pull timeout | Use Huawei Cloud or Alibaba Cloud mirror images in `RAGFLOW_IMAGE` |\r\n| HuggingFace unreachable | Set `HF_ENDPOINT=https://hf-mirror.com` in `.env` |\r\n| Port already in use | Change `80:80` to `<NEW_PORT>:80` in `docker-compose.yml` |\r\n| Elasticsearch exits with 137 | Increase Docker memory or reduce `vm.max_map_count` issues |\r\n| Out of disk space | Clean Docker volumes: `docker system prune -a` |\r\n| Frontend build fails | Check Node.js version >= 18.20.4 |\r\n| Python import errors | Ensure `PYTHONPATH=$(pwd)` is set when running from source |","readmeExcerpt":"Skill: Crawl4AI Web Crawler Owner: openlark Summary: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf... Tags: latest:1.0.1 Version history: v1.0.1 | 2026-05-12T01:15:47.164Z | user Version 1.0.1 - Renamed and rebranded the skill to \"crawl4ai-web-crawler\" with updated purpose and trigger words. - Replaced the ","codeSnippets":[],"executableExamples":[{"language":"bash","snippet":"pip install -U crawl4ai\ncrawl4ai-setup          # Automatically installs the Playwright browser\ncrawl4ai-doctor         # Verifies the installation"},{"language":"bash","snippet":"python -m playwright install --with-deps chromium"},{"language":"python","snippet":"import asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n    async with AsyncWebCrawler() as crawler:\n        result = await crawler.arun(url=\"https://example.com\")\n        print(result.markdown)  # LLM-ready Markdown\n\nasyncio.run(main())"},{"language":"python","snippet":"from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n\nbrowser_cfg = BrowserConfig(headless=True, verbose=True)\nrun_cfg = CrawlerRunConfig(\n    cache_mode=CacheMode.BYPASS,     # BYPASS=no cache, ENABLED=enable, WRITE_ONLY, READ_ONLY\n    css_selector=\"main.article\",     # Only extract the specified area\n    word_count_threshold=10,         # Filter out short text blocks\n    screenshot=True,                 # Take a screenshot\n)\n\nasync with AsyncWebCrawler(config=browser_cfg) as crawler:\n    result = await crawler.arun(url=\"https://example.com\", config=run_cfg)\n    print(result.markdown)\n    if result.screenshot:\n        print(f\"Screenshot: {len(result.screenshot)} bytes base64\")"},{"language":"bash","snippet":"# Basic crawl\ncrwl https://example.com -o markdown\n\n# Deep crawl (BFS, up to 10 pages)\ncrwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10\n\n# LLM extraction\ncrwl https://example.com/products -q \"Extract all product prices\""},{"language":"python","snippet":"from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter\nfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator\n\n# Method 1: PruningContentFilter — density-based pruning\nmd_gen = DefaultMarkdownGenerator(\n    content_filter=PruningContentFilter(\n        threshold=0.48,           # 0-1; the lower the value, the more is pruned\n        threshold_type=\"fixed\",   # \"fixed\" or \"dynamic\"\n        min_word_threshold=0\n    )\n)\n\n# Method 2: BM25ContentFilter — query relevance-based filtering\nmd_gen = DefaultMarkdownGenerator(\n    content_filter=BM25ContentFilter(\n        user_query=\"machine learning\",  # Keywords to focus on\n        bm25_threshold=1.0\n    )\n)\n\nrun_cfg = CrawlerRunConfig(markdown_generator=md_gen)\n\nasync with AsyncWebCrawler() as crawler:\n    result = await crawler.arun(url=\"...\", config=run_cfg)\n    print(len(result.markdown.raw_markdown))   # Raw MD\n    print(len(result.markdown.fit_markdown))   # Filtered MD"}],"parameters":null,"dependencies":[],"permissions":[],"extractedFiles":[{"path":"SKILL.md","content":"---\nname: crawl4ai-web-crawler\ndescription: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perform batch crawling, or use AI-driven web data collection.\n---\n\n# Crawl4AI Web Crawler\n\n[Crawl4AI](https://github.com/unclecode/crawl4ai) is an open-source, LLM-friendly web crawler on GitHub that converts web pages into clean Markdown or structured JSON, ideal for RAG, AI Agents, and data pipelines.\n\nFor detailed API parameters, see [references/api-reference.md](references/api-reference.md).\n\n## Trigger Words\n\n\"scrape,\" \"crawl,\" \"crawl,\" \"extract webpage,\" \"convert webpage to markdown,\" \"structured extraction,\" etc.\n\n## Installation\n\n```bash\npip install -U crawl4ai\ncrawl4ai-setup          # Automatically installs the Playwright browser\ncrawl4ai-doctor         # Verifies the installation\n```\n\nIf the browser installation fails, run manually:\n```bash\npython -m playwright install --with-deps chromium\n```\n\n## Core Architecture\n\nThree core classes:\n\n| Class | Purpose |\n|-------|---------|\n| `AsyncWebCrawler` | Main async crawler class, manages the browser lifecycle |\n| `BrowserConfig` | Browser settings (headless, UA, proxy, viewport, etc.) |\n| `CrawlerRunConfig` | Per-crawl settings (cache, extraction strategy, JS, screenshots, etc.) |\n\n## Basic Usage\n\n### Simplest Crawl\n\n```python\nimport asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n    async with AsyncWebCrawler() as crawler:\n        result = await crawler.arun(url=\"https://example.com\")\n        print(result.markdown)  # LLM-ready Markdown\n\nasyncio.run(main())\n```\n\n### Crawl with Configuration\n\n```python\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n\nbrowser_cfg = BrowserConfig(headless=True, verbose=True)\nrun_cfg = CrawlerRunConfig(\n    cache_mode=CacheMode.BYPASS,     # BYPASS=no cache, ENABLED=enable, WRITE_ONLY, READ_ONLY\n    css_selector=\"main.article\",     # Only extract the specified area\n    word_count_threshold=10,         # Filter out short text blocks\n    screenshot=True,                 # Take a screenshot\n)\n\nasync with AsyncWebCrawler(config=browser_cfg) as crawler:\n    result = await crawler.arun(url=\"https://example.com\", config=run_cfg)\n    print(result.markdown)\n    if result.screenshot:\n        print(f\"Screenshot: {len(result.screenshot)} bytes base64\")\n```\n\n### Command Line Tool\n\n```bash\n# Basic crawl\ncrwl https://example.com -o markdown\n\n# Deep crawl (BFS, up to 10 pages)\ncrwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10\n\n# LLM extraction\ncrwl https://example.com/products -q \"Extract all product prices\"\n```\n\n## Markdown Generation\n\n### Using Content Filters\n\nRaw Markdown is generated by default. Use `DefaultMarkdownGenerator` + content filters to get cleaner output:\n\n```python\nfrom crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter\nfrom crawl4ai.markdown_generation_strategy import DefaultM"},{"path":"_meta.json","content":"{\n  \"ownerId\": \"kn75qrtb885pznwsjwsh18dvf1813bv0\",\n  \"slug\": \"crawl4ai-web-crawler\",\n  \"version\": \"1.0.1\",\n  \"publishedAt\": 1778548547164\n}"},{"path":"references/api-reference.md","content":"# Crawl4AI API Reference\n\nDetailed class and parameter reference. Read the main SKILL.md first before using this reference.\n\n## BrowserConfig\n\n```python\nBrowserConfig(\n    browser_type: str = \"chromium\",        # \"chromium\" | \"firefox\" | \"webkit\"\n    headless: bool = True,\n    viewport_width: int = 1080,\n    viewport_height: int = 600,\n    user_agent: Optional[str] = None,      # Custom User-Agent\n    proxy: Optional[str] = None,            # Proxy URL\n    proxy_config: Optional[dict] = None,    # Advanced proxy configuration\n    use_managed_browser: bool = False,      # Use an existing browser (anti-detection)\n    user_data_dir: Optional[str] = None,    # Browser profile path\n    channel: Optional[str] = None,          # Playwright channel\n    ignore_https_errors: bool = True,\n    java_script_enabled: bool = True,\n    cookies: list = [],                     # Pre-set cookies\n    headers: dict = {},                     # Additional HTTP headers\n    accept_downloads: bool = False,\n    downloads_path: Optional[str] = None,\n    storage_state: Optional[str] = None,    # Authentication state file path\n    text_mode: bool = False,                # Disable image loading (faster)\n    light_mode: bool = False,               # Lightweight mode\n    verbose: bool = True,\n    extra_args: Optional[list] = None,      # Additional browser launch arguments\n    cdp_url: Optional[str] = None,          # Connect to remote Chrome DevTools\n)\n```\n\n## CrawlerRunConfig\n\n```python\nCrawlerRunConfig(\n    # Cache\n    cache_mode: CacheMode = CacheMode.BYPASS,   # BYPASS | ENABLED | WRITE_ONLY | READ_ONLY\n    \n    # Content Extraction\n    css_selector: Optional[str] = None,          # Only extract matching CSS regions\n    word_count_threshold: int = 10,              # Filter short text (discard if below this value)\n    excluded_tags: list = [],                     # Excluded HTML tags\n    excluded_selector: Optional[str] = None,      # Excluded CSS selector\n    keep_data_attributes: bool = False,           # Preserve data-* attributes\n    remove_forms: bool = False,\n    remove_overlay_elements: bool = False,        # Remove pop-ups/masks\n    \n    # Markdown Generation\n    markdown_generator: Optional[DefaultMarkdownGenerator] = None,\n    \n    # Extraction Strategy\n    extraction_strategy: Optional = None,         # JsonCssExtractionStrategy | LLMExtractionStrategy\n    \n    # Dynamic Pages\n    js_code: Optional[list] = None,               # List of JS code snippets to execute\n    js_only: bool = False,                        # Use JS only (do not load HTML)\n    wait_for: Optional[str] = None,               # CSS selector to wait for\n    wait_for_images: bool = False,                # Wait for images to load\n    delay_before_return_html: float = 0.0,        # Additional wait in seconds before returning\n    page_timeout: int = 60000,                    # Page load timeout (ms)\n    \n    # Media\n    screenshot: bool = False,\n    screenshot_wait_for: Optional[float] = None,  # Wait "},{"path":"skill-card.md","content":"## Description:\n\nUse Crawl4AI for web scraping and content extraction, including structured data extraction, Markdown conversion, batch crawling, and AI-driven web data collection.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[openlark](https://clawhub.ai/user/openlark)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nDevelopers and engineers use this skill to configure and run Crawl4AI for web scraping, content-to-Markdown conversion, structured JSON extraction, batch crawling, deep crawling, and optional LLM-assisted extraction.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: Crawler and browser tooling can access broad network targets, persistent browser profiles, downloads, and cached page content.\n\nMitigation: Use an isolated environment, avoid private or authenticated sites unless explicitly intended, avoid normal browser profiles, and keep downloads and caches in disposable directories.\n\nRisk: Unpinned package and Docker image installation can change behavior across releases.\n\nMitigation: Pin the Crawl4AI package version and Docker image digest before use.\n\nRisk: Remote LLM extraction can disclose crawled page content to the configured provider.\n\nMitigation: Use remote LLM extraction only for content approved for that provider, or use a local model when page content should remain local.\n\n## Reference(s):\n\n- [Crawl4AI API Reference](references/api-reference.md)\n- [Crawl4AI GitHub Repository](https://github.com/unclecode/crawl4ai)\n- [Crawl4AI Documentation](https://docs.crawl4ai.com)\n\n## Skill Output:\n\n**Output Type(s):** [text, markdown, code, shell commands, configuration, guidance]\n\n**Output Format:** [Markdown guidance with Python and bash code blocks]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [May include Crawl4AI installation commands, crawler configuration, API usage examples, and safety scoping guidance.]\n\n## Skill Version(s):\n\n1.0.1 (source: server release evidence)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment."}],"languages":[],"docsSourceLabel":"CLAWHUB","editorialOverview":"Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf... Skill: Crawl4AI Web Crawler Owner: openlark Summary: Use Crawl4AI for web scraping and content extraction. Use when users need to scrape web content, extract structured data, convert web pages to Markdown, perf... Tags: latest:1.0.1 Version history: v1.0.1 | 2026-05-12T01:15:47.164Z | user Version 1.0.1 - Renamed and rebranded the skill to \"crawl4ai-web-crawler\" with updated purpose and trigger words. - Replaced the","editorialQuality":{"score":100,"threshold":65,"status":"ready","wordCount":1274,"uniquenessScore":49,"reasons":[]}},"media":{"evidence":{"source":"no-media","verified":false,"confidence":"low","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":"No screenshots, media assets, or demo links are available."},"primaryImageUrl":null,"mediaAssetCount":0,"assets":[],"demoUrl":null},"ownerResources":{"evidence":{"source":"unclaimed","verified":false,"confidence":"low","updatedAt":"2026-10-10T18:23:08.768Z","emptyReason":"This page has not been claimed by the agent owner."},"hasCustomPage":false,"customPageUpdatedAt":null,"customLinks":[],"structuredLinks":{"docsUrl":null,"demoUrl":null,"supportUrl":null,"pricingUrl":null,"statusUrl":null},"customPage":null},"relatedAgents":{"evidence":{"source":"protocol-neighbors","verified":false,"confidence":"medium","updatedAt":"2026-10-10T21:46:30.926Z","emptyReason":null},"items":[{"id":"8ebccd8e-3863-4187-8355-c3f14e1f9edf","entityType":"agent","canonicalPath":"/agent/iofficeai-aionui","slug":"iofficeai-aionui","name":"AionUi","description":"Free, local, open-source 24/7 Cowork app and OpenClaw for Gemini CLI, Claude Code, Codex, OpenCode, Qwen Code, Goose CLI, Auggie, and more | 🌟 Star if you like it!","url":"https://github.com/iOfficeAI/AionUi","homepage":"https://www.aionui.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-10-09T19:11:12.944Z","createdAt":"2026-02-25T03:38:16.584Z","downloads":null},{"id":"b917f68a-ebff-438e-84f8-3f4b2494c0bc","entityType":"agent","canonicalPath":"/agent/activepieces-activepieces","slug":"activepieces-activepieces","name":"activepieces","description":"AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents","url":"https://github.com/activepieces/activepieces","homepage":"https://www.activepieces.com","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-15T02:22:12.426Z","createdAt":"2026-02-25T03:38:12.412Z","downloads":null},{"id":"5cb26759-3a39-483f-94cf-276a98c13bb8","entityType":"agent","canonicalPath":"/agent/cherryhq-cherry-studio","slug":"cherryhq-cherry-studio","name":"cherry-studio","description":"AI productivity studio with smart chat, autonomous agents, and 300+ assistants. Unified access to frontier LLMs","url":"https://github.com/CherryHQ/cherry-studio","homepage":"https://cherry-ai.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-11T14:38:40.986Z","createdAt":"2026-02-25T03:38:19.379Z","downloads":null},{"id":"6f6582d0-5d76-4f0f-b81d-86520247950b","entityType":"agent","canonicalPath":"/agent/copilotkit-copilotkit","slug":"copilotkit-copilotkit","name":"CopilotKit","description":"The Frontend for Agents & Generative UI. React + Angular","url":"https://github.com/CopilotKit/CopilotKit","homepage":"https://docs.copilotkit.ai","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-03-25T09:50:57.846Z","createdAt":"2026-02-25T03:39:14.617Z","downloads":null}],"links":{"hub":"/agent","source":"/agent/source/clawhub","protocols":[{"label":"OpenClaw","href":"/agent/protocol/openclew"}]}}}