{"id":"2e41f8b1-35d6-4ef5-95be-25183e133aea","entityType":"agent","slug":"clawhub-kkellyoffical-nvidia-cuda","name":"NVIDIA CUDA","canonicalUrl":"https://www.xpersona.co/agent/clawhub-kkellyoffical-nvidia-cuda","canonicalPath":"/agent/clawhub-kkellyoffical-nvidia-cuda","generatedAt":"2026-10-12T14:01:21.763Z","source":"CLAWHUB","claimStatus":"UNCLAIMED","verificationTier":"NONE","summary":{"evidence":{"source":"editorial-content","verified":true,"confidence":"high","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":null},"description":"Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c... Skill: NVIDIA CUDA Owner: kkellyoffical Summary: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c... Tags: latest:0.4.0 Version history: v0.4.0 | 2026-04-19T09:11:55.823Z | user Add official FA/DDP/FSDP tooling and broader scanner coverage v0.3.0 | 2026-04-19T09:02:50.881Z | user Add latest GPU recommendations","descriptionLabel":"Technical summary","evidenceSummary":"Capability contract not published. No trust telemetry is available yet. 973 downloads reported by the source. Last updated 10/11/2026.","installCommand":"clawhub skill install s17c3bh00scgwm0549padd6cc18547af:nvidia-cuda","sourceUrl":"https://clawhub.ai/kkellyoffical/nvidia-cuda","homepage":"https://clawhub.ai/kkellyoffical/skills/nvidia-cuda","primaryLinks":[{"label":"View on ClawHub","url":"https://clawhub.ai/kkellyoffical/nvidia-cuda","kind":"source"},{"label":"Homepage","url":"https://clawhub.ai/kkellyoffical/skills/nvidia-cuda","kind":"homepage"}],"safetyScore":84,"overallRank":62,"popularityScore":60,"trustScore":null,"claimedByName":null,"isOwner":false,"seoDescription":"Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c..."},"coverage":{"evidence":{"source":"public-profile","verified":false,"confidence":"medium","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":null},"protocols":[{"protocol":"OPENCLEW","label":"OpenClaw","status":"self-declared","notes":"Declared in the public agent profile."}],"capabilities":[],"verifiedCount":0,"selfDeclaredCount":1,"capabilityMatrix":{"rows":[{"key":"OPENCLEW","type":"protocol","support":"unknown","confidenceSource":"profile","notes":"Listed on profile"}],"flattenedTokens":"protocol:OPENCLEW|unknown|profile"}},"adoption":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":null},"stars":null,"forks":null,"downloads":973,"likes":null,"task":null,"library":null,"packageName":null,"latestVersion":"0.4.0","tractionLabel":"973 downloads"},"release":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-11T23:53:27.481Z","emptyReason":null},"lastUpdatedAt":"2026-10-11T23:53:27.615Z","lastCrawledAt":"2026-10-11T23:53:27.481Z","lastIndexedAt":null,"nextCrawlAt":"2026-10-12T23:53:27.481Z","lastVerifiedAt":null,"highlights":[{"version":"0.4.0","createdAt":"2026-04-19T09:11:55.823Z","changelog":"Add official FA/DDP/FSDP tooling and broader scanner coverage","fileCount":23,"zipByteSize":39007},{"version":"0.3.0","createdAt":"2026-04-19T09:02:50.881Z","changelog":"Add latest GPU recommendations, sample configs, CI smoke, and NCCL smoke","fileCount":21,"zipByteSize":33646},{"version":"0.2.0","createdAt":"2026-04-19T08:52:48.987Z","changelog":"Add reusable probes, scanners, and benchmark scripts","fileCount":11,"zipByteSize":24330},{"version":"0.1.0","createdAt":"2026-04-19T08:44:26.647Z","changelog":"Initial public release","fileCount":6,"zipByteSize":14656}]},"execution":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No published capability contract is available yet."},"installCommand":"clawhub skill install s17c3bh00scgwm0549padd6cc18547af:nvidia-cuda","setupComplexity":"low","setupSteps":["Setup complexity is LOW. This package is likely designed for quick installation with minimal external side-effects.","Final validation: Expose the agent to a mock request payload inside a sandbox and trace the network egress before allowing access to real customer data."],"contract":{"contractStatus":"missing","authModes":[],"requires":[],"forbidden":[],"supportsMcp":false,"supportsA2a":false,"supportsStreaming":false,"inputSchemaRef":null,"outputSchemaRef":null,"dataRegion":null,"contractUpdatedAt":null,"sourceUpdatedAt":null,"freshnessSeconds":null},"invocationGuide":{"preferredApi":{"snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/trust"},"curlExamples":["curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/snapshot\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/contract\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/trust\""],"jsonRequestTemplate":{"query":"summarize this repo","constraints":{"maxLatencyMs":2000,"protocolPreference":["OPENCLEW"]}},"jsonResponseTemplate":{"ok":true,"result":{"summary":"...","confidence":0.9},"meta":{"source":"CLAWHUB","generatedAt":"2026-10-12T14:01:21.760Z"}},"retryPolicy":{"maxAttempts":3,"backoffMs":[500,1500,3500],"retryableConditions":["HTTP_429","HTTP_503","NETWORK_TIMEOUT"]}},"endpoints":{"dossierUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/dossier","snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-kkellyoffical-nvidia-cuda/trust"}},"reliability":{"evidence":{"source":"runtime-metrics","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No trust, reliability, or runtime telemetry is available."},"trust":{"status":"unavailable","handshakeStatus":"UNKNOWN","verificationFreshnessHours":null,"reputationScore":null,"p95LatencyMs":null,"successRate30d":null,"fallbackRate":null,"attempts30d":null,"trustUpdatedAt":null,"trustConfidence":"unknown","sourceUpdatedAt":null,"freshnessSeconds":null},"decisionGuardrails":{"doNotUseIf":["Contract metadata is missing or unavailable for deterministic execution."],"safeUseWhen":[],"riskFlags":["missing_or_unavailable_contract","trust_data_unavailable","schema_references_missing"],"operationalConfidence":"low"},"executionMetrics":{"observedLatencyMsP50":null,"observedLatencyMsP95":null,"estimatedCostUsd":null,"uptime30d":null,"rateLimitRpm":null,"rateLimitBurst":null,"lastVerifiedAt":null,"verificationSource":null},"runtimeMetrics":{"successRate":null,"avgLatencyMs":null,"avgCostUsd":null,"hallucinationRate":null,"retryRate":null,"disputeRate":null,"p50Latency":null,"p95Latency":null,"lastUpdated":null}},"benchmarks":{"evidence":{"source":"no-benchmark-data","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No benchmark suites or observed failure patterns are available."},"suites":[],"failurePatterns":[]},"artifacts":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"high","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":null},"readme":"Skill: NVIDIA CUDA\n\nOwner: kkellyoffical\n\nSummary: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c...\n\nTags: latest:0.4.0\n\nVersion history:\n\nv0.4.0 | 2026-04-19T09:11:55.823Z | user\n\nAdd official FA/DDP/FSDP tooling and broader scanner coverage\n\nv0.3.0 | 2026-04-19T09:02:50.881Z | user\n\nAdd latest GPU recommendations, sample configs, CI smoke, and NCCL smoke\n\nv0.2.0 | 2026-04-19T08:52:48.987Z | user\n\nAdd reusable probes, scanners, and benchmark scripts\n\nv0.1.0 | 2026-04-19T08:44:26.647Z | user\n\nInitial public release\n\nArchive index:\n\nArchive v0.4.0: 23 files, 39007 bytes\n\nFiles: agents/openai.yaml (250b), CHANGELOG.md (2773b), examples/b200-8gpu-torchrun.env (379b), examples/dgx-b300-training.yaml (878b), examples/gb300-serving.yaml (625b), examples/h200-long-context-training.yaml (701b), examples/l4-edge-serving.yaml (643b), examples/workstation-rtx-pro6000-blackwell.env (559b), README.md (6669b), references/latest-gpu-recommendations-2026-04.md (7603b), references/official-notes.md (10217b), references/runtime-support.md (1287b), requirements.txt (12b), scripts/benchmark_attention.py (4739b), scripts/check_training_stack.py (10668b), scripts/cuda_env_probe.py (4317b), scripts/dataloader_benchmark.py (3986b), scripts/ddp_fsdp_smoke.py (3228b), scripts/nccl_smoke.py (1942b), scripts/training_step_benchmark.py (4870b), skill-card.md (2431b), SKILL.md (23111b), _meta.json (130b)\n\nFile v0.4.0:SKILL.md\n\n---\nname: nvidia-cuda\ndescription: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware code conventions for PyTorch on CUDA, including dtype policy, memory movement, NCCL/DDP/FSDP choices, profiling, benchmarking, and H100/H200/B200 optimization habits.\n---\n\n# NVIDIA CUDA\n\nUse this skill when the task involves NVIDIA GPU training or inference, multi-GPU execution, CUDA/Triton kernels, or CUDA-specific performance/debugging work.\n\nThis skill is for implementation and review, not generic theory. It should push the work toward:\n\n- correct GPU usage first\n- stable and measurable performance second\n- low-risk tuning before low-level kernel work\n- frontier algorithms only when the workload shape justifies them\n\n## Hardware stance\n\nProbe the actual machine before acting. This skill is written for modern NVIDIA Tensor Core systems and is especially opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class accelerators\n\nIt assumes the agent should actively choose precision, attention backends, sharding strategy, logging frequency, and dataloader settings instead of inheriting slow defaults.\n\n## Use this skill for\n\n- PyTorch training or inference on CUDA\n- GPU memory, throughput, or latency bottlenecks\n- DDP / FSDP / NCCL decisions\n- CUDA env var and runtime hygiene\n- Triton or custom CUDA kernel review\n- benchmark and profiler setup for NVIDIA GPUs\n\n## Do not use this skill for\n\n- CPU-only optimization\n- ROCm / AMD-specific work\n- TPU / XLA-specific work\n- vague \"make it faster\" requests without inspecting the actual GPU path first\n\n## First-pass workflow\n\nBefore changing code, inspect the actual runtime surface:\n\n1. Check hardware and driver: `nvidia-smi`\n2. Check framework build: `python -c \"import torch; print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\"`\n3. Check device visibility and topology:\n   - `CUDA_VISIBLE_DEVICES`\n   - `torch.cuda.device_count()`\n   - `torch.cuda.get_device_properties(i)`\n4. Check whether the bottleneck is:\n   - input pipeline\n   - host/device transfer\n   - eager Python overhead\n   - kernel efficiency\n   - distributed communication\n   - memory pressure / fragmentation / OOM\n\nDo not jump to kernel rewrites before ruling out bad data movement, wrong dtype, graph breaks, or poor distributed setup.\n\n## Bundled tooling\n\nUse the included scripts when you need deterministic probes instead of ad hoc snippets:\n\n- `scripts/cuda_env_probe.py`: collect CUDA, PyTorch, device, and env facts\n- `scripts/check_training_stack.py`: scan Python code for high-cost anti-patterns\n- `scripts/benchmark_attention.py`: benchmark SDPA, flash, cuDNN, and official flash-implementation activation paths\n- `scripts/training_step_benchmark.py`: benchmark a synthetic transformer training step with dtype, compile, and `.item()` logging knobs\n- `scripts/dataloader_benchmark.py`: benchmark DataLoader worker, pinning, and prefetch settings\n- `scripts/nccl_smoke.py`: run a minimal NCCL all-reduce smoke test under `torchrun`\n- `scripts/ddp_fsdp_smoke.py`: run a one-step DDP or FSDP training smoke test under `torchrun`\n\nRun the probe first, then benchmark or scan the real workload path.\n\n## Planning hardware purchases\n\nWhen the user wants current NVIDIA GPU recommendations, read:\n\n- [references/latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md)\n- the example configs under [examples/](examples)\n\nKeep recommendation output scenario-based:\n\n- cost-sensitive local prototyping\n- serious workstation development\n- enterprise server deployment\n- single-node training\n- rack-scale training or reasoning\n\nDo not recommend by peak FLOPS alone. Weight memory, interconnect, thermals, deployment form, and software maturity.\n\n## Reviewing Triton, CUDA, and distributed code\n\nWhen the target includes Triton kernels, CUDA or C++ files, or distributed launcher code:\n\n- use `scripts/check_training_stack.py` across the whole tree, not just Python subfolders\n- treat Triton kernels as first-class review targets\n- verify distributed paths with `scripts/nccl_smoke.py` or `scripts/ddp_fsdp_smoke.py` before claiming they are healthy\n- benchmark flash attention implementation changes with `scripts/benchmark_attention.py --list-flash-impls` and explicit activation when available\n\n## Non-negotiable code conventions\n\n### High-cost anti-patterns to ban\n\nTreat the following as red flags in reviews and optimization work:\n\n1. keeping transformer training or inference in full FP32 by default on Tensor Core GPUs\n2. refusing to try `torch.compile` on stable hot paths\n3. staying on a generic default SDPA path on H200/B200 when newer flash-backed implementations or cuDNN/TE attention backends are available and benchmarkable\n4. reaching for activation checkpointing before FSDP / ZeRO-style sharding when the real problem is replicated params, grads, or optimizer state\n5. calling `.item()` every step for logging and metrics\n6. leaving DataLoader workers, pinning, prefetching, and persistent workers untuned until GPU starvation shows up\n\nIf several of these exist at once, assume the GPU is underfed until proven otherwise.\n\n### 1. Dtype policy\n\n- On H100 / Hopper, prefer `bfloat16` for training and inference unless there is a measured or correctness-driven reason not to.\n- On H200 / B200-class systems, treat BF16 or FP8-capable paths as the default starting point for deep learning workloads.\n- Full FP32 by default is a correctness mode, not a performance baseline.\n- Use `float16` only when a dependency or model explicitly benefits from it.\n- If using `float16` training, use `torch.amp.GradScaler(\"cuda\")`.\n- For mixed precision, use the current API:\n  - `torch.amp.autocast(\"cuda\", dtype=...)`\n  - not deprecated `torch.cuda.amp.*`\n\n### 2. FP32 fallback policy\n\n- If the model is nominally FP32 but does not require strict IEEE-style matmul precision, prefer enabling faster matmul paths with `torch.set_float32_matmul_precision(\"high\")`.\n- Disable TF32 only for reproducibility or numerics investigations.\n- Never silently mix \"strict numerics\" and \"fast numerics\" requirements in the same change. State the choice.\n\n### 3. Device movement rules\n\n- Keep tensors on device across the hot path.\n- Move data in batches, not item-by-item.\n- Use `pin_memory=True` in `DataLoader` when feeding CUDA.\n- Use `tensor.to(\"cuda\", non_blocking=True)` or equivalent only when the source is pinned or already async-safe.\n- Avoid CPU round-trips in the training or inference loop.\n\n### 4. Avoid accidental synchronization\n\nInside hot loops, avoid or justify:\n\n- `.item()`\n- `.cpu()`\n- `.numpy()`\n- `print(cuda_tensor)`\n- timing without `torch.cuda.synchronize()`\n\nThese often force host synchronization and distort both throughput and profiling.\n\n### 5. Benchmarking rules\n\n- Warm up before measuring.\n- Use CUDA events or a framework benchmark harness, not bare wall clock alone.\n- Synchronize before reading timings.\n- Record batch size, dtype, sequence/image shape, compile state, and number of warmup / measured iterations.\n- For throughput and latency claims, include the exact command or code path used.\n\n### 6. `torch.compile` rules\n\n- Reach correctness first in eager mode.\n- Then apply `torch.compile` to stable hot paths.\n- Do not reject `torch.compile` categorically for CUDA models. Benchmark it.\n- Prefer:\n  - `mode=\"default\"` for general speedups\n  - `mode=\"reduce-overhead\"` for small-batch / latency-sensitive CUDA paths with stable shapes\n  - `mode=\"max-autotune\"` when compile overhead is acceptable and the path is hot enough\n- If shapes are unstable, be explicit about `dynamic=True` or accept recompiles intentionally.\n- Use `TORCH_LOGS=perf_hints` or `TORCH_LOGS=dynamic` when diagnosing graph breaks or overspecialization.\n\n### 6.1 Attention kernel policy\n\n- For transformer attention, prefer native `scaled_dot_product_attention` and PyTorch attention backends before custom kernels.\n- Use PyTorch SDPA / FlashAttention backends as the default fast path on CUDA.\n- On H200 / B200-class systems, do not assume the default shipped SDPA backend is the best available path. Check whether FA3 / FA4 implementations are registered and benchmark them.\n- Where Transformer Engine or cuDNN attention backends are available, benchmark them against the default path on Hopper/Blackwell instead of assuming SDPA wins.\n- If masks or score transforms are unusual, consider `torch.nn.attention.flex_attention` only after confirming SDPA cannot express the workload cleanly.\n- For variable-length packed batches, prefer varlen / packed attention paths rather than padding to worst-case sequence length.\n- Do not keep a hand-rolled eager attention implementation in a hot path if SDPA, TE attention, or TensorRT-LLM fused attention can replace it.\n\n### 6.2 CUDA Graphs policy\n\n- Use CUDA Graphs only for repeated, shape-stable hot paths where CPU launch overhead is material.\n- Graph capture is a strong fit for:\n  - small-batch inference\n  - fixed-shape decode loops\n  - stable training microbatches\n- Do not graph workloads with live shape churn, `.item()` sync points, or allocator churn in the captured region.\n- Prefer graphing after eager correctness and after compile/engine selection are already settled.\n\n### 7. Distributed rules\n\n- Use NCCL for distributed CUDA training.\n- Use DDP when the model fits on one GPU and you mainly need data parallel scale-out.\n- Use FSDP2 when the model does not fit on one GPU or optimizer state dominates memory.\n- When tensor parallelism is enabled, sequence parallelism is the default companion unless a framework limitation blocks it.\n- For long-context transformer training, consider context parallelism before blindly increasing tensor parallel degree.\n- For MoE, use expert parallelism instead of replicating all experts everywhere.\n- For large-scale runs, consider a distributed optimizer before inventing custom sharding logic.\n- Use `torchrun` and one process per GPU.\n- Set the current CUDA device from rank/local-rank early and only once.\n- Do not tune NCCL environment variables by superstition. Change them only after evidence.\n\n### 8. Memory rules\n\n- Prefer algorithmic memory wins before allocator tweaks:\n  - BF16\n  - FSDP / sharding\n  - distributed optimizer\n  - smaller activation footprints\n  - sequence or batch shaping\n  - activation checkpointing\n- When the memory problem is replicated model state, prioritize FSDP / ZeRO-2 / ZeRO-3 style sharding before gradient checkpointing.\n- Use checkpointing when activation memory is the real limiter, not as a substitute for state sharding.\n- If activation checkpointing is needed, prefer `use_reentrant=False` unless there is a specific reason otherwise.\n- Treat allocator and workspace env vars as debugging/tuning levers, not first-line fixes.\n\n### 9. Profiling rules\n\n- Start with Nsight Systems for timeline truth:\n  - CPU launch gaps\n  - H2D / D2H copies\n  - stream overlap\n  - NCCL overlap and stalls\n  - Unified Memory migrations\n- Move to Nsight Compute only after isolating the hot kernels.\n- For PyTorch-level attribution, use the profiler only if you need operator names or stack grouping.\n\n## Frontier algorithm ladder\n\nUse this section when the user explicitly wants leading-edge optimization or when the baseline is already healthy.\n\nDo not apply all of these at once. Climb the ladder in order and keep a benchmark after each step.\n\n### A. Hopper and Blackwell transformer path\n\nFor H100/H200/B200-class transformer workloads, the default advanced path is:\n\n1. BF16 baseline\n2. native SDPA / FlashAttention fast path\n3. `torch.compile`\n4. CUDA Graphs if shapes stabilize\n5. Transformer Engine FP8 if:\n   - the model is transformer-dominated\n   - BF16 numerics are already validated\n   - there is an accuracy gate\n\nFP8 is not a blanket default. It is a targeted transformer optimization with strict measurement.\n\n### B. Large-model training algorithms\n\nFor models that are too large, too long-context, or too communication-heavy, consider:\n\n- FSDP2 / Megatron FSDP for parameter, grad, and optimizer sharding\n- distributed optimizer for optimizer-state memory pressure\n- tensor parallelism for large hidden dimensions\n- sequence parallelism when TP is enabled\n- context parallelism for long sequences\n- pipeline parallelism for very deep models\n- expert parallelism for MoE\n- activation checkpointing when the above is still insufficient\n\nSelection heuristics:\n\n- if the model barely misses single-GPU fit: start with FSDP2\n- if hidden dimensions are very large: add TP\n- if sequence length is the actual problem: prefer CP over just raising TP\n- if the model is MoE: use EP rather than replicating experts\n- if optimizer state dominates: distributed optimizer or FSDP shard first\n\n### C. LLM inference algorithms\n\nFor autoregressive serving, the advanced inference ladder is:\n\n1. fused attention path\n2. paged KV cache / paged context handling\n3. continuous or inflight batching\n4. chunked prefill / context chunking for large prompts\n5. CUDA Graphs for stable decode loops\n6. speculative decoding when low-batch latency matters and acceptance rate is healthy\n7. FP8 or weight-focused quantization only after accuracy validation\n\nStrong fits:\n\n- paged context / chunked prefill: long prompts and mixed request sizes\n- speculative decoding: low-batch underutilized serving\n- inflight fused batching: multi-request serving with scheduler pressure\n- FP8 serving: Hopper/Blackwell inference paths with validated quality\n\n### D. Quantization policy\n\n- On Hopper and newer, prefer BF16 first, FP8 second, INT8/INT4 only when memory or throughput targets require it.\n- For training or finetuning quantized targets, prefer QAT or recipe-backed flows over ad hoc fake-quant code.\n- For serving, keep a clear boundary between:\n  - engine build or calibration\n  - runtime benchmarking\n  - accuracy evaluation\n\nDo not merge a quantization change without an explicit quality gate.\n\n## Training optimization playbook\n\nWhen optimizing training code, prefer this order:\n\n1. Ensure correct device placement and eliminate CPU fallbacks\n2. Fix input pipeline:\n   - `pin_memory=True`\n   - enough workers\n   - `persistent_workers=True` when the workload is steady and worker startup cost matters\n   - tuned `prefetch_factor`\n   - no CPU-heavy transforms inside the critical path unless justified\n3. Set precision policy:\n   - BF16 first on Hopper\n   - BF16 / FP8-capable path first on H200/B200\n   - FP16 only with explicit reason\n4. Remove accidental syncs and per-step Python overhead\n5. Try `torch.compile`\n6. If this is transformer-heavy, move attention to SDPA / FlashAttention / TE attention fast paths\n7. Scale to DDP or FSDP2 if single-GPU utilization or model size requires it\n8. Add TP + SP, CP, PP, or EP only according to the actual bottleneck\n9. Add activation checkpointing if memory still blocks batch size\n10. For Hopper/Blackwell transformer stacks, evaluate Transformer Engine FP8 behind an accuracy gate\n11. Profile and only then consider Triton or custom CUDA work\n\n## Inference optimization playbook\n\nWhen optimizing inference code:\n\n1. Use `model.eval()` and `torch.no_grad()` / `torch.inference_mode()`\n2. Pick dtype deliberately:\n   - BF16 or FP16 on Hopper when numerically acceptable\n   - TF32-enabled FP32 if accuracy must stay near FP32 but pure FP32 is too slow\n3. Move attention and decode loops onto fused kernels or native SDPA paths\n4. Stabilize shapes if possible\n5. Apply `torch.compile` or engine compilation only after correctness baselines\n6. For LLM serving, evaluate paged KV / paged context and inflight batching early\n7. Use CUDA Graphs only for repeated, fixed-shape paths where launch overhead matters\n8. If low-batch latency dominates, evaluate speculative decoding with acceptance-rate tracking\n9. Benchmark after warmup, with synchronization\n10. If using TensorRT or TensorRT-LLM, profile runtime separately from engine build\n\n## Multi-GPU and NCCL playbook\n\nBaseline expectations:\n\n- launch with `torchrun`\n- backend `nccl`\n- one process per GPU\n- clear `MASTER_ADDR`, `MASTER_PORT`, `RANK`, `WORLD_SIZE`, `LOCAL_RANK`\n\nParallelism heuristics:\n\n- `DP` first for ordinary scale-out\n- `FSDP2` when full replication is too expensive\n- `FSDP / ZeRO-2 / ZeRO-3` before checkpointing when optimizer, grad, or param state replication is the memory bottleneck\n- `TP + SP` for large transformer blocks\n- `CP` for long-context models\n- `PP` for deep stacks that still do not fit\n- `EP` for MoE\n- combine only what the bottleneck requires\n\nDebugging rules:\n\n- First-line NCCL debug:\n  - `NCCL_DEBUG=INFO`\n  - `NCCL_DEBUG_SUBSYS=COLL,GRAPH` when debugging hangs or topology issues\n- If interface auto-detection is wrong, set `NCCL_SOCKET_IFNAME`\n- Use `TORCH_NCCL_USE_COMM_NONBLOCKING=1` when you specifically want non-blocking NCCL error handling\n\nDo not hardcode socket or thread tuning variables unless profiling or production evidence shows the default topology tuning is wrong.\n\n## Custom CUDA / Triton kernel rules\n\nOnly drop to Triton or CUDA when operator fusion, layout cleanup, `torch.compile`, or library kernels are not enough.\n\nFor custom kernels:\n\n- favor coalesced global memory access\n- treat shared memory as a tool, not a reflex\n- watch register pressure; occupancy is a means, not the target\n- use block sizes that are multiples of 32\n- start experiments around `128` to `256` threads per block unless the kernel shape clearly suggests otherwise\n- avoid giant one-block-per-SM thinking; multiple resident blocks often help latency hiding\n- measure spills, occupancy, achieved bandwidth, and kernel time before and after changes\n\nDo not claim a kernel is faster because occupancy increased. Show end-to-end or kernel-level timing.\n\n## Transformer Engine and FP8 rules\n\nUse Transformer Engine when all of these are true:\n\n- the workload is transformer-dominated\n- the target GPUs are Hopper/Ada/Blackwell-class\n- BF16 baseline is already correct\n- you can run an accuracy gate\n\nRules:\n\n- wrap only the forward pass in TE FP8 autocast\n- use recipe-backed scaling, not home-grown FP8 metadata logic\n- start with delayed scaling or hybrid FP8 recipes\n- keep optimizer and master-weight behavior explicit\n- if FP8 helps throughput but destabilizes training, fall back to BF16 instead of layering hacks\n\nPrefer TE modules or established framework integration over custom FP8 plumbing.\n\n## Attention and sequence-shape rules\n\nFor long-sequence transformer work:\n\n- prefer fused attention over matmul-softmax-matmul Python compositions\n- on H200/B200, actively test newer flash attention implementations when available instead of passively staying with the default SDPA stack\n- prefer grouped or multi-query attention-aware fast paths when the model architecture already uses GQA or MQA\n- prefer context parallelism and packed or varlen attention before wasting memory on blanket padding\n- if custom masking is the only blocker, test FlexAttention before writing a new CUDA kernel\n\nFor decoder serving:\n\n- KV cache layout is part of the algorithm, not just a storage detail\n- paged KV and chunked context handling are first-class optimization choices\n- scheduler choice affects TTFT and ITL, so benchmark serving algorithms at the scheduler level, not only kernel level\n\n## Logging and metric collection rules\n\n- Do not call `.item()` every step for loss curves, metric dashboards, or debug prints.\n- Aggregate metrics on device and synchronize periodically:\n  - every `N` steps\n  - end of an accumulation window\n  - end of an evaluation window\n- If exact per-step host-visible metrics are required for debugging, state that this is a debug-only mode.\n- Prefer batched reductions and detached device-side buffers over step-wise host sync.\n\n## Data pipeline and prefetch rules\n\nFor CUDA training jobs, always inspect the input path before blaming kernels.\n\nRequired checks:\n\n- `num_workers`\n- `pin_memory`\n- `persistent_workers`\n- `prefetch_factor`\n- CPU decode, tokenize, or augment cost\n- storage throughput and remote filesystem latency\n\nRules:\n\n- If GPU utilization dips between steps, inspect the dataloader before touching kernels.\n- Tune `prefetch_factor` and worker count together; more workers without enough prefetch often just moves the bottleneck.\n- Keep host preprocessing out of the critical path when possible.\n- For very fast H100/H200/B200 training loops, assume untuned input pipelines are guilty until measurements say otherwise.\n\n## Reproducibility and numerics rules\n\n- Be explicit when a change trades accuracy for speed.\n- cuDNN determinism is not guaranteed across architectures.\n- Across devices or architectures, do not promise bitwise-identical outputs.\n- If NaNs or Infs appear:\n  - check data first\n  - check loss scaling or dtype policy\n  - check reduced-precision reductions\n  - inspect unstable linalg paths\n\n## Environment variable policy\n\nAllowed as targeted tools, not blanket defaults:\n\n- `CUDA_VISIBLE_DEVICES`\n- `CUDA_LAUNCH_BLOCKING=1` for debugging only\n- `PYTORCH_ALLOC_CONF`\n- `TORCH_CUDNN_V8_API_LRU_CACHE_LIMIT`\n- `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE`\n- `TORCH_NCCL_USE_COMM_NONBLOCKING=1`\n- `NCCL_DEBUG`\n- `NCCL_DEBUG_SUBSYS`\n- `NCCL_SOCKET_IFNAME`\n\nIf you set an env var in code, config, or docs, explain:\n\n1. why it is needed\n2. whether it is for debugging, correctness, or performance\n3. when it should be removed\n\n## Review checklist\n\nWhen reviewing or generating code in this domain, check for:\n\n- tensors bouncing between CPU and GPU\n- missing `pin_memory=True` on CUDA dataloaders\n- missing `non_blocking=True` on pinned copies\n- missing `persistent_workers=True` or untuned `prefetch_factor` on steady-state training jobs\n- deprecated AMP API usage\n- full FP32 training or inference on Hopper/Blackwell with no numerics justification\n- FP16 used by default on Hopper with no reason\n- no BF16 or FP8 path evaluation on H200/B200-class hardware\n- no warmup in benchmarks\n- no synchronize around timing\n- `.item()` / `.cpu()` / `.numpy()` inside the step loop\n- distributed code using non-NCCL backend for CUDA\n- DDP used where model clearly does not fit, instead of FSDP2\n- activation checkpointing used to paper over what should be FSDP or ZeRO sharding\n- TP enabled without SP where the framework expects SP\n- long-context workloads scaled by TP alone when CP is the better lever\n- transformer stack on Hopper staying on unfused attention with no reason\n- transformer stack on H200/B200 never testing newer flash attention implementations\n- speculative decoding added with no acceptance-rate measurement\n- quantization merged with no quality gate\n- FP8 added without recipe-backed scaling or accuracy validation\n- custom kernel work started before simpler wins were exhausted\n\n## Output contract\n\nWhen using this skill, report in this order:\n\n1. runtime facts\n2. bottleneck hypothesis\n3. chosen optimization order\n4. changes made\n5. verification evidence\n6. remaining risks\n\nIf you need deeper justification or exact source-backed tuning notes, read [references/official-notes.md](references/official-notes.md).\n\nFile v0.4.0:README.md\n\n# nvidia-cuda\n\n`nvidia-cuda` is an open-source skill package for Codex, OpenClaw, and similar agent systems that need to write or review deep learning code for NVIDIA GPUs.\n\nIt is opinionated on purpose. The package encodes modern AI infra practice for H100, H200, and B200-class systems:\n\n- BF16 and FP8-aware precision policy\n- `torch.compile` as a benchmarked option, not a taboo\n- attention backend selection that does not stop at default SDPA\n- FSDP and ZeRO before using gradient checkpointing as a crutch\n- `.item()` and host sync anti-pattern detection\n- dataloader, pinning, prefetch, and persistent worker checks\n- NCCL, TP/SP/CP/EP, TensorRT-LLM, speculative decoding, and TE guidance\n\n## Why this exists\n\nA lot of GPU training code still leaves major performance on the floor because it inherits conservative defaults:\n\n- full FP32 everywhere\n- no compile pass\n- no attention backend evaluation on newer hardware\n- per-step host synchronization for metrics\n- checkpointing used before state sharding\n- dataloaders that starve the GPU\n\nThose mistakes compound. This skill gives an agent a clear optimization order and a concrete review checklist.\n\n## What is inside\n\n- [SKILL.md](SKILL.md): the main operational policy\n- [references/official-notes.md](references/official-notes.md): official PyTorch and NVIDIA references\n- [references/latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md): latest GPU shortlist as of 2026-04-19\n- [references/runtime-support.md](references/runtime-support.md): tested and expected runtime surface\n- [agents/openai.yaml](agents/openai.yaml): optional UI metadata for skill-aware clients\n- [scripts/](scripts/): reusable probes, scanners, and benchmarks\n- [examples/](examples/): sample environment and planning configs\n\n## Bundled tooling\n\nThe repository now includes reusable scripts:\n\n- `scripts/cuda_env_probe.py`\n- `scripts/check_training_stack.py`\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n- `scripts/nccl_smoke.py`\n- `scripts/ddp_fsdp_smoke.py`\n\nExample commands:\n\n```bash\npython scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --list-flash-impls\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\ntorchrun --standalone --nproc_per_node=2 scripts/nccl_smoke.py --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode ddp --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode fsdp --json\n```\n\n## Install locally\n\n### Codex\n\nCopy this directory to your skills path as `~/.codex/skills/nvidia-cuda/`.\n\n### OpenClaw / ClawHub-compatible agents\n\nThe repository root is already a skill package:\n\n- `SKILL.md` at the root\n- supporting files under `references/` and `agents/`\n\nThat means you can publish this repository directly as a skill package after review.\n\n## Triggering\n\nUse the skill by name:\n\n```text\n$nvidia-cuda optimize this H100 training loop\n```\n\nExample prompts:\n\n- `$nvidia-cuda review this multi-GPU training stack for H200 underutilization`\n- `$nvidia-cuda optimize this transformer inference path on B200`\n- `$nvidia-cuda audit this dataloader, precision policy, and logging loop`\n\n## Hardware and framework stance\n\nThe skill is most opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class GPUs\n\nThe primary software target is PyTorch on CUDA, with side coverage for:\n\n- NCCL\n- Triton\n- Transformer Engine\n- TensorRT-LLM\n- Megatron Core style large-model sharding\n\n## Latest GPU recommendations\n\nAs of **2026-04-19**, the practical shortlist in this repository is:\n\n- **RTX 5090** for cost-sensitive local prototyping\n- **RTX PRO 6000 Blackwell Workstation Edition** for serious single-workstation AI development\n- **DGX Station** if you want a full deskside AI system rather than a loose card\n- **RTX PRO 6000 Blackwell Server Edition** for universal enterprise server deployment\n- **H200** for memory-first Hopper deployments\n- **DGX B300** for the newest top-end single-node training system\n- **DGX B200** as the current Blackwell step-down single-node training system\n- **GB200 NVL72** for rack-scale frontier training and trillion-parameter inference\n- **GB300 NVL72** as the newest rack-scale recommendation for AI reasoning and test-time scaling\n- **L4** for efficient low-power inference\n\nDetailed rationale and official links:\n\n- [latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md)\n\n## Sample configs\n\nThis repository now includes planning and environment examples:\n\n- [workstation-rtx-pro6000-blackwell.env](examples/workstation-rtx-pro6000-blackwell.env)\n- [b200-8gpu-torchrun.env](examples/b200-8gpu-torchrun.env)\n- [dgx-b300-training.yaml](examples/dgx-b300-training.yaml)\n- [h200-long-context-training.yaml](examples/h200-long-context-training.yaml)\n- [gb300-serving.yaml](examples/gb300-serving.yaml)\n- [l4-edge-serving.yaml](examples/l4-edge-serving.yaml)\n- [runtime-support.md](references/runtime-support.md)\n\n## Release policy\n\nThis repository uses semver. The current target release is `0.4.0`.\n\nVersion `0.4.0` establishes:\n\n- the initial optimization doctrine\n- the anti-pattern blocklist\n- H100/H200/B200 attention and precision guidance\n- training and inference optimization ladders\n- ClawHub-ready repository packaging\n- reusable probes and benchmark scripts for environment, attention, training-step, and dataloader analysis\n- current NVIDIA GPU purchase guidance with official references and sample planning configs\n- a basic CI smoke workflow and NCCL smoke-test helper\n- explicit DDP/FSDP smoke coverage and broader scanner coverage for Triton, CUDA, and C++ review surfaces\n\n## ClawHub publication note\n\nClawHub supports publishing skill packages from a local path via its CLI.\n\nAt the time this repository was prepared:\n\n- the upstream ClawHub project documents `clawhub skill publish <path>`\n- the ClawHub web UI also supports signed-in publication\n\nCLI example:\n\n```bash\nnpx clawhub@latest publish . \\\n  --slug nvidia-cuda \\\n  --name \"NVIDIA CUDA\" \\\n  --version 0.4.0 \\\n  --changelog \"Add official FA/DDP/FSDP tooling and broader scanner coverage\"\n```\n\nBecause community skill marketplaces can carry supply-chain risk, publish only reviewed, minimal, instruction-focused packages and inspect any scanner output before making a listing public.\n\n## License\n\n[MIT-0](LICENSE)\n\nFile v0.4.0:_meta.json\n\n{\n  \"ownerId\": \"kn7bcfdgsryztp2rsxyq54gydh82b5be\",\n  \"slug\": \"nvidia-cuda\",\n  \"version\": \"0.4.0\",\n  \"publishedAt\": 1776589915823\n}\n\nFile v0.4.0:references/latest-gpu-recommendations-2026-04.md\n\n# Latest NVIDIA GPU Recommendations for AI Infra\n\nLast reviewed: 2026-04-19\n\nThis note summarizes practical NVIDIA GPU recommendations for AI infrastructure based on official NVIDIA product positioning and published specifications as of 2026-04-19.\n\nImportant: these recommendations are partly **inference from official specs and positioning**, not direct vendor buying advice. Use them to choose a shortlist, then validate availability, thermals, interconnect, memory, and software fit for your workload.\n\nNVIDIA's own data center line card labels products relative to each workload column. The ranking below is a practical shortlist built from that line card plus current product pages, not an official NVIDIA rank list.\n\n## Recommendation matrix\n\n### 1. Cost-sensitive local prototyping\n\nRecommended:\n\n- **GeForce RTX 5090**\n\nUse when:\n\n- you need a powerful local GPU for experimentation\n- your models fit within **32 GB GDDR7**\n- you are doing inference, small finetunes, QLoRA-style work, or kernel benchmarking\n\nAvoid as the default choice when:\n\n- you need large-memory training\n- you need ECC-centric enterprise workflows\n- you need dense multi-GPU workstation or server deployment\n\nWhy:\n\n- NVIDIA positions RTX 5090 as its flagship GeForce GPU with **32 GB GDDR7** and Blackwell architecture.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5090/\n\n### 2. Serious single-workstation AI development\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Workstation Edition**\n- **NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition** when density or power matters more than peak board power\n\nUse when:\n\n- you want a desk-side AI workstation\n- you need **96 GB GDDR7 with ECC**\n- you want stronger enterprise and workstation ergonomics than GeForce\n\nWhy:\n\n- NVIDIA positions the Workstation Edition as its most powerful desktop GPU for professionals.\n- NVIDIA positions the Max-Q variant as a **300 W** option that scales to **up to four GPUs in one system**.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000/\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000-max-q/\n\nIf you want a full deskside AI system rather than a loose card:\n\n- **NVIDIA DGX Station**\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/dgx-station/\n\n### 3. Universal enterprise server GPU\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Server Edition**\n\nUse when:\n\n- you need a flexible enterprise server GPU for AI plus graphics or simulation\n- you want **96 GB GDDR7 with ECC**\n- you are building enterprise servers rather than bare enthusiast workstations\n\nWhy:\n\n- NVIDIA positions this card as a universal Blackwell data center GPU for AI and visual computing.\n- It is a strong fit for enterprise inference, agentic AI, multimodal systems, and mixed graphics plus AI environments.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/rtx-pro-6000-blackwell-server-edition/\n\n### 4. Memory-first Hopper node\n\nRecommended:\n\n- **NVIDIA H200**\n\nUse when:\n\n- your software stack is Hopper-optimized already\n- model memory and bandwidth are the real limit\n- you want **141 GB HBM3E** and **4.8 TB/s** bandwidth per GPU\n\nWhy:\n\n- H200 is still compelling when memory capacity and Hopper continuity matter more than jumping immediately to Blackwell systems.\n\nOfficial:\n\n- https://www.nvidia.com/en-gb/data-center/h200/\n\n### 5. Single-node frontier training\n\nRecommended:\n\n- **NVIDIA DGX B300**\n- **NVIDIA DGX B200**\n- **NVIDIA DGX H200** as the Hopper fallback\n\nUse when:\n\n- you are buying a serious training node rather than a single loose GPU\n- you want a turnkey high-end training node instead of integrating loose GPUs yourself\n\nWhy:\n\n- DGX B300 is the newest top-end single-node training system in the official lineup surfaced here.\n- NVIDIA states DGX B200 delivers **3x training** and **15x inference** performance versus DGX H100 in its published positioning.\n- DGX H200 remains the Hopper fallback when software continuity or procurement timing makes Blackwell rollout less practical.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/dgx-b300/\n- https://www.nvidia.com/en-us/data-center/dgx-b200/\n- https://www.nvidia.com/en-us/data-center/dgx-h200/\n\n### 6. Rack-scale frontier training and trillion-parameter inference\n\nRecommended:\n\n- **NVIDIA GB200 NVL72**\n- **NVIDIA HGX B300** as the scale-up platform choice when you are building around HGX systems\n\nUse when:\n\n- you need rack-scale Blackwell today\n- you are targeting massive training or real-time trillion-parameter inference\n\nWhy:\n\n- NVIDIA positions GB200 NVL72 as a **72-GPU NVLink domain** and publishes **4x training vs H100** and **30x real-time LLM inference** positioning.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/gb200-nvl72/\n- https://www.nvidia.com/en-us/data-center/hgx/\n\n### 7. Latest rack-scale reasoning and test-time scaling\n\nRecommended:\n\n- **NVIDIA GB300 NVL72**\n\nUse when:\n\n- the priority is latest-generation reasoning or test-time scaling infrastructure\n- you are planning a new rack-scale Blackwell Ultra purchase\n\nWhy:\n\n- NVIDIA positions GB300 NVL72 specifically for AI reasoning and test-time scaling.\n- If you are choosing the most current top-end NVIDIA rack-scale platform as of **2026-04-19**, this is the newest one in the official lineup surfaced here.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/gb300-nvl72/\n\n### 8. Enterprise and edge inference efficiency\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Server Edition**\n- **NVIDIA L4 Tensor Core GPU**\n\nUse when:\n\n- you need enterprise inference in a conventional server footprint\n- you need low-power inference or video-plus-AI deployment at the edge\n\nWhy:\n\n- RTX PRO 6000 Blackwell Server Edition is the enterprise universal Blackwell server option.\n- L4 remains a practical low-power inference choice when deployment density and efficiency matter more than frontier throughput.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/rtx-pro-6000-blackwell-server-edition/\n- https://www.nvidia.com/en-us/data-center/l4/\n\n## Practical shortlist by scenario\n\n- **Cheapest serious local bring-up**: RTX 5090\n- **Best serious single-workstation AI GPU**: RTX PRO 6000 Blackwell Workstation Edition\n- **Best dense workstation variant**: RTX PRO 6000 Blackwell Max-Q Workstation Edition\n- **Best deskside full system**: DGX Station\n- **Best universal enterprise server GPU**: RTX PRO 6000 Blackwell Server Edition\n- **Best memory-first Hopper choice**: H200\n- **Best single-node training system**: DGX B300\n- **Best single-node step-down training system**: DGX B200\n- **Best Hopper fallback training node**: DGX H200\n- **Best deployed rack-scale Blackwell platform**: GB200 NVL72\n- **Newest top-end rack-scale recommendation**: GB300 NVL72\n- **Best low-power inference choice**: L4\n\n## Caveats\n\n- Consumer GPUs are attractive for price and local availability, but they are not the default recommendation for multi-user or production server environments.\n- H200 remains a strong recommendation when memory capacity, Hopper software maturity, or HBM bandwidth dominate the decision.\n- DGX and NVL systems are system-level recommendations, not loose-GPU recommendations.\n- If your workload is dominated by inference memory capacity and scheduler behavior rather than raw training throughput, shortlist by **memory, KV-cache behavior, and software path**, not just peak FLOPS.\n\n## Additional official source\n\n- NVIDIA Data Center Platform Line Card:\n  - https://docs.nvidia.com/data-center-gpu/line-card.pdf\n\nFile v0.4.0:references/official-notes.md\n\n# NVIDIA CUDA Skill Notes\n\nThis file is reference material for the `nvidia-cuda` skill. Read it when you need official grounding for a tuning choice, a code review decision, or a user-facing explanation.\n\n## Official facts to use\n\n### Mixed precision\n\n- PyTorch AMP says lower precision ops such as linear layers and convolutions are often much faster in lower precision, while other ops still need FP32 range.\n- Current AMP API is `torch.amp.autocast(...)` and `torch.amp.GradScaler(...)`; old `torch.cuda.amp.*` entry points are deprecated.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/amp.html\n\n### TF32 and FP32 matmul precision\n\n- `torch.set_float32_matmul_precision(\"high\" | \"medium\")` can significantly improve performance for FP32 matmuls on CUDA.\n- It changes internal computation precision, not output dtype.\n- On Ampere and later, TF32 can improve speed but may reduce accuracy for some workloads.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html\n  - https://docs.pytorch.org/docs/stable/notes/numerical_accuracy.html\n\n### CUDA env vars and allocator/debug knobs\n\n- `PYTORCH_ALLOC_CONF`, `CUDA_LAUNCH_BLOCKING`, `CUDA_VISIBLE_DEVICES`, TF32 override knobs, cuDNN cache or workspace knobs, and NCCL non-blocking error handling are documented in one place.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/cuda_environment_variables.html\n\n### Data movement\n\n- PyTorch data docs say host-to-GPU copies are much faster from pinned memory.\n- `DataLoader(pin_memory=True)` automatically places fetched tensors in pinned memory.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/data.html\n\n### Distributed backend choice\n\n- PyTorch distributed docs say the rule of thumb for CUDA distributed training is NCCL.\n- PyTorch distributed overview says:\n  - use DDP when the model fits on one GPU\n  - use FSDP2 when the model cannot fit on one GPU\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n  - https://docs.pytorch.org/tutorials/beginner/dist_overview.html\n  - https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html\n\n### Activation checkpointing\n\n- Activation checkpointing trades compute for memory.\n- PyTorch recommends `use_reentrant=False`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/checkpoint.html\n\n### `torch.compile`\n\n- `torch.compile` supports modes `default`, `reduce-overhead`, `max-autotune`, and `max-autotune-no-cudagraphs`.\n- `reduce-overhead` and `max-autotune` can leverage CUDA Graphs for suitable GPU paths.\n- `shape_padding` exists to better align loads on GPUs, especially for tensor cores.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.compile.html\n\n### NCCL tuning and debug\n\n- PyTorch docs recommend NCCL for CUDA and point to NVIDIA NCCL docs for the full env list.\n- NCCL docs describe debug and subsystem filters such as `NCCL_DEBUG` and `NCCL_DEBUG_SUBSYS`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/nccl/archives/nccl_2265/user-guide/docs/env.html\n\n### Profiling\n\n- Nsight Systems is the first profiler to reach for when you need end-to-end CUDA timeline truth.\n- NVIDIA docs note that heavy Unified Memory migration traffic is a performance problem and recommend manual copies from pinned host memory or peer copies instead.\n- NVIDIA docs:\n  - https://docs.nvidia.com/nsight-systems/UserGuide/\n\n### Custom kernel heuristics\n\n- CUDA best practices say:\n  - threads per block should be a multiple of 32\n  - 128 to 256 threads per block is a good initial range for experimentation\n  - higher occupancy does not automatically mean higher performance\n  - coalescing and latency hiding are core concerns\n- NVIDIA docs:\n  - https://docs.nvidia.com/cuda/pdf/CUDA_C_Best_Practices_Guide.pdf\n\n### Flash attention backends and newer implementations\n\n- PyTorch attention docs expose registration and activation for newer flash attention implementations via `register_flash_attention_impl()` and `activate_flash_attention_impl()`.\n- The documented implementation identifiers include `FA3` and `FA4`.\n- PyTorch attention docs also expose `list_flash_attention_impls()`, `current_flash_attention_impl()`, and `restore_flash_attention_impl()`.\n- `scaled_dot_product_attention` says it will automatically choose the most optimal implementation when possible unless you override backend choice.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.register_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.activate_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/nn.attention.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.sdpa_kernel.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html\n\n### cuDNN and Transformer Engine attention on Hopper and Blackwell\n\n- cuDNN frontend documents unified SDPA based on the FlashAttention-2 algorithm.\n- cuDNN attention support matrix includes Hopper and Blackwell paths, with fp8, fp16, and bf16 support depending on phase and architecture.\n- Transformer Engine docs state both its flash-attention backend and cuDNN attention backend are flash-algorithm based.\n- Transformer Engine docs also note cuDNN attention had 20-50% advantages on Hopper in their benchmarks for a number of common model configurations.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/frontend/latest/operations/Attention.html\n  - https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.4/user-guide/examples/attention/attention.html\n\n### Variable-length attention\n\n- PyTorch attention docs include a `varlen` module for variable-length attention using Flash Attention.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/main/nn.attention.varlen.html\n\n### Reproducibility\n\n- cuDNN docs state bitwise reproducibility is not guaranteed across architectures.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/backend/v9.13.0/developer/misc.html\n\n### TensorRT inference note\n\n- TensorRT best practices separate engine build from runtime profiling and recommend profiling inference rather than the build phase.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/tensorrt/latest/performance/best-practices.html\n\n### TensorRT-LLM serving algorithms\n\n- TensorRT-LLM documents paged context attention as useful for large input lengths and safe to benchmark as an opt-in build flag.\n- TensorRT-LLM docs describe speculative decoding as a family of techniques that can reduce average per-token latency when the GPU is underutilized at small batch sizes.\n- TensorRT-LLM docs cover in-flight batching and scheduler runtime options separately from build-time attention choices.\n- NVIDIA docs:\n  - https://nvidia.github.io/TensorRT-LLM/performance/performance-tuning-guide/useful-build-time-flags.html\n  - https://nvidia.github.io/TensorRT-LLM/advanced/speculative-decoding.html\n  - https://nvidia.github.io/TensorRT-LLM/performance/performance-tuning-guide/useful-runtime-flags.html\n\n### Distributed optimizer and large-model sharding\n\n- Megatron Core distributed optimizer docs describe memory savings from sharding optimizer state across data parallel ranks.\n- Megatron Core parallelism docs recommend sequence parallelism with tensor parallelism and position context parallelism as the lever for long sequences.\n- Megatron Core docs also expose expert parallelism for MoE and FSDP-style sharding strategies.\n- NVIDIA docs:\n  - https://docs.nvidia.com/megatron-core/developer-guide/0.15.0/api-guide/dist_optimizer.html\n  - https://docs.nvidia.com/megatron-core/developer-guide/0.16.0/user-guide/parallelism-guide.html\n  - https://docs.nvidia.com/megatron-core/developer-guide/latest/user-guide/features/custom_fsdp.html\n\n### DDP, FSDP, and torchrun smoke guidance\n\n- `torchrun` is the official launcher for single-node and multi-node distributed training and sets `LOCAL_RANK`, `RANK`, and related env vars for the script.\n- PyTorch docs explicitly recommend NCCL for best GPU training performance.\n- DDP docs state `DistributedDataParallel` is significantly faster than `torch.nn.DataParallel` for single-node multi-GPU data parallel training.\n- DDP docs require the process group to be initialized before constructing `DistributedDataParallel`.\n- DDP docs use `device_ids=[local_rank]` and `output_device=local_rank` for the one-GPU-per-process training pattern.\n- FSDP docs say to wrap the module first and create the optimizer after wrapping.\n- FSDP docs show the canonical wrapping pattern with `FullyShardedDataParallel`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/elastic/run.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html\n  - https://docs.pytorch.org/docs/stable/fsdp.html\n\n### Triton review guidance\n\n- Triton is the official language and compiler for productively writing custom DNN compute kernels on modern GPUs.\n- Triton tutorials explicitly recommend validating and benchmarking custom ops against native reference implementations.\n- Triton docs expose compiler hints and debug ops such as `multiple_of`, `max_contiguous`, `device_print`, and `device_assert`, which are useful review surfaces.\n- Triton debugging docs recommend `TRITON_INTERPRET=1` for CPU-interpreter debugging, with caveats around `bfloat16` and indirect memory access.\n- NVIDIA Compute Sanitizer is the official NVIDIA correctness tool for CUDA kernels and recommends running `memcheck` first before `racecheck` or `synccheck`.\n- Triton docs:\n  - https://triton-lang.org/main/index.html\n  - https://triton-lang.org/main/getting-started/tutorials/01-vector-add.html\n  - https://triton-lang.org/main/python-api/triton.language.html\n  - https://triton-lang.org/main/programming-guide/chapter-3/debugging.html\n  - https://triton-lang.org/main/python-api/generated/triton.testing.assert_close.html\n  - https://triton-lang.org/main/python-api/generated/triton.testing.do_bench.html\n- NVIDIA docs:\n  - https://docs.nvidia.com/compute-sanitizer/ComputeSanitizer/index.html\n\nFile v0.4.0:references/runtime-support.md\n\n# Runtime Support\n\nThis repository is not a Python library package. It is a skill package plus runnable helper scripts.\n\n## Intended runtime\n\nTested locally during repository development on:\n\n- Python `3.13.11`\n- PyTorch `2.11.0+cu130`\n- CUDA runtime `13.0`\n- NVIDIA H100 80GB HBM3\n\n## Minimum practical expectations\n\n- Python `3.10+`\n- PyTorch `2.11+`\n- CUDA-capable PyTorch build for any GPU benchmark or NCCL smoke test\n\n## Script expectations\n\n- `scripts/check_training_stack.py`:\n  - Python standard library only\n  - no CUDA required\n\n- `scripts/cuda_env_probe.py`:\n  - requires `torch`\n  - does not require a working GPU just to report environment facts, but is most useful when CUDA is visible\n\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n- `scripts/nccl_smoke.py`\n- `scripts/ddp_fsdp_smoke.py`\n  - require `torch`\n  - require CUDA visibility\n  - `scripts/nccl_smoke.py` and `scripts/ddp_fsdp_smoke.py` also require `torchrun`-style distributed environment variables\n\n## Compatibility note\n\nSome recommendations in the skill target H200, B200, GB200, and GB300-era hardware. Those parts are based on official NVIDIA positioning and documentation, but not every path is runnable on the local H100 development host.\n\nFile v0.4.0:CHANGELOG.md\n\n# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on Keep a Changelog and the project follows Semantic Versioning.\n\n## [0.4.0] - 2026-04-19\n\n### Added\n\n- `scripts/ddp_fsdp_smoke.py` for one-step DDP and FSDP smoke validation under `torchrun`\n\n### Changed\n\n- expanded `scripts/benchmark_attention.py` to expose official flash implementation listing and activation paths\n- expanded `scripts/check_training_stack.py` to cover Triton, CUDA, C++, and launcher-adjacent text surfaces instead of only Python AST scanning\n- documented official DDP/FSDP/torchrun and Triton review references\n\n## [0.3.0] - 2026-04-19\n\n### Added\n\n- `references/latest-gpu-recommendations-2026-04.md` with scenario-based NVIDIA GPU recommendations as of 2026-04-19\n- `examples/workstation-rtx-pro6000-blackwell.env`\n- `examples/b200-8gpu-torchrun.env`\n- `examples/dgx-b300-training.yaml`\n- `examples/h200-long-context-training.yaml`\n- `examples/gb300-serving.yaml`\n- `examples/l4-edge-serving.yaml`\n- `references/runtime-support.md`\n- `scripts/nccl_smoke.py`\n- `requirements.txt`\n- `.github/workflows/smoke.yml`\n\n### Changed\n\n- documented the latest GPU shortlist directly in `README.md`\n- added hardware-purchase guidance to `SKILL.md`\n\n## [0.2.0] - 2026-04-19\n\n### Added\n\n- `scripts/cuda_env_probe.py` for deterministic CUDA and PyTorch environment probing\n- `scripts/check_training_stack.py` for heuristic scanning of high-cost training anti-patterns\n- `scripts/benchmark_attention.py` for SDPA, flash, and cuDNN attention backend benchmarking\n- `scripts/training_step_benchmark.py` for synthetic transformer training-step benchmarking\n- `scripts/dataloader_benchmark.py` for worker, pinning, and prefetch throughput checks\n\n### Changed\n\n- upgraded the repository from a documentation-only skill package into a skill package with reusable tooling\n- documented bundled tooling in both `SKILL.md` and `README.md`\n\n## [0.1.0] - 2026-04-19\n\n### Added\n\n- initial `nvidia-cuda` skill package for NVIDIA deep learning optimization and review\n- BF16, FP8, TF32, and mixed precision policy guidance for H100, H200, and B200-class hardware\n- `torch.compile` and CUDA Graphs decision rules\n- attention backend policy covering SDPA, FA3/FA4 registration, cuDNN attention, Transformer Engine, and varlen attention\n- large-model training guidance for FSDP2, ZeRO-style sharding, distributed optimizer, TP, SP, CP, PP, and EP\n- LLM serving guidance for paged KV, chunked prefill, inflight batching, speculative decoding, and TensorRT-LLM\n- hard anti-pattern rules for per-step `.item()`, untuned dataloaders, and premature checkpointing\n- official reference notes with PyTorch and NVIDIA documentation links\n- ClawHub-ready project layout with root `SKILL.md`\n\nFile v0.4.0:skill-card.md\n\n## Description:\n\nUse when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[kkellyoffical](https://clawhub.ai/user/kkellyoffical)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nDevelopers and engineers use this skill to optimize and review CUDA/PyTorch training, inference, distributed execution, and GPU infrastructure decisions on NVIDIA hardware.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: Local CUDA and PyTorch probes can reveal hardware, driver, environment variable, and topology details.\n\nMitigation: Run probes only in trusted workspaces and redact system details before sharing outputs.\n\nRisk: Benchmarks and smoke tests execute CUDA, NCCL, and PyTorch workloads that can consume GPU resources or fail on hosts without the expected runtime.\n\nMitigation: Confirm CUDA visibility, PyTorch compatibility, and distributed environment variables before running the bundled benchmark or smoke-test commands.\n\nRisk: Optional install or publication commands that use latest package versions can reduce reproducibility.\n\nMitigation: Use an isolated environment and pinned package or CLI versions for reviewed releases.\n\n## Reference(s):\n\n- [ClawHub skill page](https://clawhub.ai/kkellyoffical/skills/nvidia-cuda)\n- [README](README.md)\n- [NVIDIA CUDA Skill Notes](references/official-notes.md)\n- [Latest NVIDIA GPU Recommendations for AI Infra](references/latest-gpu-recommendations-2026-04.md)\n- [Runtime Support](references/runtime-support.md)\n\n## Skill Output:\n\n**Output Type(s):** [text, markdown, code, shell commands, configuration, guidance]\n\n**Output Format:** [Markdown with inline commands, code snippets, configuration examples, and optional JSON tool output]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [May include local CUDA/PyTorch probe results, heuristic scan findings, and benchmark summaries when the bundled scripts are used.]\n\n## Skill Version(s):\n\n0.4.0 (source: server release metadata and changelog, released 2026-04-19)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.\n\nFile v0.4.0:agents/openai.yaml\n\ninterface:\n  display_name: \"NVIDIA CUDA\"\n  short_description: \"NVIDIA GPU deep learning optimization policy\"\n  default_prompt: \"Use $nvidia-cuda to optimize or review NVIDIA GPU deep learning code on CUDA.\"\n\npolicy:\n  allow_implicit_invocation: true\n\nFile v0.4.0:examples/dgx-b300-training.yaml\n\nhardware_tier: dgx-b300\nobjective: frontier single-node training\n\nprecision:\n  baseline: bf16\n  fp8_candidate: true\n  accuracy_gate_required: true\n\ndistributed:\n  launcher: torchrun\n  backend: nccl\n  data_parallel: true\n  fsdp: true\n  fsdp_strategy: full_shard\n  tensor_parallel: evaluate\n  sequence_parallel: evaluate_if_tp\n  context_parallel: evaluate_for_long_context\n  activation_checkpointing: selective_after_sharding\n\ndataloader:\n  pin_memory: true\n  persistent_workers: true\n  prefetch_factor: 4\n  num_workers_per_rank: 8\n\nbenchmark_order:\n  - eager_bf16\n  - torch_compile\n  - fused_attention_backend_compare\n  - fp8_candidate\n  - distributed_scaling\n\nnotes:\n  - Use DGX B300 as the newest top-end single-node training reference in this repository.\n  - Shard state before broad checkpointing when memory pressure comes from replicated params, grads, or optimizer state.\n\nFile v0.4.0:examples/gb300-serving.yaml\n\nhardware_tier: gb300-rack-scale\nobjective: large-scale LLM reasoning and low-latency serving\n\nprecision:\n  baseline: bf16\n  fp8_candidate: true\n  accuracy_gate_required: true\n\nserving:\n  paged_kv_cache: true\n  chunked_prefill: true\n  inflight_batching: true\n  speculative_decoding: evaluate\n  cuda_graphs_for_decode: true\n\nattention:\n  benchmark_default_sdpa: true\n  benchmark_flash_impls: true\n  benchmark_cudnn_attention: true\n\nnotes:\n  - Treat GB300 NVL72 as a rack-scale system recommendation, not a loose-card recommendation.\n  - Prioritize scheduler-level benchmarks and TTFT/ITL measurement over isolated kernel wins.\n\nFile v0.4.0:examples/h200-long-context-training.yaml\n\nhardware_tier: h200-memory-first\nobjective: long-context transformer training\n\nprecision:\n  baseline: bf16\n  tf32_matmul: high\n  fp8_candidate: false\n\ndistributed:\n  launcher: torchrun\n  backend: nccl\n  data_parallel: true\n  fsdp: true\n  fsdp_strategy: full_shard\n  tensor_parallel: 1\n  sequence_parallel: false\n  context_parallel: 2\n  activation_checkpointing: selective\n\ndataloader:\n  pin_memory: true\n  persistent_workers: true\n  prefetch_factor: 4\n  num_workers_per_rank: 8\n\nprofiling:\n  first_tool: nsys\n  second_tool: ncu\n\nnotes:\n  - Choose H200 when model memory and bandwidth dominate the decision.\n  - Use FSDP before broadening checkpointing if state replication is the main memory problem.\n\nArchive v0.3.0: 21 files, 33646 bytes\n\nFiles: agents/openai.yaml (250b), CHANGELOG.md (2305b), examples/b200-8gpu-torchrun.env (379b), examples/dgx-b300-training.yaml (878b), examples/gb300-serving.yaml (625b), examples/h200-long-context-training.yaml (701b), examples/l4-edge-serving.yaml (643b), examples/workstation-rtx-pro6000-blackwell.env (559b), README.md (6344b), references/latest-gpu-recommendations-2026-04.md (7603b), references/official-notes.md (7436b), references/runtime-support.md (1226b), requirements.txt (12b), scripts/benchmark_attention.py (3396b), scripts/check_training_stack.py (7199b), scripts/cuda_env_probe.py (4317b), scripts/dataloader_benchmark.py (3986b), scripts/nccl_smoke.py (1942b), scripts/training_step_benchmark.py (4870b), SKILL.md (22436b), _meta.json (130b)\n\nFile v0.3.0:SKILL.md\n\n---\nname: nvidia-cuda\ndescription: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware code conventions for PyTorch on CUDA, including dtype policy, memory movement, NCCL/DDP/FSDP choices, profiling, benchmarking, and H100/H200/B200 optimization habits.\n---\n\n# NVIDIA CUDA\n\nUse this skill when the task involves NVIDIA GPU training or inference, multi-GPU execution, CUDA/Triton kernels, or CUDA-specific performance/debugging work.\n\nThis skill is for implementation and review, not generic theory. It should push the work toward:\n\n- correct GPU usage first\n- stable and measurable performance second\n- low-risk tuning before low-level kernel work\n- frontier algorithms only when the workload shape justifies them\n\n## Hardware stance\n\nProbe the actual machine before acting. This skill is written for modern NVIDIA Tensor Core systems and is especially opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class accelerators\n\nIt assumes the agent should actively choose precision, attention backends, sharding strategy, logging frequency, and dataloader settings instead of inheriting slow defaults.\n\n## Use this skill for\n\n- PyTorch training or inference on CUDA\n- GPU memory, throughput, or latency bottlenecks\n- DDP / FSDP / NCCL decisions\n- CUDA env var and runtime hygiene\n- Triton or custom CUDA kernel review\n- benchmark and profiler setup for NVIDIA GPUs\n\n## Do not use this skill for\n\n- CPU-only optimization\n- ROCm / AMD-specific work\n- TPU / XLA-specific work\n- vague \"make it faster\" requests without inspecting the actual GPU path first\n\n## First-pass workflow\n\nBefore changing code, inspect the actual runtime surface:\n\n1. Check hardware and driver: `nvidia-smi`\n2. Check framework build: `python -c \"import torch; print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\"`\n3. Check device visibility and topology:\n   - `CUDA_VISIBLE_DEVICES`\n   - `torch.cuda.device_count()`\n   - `torch.cuda.get_device_properties(i)`\n4. Check whether the bottleneck is:\n   - input pipeline\n   - host/device transfer\n   - eager Python overhead\n   - kernel efficiency\n   - distributed communication\n   - memory pressure / fragmentation / OOM\n\nDo not jump to kernel rewrites before ruling out bad data movement, wrong dtype, graph breaks, or poor distributed setup.\n\n## Bundled tooling\n\nUse the included scripts when you need deterministic probes instead of ad hoc snippets:\n\n- `scripts/cuda_env_probe.py`: collect CUDA, PyTorch, device, and env facts\n- `scripts/check_training_stack.py`: scan Python code for high-cost anti-patterns\n- `scripts/benchmark_attention.py`: benchmark SDPA, flash, and cuDNN attention backends\n- `scripts/training_step_benchmark.py`: benchmark a synthetic transformer training step with dtype, compile, and `.item()` logging knobs\n- `scripts/dataloader_benchmark.py`: benchmark DataLoader worker, pinning, and prefetch settings\n- `scripts/nccl_smoke.py`: run a minimal NCCL all-reduce smoke test under `torchrun`\n\nRun the probe first, then benchmark or scan the real workload path.\n\n## Planning hardware purchases\n\nWhen the user wants current NVIDIA GPU recommendations, read:\n\n- [references/latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md)\n- the example configs under [examples/](examples)\n\nKeep recommendation output scenario-based:\n\n- cost-sensitive local prototyping\n- serious workstation development\n- enterprise server deployment\n- single-node training\n- rack-scale training or reasoning\n\nDo not recommend by peak FLOPS alone. Weight memory, interconnect, thermals, deployment form, and software maturity.\n\n## Non-negotiable code conventions\n\n### High-cost anti-patterns to ban\n\nTreat the following as red flags in reviews and optimization work:\n\n1. keeping transformer training or inference in full FP32 by default on Tensor Core GPUs\n2. refusing to try `torch.compile` on stable hot paths\n3. staying on a generic default SDPA path on H200/B200 when newer flash-backed implementations or cuDNN/TE attention backends are available and benchmarkable\n4. reaching for activation checkpointing before FSDP / ZeRO-style sharding when the real problem is replicated params, grads, or optimizer state\n5. calling `.item()` every step for logging and metrics\n6. leaving DataLoader workers, pinning, prefetching, and persistent workers untuned until GPU starvation shows up\n\nIf several of these exist at once, assume the GPU is underfed until proven otherwise.\n\n### 1. Dtype policy\n\n- On H100 / Hopper, prefer `bfloat16` for training and inference unless there is a measured or correctness-driven reason not to.\n- On H200 / B200-class systems, treat BF16 or FP8-capable paths as the default starting point for deep learning workloads.\n- Full FP32 by default is a correctness mode, not a performance baseline.\n- Use `float16` only when a dependency or model explicitly benefits from it.\n- If using `float16` training, use `torch.amp.GradScaler(\"cuda\")`.\n- For mixed precision, use the current API:\n  - `torch.amp.autocast(\"cuda\", dtype=...)`\n  - not deprecated `torch.cuda.amp.*`\n\n### 2. FP32 fallback policy\n\n- If the model is nominally FP32 but does not require strict IEEE-style matmul precision, prefer enabling faster matmul paths with `torch.set_float32_matmul_precision(\"high\")`.\n- Disable TF32 only for reproducibility or numerics investigations.\n- Never silently mix \"strict numerics\" and \"fast numerics\" requirements in the same change. State the choice.\n\n### 3. Device movement rules\n\n- Keep tensors on device across the hot path.\n- Move data in batches, not item-by-item.\n- Use `pin_memory=True` in `DataLoader` when feeding CUDA.\n- Use `tensor.to(\"cuda\", non_blocking=True)` or equivalent only when the source is pinned or already async-safe.\n- Avoid CPU round-trips in the training or inference loop.\n\n### 4. Avoid accidental synchronization\n\nInside hot loops, avoid or justify:\n\n- `.item()`\n- `.cpu()`\n- `.numpy()`\n- `print(cuda_tensor)`\n- timing without `torch.cuda.synchronize()`\n\nThese often force host synchronization and distort both throughput and profiling.\n\n### 5. Benchmarking rules\n\n- Warm up before measuring.\n- Use CUDA events or a framework benchmark harness, not bare wall clock alone.\n- Synchronize before reading timings.\n- Record batch size, dtype, sequence/image shape, compile state, and number of warmup / measured iterations.\n- For throughput and latency claims, include the exact command or code path used.\n\n### 6. `torch.compile` rules\n\n- Reach correctness first in eager mode.\n- Then apply `torch.compile` to stable hot paths.\n- Do not reject `torch.compile` categorically for CUDA models. Benchmark it.\n- Prefer:\n  - `mode=\"default\"` for general speedups\n  - `mode=\"reduce-overhead\"` for small-batch / latency-sensitive CUDA paths with stable shapes\n  - `mode=\"max-autotune\"` when compile overhead is acceptable and the path is hot enough\n- If shapes are unstable, be explicit about `dynamic=True` or accept recompiles intentionally.\n- Use `TORCH_LOGS=perf_hints` or `TORCH_LOGS=dynamic` when diagnosing graph breaks or overspecialization.\n\n### 6.1 Attention kernel policy\n\n- For transformer attention, prefer native `scaled_dot_product_attention` and PyTorch attention backends before custom kernels.\n- Use PyTorch SDPA / FlashAttention backends as the default fast path on CUDA.\n- On H200 / B200-class systems, do not assume the default shipped SDPA backend is the best available path. Check whether FA3 / FA4 implementations are registered and benchmark them.\n- Where Transformer Engine or cuDNN attention backends are available, benchmark them against the default path on Hopper/Blackwell instead of assuming SDPA wins.\n- If masks or score transforms are unusual, consider `torch.nn.attention.flex_attention` only after confirming SDPA cannot express the workload cleanly.\n- For variable-length packed batches, prefer varlen / packed attention paths rather than padding to worst-case sequence length.\n- Do not keep a hand-rolled eager attention implementation in a hot path if SDPA, TE attention, or TensorRT-LLM fused attention can replace it.\n\n### 6.2 CUDA Graphs policy\n\n- Use CUDA Graphs only for repeated, shape-stable hot paths where CPU launch overhead is material.\n- Graph capture is a strong fit for:\n  - small-batch inference\n  - fixed-shape decode loops\n  - stable training microbatches\n- Do not graph workloads with live shape churn, `.item()` sync points, or allocator churn in the captured region.\n- Prefer graphing after eager correctness and after compile/engine selection are already settled.\n\n### 7. Distributed rules\n\n- Use NCCL for distributed CUDA training.\n- Use DDP when the model fits on one GPU and you mainly need data parallel scale-out.\n- Use FSDP2 when the model does not fit on one GPU or optimizer state dominates memory.\n- When tensor parallelism is enabled, sequence parallelism is the default companion unless a framework limitation blocks it.\n- For long-context transformer training, consider context parallelism before blindly increasing tensor parallel degree.\n- For MoE, use expert parallelism instead of replicating all experts everywhere.\n- For large-scale runs, consider a distributed optimizer before inventing custom sharding logic.\n- Use `torchrun` and one process per GPU.\n- Set the current CUDA device from rank/local-rank early and only once.\n- Do not tune NCCL environment variables by superstition. Change them only after evidence.\n\n### 8. Memory rules\n\n- Prefer algorithmic memory wins before allocator tweaks:\n  - BF16\n  - FSDP / sharding\n  - distributed optimizer\n  - smaller activation footprints\n  - sequence or batch shaping\n  - activation checkpointing\n- When the memory problem is replicated model state, prioritize FSDP / ZeRO-2 / ZeRO-3 style sharding before gradient checkpointing.\n- Use checkpointing when activation memory is the real limiter, not as a substitute for state sharding.\n- If activation checkpointing is needed, prefer `use_reentrant=False` unless there is a specific reason otherwise.\n- Treat allocator and workspace env vars as debugging/tuning levers, not first-line fixes.\n\n### 9. Profiling rules\n\n- Start with Nsight Systems for timeline truth:\n  - CPU launch gaps\n  - H2D / D2H copies\n  - stream overlap\n  - NCCL overlap and stalls\n  - Unified Memory migrations\n- Move to Nsight Compute only after isolating the hot kernels.\n- For PyTorch-level attribution, use the profiler only if you need operator names or stack grouping.\n\n## Frontier algorithm ladder\n\nUse this section when the user explicitly wants leading-edge optimization or when the baseline is already healthy.\n\nDo not apply all of these at once. Climb the ladder in order and keep a benchmark after each step.\n\n### A. Hopper and Blackwell transformer path\n\nFor H100/H200/B200-class transformer workloads, the default advanced path is:\n\n1. BF16 baseline\n2. native SDPA / FlashAttention fast path\n3. `torch.compile`\n4. CUDA Graphs if shapes stabilize\n5. Transformer Engine FP8 if:\n   - the model is transformer-dominated\n   - BF16 numerics are already validated\n   - there is an accuracy gate\n\nFP8 is not a blanket default. It is a targeted transformer optimization with strict measurement.\n\n### B. Large-model training algorithms\n\nFor models that are too large, too long-context, or too communication-heavy, consider:\n\n- FSDP2 / Megatron FSDP for parameter, grad, and optimizer sharding\n- distributed optimizer for optimizer-state memory pressure\n- tensor parallelism for large hidden dimensions\n- sequence parallelism when TP is enabled\n- context parallelism for long sequences\n- pipeline parallelism for very deep models\n- expert parallelism for MoE\n- activation checkpointing when the above is still insufficient\n\nSelection heuristics:\n\n- if the model barely misses single-GPU fit: start with FSDP2\n- if hidden dimensions are very large: add TP\n- if sequence length is the actual problem: prefer CP over just raising TP\n- if the model is MoE: use EP rather than replicating experts\n- if optimizer state dominates: distributed optimizer or FSDP shard first\n\n### C. LLM inference algorithms\n\nFor autoregressive serving, the advanced inference ladder is:\n\n1. fused attention path\n2. paged KV cache / paged context handling\n3. continuous or inflight batching\n4. chunked prefill / context chunking for large prompts\n5. CUDA Graphs for stable decode loops\n6. speculative decoding when low-batch latency matters and acceptance rate is healthy\n7. FP8 or weight-focused quantization only after accuracy validation\n\nStrong fits:\n\n- paged context / chunked prefill: long prompts and mixed request sizes\n- speculative decoding: low-batch underutilized serving\n- inflight fused batching: multi-request serving with scheduler pressure\n- FP8 serving: Hopper/Blackwell inference paths with validated quality\n\n### D. Quantization policy\n\n- On Hopper and newer, prefer BF16 first, FP8 second, INT8/INT4 only when memory or throughput targets require it.\n- For training or finetuning quantized targets, prefer QAT or recipe-backed flows over ad hoc fake-quant code.\n- For serving, keep a clear boundary between:\n  - engine build or calibration\n  - runtime benchmarking\n  - accuracy evaluation\n\nDo not merge a quantization change without an explicit quality gate.\n\n## Training optimization playbook\n\nWhen optimizing training code, prefer this order:\n\n1. Ensure correct device placement and eliminate CPU fallbacks\n2. Fix input pipeline:\n   - `pin_memory=True`\n   - enough workers\n   - `persistent_workers=True` when the workload is steady and worker startup cost matters\n   - tuned `prefetch_factor`\n   - no CPU-heavy transforms inside the critical path unless justified\n3. Set precision policy:\n   - BF16 first on Hopper\n   - BF16 / FP8-capable path first on H200/B200\n   - FP16 only with explicit reason\n4. Remove accidental syncs and per-step Python overhead\n5. Try `torch.compile`\n6. If this is transformer-heavy, move attention to SDPA / FlashAttention / TE attention fast paths\n7. Scale to DDP or FSDP2 if single-GPU utilization or model size requires it\n8. Add TP + SP, CP, PP, or EP only according to the actual bottleneck\n9. Add activation checkpointing if memory still blocks batch size\n10. For Hopper/Blackwell transformer stacks, evaluate Transformer Engine FP8 behind an accuracy gate\n11. Profile and only then consider Triton or custom CUDA work\n\n## Inference optimization playbook\n\nWhen optimizing inference code:\n\n1. Use `model.eval()` and `torch.no_grad()` / `torch.inference_mode()`\n2. Pick dtype deliberately:\n   - BF16 or FP16 on Hopper when numerically acceptable\n   - TF32-enabled FP32 if accuracy must stay near FP32 but pure FP32 is too slow\n3. Move attention and decode loops onto fused kernels or native SDPA paths\n4. Stabilize shapes if possible\n5. Apply `torch.compile` or engine compilation only after correctness baselines\n6. For LLM serving, evaluate paged KV / paged context and inflight batching early\n7. Use CUDA Graphs only for repeated, fixed-shape paths where launch overhead matters\n8. If low-batch latency dominates, evaluate speculative decoding with acceptance-rate tracking\n9. Benchmark after warmup, with synchronization\n10. If using TensorRT or TensorRT-LLM, profile runtime separately from engine build\n\n## Multi-GPU and NCCL playbook\n\nBaseline expectations:\n\n- launch with `torchrun`\n- backend `nccl`\n- one process per GPU\n- clear `MASTER_ADDR`, `MASTER_PORT`, `RANK`, `WORLD_SIZE`, `LOCAL_RANK`\n\nParallelism heuristics:\n\n- `DP` first for ordinary scale-out\n- `FSDP2` when full replication is too expensive\n- `FSDP / ZeRO-2 / ZeRO-3` before checkpointing when optimizer, grad, or param state replication is the memory bottleneck\n- `TP + SP` for large transformer blocks\n- `CP` for long-context models\n- `PP` for deep stacks that still do not fit\n- `EP` for MoE\n- combine only what the bottleneck requires\n\nDebugging rules:\n\n- First-line NCCL debug:\n  - `NCCL_DEBUG=INFO`\n  - `NCCL_DEBUG_SUBSYS=COLL,GRAPH` when debugging hangs or topology issues\n- If interface auto-detection is wrong, set `NCCL_SOCKET_IFNAME`\n- Use `TORCH_NCCL_USE_COMM_NONBLOCKING=1` when you specifically want non-blocking NCCL error handling\n\nDo not hardcode socket or thread tuning variables unless profiling or production evidence shows the default topology tuning is wrong.\n\n## Custom CUDA / Triton kernel rules\n\nOnly drop to Triton or CUDA when operator fusion, layout cleanup, `torch.compile`, or library kernels are not enough.\n\nFor custom kernels:\n\n- favor coalesced global memory access\n- treat shared memory as a tool, not a reflex\n- watch register pressure; occupancy is a means, not the target\n- use block sizes that are multiples of 32\n- start experiments around `128` to `256` threads per block unless the kernel shape clearly suggests otherwise\n- avoid giant one-block-per-SM thinking; multiple resident blocks often help latency hiding\n- measure spills, occupancy, achieved bandwidth, and kernel time before and after changes\n\nDo not claim a kernel is faster because occupancy increased. Show end-to-end or kernel-level timing.\n\n## Transformer Engine and FP8 rules\n\nUse Transformer Engine when all of these are true:\n\n- the workload is transformer-dominated\n- the target GPUs are Hopper/Ada/Blackwell-class\n- BF16 baseline is already correct\n- you can run an accuracy gate\n\nRules:\n\n- wrap only the forward pass in TE FP8 autocast\n- use recipe-backed scaling, not home-grown FP8 metadata logic\n- start with delayed scaling or hybrid FP8 recipes\n- keep optimizer and master-weight behavior explicit\n- if FP8 helps throughput but destabilizes training, fall back to BF16 instead of layering hacks\n\nPrefer TE modules or established framework integration over custom FP8 plumbing.\n\n## Attention and sequence-shape rules\n\nFor long-sequence transformer work:\n\n- prefer fused attention over matmul-softmax-matmul Python compositions\n- on H200/B200, actively test newer flash attention implementations when available instead of passively staying with the default SDPA stack\n- prefer grouped or multi-query attention-aware fast paths when the model architecture already uses GQA or MQA\n- prefer context parallelism and packed or varlen attention before wasting memory on blanket padding\n- if custom masking is the only blocker, test FlexAttention before writing a new CUDA kernel\n\nFor decoder serving:\n\n- KV cache layout is part of the algorithm, not just a storage detail\n- paged KV and chunked context handling are first-class optimization choices\n- scheduler choice affects TTFT and ITL, so benchmark serving algorithms at the scheduler level, not only kernel level\n\n## Logging and metric collection rules\n\n- Do not call `.item()` every step for loss curves, metric dashboards, or debug prints.\n- Aggregate metrics on device and synchronize periodically:\n  - every `N` steps\n  - end of an accumulation window\n  - end of an evaluation window\n- If exact per-step host-visible metrics are required for debugging, state that this is a debug-only mode.\n- Prefer batched reductions and detached device-side buffers over step-wise host sync.\n\n## Data pipeline and prefetch rules\n\nFor CUDA training jobs, always inspect the input path before blaming kernels.\n\nRequired checks:\n\n- `num_workers`\n- `pin_memory`\n- `persistent_workers`\n- `prefetch_factor`\n- CPU decode, tokenize, or augment cost\n- storage throughput and remote filesystem latency\n\nRules:\n\n- If GPU utilization dips between steps, inspect the dataloader before touching kernels.\n- Tune `prefetch_factor` and worker count together; more workers without enough prefetch often just moves the bottleneck.\n- Keep host preprocessing out of the critical path when possible.\n- For very fast H100/H200/B200 training loops, assume untuned input pipelines are guilty until measurements say otherwise.\n\n## Reproducibility and numerics rules\n\n- Be explicit when a change trades accuracy for speed.\n- cuDNN determinism is not guaranteed across architectures.\n- Across devices or architectures, do not promise bitwise-identical outputs.\n- If NaNs or Infs appear:\n  - check data first\n  - check loss scaling or dtype policy\n  - check reduced-precision reductions\n  - inspect unstable linalg paths\n\n## Environment variable policy\n\nAllowed as targeted tools, not blanket defaults:\n\n- `CUDA_VISIBLE_DEVICES`\n- `CUDA_LAUNCH_BLOCKING=1` for debugging only\n- `PYTORCH_ALLOC_CONF`\n- `TORCH_CUDNN_V8_API_LRU_CACHE_LIMIT`\n- `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE`\n- `TORCH_NCCL_USE_COMM_NONBLOCKING=1`\n- `NCCL_DEBUG`\n- `NCCL_DEBUG_SUBSYS`\n- `NCCL_SOCKET_IFNAME`\n\nIf you set an env var in code, config, or docs, explain:\n\n1. why it is needed\n2. whether it is for debugging, correctness, or performance\n3. when it should be removed\n\n## Review checklist\n\nWhen reviewing or generating code in this domain, check for:\n\n- tensors bouncing between CPU and GPU\n- missing `pin_memory=True` on CUDA dataloaders\n- missing `non_blocking=True` on pinned copies\n- missing `persistent_workers=True` or untuned `prefetch_factor` on steady-state training jobs\n- deprecated AMP API usage\n- full FP32 training or inference on Hopper/Blackwell with no numerics justification\n- FP16 used by default on Hopper with no reason\n- no BF16 or FP8 path evaluation on H200/B200-class hardware\n- no warmup in benchmarks\n- no synchronize around timing\n- `.item()` / `.cpu()` / `.numpy()` inside the step loop\n- distributed code using non-NCCL backend for CUDA\n- DDP used where model clearly does not fit, instead of FSDP2\n- activation checkpointing used to paper over what should be FSDP or ZeRO sharding\n- TP enabled without SP where the framework expects SP\n- long-context workloads scaled by TP alone when CP is the better lever\n- transformer stack on Hopper staying on unfused attention with no reason\n- transformer stack on H200/B200 never testing newer flash attention implementations\n- speculative decoding added with no acceptance-rate measurement\n- quantization merged with no quality gate\n- FP8 added without recipe-backed scaling or accuracy validation\n- custom kernel work started before simpler wins were exhausted\n\n## Output contract\n\nWhen using this skill, report in this order:\n\n1. runtime facts\n2. bottleneck hypothesis\n3. chosen optimization order\n4. changes made\n5. verification evidence\n6. remaining risks\n\nIf you need deeper justification or exact source-backed tuning notes, read [references/official-notes.md](references/official-notes.md).\n\nFile v0.3.0:README.md\n\n# nvidia-cuda\n\n`nvidia-cuda` is an open-source skill package for Codex, OpenClaw, and similar agent systems that need to write or review deep learning code for NVIDIA GPUs.\n\nIt is opinionated on purpose. The package encodes modern AI infra practice for H100, H200, and B200-class systems:\n\n- BF16 and FP8-aware precision policy\n- `torch.compile` as a benchmarked option, not a taboo\n- attention backend selection that does not stop at default SDPA\n- FSDP and ZeRO before using gradient checkpointing as a crutch\n- `.item()` and host sync anti-pattern detection\n- dataloader, pinning, prefetch, and persistent worker checks\n- NCCL, TP/SP/CP/EP, TensorRT-LLM, speculative decoding, and TE guidance\n\n## Why this exists\n\nA lot of GPU training code still leaves major performance on the floor because it inherits conservative defaults:\n\n- full FP32 everywhere\n- no compile pass\n- no attention backend evaluation on newer hardware\n- per-step host synchronization for metrics\n- checkpointing used before state sharding\n- dataloaders that starve the GPU\n\nThose mistakes compound. This skill gives an agent a clear optimization order and a concrete review checklist.\n\n## What is inside\n\n- [SKILL.md](SKILL.md): the main operational policy\n- [references/official-notes.md](references/official-notes.md): official PyTorch and NVIDIA references\n- [references/latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md): latest GPU shortlist as of 2026-04-19\n- [references/runtime-support.md](references/runtime-support.md): tested and expected runtime surface\n- [agents/openai.yaml](agents/openai.yaml): optional UI metadata for skill-aware clients\n- [scripts/](scripts/): reusable probes, scanners, and benchmarks\n- [examples/](examples/): sample environment and planning configs\n\n## Bundled tooling\n\nThe `0.2.0` upgrade turns this repository into more than a text-only skill. It now includes reusable scripts:\n\n- `scripts/cuda_env_probe.py`\n- `scripts/check_training_stack.py`\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n- `scripts/nccl_smoke.py`\n\nExample commands:\n\n```bash\npython scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\ntorchrun --nproc_per_node=2 scripts/nccl_smoke.py --json\n```\n\n## Install locally\n\n### Codex\n\nCopy this directory to your skills path as `~/.codex/skills/nvidia-cuda/`.\n\n### OpenClaw / ClawHub-compatible agents\n\nThe repository root is already a skill package:\n\n- `SKILL.md` at the root\n- supporting files under `references/` and `agents/`\n\nThat means you can publish this repository directly as a skill package after review.\n\n## Triggering\n\nUse the skill by name:\n\n```text\n$nvidia-cuda optimize this H100 training loop\n```\n\nExample prompts:\n\n- `$nvidia-cuda review this multi-GPU training stack for H200 underutilization`\n- `$nvidia-cuda optimize this transformer inference path on B200`\n- `$nvidia-cuda audit this dataloader, precision policy, and logging loop`\n\n## Hardware and framework stance\n\nThe skill is most opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class GPUs\n\nThe primary software target is PyTorch on CUDA, with side coverage for:\n\n- NCCL\n- Triton\n- Transformer Engine\n- TensorRT-LLM\n- Megatron Core style large-model sharding\n\n## Latest GPU recommendations\n\nAs of **2026-04-19**, the practical shortlist in this repository is:\n\n- **RTX 5090** for cost-sensitive local prototyping\n- **RTX PRO 6000 Blackwell Workstation Edition** for serious single-workstation AI development\n- **DGX Station** if you want a full deskside AI system rather than a loose card\n- **RTX PRO 6000 Blackwell Server Edition** for universal enterprise server deployment\n- **H200** for memory-first Hopper deployments\n- **DGX B300** for the newest top-end single-node training system\n- **DGX B200** as the current Blackwell step-down single-node training system\n- **GB200 NVL72** for rack-scale frontier training and trillion-parameter inference\n- **GB300 NVL72** as the newest rack-scale recommendation for AI reasoning and test-time scaling\n- **L4** for efficient low-power inference\n\nDetailed rationale and official links:\n\n- [latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md)\n\n## Sample configs\n\nThis repository now includes planning and environment examples:\n\n- [workstation-rtx-pro6000-blackwell.env](examples/workstation-rtx-pro6000-blackwell.env)\n- [b200-8gpu-torchrun.env](examples/b200-8gpu-torchrun.env)\n- [dgx-b300-training.yaml](examples/dgx-b300-training.yaml)\n- [h200-long-context-training.yaml](examples/h200-long-context-training.yaml)\n- [gb300-serving.yaml](examples/gb300-serving.yaml)\n- [l4-edge-serving.yaml](examples/l4-edge-serving.yaml)\n- [runtime-support.md](references/runtime-support.md)\n\n## Release policy\n\nThis repository uses semver. The current target release is `0.3.0`.\n\nVersion `0.3.0` establishes:\n\n- the initial optimization doctrine\n- the anti-pattern blocklist\n- H100/H200/B200 attention and precision guidance\n- training and inference optimization ladders\n- ClawHub-ready repository packaging\n- reusable probes and benchmark scripts for environment, attention, training-step, and dataloader analysis\n- current NVIDIA GPU purchase guidance with official references and sample planning configs\n- a basic CI smoke workflow and NCCL smoke-test helper\n\n## ClawHub publication note\n\nClawHub supports publishing skill packages from a local path via its CLI.\n\nAt the time this repository was prepared:\n\n- the upstream ClawHub project documents `clawhub skill publish <path>`\n- the ClawHub web UI also supports signed-in publication\n\nCLI example:\n\n```bash\nnpx clawhub@latest publish . \\\n  --slug nvidia-cuda \\\n  --name \"NVIDIA CUDA\" \\\n  --version 0.3.0 \\\n  --changelog \"Add latest GPU recommendations and sample configs\"\n```\n\nBecause community skill marketplaces can carry supply-chain risk, publish only reviewed, minimal, instruction-focused packages and inspect any scanner output before making a listing public.\n\n## License\n\n[MIT-0](LICENSE)\n\nFile v0.3.0:_meta.json\n\n{\n  \"ownerId\": \"kn7bcfdgsryztp2rsxyq54gydh82b5be\",\n  \"slug\": \"nvidia-cuda\",\n  \"version\": \"0.3.0\",\n  \"publishedAt\": 1776589370881\n}\n\nFile v0.3.0:references/latest-gpu-recommendations-2026-04.md\n\n# Latest NVIDIA GPU Recommendations for AI Infra\n\nLast reviewed: 2026-04-19\n\nThis note summarizes practical NVIDIA GPU recommendations for AI infrastructure based on official NVIDIA product positioning and published specifications as of 2026-04-19.\n\nImportant: these recommendations are partly **inference from official specs and positioning**, not direct vendor buying advice. Use them to choose a shortlist, then validate availability, thermals, interconnect, memory, and software fit for your workload.\n\nNVIDIA's own data center line card labels products relative to each workload column. The ranking below is a practical shortlist built from that line card plus current product pages, not an official NVIDIA rank list.\n\n## Recommendation matrix\n\n### 1. Cost-sensitive local prototyping\n\nRecommended:\n\n- **GeForce RTX 5090**\n\nUse when:\n\n- you need a powerful local GPU for experimentation\n- your models fit within **32 GB GDDR7**\n- you are doing inference, small finetunes, QLoRA-style work, or kernel benchmarking\n\nAvoid as the default choice when:\n\n- you need large-memory training\n- you need ECC-centric enterprise workflows\n- you need dense multi-GPU workstation or server deployment\n\nWhy:\n\n- NVIDIA positions RTX 5090 as its flagship GeForce GPU with **32 GB GDDR7** and Blackwell architecture.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5090/\n\n### 2. Serious single-workstation AI development\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Workstation Edition**\n- **NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition** when density or power matters more than peak board power\n\nUse when:\n\n- you want a desk-side AI workstation\n- you need **96 GB GDDR7 with ECC**\n- you want stronger enterprise and workstation ergonomics than GeForce\n\nWhy:\n\n- NVIDIA positions the Workstation Edition as its most powerful desktop GPU for professionals.\n- NVIDIA positions the Max-Q variant as a **300 W** option that scales to **up to four GPUs in one system**.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000/\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000-max-q/\n\nIf you want a full deskside AI system rather than a loose card:\n\n- **NVIDIA DGX Station**\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/dgx-station/\n\n### 3. Universal enterprise server GPU\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Server Edition**\n\nUse when:\n\n- you need a flexible enterprise server GPU for AI plus graphics or simulation\n- you want **96 GB GDDR7 with ECC**\n- you are building enterprise servers rather than bare enthusiast workstations\n\nWhy:\n\n- NVIDIA positions this card as a universal Blackwell data center GPU for AI and visual computing.\n- It is a strong fit for enterprise inference, agentic AI, multimodal systems, and mixed graphics plus AI environments.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/rtx-pro-6000-blackwell-server-edition/\n\n### 4. Memory-first Hopper node\n\nRecommended:\n\n- **NVIDIA H200**\n\nUse when:\n\n- your software stack is Hopper-optimized already\n- model memory and bandwidth are the real limit\n- you want **141 GB HBM3E** and **4.8 TB/s** bandwidth per GPU\n\nWhy:\n\n- H200 is still compelling when memory capacity and Hopper continuity matter more than jumping immediately to Blackwell systems.\n\nOfficial:\n\n- https://www.nvidia.com/en-gb/data-center/h200/\n\n### 5. Single-node frontier training\n\nRecommended:\n\n- **NVIDIA DGX B300**\n- **NVIDIA DGX B200**\n- **NVIDIA DGX H200** as the Hopper fallback\n\nUse when:\n\n- you are buying a serious training node rather than a single loose GPU\n- you want a turnkey high-end training node instead of integrating loose GPUs yourself\n\nWhy:\n\n- DGX B300 is the newest top-end single-node training system in the official lineup surfaced here.\n- NVIDIA states DGX B200 delivers **3x training** and **15x inference** performance versus DGX H100 in its published positioning.\n- DGX H200 remains the Hopper fallback when software continuity or procurement timing makes Blackwell rollout less practical.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/dgx-b300/\n- https://www.nvidia.com/en-us/data-center/dgx-b200/\n- https://www.nvidia.com/en-us/data-center/dgx-h200/\n\n### 6. Rack-scale frontier training and trillion-parameter inference\n\nRecommended:\n\n- **NVIDIA GB200 NVL72**\n- **NVIDIA HGX B300** as the scale-up platform choice when you are building around HGX systems\n\nUse when:\n\n- you need rack-scale Blackwell today\n- you are targeting massive training or real-time trillion-parameter inference\n\nWhy:\n\n- NVIDIA positions GB200 NVL72 as a **72-GPU NVLink domain** and publishes **4x training vs H100** and **30x real-time LLM inference** positioning.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/gb200-nvl72/\n- https://www.nvidia.com/en-us/data-center/hgx/\n\n### 7. Latest rack-scale reasoning and test-time scaling\n\nRecommended:\n\n- **NVIDIA GB300 NVL72**\n\nUse when:\n\n- the priority is latest-generation reasoning or test-time scaling infrastructure\n- you are planning a new rack-scale Blackwell Ultra purchase\n\nWhy:\n\n- NVIDIA positions GB300 NVL72 specifically for AI reasoning and test-time scaling.\n- If you are choosing the most current top-end NVIDIA rack-scale platform as of **2026-04-19**, this is the newest one in the official lineup surfaced here.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/gb300-nvl72/\n\n### 8. Enterprise and edge inference efficiency\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Server Edition**\n- **NVIDIA L4 Tensor Core GPU**\n\nUse when:\n\n- you need enterprise inference in a conventional server footprint\n- you need low-power inference or video-plus-AI deployment at the edge\n\nWhy:\n\n- RTX PRO 6000 Blackwell Server Edition is the enterprise universal Blackwell server option.\n- L4 remains a practical low-power inference choice when deployment density and efficiency matter more than frontier throughput.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/rtx-pro-6000-blackwell-server-edition/\n- https://www.nvidia.com/en-us/data-center/l4/\n\n## Practical shortlist by scenario\n\n- **Cheapest serious local bring-up**: RTX 5090\n- **Best serious single-workstation AI GPU**: RTX PRO 6000 Blackwell Workstation Edition\n- **Best dense workstation variant**: RTX PRO 6000 Blackwell Max-Q Workstation Edition\n- **Best deskside full system**: DGX Station\n- **Best universal enterprise server GPU**: RTX PRO 6000 Blackwell Server Edition\n- **Best memory-first Hopper choice**: H200\n- **Best single-node training system**: DGX B300\n- **Best single-node step-down training system**: DGX B200\n- **Best Hopper fallback training node**: DGX H200\n- **Best deployed rack-scale Blackwell platform**: GB200 NVL72\n- **Newest top-end rack-scale recommendation**: GB300 NVL72\n- **Best low-power inference choice**: L4\n\n## Caveats\n\n- Consumer GPUs are attractive for price and local availability, but they are not the default recommendation for multi-user or production server environments.\n- H200 remains a strong recommendation when memory capacity, Hopper software maturity, or HBM bandwidth dominate the decision.\n- DGX and NVL systems are system-level recommendations, not loose-GPU recommendations.\n- If your workload is dominated by inference memory capacity and scheduler behavior rather than raw training throughput, shortlist by **memory, KV-cache behavior, and software path**, not just peak FLOPS.\n\n## Additional official source\n\n- NVIDIA Data Center Platform Line Card:\n  - https://docs.nvidia.com/data-center-gpu/line-card.pdf\n\nFile v0.3.0:references/official-notes.md\n\n# NVIDIA CUDA Skill Notes\n\nThis file is reference material for the `nvidia-cuda` skill. Read it when you need official grounding for a tuning choice, a code review decision, or a user-facing explanation.\n\n## Official facts to use\n\n### Mixed precision\n\n- PyTorch AMP says lower precision ops such as linear layers and convolutions are often much faster in lower precision, while other ops still need FP32 range.\n- Current AMP API is `torch.amp.autocast(...)` and `torch.amp.GradScaler(...)`; old `torch.cuda.amp.*` entry points are deprecated.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/amp.html\n\n### TF32 and FP32 matmul precision\n\n- `torch.set_float32_matmul_precision(\"high\" | \"medium\")` can significantly improve performance for FP32 matmuls on CUDA.\n- It changes internal computation precision, not output dtype.\n- On Ampere and later, TF32 can improve speed but may reduce accuracy for some workloads.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html\n  - https://docs.pytorch.org/docs/stable/notes/numerical_accuracy.html\n\n### CUDA env vars and allocator/debug knobs\n\n- `PYTORCH_ALLOC_CONF`, `CUDA_LAUNCH_BLOCKING`, `CUDA_VISIBLE_DEVICES`, TF32 override knobs, cuDNN cache or workspace knobs, and NCCL non-blocking error handling are documented in one place.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/cuda_environment_variables.html\n\n### Data movement\n\n- PyTorch data docs say host-to-GPU copies are much faster from pinned memory.\n- `DataLoader(pin_memory=True)` automatically places fetched tensors in pinned memory.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/data.html\n\n### Distributed backend choice\n\n- PyTorch distributed docs say the rule of thumb for CUDA distributed training is NCCL.\n- PyTorch distributed overview says:\n  - use DDP when the model fits on one GPU\n  - use FSDP2 when the model cannot fit on one GPU\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n  - https://docs.pytorch.org/tutorials/beginner/dist_overview.html\n  - https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html\n\n### Activation checkpointing\n\n- Activation checkpointing trades compute for memory.\n- PyTorch recommends `use_reentrant=False`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/checkpoint.html\n\n### `torch.compile`\n\n- `torch.compile` supports modes `default`, `reduce-overhead`, `max-autotune`, and `max-autotune-no-cudagraphs`.\n- `reduce-overhead` and `max-autotune` can leverage CUDA Graphs for suitable GPU paths.\n- `shape_padding` exists to better align loads on GPUs, especially for tensor cores.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.compile.html\n\n### NCCL tuning and debug\n\n- PyTorch docs recommend NCCL for CUDA and point to NVIDIA NCCL docs for the full env list.\n- NCCL docs describe debug and subsystem filters such as `NCCL_DEBUG` and `NCCL_DEBUG_SUBSYS`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/nccl/archives/nccl_2265/user-guide/docs/env.html\n\n### Profiling\n\n- Nsight Systems is the first profiler to reach for when you need end-to-end CUDA timeline truth.\n- NVIDIA docs note that heavy Unified Memory migration traffic is a performance problem and recommend manual copies from pinned host memory or peer copies instead.\n- NVIDIA docs:\n  - https://docs.nvidia.com/nsight-systems/UserGuide/\n\n### Custom kernel heuristics\n\n- CUDA best practices say:\n  - threads per block should be a multiple of 32\n  - 128 to 256 threads per block is a good initial range for experimentation\n  - higher occupancy does not automatically mean higher performance\n  - coalescing and latency hiding are core concerns\n- NVIDIA docs:\n  - https://docs.nvidia.com/cuda/pdf/CUDA_C_Best_Practices_Guide.pdf\n\n### Flash attention backends and newer implementations\n\n- PyTorch attention docs expose registration and activation for newer flash attention implementations via `register_flash_attention_impl()` and `activate_flash_attention_impl()`.\n- The documented implementation identifiers include `FA3` and `FA4`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.register_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.activate_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/nn.attention.html\n\n### cuDNN and Transformer Engine attention on Hopper and Blackwell\n\n- cuDNN frontend documents unified SDPA based on the FlashAttention-2 algorithm.\n- cuDNN attention support matrix includes Hopper and Blackwell paths, with fp8, fp16, and bf16 support depending on phase and architecture.\n- Transformer Engine docs state both its flash-attention backend and cuDNN attention backend are flash-algorithm based.\n- Transformer Engine docs also note cuDNN attention had 20-50% advantages on Hopper in their benchmarks for a number of common model configurations.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/frontend/latest/operations/Attention.html\n  - https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.4/user-guide/examples/attention/attention.html\n\n### Variable-length attention\n\n- PyTorch attention docs include a `varlen` module for variable-length attention using Flash Attention.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/main/nn.attention.varlen.html\n\n### Reproducibility\n\n- cuDNN docs state bitwise reproducibility is not guaranteed across architectures.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/backend/v9.13.0/developer/misc.html\n\n### TensorRT inference note\n\n- TensorRT best practices separate engine build from runtime profiling and recommend profiling inference rather than the build phase.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/tensorrt/latest/performance/best-practices.html\n\n### TensorRT-LLM serving algorithms\n\n- TensorRT-LLM documents paged context attention as useful for large input lengths and safe to benchmark as an opt-in build flag.\n- TensorRT-LLM docs describe speculative decoding as a family of techniques that can reduce average per-token latency when the GPU is underutilized at small batch sizes.\n- TensorRT-LLM docs cover in-flight batching and scheduler runtime options separately from build-time attention choices.\n- NVIDIA docs:\n  - https://nvidia.github.io/TensorRT-LLM/performance/performance-tuning-guide/useful-build-time-flags.html\n  - https://nvidia.github.io/TensorRT-LLM/advanced/speculative-decoding.html\n  - https://nvidia.github.io/TensorRT-LLM/performance/performance-tuning-guide/useful-runtime-flags.html\n\n### Distributed optimizer and large-model sharding\n\n- Megatron Core distributed optimizer docs describe memory savings from sharding optimizer state across data parallel ranks.\n- Megatron Core parallelism docs recommend sequence parallelism with tensor parallelism and position context parallelism as the lever for long sequences.\n- Megatron Core docs also expose expert parallelism for MoE and FSDP-style sharding strategies.\n- NVIDIA docs:\n  - https://docs.nvidia.com/megatron-core/developer-guide/0.15.0/api-guide/dist_optimizer.html\n  - https://docs.nvidia.com/megatron-core/developer-guide/0.16.0/user-guide/parallelism-guide.html\n  - https://docs.nvidia.com/megatron-core/developer-guide/latest/user-guide/features/custom_fsdp.html\n\nFile v0.3.0:references/runtime-support.md\n\n# Runtime Support\n\nThis repository is not a Python library package. It is a skill package plus runnable helper scripts.\n\n## Intended runtime\n\nTested locally during repository development on:\n\n- Python `3.13.11`\n- PyTorch `2.11.0+cu130`\n- CUDA runtime `13.0`\n- NVIDIA H100 80GB HBM3\n\n## Minimum practical expectations\n\n- Python `3.10+`\n- PyTorch `2.11+`\n- CUDA-capable PyTorch build for any GPU benchmark or NCCL smoke test\n\n## Script expectations\n\n- `scripts/check_training_stack.py`:\n  - Python standard library only\n  - no CUDA required\n\n- `scripts/cuda_env_probe.py`:\n  - requires `torch`\n  - does not require a working GPU just to report environment facts, but is most useful when CUDA is visible\n\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n- `scripts/nccl_smoke.py`\n  - require `torch`\n  - require CUDA visibility\n  - `scripts/nccl_smoke.py` also requires `torchrun`-style distributed environment variables\n\n## Compatibility note\n\nSome recommendations in the skill target H200, B200, GB200, and GB300-era hardware. Those parts are based on official NVIDIA positioning and documentation, but not every path is runnable on the local H100 development host.\n\nFile v0.3.0:CHANGELOG.md\n\n# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on Keep a Changelog and the project follows Semantic Versioning.\n\n## [0.3.0] - 2026-04-19\n\n### Added\n\n- `references/latest-gpu-recommendations-2026-04.md` with scenario-based NVIDIA GPU recommendations as of 2026-04-19\n- `examples/workstation-rtx-pro6000-blackwell.env`\n- `examples/b200-8gpu-torchrun.env`\n- `examples/dgx-b300-training.yaml`\n- `examples/h200-long-context-training.yaml`\n- `examples/gb300-serving.yaml`\n- `examples/l4-edge-serving.yaml`\n- `references/runtime-support.md`\n- `scripts/nccl_smoke.py`\n- `requirements.txt`\n- `.github/workflows/smoke.yml`\n\n### Changed\n\n- documented the latest GPU shortlist directly in `README.md`\n- added hardware-purchase guidance to `SKILL.md`\n\n## [0.2.0] - 2026-04-19\n\n### Added\n\n- `scripts/cuda_env_probe.py` for deterministic CUDA and PyTorch environment probing\n- `scripts/check_training_stack.py` for heuristic scanning of high-cost training anti-patterns\n- `scripts/benchmark_attention.py` for SDPA, flash, and cuDNN attention backend benchmarking\n- `scripts/training_step_benchmark.py` for synthetic transformer training-step benchmarking\n- `scripts/dataloader_benchmark.py` for worker, pinning, and prefetch throughput checks\n\n### Changed\n\n- upgraded the repository from a documentation-only skill package into a skill package with reusable tooling\n- documented bundled tooling in both `SKILL.md` and `README.md`\n\n## [0.1.0] - 2026-04-19\n\n### Added\n\n- initial `nvidia-cuda` skill package for NVIDIA deep learning optimization and review\n- BF16, FP8, TF32, and mixed precision policy guidance for H100, H200, and B200-class hardware\n- `torch.compile` and CUDA Graphs decision rules\n- attention backend policy covering SDPA, FA3/FA4 registration, cuDNN attention, Transformer Engine, and varlen attention\n- large-model training guidance for FSDP2, ZeRO-style sharding, distributed optimizer, TP, SP, CP, PP, and EP\n- LLM serving guidance for paged KV, chunked prefill, inflight batching, speculative decoding, and TensorRT-LLM\n- hard anti-pattern rules for per-step `.item()`, untuned dataloaders, and premature checkpointing\n- official reference notes with PyTorch and NVIDIA documentation links\n- ClawHub-ready project layout with root `SKILL.md`\n\nFile v0.3.0:agents/openai.yaml\n\ninterface:\n  display_name: \"NVIDIA CUDA\"\n  short_description: \"NVIDIA GPU deep learning optimization policy\"\n  default_prompt: \"Use $nvidia-cuda to optimize or review NVIDIA GPU deep learning code on CUDA.\"\n\npolicy:\n  allow_implicit_invocation: true\n\nFile v0.3.0:examples/dgx-b300-training.yaml\n\nhardware_tier: dgx-b300\nobjective: frontier single-node training\n\nprecision:\n  baseline: bf16\n  fp8_candidate: true\n  accuracy_gate_required: true\n\ndistributed:\n  launcher: torchrun\n  backend: nccl\n  data_parallel: true\n  fsdp: true\n  fsdp_strategy: full_shard\n  tensor_parallel: evaluate\n  sequence_parallel: evaluate_if_tp\n  context_parallel: evaluate_for_long_context\n  activation_checkpointing: selective_after_sharding\n\ndataloader:\n  pin_memory: true\n  persistent_workers: true\n  prefetch_factor: 4\n  num_workers_per_rank: 8\n\nbenchmark_order:\n  - eager_bf16\n  - torch_compile\n  - fused_attention_backend_compare\n  - fp8_candidate\n  - distributed_scaling\n\nnotes:\n  - Use DGX B300 as the newest top-end single-node training reference in this repository.\n  - Shard state before broad checkpointing when memory pressure comes from replicated params, grads, or optimizer state.\n\nFile v0.3.0:examples/gb300-serving.yaml\n\nhardware_tier: gb300-rack-scale\nobjective: large-scale LLM reasoning and low-latency serving\n\nprecision:\n  baseline: bf16\n  fp8_candidate: true\n  accuracy_gate_required: true\n\nserving:\n  paged_kv_cache: true\n  chunked_prefill: true\n  inflight_batching: true\n  speculative_decoding: evaluate\n  cuda_graphs_for_decode: true\n\nattention:\n  benchmark_default_sdpa: true\n  benchmark_flash_impls: true\n  benchmark_cudnn_attention: true\n\nnotes:\n  - Treat GB300 NVL72 as a rack-scale system recommendation, not a loose-card recommendation.\n  - Prioritize scheduler-level benchmarks and TTFT/ITL measurement over isolated kernel wins.\n\nFile v0.3.0:examples/h200-long-context-training.yaml\n\nhardware_tier: h200-memory-first\nobjective: long-context transformer training\n\nprecision:\n  baseline: bf16\n  tf32_matmul: high\n  fp8_candidate: false\n\ndistributed:\n  launcher: torchrun\n  backend: nccl\n  data_parallel: true\n  fsdp: true\n  fsdp_strategy: full_shard\n  tensor_parallel: 1\n  sequence_parallel: false\n  context_parallel: 2\n  activation_checkpointing: selective\n\ndataloader:\n  pin_memory: true\n  persistent_workers: true\n  prefetch_factor: 4\n  num_workers_per_rank: 8\n\nprofiling:\n  first_tool: nsys\n  second_tool: ncu\n\nnotes:\n  - Choose H200 when model memory and bandwidth dominate the decision.\n  - Use FSDP before broadening checkpointing if state replication is the main memory problem.\n\nFile v0.3.0:examples/l4-edge-serving.yaml\n\nhardware_tier: l4-edge-inference\nobjective: efficient low-power inference\n\nprecision:\n  baseline: fp16_or_bf16_if_supported_by_stack\n  tf32_matmul: high\n  fp8_candidate: false\n\nserving:\n  paged_kv_cache: evaluate\n  chunked_prefill: evaluate\n  inflight_batching: true\n  speculative_decoding: only_if_acceptance_rate_is_healthy\n\nconstraints:\n  prioritize_power_efficiency: true\n  prioritize_density: true\n  prioritize_memory_fit: true\n\nnotes:\n  - Use L4 when deployment density and power efficiency matter more than frontier-scale throughput.\n  - Validate latency under the actual scheduler and prompt mix, not only with single-request kernels.\n\nArchive v0.2.0: 11 files, 24330 bytes\n\nFiles: agents/openai.yaml (250b), CHANGELOG.md (1679b), README.md (4277b), references/official-notes.md (7436b), scripts/benchmark_attention.py (3396b), scripts/check_training_stack.py (7199b), scripts/cuda_env_probe.py (4317b), scripts/dataloader_benchmark.py (3986b), scripts/training_step_benchmark.py (4870b), SKILL.md (21781b), _meta.json (130b)\n\nFile v0.2.0:SKILL.md\n\n---\nname: nvidia-cuda\ndescription: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware code conventions for PyTorch on CUDA, including dtype policy, memory movement, NCCL/DDP/FSDP choices, profiling, benchmarking, and H100/H200/B200 optimization habits.\n---\n\n# NVIDIA CUDA\n\nUse this skill when the task involves NVIDIA GPU training or inference, multi-GPU execution, CUDA/Triton kernels, or CUDA-specific performance/debugging work.\n\nThis skill is for implementation and review, not generic theory. It should push the work toward:\n\n- correct GPU usage first\n- stable and measurable performance second\n- low-risk tuning before low-level kernel work\n- frontier algorithms only when the workload shape justifies them\n\n## Hardware stance\n\nProbe the actual machine before acting. This skill is written for modern NVIDIA Tensor Core systems and is especially opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class accelerators\n\nIt assumes the agent should actively choose precision, attention backends, sharding strategy, logging frequency, and dataloader settings instead of inheriting slow defaults.\n\n## Use this skill for\n\n- PyTorch training or inference on CUDA\n- GPU memory, throughput, or latency bottlenecks\n- DDP / FSDP / NCCL decisions\n- CUDA env var and runtime hygiene\n- Triton or custom CUDA kernel review\n- benchmark and profiler setup for NVIDIA GPUs\n\n## Do not use this skill for\n\n- CPU-only optimization\n- ROCm / AMD-specific work\n- TPU / XLA-specific work\n- vague \"make it faster\" requests without inspecting the actual GPU path first\n\n## First-pass workflow\n\nBefore changing code, inspect the actual runtime surface:\n\n1. Check hardware and driver: `nvidia-smi`\n2. Check framework build: `python -c \"import torch; print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\"`\n3. Check device visibility and topology:\n   - `CUDA_VISIBLE_DEVICES`\n   - `torch.cuda.device_count()`\n   - `torch.cuda.get_device_properties(i)`\n4. Check whether the bottleneck is:\n   - input pipeline\n   - host/device transfer\n   - eager Python overhead\n   - kernel efficiency\n   - distributed communication\n   - memory pressure / fragmentation / OOM\n\nDo not jump to kernel rewrites before ruling out bad data movement, wrong dtype, graph breaks, or poor distributed setup.\n\n## Bundled tooling\n\nUse the included scripts when you need deterministic probes instead of ad hoc snippets:\n\n- `scripts/cuda_env_probe.py`: collect CUDA, PyTorch, device, and env facts\n- `scripts/check_training_stack.py`: scan Python code for high-cost anti-patterns\n- `scripts/benchmark_attention.py`: benchmark SDPA, flash, and cuDNN attention backends\n- `scripts/training_step_benchmark.py`: benchmark a synthetic transformer training step with dtype, compile, and `.item()` logging knobs\n- `scripts/dataloader_benchmark.py`: benchmark DataLoader worker, pinning, and prefetch settings\n\nRun the probe first, then benchmark or scan the real workload path.\n\n## Non-negotiable code conventions\n\n### High-cost anti-patterns to ban\n\nTreat the following as red flags in reviews and optimization work:\n\n1. keeping transformer training or inference in full FP32 by default on Tensor Core GPUs\n2. refusing to try `torch.compile` on stable hot paths\n3. staying on a generic default SDPA path on H200/B200 when newer flash-backed implementations or cuDNN/TE attention backends are available and benchmarkable\n4. reaching for activation checkpointing before FSDP / ZeRO-style sharding when the real problem is replicated params, grads, or optimizer state\n5. calling `.item()` every step for logging and metrics\n6. leaving DataLoader workers, pinning, prefetching, and persistent workers untuned until GPU starvation shows up\n\nIf several of these exist at once, assume the GPU is underfed until proven otherwise.\n\n### 1. Dtype policy\n\n- On H100 / Hopper, prefer `bfloat16` for training and inference unless there is a measured or correctness-driven reason not to.\n- On H200 / B200-class systems, treat BF16 or FP8-capable paths as the default starting point for deep learning workloads.\n- Full FP32 by default is a correctness mode, not a performance baseline.\n- Use `float16` only when a dependency or model explicitly benefits from it.\n- If using `float16` training, use `torch.amp.GradScaler(\"cuda\")`.\n- For mixed precision, use the current API:\n  - `torch.amp.autocast(\"cuda\", dtype=...)`\n  - not deprecated `torch.cuda.amp.*`\n\n### 2. FP32 fallback policy\n\n- If the model is nominally FP32 but does not require strict IEEE-style matmul precision, prefer enabling faster matmul paths with `torch.set_float32_matmul_precision(\"high\")`.\n- Disable TF32 only for reproducibility or numerics investigations.\n- Never silently mix \"strict numerics\" and \"fast numerics\" requirements in the same change. State the choice.\n\n### 3. Device movement rules\n\n- Keep tensors on device across the hot path.\n- Move data in batches, not item-by-item.\n- Use `pin_memory=True` in `DataLoader` when feeding CUDA.\n- Use `tensor.to(\"cuda\", non_blocking=True)` or equivalent only when the source is pinned or already async-safe.\n- Avoid CPU round-trips in the training or inference loop.\n\n### 4. Avoid accidental synchronization\n\nInside hot loops, avoid or justify:\n\n- `.item()`\n- `.cpu()`\n- `.numpy()`\n- `print(cuda_tensor)`\n- timing without `torch.cuda.synchronize()`\n\nThese often force host synchronization and distort both throughput and profiling.\n\n### 5. Benchmarking rules\n\n- Warm up before measuring.\n- Use CUDA events or a framework benchmark harness, not bare wall clock alone.\n- Synchronize before reading timings.\n- Record batch size, dtype, sequence/image shape, compile state, and number of warmup / measured iterations.\n- For throughput and latency claims, include the exact command or code path used.\n\n### 6. `torch.compile` rules\n\n- Reach correctness first in eager mode.\n- Then apply `torch.compile` to stable hot paths.\n- Do not reject `torch.compile` categorically for CUDA models. Benchmark it.\n- Prefer:\n  - `mode=\"default\"` for general speedups\n  - `mode=\"reduce-overhead\"` for small-batch / latency-sensitive CUDA paths with stable shapes\n  - `mode=\"max-autotune\"` when compile overhead is acceptable and the path is hot enough\n- If shapes are unstable, be explicit about `dynamic=True` or accept recompiles intentionally.\n- Use `TORCH_LOGS=perf_hints` or `TORCH_LOGS=dynamic` when diagnosing graph breaks or overspecialization.\n\n### 6.1 Attention kernel policy\n\n- For transformer attention, prefer native `scaled_dot_product_attention` and PyTorch attention backends before custom kernels.\n- Use PyTorch SDPA / FlashAttention backends as the default fast path on CUDA.\n- On H200 / B200-class systems, do not assume the default shipped SDPA backend is the best available path. Check whether FA3 / FA4 implementations are registered and benchmark them.\n- Where Transformer Engine or cuDNN attention backends are available, benchmark them against the default path on Hopper/Blackwell instead of assuming SDPA wins.\n- If masks or score transforms are unusual, consider `torch.nn.attention.flex_attention` only after confirming SDPA cannot express the workload cleanly.\n- For variable-length packed batches, prefer varlen / packed attention paths rather than padding to worst-case sequence length.\n- Do not keep a hand-rolled eager attention implementation in a hot path if SDPA, TE attention, or TensorRT-LLM fused attention can replace it.\n\n### 6.2 CUDA Graphs policy\n\n- Use CUDA Graphs only for repeated, shape-stable hot paths where CPU launch overhead is material.\n- Graph capture is a strong fit for:\n  - small-batch inference\n  - fixed-shape decode loops\n  - stable training microbatches\n- Do not graph workloads with live shape churn, `.item()` sync points, or allocator churn in the captured region.\n- Prefer graphing after eager correctness and after compile/engine selection are already settled.\n\n### 7. Distributed rules\n\n- Use NCCL for distributed CUDA training.\n- Use DDP when the model fits on one GPU and you mainly need data parallel scale-out.\n- Use FSDP2 when the model does not fit on one GPU or optimizer state dominates memory.\n- When tensor parallelism is enabled, sequence parallelism is the default companion unless a framework limitation blocks it.\n- For long-context transformer training, consider context parallelism before blindly increasing tensor parallel degree.\n- For MoE, use expert parallelism instead of replicating all experts everywhere.\n- For large-scale runs, consider a distributed optimizer before inventing custom sharding logic.\n- Use `torchrun` and one process per GPU.\n- Set the current CUDA device from rank/local-rank early and only once.\n- Do not tune NCCL environment variables by superstition. Change them only after evidence.\n\n### 8. Memory rules\n\n- Prefer algorithmic memory wins before allocator tweaks:\n  - BF16\n  - FSDP / sharding\n  - distributed optimizer\n  - smaller activation footprints\n  - sequence or batch shaping\n  - activation checkpointing\n- When the memory problem is replicated model state, prioritize FSDP / ZeRO-2 / ZeRO-3 style sharding before gradient checkpointing.\n- Use checkpointing when activation memory is the real limiter, not as a substitute for state sharding.\n- If activation checkpointing is needed, prefer `use_reentrant=False` unless there is a specific reason otherwise.\n- Treat allocator and workspace env vars as debugging/tuning levers, not first-line fixes.\n\n### 9. Profiling rules\n\n- Start with Nsight Systems for timeline truth:\n  - CPU launch gaps\n  - H2D / D2H copies\n  - stream overlap\n  - NCCL overlap and stalls\n  - Unified Memory migrations\n- Move to Nsight Compute only after isolating the hot kernels.\n- For PyTorch-level attribution, use the profiler only if you need operator names or stack grouping.\n\n## Frontier algorithm ladder\n\nUse this section when the user explicitly wants leading-edge optimization or when the baseline is already healthy.\n\nDo not apply all of these at once. Climb the ladder in order and keep a benchmark after each step.\n\n### A. Hopper and Blackwell transformer path\n\nFor H100/H200/B200-class transformer workloads, the default advanced path is:\n\n1. BF16 baseline\n2. native SDPA / FlashAttention fast path\n3. `torch.compile`\n4. CUDA Graphs if shapes stabilize\n5. Transformer Engine FP8 if:\n   - the model is transformer-dominated\n   - BF16 numerics are already validated\n   - there is an accuracy gate\n\nFP8 is not a blanket default. It is a targeted transformer optimization with strict measurement.\n\n### B. Large-model training algorithms\n\nFor models that are too large, too long-context, or too communication-heavy, consider:\n\n- FSDP2 / Megatron FSDP for parameter, grad, and optimizer sharding\n- distributed optimizer for optimizer-state memory pressure\n- tensor parallelism for large hidden dimensions\n- sequence parallelism when TP is enabled\n- context parallelism for long sequences\n- pipeline parallelism for very deep models\n- expert parallelism for MoE\n- activation checkpointing when the above is still insufficient\n\nSelection heuristics:\n\n- if the model barely misses single-GPU fit: start with FSDP2\n- if hidden dimensions are very large: add TP\n- if sequence length is the actual problem: prefer CP over just raising TP\n- if the model is MoE: use EP rather than replicating experts\n- if optimizer state dominates: distributed optimizer or FSDP shard first\n\n### C. LLM inference algorithms\n\nFor autoregressive serving, the advanced inference ladder is:\n\n1. fused attention path\n2. paged KV cache / paged context handling\n3. continuous or inflight batching\n4. chunked prefill / context chunking for large prompts\n5. CUDA Graphs for stable decode loops\n6. speculative decoding when low-batch latency matters and acceptance rate is healthy\n7. FP8 or weight-focused quantization only after accuracy validation\n\nStrong fits:\n\n- paged context / chunked prefill: long prompts and mixed request sizes\n- speculative decoding: low-batch underutilized serving\n- inflight fused batching: multi-request serving with scheduler pressure\n- FP8 serving: Hopper/Blackwell inference paths with validated quality\n\n### D. Quantization policy\n\n- On Hopper and newer, prefer BF16 first, FP8 second, INT8/INT4 only when memory or throughput targets require it.\n- For training or finetuning quantized targets, prefer QAT or recipe-backed flows over ad hoc fake-quant code.\n- For serving, keep a clear boundary between:\n  - engine build or calibration\n  - runtime benchmarking\n  - accuracy evaluation\n\nDo not merge a quantization change without an explicit quality gate.\n\n## Training optimization playbook\n\nWhen optimizing training code, prefer this order:\n\n1. Ensure correct device placement and eliminate CPU fallbacks\n2. Fix input pipeline:\n   - `pin_memory=True`\n   - enough workers\n   - `persistent_workers=True` when the workload is steady and worker startup cost matters\n   - tuned `prefetch_factor`\n   - no CPU-heavy transforms inside the critical path unless justified\n3. Set precision policy:\n   - BF16 first on Hopper\n   - BF16 / FP8-capable path first on H200/B200\n   - FP16 only with explicit reason\n4. Remove accidental syncs and per-step Python overhead\n5. Try `torch.compile`\n6. If this is transformer-heavy, move attention to SDPA / FlashAttention / TE attention fast paths\n7. Scale to DDP or FSDP2 if single-GPU utilization or model size requires it\n8. Add TP + SP, CP, PP, or EP only according to the actual bottleneck\n9. Add activation checkpointing if memory still blocks batch size\n10. For Hopper/Blackwell transformer stacks, evaluate Transformer Engine FP8 behind an accuracy gate\n11. Profile and only then consider Triton or custom CUDA work\n\n## Inference optimization playbook\n\nWhen optimizing inference code:\n\n1. Use `model.eval()` and `torch.no_grad()` / `torch.inference_mode()`\n2. Pick dtype deliberately:\n   - BF16 or FP16 on Hopper when numerically acceptable\n   - TF32-enabled FP32 if accuracy must stay near FP32 but pure FP32 is too slow\n3. Move attention and decode loops onto fused kernels or native SDPA paths\n4. Stabilize shapes if possible\n5. Apply `torch.compile` or engine compilation only after correctness baselines\n6. For LLM serving, evaluate paged KV / paged context and inflight batching early\n7. Use CUDA Graphs only for repeated, fixed-shape paths where launch overhead matters\n8. If low-batch latency dominates, evaluate speculative decoding with acceptance-rate tracking\n9. Benchmark after warmup, with synchronization\n10. If using TensorRT or TensorRT-LLM, profile runtime separately from engine build\n\n## Multi-GPU and NCCL playbook\n\nBaseline expectations:\n\n- launch with `torchrun`\n- backend `nccl`\n- one process per GPU\n- clear `MASTER_ADDR`, `MASTER_PORT`, `RANK`, `WORLD_SIZE`, `LOCAL_RANK`\n\nParallelism heuristics:\n\n- `DP` first for ordinary scale-out\n- `FSDP2` when full replication is too expensive\n- `FSDP / ZeRO-2 / ZeRO-3` before checkpointing when optimizer, grad, or param state replication is the memory bottleneck\n- `TP + SP` for large transformer blocks\n- `CP` for long-context models\n- `PP` for deep stacks that still do not fit\n- `EP` for MoE\n- combine only what the bottleneck requires\n\nDebugging rules:\n\n- First-line NCCL debug:\n  - `NCCL_DEBUG=INFO`\n  - `NCCL_DEBUG_SUBSYS=COLL,GRAPH` when debugging hangs or topology issues\n- If interface auto-detection is wrong, set `NCCL_SOCKET_IFNAME`\n- Use `TORCH_NCCL_USE_COMM_NONBLOCKING=1` when you specifically want non-blocking NCCL error handling\n\nDo not hardcode socket or thread tuning variables unless profiling or production evidence shows the default topology tuning is wrong.\n\n## Custom CUDA / Triton kernel rules\n\nOnly drop to Triton or CUDA when operator fusion, layout cleanup, `torch.compile`, or library kernels are not enough.\n\nFor custom kernels:\n\n- favor coalesced global memory access\n- treat shared memory as a tool, not a reflex\n- watch register pressure; occupancy is a means, not the target\n- use block sizes that are multiples of 32\n- start experiments around `128` to `256` threads per block unless the kernel shape clearly suggests otherwise\n- avoid giant one-block-per-SM thinking; multiple resident blocks often help latency hiding\n- measure spills, occupancy, achieved bandwidth, and kernel time before and after changes\n\nDo not claim a kernel is faster because occupancy increased. Show end-to-end or kernel-level timing.\n\n## Transformer Engine and FP8 rules\n\nUse Transformer Engine when all of these are true:\n\n- the workload is transformer-dominated\n- the target GPUs are Hopper/Ada/Blackwell-class\n- BF16 baseline is already correct\n- you can run an accuracy gate\n\nRules:\n\n- wrap only the forward pass in TE FP8 autocast\n- use recipe-backed scaling, not home-grown FP8 metadata logic\n- start with delayed scaling or hybrid FP8 recipes\n- keep optimizer and master-weight behavior explicit\n- if FP8 helps throughput but destabilizes training, fall back to BF16 instead of layering hacks\n\nPrefer TE modules or established framework integration over custom FP8 plumbing.\n\n## Attention and sequence-shape rules\n\nFor long-sequence transformer work:\n\n- prefer fused attention over matmul-softmax-matmul Python compositions\n- on H200/B200, actively test newer flash attention implementations when available instead of passively staying with the default SDPA stack\n- prefer grouped or multi-query attention-aware fast paths when the model architecture already uses GQA or MQA\n- prefer context parallelism and packed or varlen attention before wasting memory on blanket padding\n- if custom masking is the only blocker, test FlexAttention before writing a new CUDA kernel\n\nFor decoder serving:\n\n- KV cache layout is part of the algorithm, not just a storage detail\n- paged KV and chunked context handling are first-class optimization choices\n- scheduler choice affects TTFT and ITL, so benchmark serving algorithms at the scheduler level, not only kernel level\n\n## Logging and metric collection rules\n\n- Do not call `.item()` every step for loss curves, metric dashboards, or debug prints.\n- Aggregate metrics on device and synchronize periodically:\n  - every `N` steps\n  - end of an accumulation window\n  - end of an evaluation window\n- If exact per-step host-visible metrics are required for debugging, state that this is a debug-only mode.\n- Prefer batched reductions and detached device-side buffers over step-wise host sync.\n\n## Data pipeline and prefetch rules\n\nFor CUDA training jobs, always inspect the input path before blaming kernels.\n\nRequired checks:\n\n- `num_workers`\n- `pin_memory`\n- `persistent_workers`\n- `prefetch_factor`\n- CPU decode, tokenize, or augment cost\n- storage throughput and remote filesystem latency\n\nRules:\n\n- If GPU utilization dips between steps, inspect the dataloader before touching kernels.\n- Tune `prefetch_factor` and worker count together; more workers without enough prefetch often just moves the bottleneck.\n- Keep host preprocessing out of the critical path when possible.\n- For very fast H100/H200/B200 training loops, assume untuned input pipelines are guilty until measurements say otherwise.\n\n## Reproducibility and numerics rules\n\n- Be explicit when a change trades accuracy for speed.\n- cuDNN determinism is not guaranteed across architectures.\n- Across devices or architectures, do not promise bitwise-identical outputs.\n- If NaNs or Infs appear:\n  - check data first\n  - check loss scaling or dtype policy\n  - check reduced-precision reductions\n  - inspect unstable linalg paths\n\n## Environment variable policy\n\nAllowed as targeted tools, not blanket defaults:\n\n- `CUDA_VISIBLE_DEVICES`\n- `CUDA_LAUNCH_BLOCKING=1` for debugging only\n- `PYTORCH_ALLOC_CONF`\n- `TORCH_CUDNN_V8_API_LRU_CACHE_LIMIT`\n- `TORCH_ALLOW_TF32_CUBLAS_OVERRIDE`\n- `TORCH_NCCL_USE_COMM_NONBLOCKING=1`\n- `NCCL_DEBUG`\n- `NCCL_DEBUG_SUBSYS`\n- `NCCL_SOCKET_IFNAME`\n\nIf you set an env var in code, config, or docs, explain:\n\n1. why it is needed\n2. whether it is for debugging, correctness, or performance\n3. when it should be removed\n\n## Review checklist\n\nWhen reviewing or generating code in this domain, check for:\n\n- tensors bouncing between CPU and GPU\n- missing `pin_memory=True` on CUDA dataloaders\n- missing `non_blocking=True` on pinned copies\n- missing `persistent_workers=True` or untuned `prefetch_factor` on steady-state training jobs\n- deprecated AMP API usage\n- full FP32 training or inference on Hopper/Blackwell with no numerics justification\n- FP16 used by default on Hopper with no reason\n- no BF16 or FP8 path evaluation on H200/B200-class hardware\n- no warmup in benchmarks\n- no synchronize around timing\n- `.item()` / `.cpu()` / `.numpy()` inside the step loop\n- distributed code using non-NCCL backend for CUDA\n- DDP used where model clearly does not fit, instead of FSDP2\n- activation checkpointing used to paper over what should be FSDP or ZeRO sharding\n- TP enabled without SP where the framework expects SP\n- long-context workloads scaled by TP alone when CP is the better lever\n- transformer stack on Hopper staying on unfused attention with no reason\n- transformer stack on H200/B200 never testing newer flash attention implementations\n- speculative decoding added with no acceptance-rate measurement\n- quantization merged with no quality gate\n- FP8 added without recipe-backed scaling or accuracy validation\n- custom kernel work started before simpler wins were exhausted\n\n## Output contract\n\nWhen using this skill, report in this order:\n\n1. runtime facts\n2. bottleneck hypothesis\n3. chosen optimization order\n4. changes made\n5. verification evidence\n6. remaining risks\n\nIf you need deeper justification or exact source-backed tuning notes, read [references/official-notes.md](references/official-notes.md).\n\nFile v0.2.0:README.md\n\n# nvidia-cuda\n\n`nvidia-cuda` is an open-source skill package for Codex, OpenClaw, and similar agent systems that need to write or review deep learning code for NVIDIA GPUs.\n\nIt is opinionated on purpose. The package encodes modern AI infra practice for H100, H200, and B200-class systems:\n\n- BF16 and FP8-aware precision policy\n- `torch.compile` as a benchmarked option, not a taboo\n- attention backend selection that does not stop at default SDPA\n- FSDP and ZeRO before using gradient checkpointing as a crutch\n- `.item()` and host sync anti-pattern detection\n- dataloader, pinning, prefetch, and persistent worker checks\n- NCCL, TP/SP/CP/EP, TensorRT-LLM, speculative decoding, and TE guidance\n\n## Why this exists\n\nA lot of GPU training code still leaves major performance on the floor because it inherits conservative defaults:\n\n- full FP32 everywhere\n- no compile pass\n- no attention backend evaluation on newer hardware\n- per-step host synchronization for metrics\n- checkpointing used before state sharding\n- dataloaders that starve the GPU\n\nThose mistakes compound. This skill gives an agent a clear optimization order and a concrete review checklist.\n\n## What is inside\n\n- [SKILL.md](SKILL.md): the main operational policy\n- [references/official-notes.md](references/official-notes.md): official PyTorch and NVIDIA references\n- [agents/openai.yaml](agents/openai.yaml): optional UI metadata for skill-aware clients\n- [scripts/](scripts/): reusable probes, scanners, and benchmarks\n\n## Bundled tooling\n\nThe `0.2.0` upgrade turns this repository into more than a text-only skill. It now includes reusable scripts:\n\n- `scripts/cuda_env_probe.py`\n- `scripts/check_training_stack.py`\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n\nExample commands:\n\n```bash\npython scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\n```\n\n## Install locally\n\n### Codex\n\nCopy this directory to your skills path as `~/.codex/skills/nvidia-cuda/`.\n\n### OpenClaw / ClawHub-compatible agents\n\nThe repository root is already a skill package:\n\n- `SKILL.md` at the root\n- supporting files under `references/` and `agents/`\n\nThat means you can publish this repository directly as a skill package after review.\n\n## Triggering\n\nUse the skill by name:\n\n```text\n$nvidia-cuda optimize this H100 training loop\n```\n\nExample prompts:\n\n- `$nvidia-cuda review this multi-GPU training stack for H200 underutilization`\n- `$nvidia-cuda optimize this transformer inference path on B200`\n- `$nvidia-cuda audit this dataloader, precision policy, and logging loop`\n\n## Hardware and framework stance\n\nThe skill is most opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class GPUs\n\nThe primary software target is PyTorch on CUDA, with side coverage for:\n\n- NCCL\n- Triton\n- Transformer Engine\n- TensorRT-LLM\n- Megatron Core style large-model sharding\n\n## Release policy\n\nThis repository uses semver. The current target release is `0.2.0`.\n\nVersion `0.2.0` establishes:\n\n- the initial optimization doctrine\n- the anti-pattern blocklist\n- H100/H200/B200 attention and precision guidance\n- training and inference optimization ladders\n- ClawHub-ready repository packaging\n- reusable probes and benchmark scripts for environment, attention, training-step, and dataloader analysis\n\n## ClawHub publication note\n\nClawHub supports publishing skill packages from a local path via its CLI.\n\nAt the time this repository was prepared:\n\n- the upstream ClawHub project documents `clawhub skill publish <path>`\n- the ClawHub web UI also supports signed-in publication\n\nCLI example:\n\n```bash\nnpx clawhub@latest publish . \\\n  --slug nvidia-cuda \\\n  --name \"NVIDIA CUDA\" \\\n  --version 0.1.0 \\\n  --changelog \"Initial public release\"\n```\n\nBecause community skill marketplaces can carry supply-chain risk, publish only reviewed, minimal, instruction-focused packages and inspect any scanner output before making a listing public.\n\n## License\n\n[MIT-0](LICENSE)\n\nFile v0.2.0:_meta.json\n\n{\n  \"ownerId\": \"kn7bcfdgsryztp2rsxyq54gydh82b5be\",\n  \"slug\": \"nvidia-cuda\",\n  \"version\": \"0.2.0\",\n  \"publishedAt\": 1776588768987\n}\n\nFile v0.2.0:references/official-notes.md\n\n# NVIDIA CUDA Skill Notes\n\nThis file is reference material for the `nvidia-cuda` skill. Read it when you need official grounding for a tuning choice, a code review decision, or a user-facing explanation.\n\n## Official facts to use\n\n### Mixed precision\n\n- PyTorch AMP says lower precision ops such as linear layers and convolutions are often much faster in lower precision, while other ops still need FP32 range.\n- Current AMP API is `torch.amp.autocast(...)` and `torch.amp.GradScaler(...)`; old `torch.cuda.amp.*` entry points are deprecated.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/amp.html\n\n### TF32 and FP32 matmul precision\n\n- `torch.set_float32_matmul_precision(\"high\" | \"medium\")` can significantly improve performance for FP32 matmuls on CUDA.\n- It changes internal computation precision, not output dtype.\n- On Ampere and later, TF32 can improve speed but may reduce accuracy for some workloads.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html\n  - https://docs.pytorch.org/docs/stable/notes/numerical_accuracy.html\n\n### CUDA env vars and allocator/debug knobs\n\n- `PYTORCH_ALLOC_CONF`, `CUDA_LAUNCH_BLOCKING`, `CUDA_VISIBLE_DEVICES`, TF32 override knobs, cuDNN cache or workspace knobs, and NCCL non-blocking error handling are documented in one place.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/cuda_environment_variables.html\n\n### Data movement\n\n- PyTorch data docs say host-to-GPU copies are much faster from pinned memory.\n- `DataLoader(pin_memory=True)` automatically places fetched tensors in pinned memory.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/data.html\n\n### Distributed backend choice\n\n- PyTorch distributed docs say the rule of thumb for CUDA distributed training is NCCL.\n- PyTorch distributed overview says:\n  - use DDP when the model fits on one GPU\n  - use FSDP2 when the model cannot fit on one GPU\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n  - https://docs.pytorch.org/tutorials/beginner/dist_overview.html\n  - https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html\n\n### Activation checkpointing\n\n- Activation checkpointing trades compute for memory.\n- PyTorch recommends `use_reentrant=False`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/checkpoint.html\n\n### `torch.compile`\n\n- `torch.compile` supports modes `default`, `reduce-overhead`, `max-autotune`, and `max-autotune-no-cudagraphs`.\n- `reduce-overhead` and `max-autotune` can leverage CUDA Graphs for suitable GPU paths.\n- `shape_padding` exists to better align loads on GPUs, especially for tensor cores.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.compile.html\n\n### NCCL tuning and debug\n\n- PyTorch docs recommend NCCL for CUDA and point to NVIDIA NCCL docs for the full env list.\n- NCCL docs describe debug and subsystem filters such as `NCCL_DEBUG` and `NCCL_DEBUG_SUBSYS`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/nccl/archives/nccl_2265/user-guide/docs/env.html\n\n### Profiling\n\n- Nsight Systems is the first profiler to reach for when you need end-to-end CUDA timeline truth.\n- NVIDIA docs note that heavy Unified Memory migration traffic is a performance problem and recommend manual copies from pinned host memory or peer copies instead.\n- NVIDIA docs:\n  - https://docs.nvidia.com/nsight-systems/UserGuide/\n\n### Custom kernel heuristics\n\n- CUDA best practices say:\n  - threads per block should be a multiple of 32\n  - 128 to 256 threads per block is a good initial range for experimentation\n  - higher occupancy does not automatically mean higher performance\n  - coalescing and latency hiding are core concerns\n- NVIDIA docs:\n  - https://docs.nvidia.com/cuda/pdf/CUDA_C_Best_Practices_Guide.pdf\n\n### Flash attention backends and newer implementations\n\n- PyTorch attention docs expose registration and activation for newer flash attention implementations via `register_flash_attention_impl()` and `activate_flash_attention_impl()`.\n- The documented implementation identifiers include `FA3` and `FA4`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.register_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/generated/torch.nn.attention.activate_flash_attention_impl.html\n  - https://docs.pytorch.org/docs/stable/nn.attention.html\n\n### cuDNN and Transformer Engine attention on Hopper and Blackwell\n\n- cuDNN frontend documents unified SDPA based on the FlashAttention-2 algorithm.\n- cuDNN attention support matrix includes Hopper and Blackwell paths, with fp8, fp16, and bf16 support depending on phase and architecture.\n- Transformer Engine docs state both its flash-attention backend and cuDNN attention backend are flash-algorithm based.\n- Transformer Engine docs also note cuDNN attention had 20-50% advantages on Hopper in their benchmarks for a number of common model configurations.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/frontend/latest/operations/Attention.html\n  - https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.4/user-guide/examples/attention/attention.html\n\n### Variable-length attention\n\n- PyTorch attention docs include a `varlen` module for variable-length attention using Flash Attention.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/main/nn.attention.varlen.html\n\n### Reproducibility\n\n- cuDNN docs state bitwise reproducibility is not guaranteed across architectures.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/cudnn/backend/v9.13.0/developer/misc.html\n\n### TensorRT inference note\n\n- TensorRT best practices separate engine build from runtime profiling and recommend profiling inference rather than the build phase.\n- NVIDIA docs:\n  - https://docs.nvidia.com/deeplearning/tensorrt/latest/performance/best-practices.html\n\n### TensorRT-LLM serving algorithms\n\n- TensorRT-LLM documents paged context attention as useful for large input lengths and safe to benchmark as an opt-in build flag.\n- TensorRT-LLM docs describe speculative decoding as a family of techniques that can reduce average per-token latency when the GPU is underutilized at small batch sizes.\n- TensorRT-LLM docs cover in-flight batching and scheduler runtime options separately from build-time attention choices.\n- NVIDIA docs:\n  - https://nvidia.github.io/TensorRT-LLM/performance/performance-tuning-guide/useful-build-time-flags.html\n  - https://nvidia.github.io/TensorRT-LLM/advanced/speculative-decodi\n\nArchive v0.1.0: 6 files, 14656 bytes\n\nFiles: agents/openai.yaml (250b), CHANGELOG.md (1002b), README.md (3394b), references/official-notes.md (7436b), SKILL.md (21122b), _meta.json (130b)","readmeExcerpt":"Skill: NVIDIA CUDA Owner: kkellyoffical Summary: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c... Tags: latest:0.4.0 Version history: v0.4.0 | 2026-04-19T09:11:55.823Z | user Add official FA/DDP/FSDP tooling and broader scanner coverage v0.3.0 | 2026-04-19T09:02:50.881Z | user Add latest GPU recommendations","codeSnippets":[],"executableExamples":[{"language":"bash","snippet":"python scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --list-flash-impls\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\ntorchrun --standalone --nproc_per_node=2 scripts/nccl_smoke.py --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode ddp --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode fsdp --json"},{"language":"text","snippet":"$nvidia-cuda optimize this H100 training loop"},{"language":"bash","snippet":"npx clawhub@latest publish . \\\n  --slug nvidia-cuda \\\n  --name \"NVIDIA CUDA\" \\\n  --version 0.4.0 \\\n  --changelog \"Add official FA/DDP/FSDP tooling and broader scanner coverage\""},{"language":"bash","snippet":"python scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\ntorchrun --nproc_per_node=2 scripts/nccl_smoke.py --json"},{"language":"text","snippet":"$nvidia-cuda optimize this H100 training loop"},{"language":"bash","snippet":"npx clawhub@latest publish . \\\n  --slug nvidia-cuda \\\n  --name \"NVIDIA CUDA\" \\\n  --version 0.3.0 \\\n  --changelog \"Add latest GPU recommendations and sample configs\""}],"parameters":null,"dependencies":[],"permissions":[],"extractedFiles":[{"path":"SKILL.md","content":"---\nname: nvidia-cuda\ndescription: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware code conventions for PyTorch on CUDA, including dtype policy, memory movement, NCCL/DDP/FSDP choices, profiling, benchmarking, and H100/H200/B200 optimization habits.\n---\n\n# NVIDIA CUDA\n\nUse this skill when the task involves NVIDIA GPU training or inference, multi-GPU execution, CUDA/Triton kernels, or CUDA-specific performance/debugging work.\n\nThis skill is for implementation and review, not generic theory. It should push the work toward:\n\n- correct GPU usage first\n- stable and measurable performance second\n- low-risk tuning before low-level kernel work\n- frontier algorithms only when the workload shape justifies them\n\n## Hardware stance\n\nProbe the actual machine before acting. This skill is written for modern NVIDIA Tensor Core systems and is especially opinionated for:\n\n- H100 / Hopper\n- H200\n- B200 / Blackwell-class accelerators\n\nIt assumes the agent should actively choose precision, attention backends, sharding strategy, logging frequency, and dataloader settings instead of inheriting slow defaults.\n\n## Use this skill for\n\n- PyTorch training or inference on CUDA\n- GPU memory, throughput, or latency bottlenecks\n- DDP / FSDP / NCCL decisions\n- CUDA env var and runtime hygiene\n- Triton or custom CUDA kernel review\n- benchmark and profiler setup for NVIDIA GPUs\n\n## Do not use this skill for\n\n- CPU-only optimization\n- ROCm / AMD-specific work\n- TPU / XLA-specific work\n- vague \"make it faster\" requests without inspecting the actual GPU path first\n\n## First-pass workflow\n\nBefore changing code, inspect the actual runtime surface:\n\n1. Check hardware and driver: `nvidia-smi`\n2. Check framework build: `python -c \"import torch; print(torch.__version__, torch.version.cuda, torch.cuda.is_available())\"`\n3. Check device visibility and topology:\n   - `CUDA_VISIBLE_DEVICES`\n   - `torch.cuda.device_count()`\n   - `torch.cuda.get_device_properties(i)`\n4. Check whether the bottleneck is:\n   - input pipeline\n   - host/device transfer\n   - eager Python overhead\n   - kernel efficiency\n   - distributed communication\n   - memory pressure / fragmentation / OOM\n\nDo not jump to kernel rewrites before ruling out bad data movement, wrong dtype, graph breaks, or poor distributed setup.\n\n## Bundled tooling\n\nUse the included scripts when you need deterministic probes instead of ad hoc snippets:\n\n- `scripts/cuda_env_probe.py`: collect CUDA, PyTorch, device, and env facts\n- `scripts/check_training_stack.py`: scan Python code for high-cost anti-patterns\n- `scripts/benchmark_attention.py`: benchmark SDPA, flash, cuDNN, and official flash-implementation activation paths\n- `scripts/training_step_benchmark.py`: benchmark a synthetic transformer training step with dtype, compile, and `.item()` logging knobs\n- `scripts/dataloader_benchmark.py`: benchmark DataLoader worker, pinning, and prefetch se"},{"path":"README.md","content":"# nvidia-cuda\n\n`nvidia-cuda` is an open-source skill package for Codex, OpenClaw, and similar agent systems that need to write or review deep learning code for NVIDIA GPUs.\n\nIt is opinionated on purpose. The package encodes modern AI infra practice for H100, H200, and B200-class systems:\n\n- BF16 and FP8-aware precision policy\n- `torch.compile` as a benchmarked option, not a taboo\n- attention backend selection that does not stop at default SDPA\n- FSDP and ZeRO before using gradient checkpointing as a crutch\n- `.item()` and host sync anti-pattern detection\n- dataloader, pinning, prefetch, and persistent worker checks\n- NCCL, TP/SP/CP/EP, TensorRT-LLM, speculative decoding, and TE guidance\n\n## Why this exists\n\nA lot of GPU training code still leaves major performance on the floor because it inherits conservative defaults:\n\n- full FP32 everywhere\n- no compile pass\n- no attention backend evaluation on newer hardware\n- per-step host synchronization for metrics\n- checkpointing used before state sharding\n- dataloaders that starve the GPU\n\nThose mistakes compound. This skill gives an agent a clear optimization order and a concrete review checklist.\n\n## What is inside\n\n- [SKILL.md](SKILL.md): the main operational policy\n- [references/official-notes.md](references/official-notes.md): official PyTorch and NVIDIA references\n- [references/latest-gpu-recommendations-2026-04.md](references/latest-gpu-recommendations-2026-04.md): latest GPU shortlist as of 2026-04-19\n- [references/runtime-support.md](references/runtime-support.md): tested and expected runtime surface\n- [agents/openai.yaml](agents/openai.yaml): optional UI metadata for skill-aware clients\n- [scripts/](scripts/): reusable probes, scanners, and benchmarks\n- [examples/](examples/): sample environment and planning configs\n\n## Bundled tooling\n\nThe repository now includes reusable scripts:\n\n- `scripts/cuda_env_probe.py`\n- `scripts/check_training_stack.py`\n- `scripts/benchmark_attention.py`\n- `scripts/training_step_benchmark.py`\n- `scripts/dataloader_benchmark.py`\n- `scripts/nccl_smoke.py`\n- `scripts/ddp_fsdp_smoke.py`\n\nExample commands:\n\n```bash\npython scripts/cuda_env_probe.py --json\npython scripts/check_training_stack.py path/to/project\npython scripts/benchmark_attention.py --list-flash-impls\npython scripts/benchmark_attention.py --backend cudnn --dtype bf16 --seq 4096 --compile\npython scripts/training_step_benchmark.py --dtype bf16 --compile --iters 10\npython scripts/dataloader_benchmark.py --pin-memory --num-workers 8 --persistent-workers --prefetch-factor 4\ntorchrun --standalone --nproc_per_node=2 scripts/nccl_smoke.py --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode ddp --json\ntorchrun --standalone --nproc_per_node=2 scripts/ddp_fsdp_smoke.py --mode fsdp --json\n```\n\n## Install locally\n\n### Codex\n\nCopy this directory to your skills path as `~/.codex/skills/nvidia-cuda/`.\n\n### OpenClaw / ClawHub-compatible agents\n\nThe repository root is already a skill package:\n\n- `SK"},{"path":"_meta.json","content":"{\n  \"ownerId\": \"kn7bcfdgsryztp2rsxyq54gydh82b5be\",\n  \"slug\": \"nvidia-cuda\",\n  \"version\": \"0.4.0\",\n  \"publishedAt\": 1776589915823\n}"},{"path":"references/latest-gpu-recommendations-2026-04.md","content":"# Latest NVIDIA GPU Recommendations for AI Infra\n\nLast reviewed: 2026-04-19\n\nThis note summarizes practical NVIDIA GPU recommendations for AI infrastructure based on official NVIDIA product positioning and published specifications as of 2026-04-19.\n\nImportant: these recommendations are partly **inference from official specs and positioning**, not direct vendor buying advice. Use them to choose a shortlist, then validate availability, thermals, interconnect, memory, and software fit for your workload.\n\nNVIDIA's own data center line card labels products relative to each workload column. The ranking below is a practical shortlist built from that line card plus current product pages, not an official NVIDIA rank list.\n\n## Recommendation matrix\n\n### 1. Cost-sensitive local prototyping\n\nRecommended:\n\n- **GeForce RTX 5090**\n\nUse when:\n\n- you need a powerful local GPU for experimentation\n- your models fit within **32 GB GDDR7**\n- you are doing inference, small finetunes, QLoRA-style work, or kernel benchmarking\n\nAvoid as the default choice when:\n\n- you need large-memory training\n- you need ECC-centric enterprise workflows\n- you need dense multi-GPU workstation or server deployment\n\nWhy:\n\n- NVIDIA positions RTX 5090 as its flagship GeForce GPU with **32 GB GDDR7** and Blackwell architecture.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5090/\n\n### 2. Serious single-workstation AI development\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Workstation Edition**\n- **NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition** when density or power matters more than peak board power\n\nUse when:\n\n- you want a desk-side AI workstation\n- you need **96 GB GDDR7 with ECC**\n- you want stronger enterprise and workstation ergonomics than GeForce\n\nWhy:\n\n- NVIDIA positions the Workstation Edition as its most powerful desktop GPU for professionals.\n- NVIDIA positions the Max-Q variant as a **300 W** option that scales to **up to four GPUs in one system**.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000/\n- https://www.nvidia.com/en-us/products/workstations/professional-desktop-gpus/rtx-pro-6000-max-q/\n\nIf you want a full deskside AI system rather than a loose card:\n\n- **NVIDIA DGX Station**\n\nOfficial:\n\n- https://www.nvidia.com/en-us/products/workstations/dgx-station/\n\n### 3. Universal enterprise server GPU\n\nRecommended:\n\n- **NVIDIA RTX PRO 6000 Blackwell Server Edition**\n\nUse when:\n\n- you need a flexible enterprise server GPU for AI plus graphics or simulation\n- you want **96 GB GDDR7 with ECC**\n- you are building enterprise servers rather than bare enthusiast workstations\n\nWhy:\n\n- NVIDIA positions this card as a universal Blackwell data center GPU for AI and visual computing.\n- It is a strong fit for enterprise inference, agentic AI, multimodal systems, and mixed graphics plus AI environments.\n\nOfficial:\n\n- https://www.nvidia.com/en-us/data-center/rtx-pro-6000-blackwell-server-edition"},{"path":"references/official-notes.md","content":"# NVIDIA CUDA Skill Notes\n\nThis file is reference material for the `nvidia-cuda` skill. Read it when you need official grounding for a tuning choice, a code review decision, or a user-facing explanation.\n\n## Official facts to use\n\n### Mixed precision\n\n- PyTorch AMP says lower precision ops such as linear layers and convolutions are often much faster in lower precision, while other ops still need FP32 range.\n- Current AMP API is `torch.amp.autocast(...)` and `torch.amp.GradScaler(...)`; old `torch.cuda.amp.*` entry points are deprecated.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/amp.html\n\n### TF32 and FP32 matmul precision\n\n- `torch.set_float32_matmul_precision(\"high\" | \"medium\")` can significantly improve performance for FP32 matmuls on CUDA.\n- It changes internal computation precision, not output dtype.\n- On Ampere and later, TF32 can improve speed but may reduce accuracy for some workloads.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html\n  - https://docs.pytorch.org/docs/stable/notes/numerical_accuracy.html\n\n### CUDA env vars and allocator/debug knobs\n\n- `PYTORCH_ALLOC_CONF`, `CUDA_LAUNCH_BLOCKING`, `CUDA_VISIBLE_DEVICES`, TF32 override knobs, cuDNN cache or workspace knobs, and NCCL non-blocking error handling are documented in one place.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/cuda_environment_variables.html\n\n### Data movement\n\n- PyTorch data docs say host-to-GPU copies are much faster from pinned memory.\n- `DataLoader(pin_memory=True)` automatically places fetched tensors in pinned memory.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/data.html\n\n### Distributed backend choice\n\n- PyTorch distributed docs say the rule of thumb for CUDA distributed training is NCCL.\n- PyTorch distributed overview says:\n  - use DDP when the model fits on one GPU\n  - use FSDP2 when the model cannot fit on one GPU\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/distributed.html\n  - https://docs.pytorch.org/tutorials/beginner/dist_overview.html\n  - https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html\n\n### Activation checkpointing\n\n- Activation checkpointing trades compute for memory.\n- PyTorch recommends `use_reentrant=False`.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/checkpoint.html\n\n### `torch.compile`\n\n- `torch.compile` supports modes `default`, `reduce-overhead`, `max-autotune`, and `max-autotune-no-cudagraphs`.\n- `reduce-overhead` and `max-autotune` can leverage CUDA Graphs for suitable GPU paths.\n- `shape_padding` exists to better align loads on GPUs, especially for tensor cores.\n- PyTorch docs:\n  - https://docs.pytorch.org/docs/stable/generated/torch.compile.html\n\n### NCCL tuning and debug\n\n- PyTorch docs recommend NCCL for CUDA and point to NVIDIA NCCL docs for the full env list.\n- NCCL docs describe debug and subsystem filters such as `NCCL_DEBUG` and `NCCL_DEBUG_SUBSYS`.\n- PyTorch docs:\n  - https://docs.pytorch.org/do"}],"languages":[],"docsSourceLabel":"CLAWHUB","editorialOverview":"Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c... Skill: NVIDIA CUDA Owner: kkellyoffical Summary: Use when work targets NVIDIA GPUs for deep learning training, inference, distributed execution, CUDA/Triton kernels, or AI infra tuning. Enforces GPU-aware c... Tags: latest:0.4.0 Version history: v0.4.0 | 2026-04-19T09:11:55.823Z | user Add official FA/DDP/FSDP tooling and broader scanner coverage v0.3.0 | 2026-04-19T09:02:50.881Z | user Add latest GPU recommendations","editorialQuality":{"score":100,"threshold":65,"status":"ready","wordCount":2008,"uniquenessScore":45,"reasons":[]}},"media":{"evidence":{"source":"no-media","verified":false,"confidence":"low","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":"No screenshots, media assets, or demo links are available."},"primaryImageUrl":null,"mediaAssetCount":0,"assets":[],"demoUrl":null},"ownerResources":{"evidence":{"source":"unclaimed","verified":false,"confidence":"low","updatedAt":"2026-10-11T23:53:27.615Z","emptyReason":"This page has not been claimed by the agent owner."},"hasCustomPage":false,"customPageUpdatedAt":null,"customLinks":[],"structuredLinks":{"docsUrl":null,"demoUrl":null,"supportUrl":null,"pricingUrl":null,"statusUrl":null},"customPage":null},"relatedAgents":{"evidence":{"source":"protocol-neighbors","verified":false,"confidence":"medium","updatedAt":"2026-10-12T14:01:21.763Z","emptyReason":null},"items":[{"id":"8ebccd8e-3863-4187-8355-c3f14e1f9edf","entityType":"agent","canonicalPath":"/agent/iofficeai-aionui","slug":"iofficeai-aionui","name":"AionUi","description":"Free, local, open-source 24/7 Cowork app and OpenClaw for Gemini CLI, Claude Code, Codex, OpenCode, Qwen Code, Goose CLI, Auggie, and more | 🌟 Star if you like it!","url":"https://github.com/iOfficeAI/AionUi","homepage":"https://www.aionui.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-10-09T19:11:12.944Z","createdAt":"2026-02-25T03:38:16.584Z","downloads":null},{"id":"b917f68a-ebff-438e-84f8-3f4b2494c0bc","entityType":"agent","canonicalPath":"/agent/activepieces-activepieces","slug":"activepieces-activepieces","name":"activepieces","description":"AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents","url":"https://github.com/activepieces/activepieces","homepage":"https://www.activepieces.com","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-15T02:22:12.426Z","createdAt":"2026-02-25T03:38:12.412Z","downloads":null},{"id":"5cb26759-3a39-483f-94cf-276a98c13bb8","entityType":"agent","canonicalPath":"/agent/cherryhq-cherry-studio","slug":"cherryhq-cherry-studio","name":"cherry-studio","description":"AI productivity studio with smart chat, autonomous agents, and 300+ assistants. Unified access to frontier LLMs","url":"https://github.com/CherryHQ/cherry-studio","homepage":"https://cherry-ai.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-11T14:38:40.986Z","createdAt":"2026-02-25T03:38:19.379Z","downloads":null},{"id":"6f6582d0-5d76-4f0f-b81d-86520247950b","entityType":"agent","canonicalPath":"/agent/copilotkit-copilotkit","slug":"copilotkit-copilotkit","name":"CopilotKit","description":"The Frontend for Agents & Generative UI. React + Angular","url":"https://github.com/CopilotKit/CopilotKit","homepage":"https://docs.copilotkit.ai","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-03-25T09:50:57.846Z","createdAt":"2026-02-25T03:39:14.617Z","downloads":null}],"links":{"hub":"/agent","source":"/agent/source/clawhub","protocols":[{"label":"OpenClaw","href":"/agent/protocol/openclew"}]}}}