{"id":"359d783b-0d68-4ee9-a257-1a9951f0f0cd","entityType":"agent","slug":"clawhub-deciqai-probabilistic-thinking","name":"Probabilistic Thinking","canonicalUrl":"https://www.xpersona.co/agent/clawhub-deciqai-probabilistic-thinking","canonicalPath":"/agent/clawhub-deciqai-probabilistic-thinking","generatedAt":"2026-10-12T14:48:42.152Z","source":"CLAWHUB","claimStatus":"UNCLAIMED","verificationTier":"NONE","summary":{"evidence":{"source":"editorial-content","verified":true,"confidence":"high","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":null},"description":"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima... Skill: Probabilistic Thinking Owner: deciqai Summary: Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima... Tags: latest:1.0.4 Version history: v1.0.4 | 2026-07-16T18:11:57.587Z | user Description tail link + agents machine-readable metadata line (deciqai.com/s/probabilistic-thinking.json) v1.0.3 | 2026-07-08T11","descriptionLabel":"Technical summary","evidenceSummary":"Capability contract not published. No trust telemetry is available yet. 969 downloads reported by the source. Last updated 10/12/2026.","installCommand":"clawhub skill install s17a4mqcnk515kvaca5ze55d0x88pfpx:probabilistic-thinking","sourceUrl":"https://clawhub.ai/deciqai/probabilistic-thinking","homepage":"https://clawhub.ai/deciqai/skills/probabilistic-thinking","primaryLinks":[{"label":"View on ClawHub","url":"https://clawhub.ai/deciqai/probabilistic-thinking","kind":"source"},{"label":"Homepage","url":"https://clawhub.ai/deciqai/skills/probabilistic-thinking","kind":"homepage"}],"safetyScore":84,"overallRank":62,"popularityScore":60,"trustScore":null,"claimedByName":null,"isOwner":false,"seoDescription":"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima..."},"coverage":{"evidence":{"source":"public-profile","verified":false,"confidence":"medium","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":null},"protocols":[{"protocol":"OPENCLEW","label":"OpenClaw","status":"self-declared","notes":"Declared in the public agent profile."}],"capabilities":[],"verifiedCount":0,"selfDeclaredCount":1,"capabilityMatrix":{"rows":[{"key":"OPENCLEW","type":"protocol","support":"unknown","confidenceSource":"profile","notes":"Listed on profile"}],"flattenedTokens":"protocol:OPENCLEW|unknown|profile"}},"adoption":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":null},"stars":null,"forks":null,"downloads":969,"likes":null,"task":null,"library":null,"packageName":null,"latestVersion":"1.0.4","tractionLabel":"969 downloads"},"release":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"medium","updatedAt":"2026-10-12T00:17:08.300Z","emptyReason":null},"lastUpdatedAt":"2026-10-12T00:17:08.312Z","lastCrawledAt":"2026-10-12T00:17:08.300Z","lastIndexedAt":null,"nextCrawlAt":"2026-10-13T00:17:08.300Z","lastVerifiedAt":null,"highlights":[{"version":"1.0.4","createdAt":"2026-07-16T18:11:57.587Z","changelog":"Description tail link + agents machine-readable metadata line (deciqai.com/s/probabilistic-thinking.json)","fileCount":6,"zipByteSize":14229},{"version":"1.0.3","createdAt":"2026-07-08T11:15:58.993Z","changelog":"Footer now uses /c/<slug> short link (fixes UTM truncation when SKILL.md is read in a terminal)","fileCount":6,"zipByteSize":14261},{"version":"1.0.2","createdAt":"2026-07-08T01:00:34.910Z","changelog":"Refreshed content + GitHub star link in footer","fileCount":5,"zipByteSize":9976},{"version":"1.0.1","createdAt":"2026-07-07T22:32:17.332Z","changelog":"Add catalog categories and topics","fileCount":5,"zipByteSize":9993},{"version":"1.0.0","createdAt":"2026-07-01T11:18:03.117Z","changelog":"Initial publish","fileCount":5,"zipByteSize":10174}]},"execution":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No published capability contract is available yet."},"installCommand":"clawhub skill install s17a4mqcnk515kvaca5ze55d0x88pfpx:probabilistic-thinking","setupComplexity":"low","setupSteps":["Setup complexity is LOW. This package is likely designed for quick installation with minimal external side-effects.","Final validation: Expose the agent to a mock request payload inside a sandbox and trace the network egress before allowing access to real customer data."],"contract":{"contractStatus":"missing","authModes":[],"requires":[],"forbidden":[],"supportsMcp":false,"supportsA2a":false,"supportsStreaming":false,"inputSchemaRef":null,"outputSchemaRef":null,"dataRegion":null,"contractUpdatedAt":null,"sourceUpdatedAt":null,"freshnessSeconds":null},"invocationGuide":{"preferredApi":{"snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/trust"},"curlExamples":["curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/snapshot\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/contract\"","curl -s \"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/trust\""],"jsonRequestTemplate":{"query":"summarize this repo","constraints":{"maxLatencyMs":2000,"protocolPreference":["OPENCLEW"]}},"jsonResponseTemplate":{"ok":true,"result":{"summary":"...","confidence":0.9},"meta":{"source":"CLAWHUB","generatedAt":"2026-10-12T14:48:42.148Z"}},"retryPolicy":{"maxAttempts":3,"backoffMs":[500,1500,3500],"retryableConditions":["HTTP_429","HTTP_503","NETWORK_TIMEOUT"]}},"endpoints":{"dossierUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/dossier","snapshotUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/snapshot","contractUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/contract","trustUrl":"https://www.xpersona.co/api/v1/agents/clawhub-deciqai-probabilistic-thinking/trust"}},"reliability":{"evidence":{"source":"runtime-metrics","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No trust, reliability, or runtime telemetry is available."},"trust":{"status":"unavailable","handshakeStatus":"UNKNOWN","verificationFreshnessHours":null,"reputationScore":null,"p95LatencyMs":null,"successRate30d":null,"fallbackRate":null,"attempts30d":null,"trustUpdatedAt":null,"trustConfidence":"unknown","sourceUpdatedAt":null,"freshnessSeconds":null},"decisionGuardrails":{"doNotUseIf":["Contract metadata is missing or unavailable for deterministic execution."],"safeUseWhen":[],"riskFlags":["missing_or_unavailable_contract","trust_data_unavailable","schema_references_missing"],"operationalConfidence":"low"},"executionMetrics":{"observedLatencyMsP50":null,"observedLatencyMsP95":null,"estimatedCostUsd":null,"uptime30d":null,"rateLimitRpm":null,"rateLimitBurst":null,"lastVerifiedAt":null,"verificationSource":null},"runtimeMetrics":{"successRate":null,"avgLatencyMs":null,"avgCostUsd":null,"hallucinationRate":null,"retryRate":null,"disputeRate":null,"p50Latency":null,"p95Latency":null,"lastUpdated":null}},"benchmarks":{"evidence":{"source":"no-benchmark-data","verified":false,"confidence":"low","updatedAt":null,"emptyReason":"No benchmark suites or observed failure patterns are available."},"suites":[],"failurePatterns":[]},"artifacts":{"evidence":{"source":"CLAWHUB","verified":false,"confidence":"high","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":null},"readme":"Skill: Probabilistic Thinking\n\nOwner: deciqai\n\nSummary: Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima...\n\nTags: latest:1.0.4\n\nVersion history:\n\nv1.0.4 | 2026-07-16T18:11:57.587Z | user\n\nDescription tail link + agents machine-readable metadata line (deciqai.com/s/probabilistic-thinking.json)\n\nv1.0.3 | 2026-07-08T11:15:58.993Z | user\n\nFooter now uses /c/<slug> short link (fixes UTM truncation when SKILL.md is read in a terminal)\n\nv1.0.2 | 2026-07-08T01:00:34.910Z | user\n\nRefreshed content + GitHub star link in footer\n\nv1.0.1 | 2026-07-07T22:32:17.332Z | user\n\nAdd catalog categories and topics\n\nv1.0.0 | 2026-07-01T11:18:03.117Z | user\n\nInitial publish\n\nArchive index:\n\nArchive v1.0.4: 6 files, 14229 bytes\n\nFiles: examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md (7262b), examples/tetlock-iarpa-good-judgment-project-2011-2015.md (4827b), references/sources.md (2784b), skill-card.md (2783b), SKILL.md (9711b), _meta.json (141b)\n\nFile v1.0.4:SKILL.md\n\n---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact. More: deciqai.com/c/probabilistic-thinking\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: first-principles · occams-razor · second-order-thinking · inversion · regret-minimization · expected-value-and-kelly. This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence; when forecasting AI timelines / AGI arrival / agentic reliability, or judging whether AI capex, AI valuations, or AI adoption rates justify a point-estimate bet amid genuine uncertainty.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ regret-minimization).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of success\" is vague; \"probability customer X signs by Q3 given yesterday's call\" is a question.\n> **[WAIT — do not advance until user responds]**\n4. **Walk The Process one step per turn.** Base rate first, then evidence, then update *with* them.\n> **[WAIT — do not advance until user responds]**\n5. **Close.** State the probability number and the one piece of evidence that would move it most.\n> **[WAIT — do not advance until user responds]**\n\n## The Process\n\nRun the **Probability Estimate**. Base rate first, then evidence, then update, then calibration check.\n\n1. **Precise question + deadline.** \"Will the deal close?\" → \"Will customer X sign ≥$50K by 2026-09-30?\"\n2. **Anchor in a base rate.** Historical fraction of similar situations. No base rate = Knightian territory → report range, not point.\n3. **Evidence for and against.** Each signal moves estimate ↑ or ↓. Be uncharitable about both sides.\n4. **Bayesian update (plain language).** For each signal: P(evidence | outcome happens) vs P(evidence | doesn't happen). The ratio drives the shift.\n5. **Number + confidence interval.** Not \"70-ish\" — \"68%, 80% CI 55–80%.\"\n6. **Most-informative next evidence.** If nothing would move your estimate, you have a belief, not an estimate.\n7. **Calibration log.** Record estimate, date, resolution criteria. Score after: did 70%-calls land 70% of the time?\n\n### Output: the Probability Estimate\n\n```\nQuestion (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after\n```\n\n*→ Method in Action: [Tetlock, IARPA, and the Good Judgment Project (2011–2015)](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)*\n*→ 2026 lens: [Forecasting AI Timelines and Agentic Reliability (2023–2026)](examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md)*\n\n## Calibration Packs\n\n| Domain | Base rate source | Classic failure |\n|---|---|---|\n| Medical | Disease prevalence in population | Base-rate neglect → false positives |\n| Sales | Conversion-by-stage history | Anchoring on preferred deal |\n| Legal | Crime/suspect-pool frequencies | Prosecutor's fallacy |\n| Product/startup | Cohort retention, vintage distributions | Survivorship bias |\n\n## Applying It Well\n\n- **Base rate first.** A story without a base rate is fiction with a number attached.\n- **Incremental updates.** Superforecasters update *more often* but *less drastically* than amateurs.\n- **Risk ≠ uncertainty (Knight 1921).** For Knightian situations give a range + humility statement, not false precision.\n- **Name what would change your mind.** If nothing would move your estimate, you have a position, not an estimate.\n- **Score yourself.** Write forecasts down and check them — calibration is trainable only this way.\n\n*→ Primary sources: [references/sources.md](references/sources.md)*\n\n## Common Rationalizations\n\n**[D] = designed upfront | [O] = observed in real use. [O] entries are more valuable.**\n\n| Fake move | Reality |\n|---|---|\n| [D] **Base-rate neglect** | Reaching for a vivid story while ignoring the prior frequency of the outcome class. *Always anchor in a base rate first; updates come from there.* |\n| [D] Confusing **P(evidence \\| outcome)** with **P(outcome \\| evidence)** (prosecutor's fallacy) | A test triggering on 99% of cases of a rare disease will, in a low-prevalence population, produce mostly false positives. Bayes' theorem connects them; they are not interchangeable. |\n| [D] Treating \"very likely\" as a **binary** | \"I'm 90% sure\" with no calibration history and no number for \"what would make it 50%\" is a vibe, not an estimate. |\n| [D] Confusing **risk** with **uncertainty** (Knight 1921) | An actuarial-table problem and a geopolitical-forecasting problem differ in kind. Inventing a precise number for genuine Knightian uncertainty manufactures false confidence. |\n| [D] **One-shot probability fallacy** | \"The probability of *this* event is X\" implicitly invokes a reference class. Name it; otherwise the probability is undefined. |\n| [D] **Survivorship bias** in base-rate construction | Reasoning from winners without including losers gives an inflated base rate. The reference class must include the failures. |\n| [D] **Anchoring** on the first number that appears | Even random numbers shift estimates (Tversky & Kahneman 1974). Notice when you are anchoring on the latest news rather than the base rate. |\n| [D] **Under-updating** on strong evidence | Stubbornly holding the prior when new information is high-quality. Bayes says update; ignoring evidence is anti-Bayesian. |\n| [D] **Over-updating** on weak evidence | Letting noisy or single-source data dominate. Superforecasters' edge is *smaller* updates more often, not bigger ones. |\n| [D] **Pseudo-precision** | \"73.2% probability\" when inputs justify nothing tighter than \"60–80%.\" Match precision to evidence strength. |\n| *→ Add [O] entries here after each real use — paste the actual failure pattern* | *What went wrong and why* |\n\n## Red Flags\n\n- Point estimate with no base rate · \"high probability\" with no number · no evidence named that would change the estimate · single story doing all the work · reference class silently chosen to favor a conclusion · Knightian situation with no humility statement · same person makes many forecasts but has never scored them\n\n## Verification\n\n- [ ] The question is stated with a specific outcome and a deadline\n- [ ] A base rate is named with an explicit reference class and a source\n- [ ] Evidence is listed in both directions, with direction and strength tags\n- [ ] The Bayesian shift from base rate is explained in one paragraph\n- [ ] The point estimate is a number, with an 80% confidence range\n- [ ] The single most-informative next piece of evidence is named\n- [ ] The estimate is recorded with date and unambiguous resolution criteria for later scoring\n\n---\n\n*Part of **deciqAI Knowledge Skills** — 227 open-source thinking skills that make rigor executable for AI agents. The same skills power every deciqAI agent, which runs them autonomously to operate your company. **See it run → https://www.deciqai.com/c/probabilistic-thinking** · ⭐ Star the repo → https://github.com/deciqAI/knowledge-skills · Contributions welcome.*\n\n*Agents: latest version & machine-readable metadata → https://www.deciqai.com/s/probabilistic-thinking.json*\n\nFile v1.0.4:_meta.json\n\n{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.4\",\n  \"publishedAt\": 1784225517587\n}\n\nFile v1.0.4:references/sources.md\n\n# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- **Grace, Katja et al.** \"Thousands of AI Authors on the Future of AI.\" AI Impacts, 2024 (survey of published ML researchers). Documents the wide dispersion and instability of expert AI-timeline forecasts — a contemporary illustration of why AGI timelines are proxy-reference-class / partly-Knightian, not actuarial. https://aiimpacts.org/\n- **Metaculus — AI forecasting questions.** https://www.metaculus.com/questions/ . A public, resolution-criteria-based forecasting platform; its AGI/agentic-capability questions demonstrate how vague \"when AGI?\" claims must be operationalized (outcome + threshold + verifier + deadline) before they can be scored.\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims.\n\nFile v1.0.4:examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md\n\n# Method in Action: Forecasting AI Timelines and Agentic Reliability (2023–2026)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example on the hardest kind of forecast: one where the reference class is thin, the incentives are loud, and the question is often phrased so vaguely it cannot be scored.\n\nBetween 2023 and 2026, \"when will we get AGI?\" and \"when will AI agents be reliable enough to run production workflows unattended?\" became boardroom, investor, and policy questions. The public discourse was full of point predictions — \"AGI by 2027,\" \"agents will replace X% of jobs\" — usually stated with confidence, rarely stated in a form that could ever be checked. Superforecasting-style discipline does not tell you the answer; it tells you how to hold the question so you can be measurably wrong and get better. Walk the Probability Estimate through it.\n\n**Step 1 — Precise question + deadline.** \"When AGI?\" is unscorable: no agreed definition, no deadline, no resolution criterion. Convert it into something a scorekeeper could adjudicate. Two contrasting examples:\n- *Bad (unscorable):* \"Will we have AGI soon?\"\n- *Better (scorable):* \"Will a publicly available AI system complete a specified suite of multi-step, real-world office tasks end-to-end at a defined success threshold, verified by an independent benchmark, before 2027-12-31?\"\n\nThe second version names the outcome, the bar, the verifier, and the deadline. Note that even here the definition (which benchmark, what threshold) does most of the work — an honest forecaster surfaces that the disagreement is often about *definitions*, not *odds*.\n\n**Step 2 — Anchor in a base rate.** There is no clean reference class of prior AGI arrivals, so this is closer to Knightian uncertainty than to an actuarial table. Use *proxy* reference classes and say so:\n- Historical accuracy of expert technology-timeline forecasts (generally poor and over-optimistic near hype peaks).\n- The base rate of \"the last 10% of reliability takes most of the effort\" in prior autonomy pushes — self-driving cars are the canonical cautionary case: impressive demos from the mid-2010s did not translate into broad unsupervised deployment on the timelines predicted.\n- Benchmark-saturation cadence: through 2023–2025, frontier models saturated many static benchmarks faster than expected, which pushes *some* estimates earlier — a signal in the opposite direction.\n\nThe honest output of Step 2 is a range and a caveat, not a point. Naming the reference class is the load-bearing act; a probability without one is undefined (the one-shot probability fallacy).\n\n**Step 3 — Evidence for and against, each tagged.**\n- Rapid, repeated capability jumps on reasoning and coding tasks across model generations (2023–2025). ↑ moderate.\n- Real, growing enterprise adoption and large capital commitments to AI infrastructure. ↑ weak for *capability*, because spending reflects belief, not demonstrated reliability — a signal about sentiment, not about the physics of the problem.\n- The gap between demo success and reliable unattended operation: agents that work in a scripted demo still fail on long-horizon, high-variance real tasks. ↓ strong.\n- Compounding error over multi-step chains: a per-step success rate that looks high still collapses over long horizons. ↓ strong, and structural.\n- Verification/evaluation itself is unsolved for open-ended tasks — you cannot deploy unattended what you cannot cheaply check. ↓ moderate.\n\nBe uncharitable in *both* directions. The bull case must confront error compounding; the bear case must confront the genuine, repeated benchmark surprises.\n\n**Step 4 — Bayesian update (plain language).** For each signal ask P(we'd see this | the milestone lands by the deadline) vs P(we'd see this | it doesn't). Capability jumps are more likely in the \"lands\" world, so they shift up — but only moderately, because they were *also* fairly likely in the \"doesn't land soon\" world (impressive demos preceded the self-driving disappointment too). Persistent long-horizon failure and error compounding are far more likely in the \"doesn't land\" world, so they pull down harder. The superforecaster's characteristic move: update *often* (each model release, each benchmark result) but *less drastically* than the headlines — resist both the launch-day euphoria and the failure-mode dunk. Capex and valuations move sentiment, not the underlying reliability curve, so they should barely move a *capability* forecast even as they dominate the news.\n\n**Step 5 — Number + confidence interval.** State a number with a wide, honest band, e.g. *\"~35%, 80% CI 15–60%, that the defined benchmark-verified milestone is met before 2027-12-31\"* — with the explicit caveat that the band is wide because this is proxy-reference-class / partly-Knightian territory, and that the estimate is hostage to how the milestone is defined. The illustrative number matters far less than the discipline: a point with a wide CI and a named reference class beats a confident \"2027\" with neither. Pseudo-precision (\"62.4% AGI by 2027\") is the tell of someone who has not done Step 2.\n\n**Step 6 — Most-informative next evidence.** Name observables that would move the estimate most: an *independent, contamination-controlled* benchmark on long-horizon agentic tasks (not a vendor's own demo); measured multi-step task-completion rates over long horizons rather than single-shot accuracy; evidence on whether verification/evaluation scales. If a piece of news would not move your number — another funding round, another slogan — it is sentiment, not evidence.\n\n**Step 7 — Calibration log.** Write the estimate down with date and resolution criteria, and *score it later*. This is the antidote to the discourse: pundits who said \"AGI by 2025\" in 2023 rarely revisit the claim. A forecaster who logged \"35%, resolves 2027-12-31, criterion = benchmark X at threshold Y\" can compute a Brier score and improve. Being scorable is the point.\n\nThe payoff mirrors the Good Judgment finding: the win is not calling the date. It is thinking in distributions instead of point estimates, separating the capability question from the capex/valuation noise wrapped around it, and producing forecasts you can be measurably wrong about — the only kind you can get better at.\n\n*Sources: Tetlock, Philip E. & Gardner, Dan. *Superforecasting: The Art and Science of Prediction*. Crown, 2015 — the base-rate-first, incremental-update discipline applied here. Grace, Katja et al. \"Thousands of AI Authors on the Future of AI\" (AI Impacts / Katja Grace et al., 2024 survey of published ML researchers) — documents the wide dispersion and instability of expert AI-timeline forecasts. Metaculus AI forecasting questions (https://www.metaculus.com/questions/) — a public, resolution-criteria-based platform illustrating how AGI/agentic-capability questions must be operationalized to be scorable. Waymo/Cruise self-driving deployment history (widely reported 2016–2024) — the canonical \"last 10% of reliability\" cautionary reference class. Knight, Frank H. *Risk, Uncertainty, and Profit*, 1921 — the risk-vs-Knightian-uncertainty distinction invoked in Step 2.*\n\nFile v1.0.4:examples/tetlock-iarpa-good-judgment-project-2011-2015.md\n\n# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typically by less than novice forecasters shifted. Superforecasters' characteristic move was *smaller updates more frequently* — they avoided both the journalistic over-reaction and the ostrich-style under-reaction.\n- **Estimate (Step 5):** A point estimate with a confidence range, updated as evidence arrived. (Superforecasters typically gave probabilities like \"37%\" rather than \"around a third.\")\n- **Most-informative next evidence (Step 6):** Often a specific date — IMF-payment-due dates, ECB meetings, election dates — at which a binary signal would arrive.\n- **Calibration (Step 7):** Every forecast was Brier-scored after resolution. The discipline of *being scored* was itself part of what produced calibrated forecasters.\n\nThe framework's payoff is not \"predict the future\" — it is **producing forecasts you can be measurably wrong about**, so that you can become measurably better.\n\n**Sources:** Tetlock, Philip E. & Gardner, Dan. ***Superforecasting: The Art and Science of Prediction***. Crown, 2015 — the primary popular account, written by the project's principal investigator. Mellers, Barbara et al. **\"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\"** *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115 — peer-reviewed evidence on the CHAMP training intervention. https://journals.sagepub.com/doi/10.1177/0956797614524255 . Tetlock, Philip E. ***Expert Political Judgment: How Good Is It? How Can We Know?*** Princeton, 2005; revised 2017 — the earlier, foundational work on expert prediction accuracy. IARPA Aggregative Contingent Estimation (ACE) program: https://www.iarpa.gov/research-programs/ace . Good Judgment Inc. (the post-project commercial successor) maintains active forecasting tournaments: https://goodjudgment.com .\n\nFile v1.0.4:skill-card.md\n\n## Description:\n\nHelps agents structure uncertain forecasts as calibrated probability estimates anchored in base rates, updated with evidence, and recorded for later scoring.\n\nThis skill is ready for commercial/non-commercial use.\n\n## Publisher:\n\n[deciqai](https://clawhub.ai/user/deciqai)\n\n### License/Terms of Use:\n\nMIT-0\n\n## Use Case:\n\nExternal users and developers use this skill when an agent needs to estimate uncertain outcomes, avoid binary will/won't reasoning, anchor on base rates, update with evidence, and produce a forecast that can be scored later.\n\n### Deployment Geography for Use:\n\nGlobal\n\n## Known Risks and Mitigations:\n\nRisk: Probability estimates can mislead when base rates, evidence quality, or current sources are weak.\n\nMitigation: Require the agent to name the reference class and source, include an uncertainty range, and state the next evidence that would materially change the estimate.\n\nRisk: The skill can create false precision in thin-reference-class or Knightian uncertainty cases.\n\nMitigation: Use ranges and explicit humility caveats when no usable base rate exists instead of forcing a point estimate.\n\n## Reference(s):\n\n- [Sources - probabilistic-thinking](references/sources.md)\n- [Tetlock, IARPA, and the Good Judgment Project example](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)\n- [Forecasting AI Timelines and Agentic Reliability example](examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md)\n- [Bayes, An Essay towards solving a Problem in the Doctrine of Chances](https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053)\n- [Knight, Risk, Uncertainty, and Profit](https://www.econlib.org/library/Knight/knRUP.html)\n- [Mellers et al., Psychological Strategies for Winning a Geopolitical Forecasting Tournament](https://journals.sagepub.com/doi/10.1177/0956797614524255)\n- [IARPA ACE program](https://www.iarpa.gov/research-programs/ace)\n- [AI Impacts](https://aiimpacts.org/)\n- [Metaculus AI forecasting questions](https://www.metaculus.com/questions/)\n\n## Skill Output:\n\n**Output Type(s):** [text, markdown, guidance]\n\n**Output Format:** [Markdown probability estimate with structured fields for question, base rate, evidence, Bayesian shift, estimate, next evidence, and calibration log.]\n\n**Output Parameters:** [1D]\n\n**Other Properties Related to Output:** [May include point probabilities, confidence intervals, cited base rates, and caveats for Knightian uncertainty.]\n\n## Skill Version(s):\n\n1.0.4 (source: server release evidence)\n\n## Ethical Considerations:\n\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.\n\nArchive v1.0.3: 6 files, 14261 bytes\n\nFiles: examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md (7262b), examples/tetlock-iarpa-good-judgment-project-2011-2015.md (4827b), references/sources.md (2784b), skill-card.md (3009b), SKILL.md (9556b), _meta.json (141b)\n\nFile v1.0.3:SKILL.md\n\n---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact.\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: first-principles · occams-razor · second-order-thinking · inversion · regret-minimization · expected-value-and-kelly. This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence; when forecasting AI timelines / AGI arrival / agentic reliability, or judging whether AI capex, AI valuations, or AI adoption rates justify a point-estimate bet amid genuine uncertainty.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ regret-minimization).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of success\" is vague; \"probability customer X signs by Q3 given yesterday's call\" is a question.\n> **[WAIT — do not advance until user responds]**\n4. **Walk The Process one step per turn.** Base rate first, then evidence, then update *with* them.\n> **[WAIT — do not advance until user responds]**\n5. **Close.** State the probability number and the one piece of evidence that would move it most.\n> **[WAIT — do not advance until user responds]**\n\n## The Process\n\nRun the **Probability Estimate**. Base rate first, then evidence, then update, then calibration check.\n\n1. **Precise question + deadline.** \"Will the deal close?\" → \"Will customer X sign ≥$50K by 2026-09-30?\"\n2. **Anchor in a base rate.** Historical fraction of similar situations. No base rate = Knightian territory → report range, not point.\n3. **Evidence for and against.** Each signal moves estimate ↑ or ↓. Be uncharitable about both sides.\n4. **Bayesian update (plain language).** For each signal: P(evidence | outcome happens) vs P(evidence | doesn't happen). The ratio drives the shift.\n5. **Number + confidence interval.** Not \"70-ish\" — \"68%, 80% CI 55–80%.\"\n6. **Most-informative next evidence.** If nothing would move your estimate, you have a belief, not an estimate.\n7. **Calibration log.** Record estimate, date, resolution criteria. Score after: did 70%-calls land 70% of the time?\n\n### Output: the Probability Estimate\n\n```\nQuestion (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after\n```\n\n*→ Method in Action: [Tetlock, IARPA, and the Good Judgment Project (2011–2015)](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)*\n*→ 2026 lens: [Forecasting AI Timelines and Agentic Reliability (2023–2026)](examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md)*\n\n## Calibration Packs\n\n| Domain | Base rate source | Classic failure |\n|---|---|---|\n| Medical | Disease prevalence in population | Base-rate neglect → false positives |\n| Sales | Conversion-by-stage history | Anchoring on preferred deal |\n| Legal | Crime/suspect-pool frequencies | Prosecutor's fallacy |\n| Product/startup | Cohort retention, vintage distributions | Survivorship bias |\n\n## Applying It Well\n\n- **Base rate first.** A story without a base rate is fiction with a number attached.\n- **Incremental updates.** Superforecasters update *more often* but *less drastically* than amateurs.\n- **Risk ≠ uncertainty (Knight 1921).** For Knightian situations give a range + humility statement, not false precision.\n- **Name what would change your mind.** If nothing would move your estimate, you have a position, not an estimate.\n- **Score yourself.** Write forecasts down and check them — calibration is trainable only this way.\n\n*→ Primary sources: [references/sources.md](references/sources.md)*\n\n## Common Rationalizations\n\n**[D] = designed upfront | [O] = observed in real use. [O] entries are more valuable.**\n\n| Fake move | Reality |\n|---|---|\n| [D] **Base-rate neglect** | Reaching for a vivid story while ignoring the prior frequency of the outcome class. *Always anchor in a base rate first; updates come from there.* |\n| [D] Confusing **P(evidence \\| outcome)** with **P(outcome \\| evidence)** (prosecutor's fallacy) | A test triggering on 99% of cases of a rare disease will, in a low-prevalence population, produce mostly false positives. Bayes' theorem connects them; they are not interchangeable. |\n| [D] Treating \"very likely\" as a **binary** | \"I'm 90% sure\" with no calibration history and no number for \"what would make it 50%\" is a vibe, not an estimate. |\n| [D] Confusing **risk** with **uncertainty** (Knight 1921) | An actuarial-table problem and a geopolitical-forecasting problem differ in kind. Inventing a precise number for genuine Knightian uncertainty manufactures false confidence. |\n| [D] **One-shot probability fallacy** | \"The probability of *this* event is X\" implicitly invokes a reference class. Name it; otherwise the probability is undefined. |\n| [D] **Survivorship bias** in base-rate construction | Reasoning from winners without including losers gives an inflated base rate. The reference class must include the failures. |\n| [D] **Anchoring** on the first number that appears | Even random numbers shift estimates (Tversky & Kahneman 1974). Notice when you are anchoring on the latest news rather than the base rate. |\n| [D] **Under-updating** on strong evidence | Stubbornly holding the prior when new information is high-quality. Bayes says update; ignoring evidence is anti-Bayesian. |\n| [D] **Over-updating** on weak evidence | Letting noisy or single-source data dominate. Superforecasters' edge is *smaller* updates more often, not bigger ones. |\n| [D] **Pseudo-precision** | \"73.2% probability\" when inputs justify nothing tighter than \"60–80%.\" Match precision to evidence strength. |\n| *→ Add [O] entries here after each real use — paste the actual failure pattern* | *What went wrong and why* |\n\n## Red Flags\n\n- Point estimate with no base rate · \"high probability\" with no number · no evidence named that would change the estimate · single story doing all the work · reference class silently chosen to favor a conclusion · Knightian situation with no humility statement · same person makes many forecasts but has never scored them\n\n## Verification\n\n- [ ] The question is stated with a specific outcome and a deadline\n- [ ] A base rate is named with an explicit reference class and a source\n- [ ] Evidence is listed in both directions, with direction and strength tags\n- [ ] The Bayesian shift from base rate is explained in one paragraph\n- [ ] The point estimate is a number, with an 80% confidence range\n- [ ] The single most-informative next piece of evidence is named\n- [ ] The estimate is recorded with date and unambiguous resolution criteria for later scoring\n\n---\n\n*Part of **deciqAI Knowledge Skills** — 164 open-source thinking skills that make rigor executable for AI agents. The same skills power every deciqAI agent, which runs them autonomously to operate your company. **See it run → https://www.deciqai.com/c/probabilistic-thinking** · ⭐ Star the repo → https://github.com/deciqAI/knowledge-skills · Contributions welcome.*\n\nFile v1.0.3:_meta.json\n\n{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.3\",\n  \"publishedAt\": 1783509358993\n}\n\nFile v1.0.3:references/sources.md\n\n# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- **Grace, Katja et al.** \"Thousands of AI Authors on the Future of AI.\" AI Impacts, 2024 (survey of published ML researchers). Documents the wide dispersion and instability of expert AI-timeline forecasts — a contemporary illustration of why AGI timelines are proxy-reference-class / partly-Knightian, not actuarial. https://aiimpacts.org/\n- **Metaculus — AI forecasting questions.** https://www.metaculus.com/questions/ . A public, resolution-criteria-based forecasting platform; its AGI/agentic-capability questions demonstrate how vague \"when AGI?\" claims must be operationalized (outcome + threshold + verifier + deadline) before they can be scored.\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims.\n\nFile v1.0.3:examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md\n\n# Method in Action: Forecasting AI Timelines and Agentic Reliability (2023–2026)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example on the hardest kind of forecast: one where the reference class is thin, the incentives are loud, and the question is often phrased so vaguely it cannot be scored.\n\nBetween 2023 and 2026, \"when will we get AGI?\" and \"when will AI agents be reliable enough to run production workflows unattended?\" became boardroom, investor, and policy questions. The public discourse was full of point predictions — \"AGI by 2027,\" \"agents will replace X% of jobs\" — usually stated with confidence, rarely stated in a form that could ever be checked. Superforecasting-style discipline does not tell you the answer; it tells you how to hold the question so you can be measurably wrong and get better. Walk the Probability Estimate through it.\n\n**Step 1 — Precise question + deadline.** \"When AGI?\" is unscorable: no agreed definition, no deadline, no resolution criterion. Convert it into something a scorekeeper could adjudicate. Two contrasting examples:\n- *Bad (unscorable):* \"Will we have AGI soon?\"\n- *Better (scorable):* \"Will a publicly available AI system complete a specified suite of multi-step, real-world office tasks end-to-end at a defined success threshold, verified by an independent benchmark, before 2027-12-31?\"\n\nThe second version names the outcome, the bar, the verifier, and the deadline. Note that even here the definition (which benchmark, what threshold) does most of the work — an honest forecaster surfaces that the disagreement is often about *definitions*, not *odds*.\n\n**Step 2 — Anchor in a base rate.** There is no clean reference class of prior AGI arrivals, so this is closer to Knightian uncertainty than to an actuarial table. Use *proxy* reference classes and say so:\n- Historical accuracy of expert technology-timeline forecasts (generally poor and over-optimistic near hype peaks).\n- The base rate of \"the last 10% of reliability takes most of the effort\" in prior autonomy pushes — self-driving cars are the canonical cautionary case: impressive demos from the mid-2010s did not translate into broad unsupervised deployment on the timelines predicted.\n- Benchmark-saturation cadence: through 2023–2025, frontier models saturated many static benchmarks faster than expected, which pushes *some* estimates earlier — a signal in the opposite direction.\n\nThe honest output of Step 2 is a range and a caveat, not a point. Naming the reference class is the load-bearing act; a probability without one is undefined (the one-shot probability fallacy).\n\n**Step 3 — Evidence for and against, each tagged.**\n- Rapid, repeated capability jumps on reasoning and coding tasks across model generations (2023–2025). ↑ moderate.\n- Real, growing enterprise adoption and large capital commitments to AI infrastructure. ↑ weak for *capability*, because spending reflects belief, not demonstrated reliability — a signal about sentiment, not about the physics of the problem.\n- The gap between demo success and reliable unattended operation: agents that work in a scripted demo still fail on long-horizon, high-variance real tasks. ↓ strong.\n- Compounding error over multi-step chains: a per-step success rate that looks high still collapses over long horizons. ↓ strong, and structural.\n- Verification/evaluation itself is unsolved for open-ended tasks — you cannot deploy unattended what you cannot cheaply check. ↓ moderate.\n\nBe uncharitable in *both* directions. The bull case must confront error compounding; the bear case must confront the genuine, repeated benchmark surprises.\n\n**Step 4 — Bayesian update (plain language).** For each signal ask P(we'd see this | the milestone lands by the deadline) vs P(we'd see this | it doesn't). Capability jumps are more likely in the \"lands\" world, so they shift up — but only moderately, because they were *also* fairly likely in the \"doesn't land soon\" world (impressive demos preceded the self-driving disappointment too). Persistent long-horizon failure and error compounding are far more likely in the \"doesn't land\" world, so they pull down harder. The superforecaster's characteristic move: update *often* (each model release, each benchmark result) but *less drastically* than the headlines — resist both the launch-day euphoria and the failure-mode dunk. Capex and valuations move sentiment, not the underlying reliability curve, so they should barely move a *capability* forecast even as they dominate the news.\n\n**Step 5 — Number + confidence interval.** State a number with a wide, honest band, e.g. *\"~35%, 80% CI 15–60%, that the defined benchmark-verified milestone is met before 2027-12-31\"* — with the explicit caveat that the band is wide because this is proxy-reference-class / partly-Knightian territory, and that the estimate is hostage to how the milestone is defined. The illustrative number matters far less than the discipline: a point with a wide CI and a named reference class beats a confident \"2027\" with neither. Pseudo-precision (\"62.4% AGI by 2027\") is the tell of someone who has not done Step 2.\n\n**Step 6 — Most-informative next evidence.** Name observables that would move the estimate most: an *independent, contamination-controlled* benchmark on long-horizon agentic tasks (not a vendor's own demo); measured multi-step task-completion rates over long horizons rather than single-shot accuracy; evidence on whether verification/evaluation scales. If a piece of news would not move your number — another funding round, another slogan — it is sentiment, not evidence.\n\n**Step 7 — Calibration log.** Write the estimate down with date and resolution criteria, and *score it later*. This is the antidote to the discourse: pundits who said \"AGI by 2025\" in 2023 rarely revisit the claim. A forecaster who logged \"35%, resolves 2027-12-31, criterion = benchmark X at threshold Y\" can compute a Brier score and improve. Being scorable is the point.\n\nThe payoff mirrors the Good Judgment finding: the win is not calling the date. It is thinking in distributions instead of point estimates, separating the capability question from the capex/valuation noise wrapped around it, and producing forecasts you can be measurably wrong about — the only kind you can get better at.\n\n*Sources: Tetlock, Philip E. & Gardner, Dan. *Superforecasting: The Art and Science of Prediction*. Crown, 2015 — the base-rate-first, incremental-update discipline applied here. Grace, Katja et al. \"Thousands of AI Authors on the Future of AI\" (AI Impacts / Katja Grace et al., 2024 survey of published ML researchers) — documents the wide dispersion and instability of expert AI-timeline forecasts. Metaculus AI forecasting questions (https://www.metaculus.com/questions/) — a public, resolution-criteria-based platform illustrating how AGI/agentic-capability questions must be operationalized to be scorable. Waymo/Cruise self-driving deployment history (widely reported 2016–2024) — the canonical \"last 10% of reliability\" cautionary reference class. Knight, Frank H. *Risk, Uncertainty, and Profit*, 1921 — the risk-vs-Knightian-uncertainty distinction invoked in Step 2.*\n\nFile v1.0.3:examples/tetlock-iarpa-good-judgment-project-2011-2015.md\n\n# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typically by less than novice forecasters shifted. Superforecasters' characteristic move was *smaller updates more frequently* — they avoided both the journalistic over-reaction and the ostrich-style under-reaction.\n- **Estimate (Step 5):** A point estimate with a confidence range, updated as evidence arrived. (Superforecasters typically gave probabilities like \"37%\" rather than \"around a third.\")\n- **Most-informative next evidence (Step 6):** Often a specific date — IMF-payment-due dates, ECB meetings, election dates — at which a binary signal would arrive.\n- **Calibration (Step 7):** Every forecast was Brier-scored after resolution. The discipline of *being scored* was itself part of what produced calibrated forecasters.\n\nThe framework's payoff is not \"predict the future\" — it is **producing forecasts you can be measurably wrong about**, so that you can become measurably better.\n\n**Sources:** Tetlock, Philip E. & Gardner, Dan. ***Superforecasting: The Art and Science of Prediction***. Crown, 2015 — the primary popular account, written by the project's principal investigator. Mellers, Barbara et al. **\"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\"** *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115 — peer-reviewed evidence on the CHAMP training intervention. https://journals.sagepub.com/doi/10.1177/0956797614524255 . Tetlock, Philip E. ***Expert Political Judgment: How Good Is It? How Can We Know?*** Princeton, 2005; revised 2017 — the earlier, foundational work on expert prediction accuracy. IARPA Aggregative Contingent Estimation (ACE) program: https://www.iarpa.gov/research-programs/ace . Good Judgment Inc. (the post-project commercial successor) maintains active forecasting tournaments: https://goodjudgment.com .\n\nFile v1.0.3:skill-card.md\n\n## Description: <br>\nHelps agents turn uncertain outcomes into calibrated probability estimates anchored in base rates, evidence updates, confidence ranges, and later scoring. <br>\n\nThis skill is ready for commercial/non-commercial use. <br>\n\n## Publisher: <br>\n[deciqai](https://clawhub.ai/user/deciqai) <br>\n\n### License/Terms of Use: <br>\nMIT-0 <br>\n\n\n## Use Case: <br>\nAgents and users use this skill when they need to reason about forecasts, diagnoses, pipeline conversions, hiring decisions, deal outcomes, or similar uncertain events. It guides the agent to state a precise question, identify a base rate, update with evidence, provide a probability and confidence range, and record resolution criteria for calibration. <br>\n\n### Deployment Geography for Use: <br>\nGlobal <br>\n\n## Known Risks and Mitigations: <br>\nRisk: Forecasts and calibration logs may contain private or sensitive business details. <br>\nMitigation: Avoid putting private or sensitive details into shared calibration logs; redact confidential context before storing or sharing estimates. <br>\nRisk: A weak base rate or genuinely uncertain situation can be turned into a falsely precise point estimate. <br>\nMitigation: Use ranges, confidence intervals, and explicit Knightian uncertainty caveats when the reference class is thin or unavailable. <br>\n\n\n## Reference(s): <br>\n- [ClawHub Skill Page](https://clawhub.ai/deciqai/skills/probabilistic-thinking) <br>\n- [Sources - probabilistic-thinking](references/sources.md) <br>\n- [Tetlock, IARPA, and the Good Judgment Project example](examples/tetlock-iarpa-good-judgment-project-2011-2015.md) <br>\n- [Forecasting AI Timelines and Agentic Reliability example](examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md) <br>\n- [Bayes, An Essay towards solving a Problem in the Doctrine of Chances](https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053) <br>\n- [Knight, Risk, Uncertainty, and Profit](https://www.econlib.org/library/Knight/knRUP.html) <br>\n- [Mellers et al., Psychological Strategies for Winning a Geopolitical Forecasting Tournament](https://journals.sagepub.com/doi/10.1177/0956797614524255) <br>\n- [IARPA Aggregative Contingent Estimation program](https://www.iarpa.gov/research-programs/ace) <br>\n- [Metaculus forecasting questions](https://www.metaculus.com/questions/) <br>\n\n\n## Skill Output: <br>\n**Output Type(s):** [text, markdown, guidance] <br>\n**Output Format:** [Markdown guidance with a structured Probability Estimate template] <br>\n**Output Parameters:** [1D] <br>\n**Other Properties Related to Output:** [May ask one question at a time in coach mode before producing the estimate.] <br>\n\n## Skill Version(s): <br>\n1.0.3 (source: server release evidence) <br>\n\n## Ethical Considerations: <br>\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment. <br>\n\nArchive v1.0.2: 5 files, 9976 bytes\n\nFiles: examples/tetlock-iarpa-good-judgment-project-2011-2015.md (4827b), references/sources.md (2125b), skill-card.md (2806b), SKILL.md (9331b), _meta.json (141b)\n\nFile v1.0.2:SKILL.md\n\n---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact.\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: first-principles · occams-razor · second-order-thinking · inversion · regret-minimization · expected-value-and-kelly. This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ regret-minimization).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of success\" is vague; \"probability customer X signs by Q3 given yesterday's call\" is a question.\n> **[WAIT — do not advance until user responds]**\n4. **Walk The Process one step per turn.** Base rate first, then evidence, then update *with* them.\n> **[WAIT — do not advance until user responds]**\n5. **Close.** State the probability number and the one piece of evidence that would move it most.\n> **[WAIT — do not advance until user responds]**\n\n## The Process\n\nRun the **Probability Estimate**. Base rate first, then evidence, then update, then calibration check.\n\n1. **Precise question + deadline.** \"Will the deal close?\" → \"Will customer X sign ≥$50K by 2026-09-30?\"\n2. **Anchor in a base rate.** Historical fraction of similar situations. No base rate = Knightian territory → report range, not point.\n3. **Evidence for and against.** Each signal moves estimate ↑ or ↓. Be uncharitable about both sides.\n4. **Bayesian update (plain language).** For each signal: P(evidence | outcome happens) vs P(evidence | doesn't happen). The ratio drives the shift.\n5. **Number + confidence interval.** Not \"70-ish\" — \"68%, 80% CI 55–80%.\"\n6. **Most-informative next evidence.** If nothing would move your estimate, you have a belief, not an estimate.\n7. **Calibration log.** Record estimate, date, resolution criteria. Score after: did 70%-calls land 70% of the time?\n\n### Output: the Probability Estimate\n\n```\nQuestion (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after\n```\n\n*→ Method in Action: [Tetlock, IARPA, and the Good Judgment Project (2011–2015)](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)*\n\n## Calibration Packs\n\n| Domain | Base rate source | Classic failure |\n|---|---|---|\n| Medical | Disease prevalence in population | Base-rate neglect → false positives |\n| Sales | Conversion-by-stage history | Anchoring on preferred deal |\n| Legal | Crime/suspect-pool frequencies | Prosecutor's fallacy |\n| Product/startup | Cohort retention, vintage distributions | Survivorship bias |\n\n## Applying It Well\n\n- **Base rate first.** A story without a base rate is fiction with a number attached.\n- **Incremental updates.** Superforecasters update *more often* but *less drastically* than amateurs.\n- **Risk ≠ uncertainty (Knight 1921).** For Knightian situations give a range + humility statement, not false precision.\n- **Name what would change your mind.** If nothing would move your estimate, you have a position, not an estimate.\n- **Score yourself.** Write forecasts down and check them — calibration is trainable only this way.\n\n*→ Primary sources: [references/sources.md](references/sources.md)*\n\n## Common Rationalizations\n\n**[D] = designed upfront | [O] = observed in real use. [O] entries are more valuable.**\n\n| Fake move | Reality |\n|---|---|\n| [D] **Base-rate neglect** | Reaching for a vivid story while ignoring the prior frequency of the outcome class. *Always anchor in a base rate first; updates come from there.* |\n| [D] Confusing **P(evidence \\| outcome)** with **P(outcome \\| evidence)** (prosecutor's fallacy) | A test triggering on 99% of cases of a rare disease will, in a low-prevalence population, produce mostly false positives. Bayes' theorem connects them; they are not interchangeable. |\n| [D] Treating \"very likely\" as a **binary** | \"I'm 90% sure\" with no calibration history and no number for \"what would make it 50%\" is a vibe, not an estimate. |\n| [D] Confusing **risk** with **uncertainty** (Knight 1921) | An actuarial-table problem and a geopolitical-forecasting problem differ in kind. Inventing a precise number for genuine Knightian uncertainty manufactures false confidence. |\n| [D] **One-shot probability fallacy** | \"The probability of *this* event is X\" implicitly invokes a reference class. Name it; otherwise the probability is undefined. |\n| [D] **Survivorship bias** in base-rate construction | Reasoning from winners without including losers gives an inflated base rate. The reference class must include the failures. |\n| [D] **Anchoring** on the first number that appears | Even random numbers shift estimates (Tversky & Kahneman 1974). Notice when you are anchoring on the latest news rather than the base rate. |\n| [D] **Under-updating** on strong evidence | Stubbornly holding the prior when new information is high-quality. Bayes says update; ignoring evidence is anti-Bayesian. |\n| [D] **Over-updating** on weak evidence | Letting noisy or single-source data dominate. Superforecasters' edge is *smaller* updates more often, not bigger ones. |\n| [D] **Pseudo-precision** | \"73.2% probability\" when inputs justify nothing tighter than \"60–80%.\" Match precision to evidence strength. |\n| *→ Add [O] entries here after each real use — paste the actual failure pattern* | *What went wrong and why* |\n\n## Red Flags\n\n- Point estimate with no base rate · \"high probability\" with no number · no evidence named that would change the estimate · single story doing all the work · reference class silently chosen to favor a conclusion · Knightian situation with no humility statement · same person makes many forecasts but has never scored them\n\n## Verification\n\n- [ ] The question is stated with a specific outcome and a deadline\n- [ ] A base rate is named with an explicit reference class and a source\n- [ ] Evidence is listed in both directions, with direction and strength tags\n- [ ] The Bayesian shift from base rate is explained in one paragraph\n- [ ] The point estimate is a number, with an 80% confidence range\n- [ ] The single most-informative next piece of evidence is named\n- [ ] The estimate is recorded with date and unambiguous resolution criteria for later scoring\n\n---\n\n*Part of **deciqAI Knowledge Skills** — 163 open-source thinking skills that make rigor executable for AI agents. The same skills power every deciqAI agent, which runs them autonomously to operate your company. **See it run → https://www.deciqai.com/skills/probabilistic-thinking?utm_source=clawhub&utm_medium=marketplace&utm_campaign=knowledge-skills&utm_content=probabilistic-thinking** · ⭐ Star the repo → https://github.com/deciqAI/knowledge-skills · Contributions welcome.*\n\nFile v1.0.2:_meta.json\n\n{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.2\",\n  \"publishedAt\": 1783472434910\n}\n\nFile v1.0.2:references/sources.md\n\n# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims.\n\nFile v1.0.2:examples/tetlock-iarpa-good-judgment-project-2011-2015.md\n\n# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typically by less than novice forecasters shifted. Superforecasters' characteristic move was *smaller updates more frequently* — they avoided both the journalistic over-reaction and the ostrich-style under-reaction.\n- **Estimate (Step 5):** A point estimate with a confidence range, updated as evidence arrived. (Superforecasters typically gave probabilities like \"37%\" rather than \"around a third.\")\n- **Most-informative next evidence (Step 6):** Often a specific date — IMF-payment-due dates, ECB meetings, election dates — at which a binary signal would arrive.\n- **Calibration (Step 7):** Every forecast was Brier-scored after resolution. The discipline of *being scored* was itself part of what produced calibrated forecasters.\n\nThe framework's payoff is not \"predict the future\" — it is **producing forecasts you can be measurably wrong about**, so that you can become measurably better.\n\n**Sources:** Tetlock, Philip E. & Gardner, Dan. ***Superforecasting: The Art and Science of Prediction***. Crown, 2015 — the primary popular account, written by the project's principal investigator. Mellers, Barbara et al. **\"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\"** *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115 — peer-reviewed evidence on the CHAMP training intervention. https://journals.sagepub.com/doi/10.1177/0956797614524255 . Tetlock, Philip E. ***Expert Political Judgment: How Good Is It? How Can We Know?*** Princeton, 2005; revised 2017 — the earlier, foundational work on expert prediction accuracy. IARPA Aggregative Contingent Estimation (ACE) program: https://www.iarpa.gov/research-programs/ace . Good Judgment Inc. (the post-project commercial successor) maintains active forecasting tournaments: https://goodjudgment.com .\n\nFile v1.0.2:skill-card.md\n\n## Description: <br>\nHelps an agent turn uncertain outcomes into calibrated probability estimates anchored in base rates, updated with evidence, and recorded for later scoring. <br>\n\nThis skill is ready for commercial/non-commercial use. <br>\n\n## Publisher: <br>\n[deciqai](https://clawhub.ai/user/deciqai) <br>\n\n### License/Terms of Use: <br>\nMIT-0 <br>\n\n\n## Use Case: <br>\nEmployees, external users, and agents use this skill when forecasting outcomes, grounding confidence claims in base rates, updating estimates with evidence, and logging probabilities for calibration. <br>\n\n### Deployment Geography for Use: <br>\nGlobal <br>\n\n## Known Risks and Mitigations: <br>\nRisk: Probability estimates may be treated as guaranteed predictions. <br>\nMitigation: Use the skill as decision-support guidance, include uncertainty ranges, and review estimates before relying on them. <br>\nRisk: External links may lead users outside the skill package. <br>\nMitigation: Review external links before opening them. <br>\n\n\n## Reference(s): <br>\n- [ClawHub skill page](https://clawhub.ai/deciqai/skills/probabilistic-thinking) <br>\n- [Sources - probabilistic-thinking](references/sources.md) <br>\n- [Tetlock, IARPA, and the Good Judgment Project example](examples/tetlock-iarpa-good-judgment-project-2011-2015.md) <br>\n- [Bayes, An Essay towards solving a Problem in the Doctrine of Chances](https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053) <br>\n- [Knight, Risk, Uncertainty, and Profit](https://www.econlib.org/library/Knight/knRUP.html) <br>\n- [Mellers et al., Psychological Strategies for Winning a Geopolitical Forecasting Tournament](https://journals.sagepub.com/doi/10.1177/0956797614524255) <br>\n- [IARPA ACE program](https://www.iarpa.gov/research-programs/ace) <br>\n- [deciqAI skill page](https://www.deciqai.com/skills/probabilistic-thinking?utm_source=clawhub&utm_medium=marketplace&utm_campaign=knowledge-skills&utm_content=probabilistic-thinking) <br>\n- [deciqAI knowledge-skills repository](https://github.com/deciqAI/knowledge-skills) <br>\n\n\n## Skill Output: <br>\n**Output Type(s):** [text, markdown, guidance] <br>\n**Output Format:** [Markdown probability estimate with question, base rate, evidence, Bayesian shift, estimate, next evidence, and calibration log] <br>\n**Output Parameters:** [1D] <br>\n**Other Properties Related to Output:** [Decision-support guidance only; estimates should be reviewed and scored against later outcomes.] <br>\n\n## Skill Version(s): <br>\n1.0.2 (source: server release metadata) <br>\n\n## Ethical Considerations: <br>\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment. <br>\n\nArchive v1.0.1: 5 files, 9993 bytes\n\nFiles: examples/tetlock-iarpa-good-judgment-project-2011-2015.md (4827b), references/sources.md (2125b), skill-card.md (2926b), SKILL.md (9291b), _meta.json (141b)\n\nFile v1.0.1:SKILL.md\n\n---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact.\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: [first-principles](../first-principles/SKILL.md) · [occams-razor](../occams-razor/SKILL.md) · [second-order-thinking](../second-order-thinking/SKILL.md) · [inversion](../inversion/SKILL.md) · [regret-minimization](../regret-minimization/SKILL.md) · [expected-value-and-kelly](../expected-value-and-kelly/SKILL.md). This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ [regret-minimization](../regret-minimization/SKILL.md)).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of success\" is vague; \"probability customer X signs by Q3 given yesterday's call\" is a question.\n> **[WAIT — do not advance until user responds]**\n4. **Walk The Process one step per turn.** Base rate first, then evidence, then update *with* them.\n> **[WAIT — do not advance until user responds]**\n5. **Close.** State the probability number and the one piece of evidence that would move it most.\n> **[WAIT — do not advance until user responds]**\n\n## The Process\n\nRun the **Probability Estimate**. Base rate first, then evidence, then update, then calibration check.\n\n1. **Precise question + deadline.** \"Will the deal close?\" → \"Will customer X sign ≥$50K by 2026-09-30?\"\n2. **Anchor in a base rate.** Historical fraction of similar situations. No base rate = Knightian territory → report range, not point.\n3. **Evidence for and against.** Each signal moves estimate ↑ or ↓. Be uncharitable about both sides.\n4. **Bayesian update (plain language).** For each signal: P(evidence | outcome happens) vs P(evidence | doesn't happen). The ratio drives the shift.\n5. **Number + confidence interval.** Not \"70-ish\" — \"68%, 80% CI 55–80%.\"\n6. **Most-informative next evidence.** If nothing would move your estimate, you have a belief, not an estimate.\n7. **Calibration log.** Record estimate, date, resolution criteria. Score after: did 70%-calls land 70% of the time?\n\n### Output: the Probability Estimate\n\n```\nQuestion (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after\n```\n\n*→ Method in Action: [Tetlock, IARPA, and the Good Judgment Project (2011–2015)](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)*\n\n## Calibration Packs\n\n| Domain | Base rate source | Classic failure |\n|---|---|---|\n| Medical | Disease prevalence in population | Base-rate neglect → false positives |\n| Sales | Conversion-by-stage history | Anchoring on preferred deal |\n| Legal | Crime/suspect-pool frequencies | Prosecutor's fallacy |\n| Product/startup | Cohort retention, vintage distributions | Survivorship bias |\n\n## Applying It Well\n\n- **Base rate first.** A story without a base rate is fiction with a number attached.\n- **Incremental updates.** Superforecasters update *more often* but *less drastically* than amateurs.\n- **Risk ≠ uncertainty (Knight 1921).** For Knightian situations give a range + humility statement, not false precision.\n- **Name what would change your mind.** If nothing would move your estimate, you have a position, not an estimate.\n- **Score yourself.** Write forecasts down and check them — calibration is trainable only this way.\n\n*→ Primary sources: [references/sources.md](references/sources.md)*\n\n## Common Rationalizations\n\n**[D] = designed upfront | [O] = observed in real use. [O] entries are more valuable.**\n\n| Fake move | Reality |\n|---|---|\n| [D] **Base-rate neglect** | Reaching for a vivid story while ignoring the prior frequency of the outcome class. *Always anchor in a base rate first; updates come from there.* |\n| [D] Confusing **P(evidence \\| outcome)** with **P(outcome \\| evidence)** (prosecutor's fallacy) | A test triggering on 99% of cases of a rare disease will, in a low-prevalence population, produce mostly false positives. Bayes' theorem connects them; they are not interchangeable. |\n| [D] Treating \"very likely\" as a **binary** | \"I'm 90% sure\" with no calibration history and no number for \"what would make it 50%\" is a vibe, not an estimate. |\n| [D] Confusing **risk** with **uncertainty** (Knight 1921) | An actuarial-table problem and a geopolitical-forecasting problem differ in kind. Inventing a precise number for genuine Knightian uncertainty manufactures false confidence. |\n| [D] **One-shot probability fallacy** | \"The probability of *this* event is X\" implicitly invokes a reference class. Name it; otherwise the probability is undefined. |\n| [D] **Survivorship bias** in base-rate construction | Reasoning from winners without including losers gives an inflated base rate. The reference class must include the failures. |\n| [D] **Anchoring** on the first number that appears | Even random numbers shift estimates (Tversky & Kahneman 1974). Notice when you are anchoring on the latest news rather than the base rate. |\n| [D] **Under-updating** on strong evidence | Stubbornly holding the prior when new information is high-quality. Bayes says update; ignoring evidence is anti-Bayesian. |\n| [D] **Over-updating** on weak evidence | Letting noisy or single-source data dominate. Superforecasters' edge is *smaller* updates more often, not bigger ones. |\n| [D] **Pseudo-precision** | \"73.2% probability\" when inputs justify nothing tighter than \"60–80%.\" Match precision to evidence strength. |\n| *→ Add [O] entries here after each real use — paste the actual failure pattern* | *What went wrong and why* |\n\n## Red Flags\n\n- Point estimate with no base rate · \"high probability\" with no number · no evidence named that would change the estimate · single story doing all the work · reference class silently chosen to favor a conclusion · Knightian situation with no humility statement · same person makes many forecasts but has never scored them\n\n## Verification\n\n- [ ] The question is stated with a specific outcome and a deadline\n- [ ] A base rate is named with an explicit reference class and a source\n- [ ] Evidence is listed in both directions, with direction and strength tags\n- [ ] The Bayesian shift from base rate is explained in one paragraph\n- [ ] The point estimate is a number, with an 80% confidence range\n- [ ] The single most-informative next piece of evidence is named\n- [ ] The estimate is recorded with date and unambiguous resolution criteria for later scoring\n\n---\n\n*Part of **deciqAI Knowledge Skills** — open-source thinking skills that make rigor executable for AI agents. Built by deciqAI · https://deciqai.com · Contributions welcome — see the template at the repo root.*\n\nFile v1.0.1:_meta.json\n\n{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.1\",\n  \"publishedAt\": 1783463537332\n}\n\nFile v1.0.1:references/sources.md\n\n# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims.\n\nFile v1.0.1:examples/tetlock-iarpa-good-judgment-project-2011-2015.md\n\n# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typically by less than novice forecasters shifted. Superforecasters' characteristic move was *smaller updates more frequently* — they avoided both the journalistic over-reaction and the ostrich-style under-reaction.\n- **Estimate (Step 5):** A point estimate with a confidence range, updated as evidence arrived. (Superforecasters typically gave probabilities like \"37%\" rather than \"around a third.\")\n- **Most-informative next evidence (Step 6):** Often a specific date — IMF-payment-due dates, ECB meetings, election dates — at which a binary signal would arrive.\n- **Calibration (Step 7):** Every forecast was Brier-scored after resolution. The discipline of *being scored* was itself part of what produced calibrated forecasters.\n\nThe framework's payoff is not \"predict the future\" — it is **producing forecasts you can be measurably wrong about**, so that you can become measurably better.\n\n**Sources:** Tetlock, Philip E. & Gardner, Dan. ***Superforecasting: The Art and Science of Prediction***. Crown, 2015 — the primary popular account, written by the project's principal investigator. Mellers, Barbara et al. **\"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\"** *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115 — peer-reviewed evidence on the CHAMP training intervention. https://journals.sagepub.com/doi/10.1177/0956797614524255 . Tetlock, Philip E. ***Expert Political Judgment: How Good Is It? How Can We Know?*** Princeton, 2005; revised 2017 — the earlier, foundational work on expert prediction accuracy. IARPA Aggregative Contingent Estimation (ACE) program: https://www.iarpa.gov/research-programs/ace . Good Judgment Inc. (the post-project commercial successor) maintains active forecasting tournaments: https://goodjudgment.com .\n\nFile v1.0.1:skill-card.md\n\n## Description: <br>\nGuides an agent to produce calibrated probability estimates grounded in base rates, evidence updates, confidence intervals, and calibration logs. <br>\n\nThis skill is ready for commercial/non-commercial use. <br>\n\n## Publisher: <br>\n[deciqai](https://clawhub.ai/user/deciqai) <br>\n\n### License/Terms of Use: <br>\nMIT-0 <br>\n\n\n## Use Case: <br>\nDevelopers, analysts, and decision makers use this skill when an agent needs to reason about uncertain outcomes, avoid base-rate neglect, and produce forecasts that can be updated and scored. <br>\n\n### Deployment Geography for Use: <br>\nGlobal <br>\n\n## Known Risks and Mitigations: <br>\nRisk: Users may over-rely on probability estimates for medical, legal, financial, business, or other high-stakes decisions. <br>\nMitigation: Treat outputs as decision support, not professional advice, and require qualified review before acting on high-stakes forecasts. <br>\nRisk: Calibration logs may contain sensitive forecasts, business details, or personal decision information. <br>\nMitigation: Store calibration logs only in approved locations and avoid including sensitive details unless the user explicitly needs them. <br>\nRisk: A point estimate can create false precision when no usable base rate exists. <br>\nMitigation: Use a range and an explicit uncertainty caveat for Knightian situations instead of forcing a precise probability. <br>\n\n\n## Reference(s): <br>\n- [Primary sources for probabilistic thinking](references/sources.md) <br>\n- [Tetlock, IARPA, and the Good Judgment Project example](examples/tetlock-iarpa-good-judgment-project-2011-2015.md) <br>\n- [Bayes, An Essay towards solving a Problem in the Doctrine of Chances](https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053) <br>\n- [Knight, Risk, Uncertainty, and Profit](https://www.econlib.org/library/Knight/knRUP.html) <br>\n- [Mellers et al., Psychological Strategies for Winning a Geopolitical Forecasting Tournament](https://journals.sagepub.com/doi/10.1177/0956797614524255) <br>\n- [IARPA Aggregative Contingent Estimation program](https://www.iarpa.gov/research-programs/ace) <br>\n\n\n## Skill Output: <br>\n**Output Type(s):** [text, markdown, guidance] <br>\n**Output Format:** [Markdown probability estimate with structured fields for question, base rate, evidence, Bayesian shift, estimate, next evidence, and calibration log] <br>\n**Output Parameters:** [1D] <br>\n**Other Properties Related to Output:** [May include confidence intervals, resolution criteria, and a Knightian uncertainty caveat when no usable base rate exists.] <br>\n\n## Skill Version(s): <br>\n1.0.1 (source: ClawHub release evidence) <br>\n\n## Ethical Considerations: <br>\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment. <br>\n\nArchive v1.0.0: 5 files, 10174 bytes\n\nFiles: examples/tetlock-iarpa-good-judgment-project-2011-2015.md (4827b), references/sources.md (2125b), skill-card.md (3359b), SKILL.md (9291b), _meta.json (141b)\n\nFile v1.0.0:SKILL.md\n\n---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact.\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: [first-principles](../first-principles/SKILL.md) · [occams-razor](../occams-razor/SKILL.md) · [second-order-thinking](../second-order-thinking/SKILL.md) · [inversion](../inversion/SKILL.md) · [regret-minimization](../regret-minimization/SKILL.md) · [expected-value-and-kelly](../expected-value-and-kelly/SKILL.md). This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ [regret-minimization](../regret-minimization/SKILL.md)).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of success\" is vague; \"probability customer X signs by Q3 given yesterday's call\" is a question.\n> **[WAIT — do not advance until user responds]**\n4. **Walk The Process one step per turn.** Base rate first, then evidence, then update *with* them.\n> **[WAIT — do not advance until user responds]**\n5. **Close.** State the probability number and the one piece of evidence that would move it most.\n> **[WAIT — do not advance until user responds]**\n\n## The Process\n\nRun the **Probability Estimate**. Base rate first, then evidence, then update, then calibration check.\n\n1. **Precise question + deadline.** \"Will the deal close?\" → \"Will customer X sign ≥$50K by 2026-09-30?\"\n2. **Anchor in a base rate.** Historical fraction of similar situations. No base rate = Knightian territory → report range, not point.\n3. **Evidence for and against.** Each signal moves estimate ↑ or ↓. Be uncharitable about both sides.\n4. **Bayesian update (plain language).** For each signal: P(evidence | outcome happens) vs P(evidence | doesn't happen). The ratio drives the shift.\n5. **Number + confidence interval.** Not \"70-ish\" — \"68%, 80% CI 55–80%.\"\n6. **Most-informative next evidence.** If nothing would move your estimate, you have a belief, not an estimate.\n7. **Calibration log.** Record estimate, date, resolution criteria. Score after: did 70%-calls land 70% of the time?\n\n### Output: the Probability Estimate\n\n```\nQuestion (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after\n```\n\n*→ Method in Action: [Tetlock, IARPA, and the Good Judgment Project (2011–2015)](examples/tetlock-iarpa-good-judgment-project-2011-2015.md)*\n\n## Calibration Packs\n\n| Domain | Base rate source | Classic failure |\n|---|---|---|\n| Medical | Disease prevalence in population | Base-rate neglect → false positives |\n| Sales | Conversion-by-stage history | Anchoring on preferred deal |\n| Legal | Crime/suspect-pool frequencies | Prosecutor's fallacy |\n| Product/startup | Cohort retention, vintage distributions | Survivorship bias |\n\n## Applying It Well\n\n- **Base rate first.** A story without a base rate is fiction with a number attached.\n- **Incremental updates.** Superforecasters update *more often* but *less drastically* than amateurs.\n- **Risk ≠ uncertainty (Knight 1921).** For Knightian situations give a range + humility statement, not false precision.\n- **Name what would change your mind.** If nothing would move your estimate, you have a position, not an estimate.\n- **Score yourself.** Write forecasts down and check them — calibration is trainable only this way.\n\n*→ Primary sources: [references/sources.md](references/sources.md)*\n\n## Common Rationalizations\n\n**[D] = designed upfront | [O] = observed in real use. [O] entries are more valuable.**\n\n| Fake move | Reality |\n|---|---|\n| [D] **Base-rate neglect** | Reaching for a vivid story while ignoring the prior frequency of the outcome class. *Always anchor in a base rate first; updates come from there.* |\n| [D] Confusing **P(evidence \\| outcome)** with **P(outcome \\| evidence)** (prosecutor's fallacy) | A test triggering on 99% of cases of a rare disease will, in a low-prevalence population, produce mostly false positives. Bayes' theorem connects them; they are not interchangeable. |\n| [D] Treating \"very likely\" as a **binary** | \"I'm 90% sure\" with no calibration history and no number for \"what would make it 50%\" is a vibe, not an estimate. |\n| [D] Confusing **risk** with **uncertainty** (Knight 1921) | An actuarial-table problem and a geopolitical-forecasting problem differ in kind. Inventing a precise number for genuine Knightian uncertainty manufactures false confidence. |\n| [D] **One-shot probability fallacy** | \"The probability of *this* event is X\" implicitly invokes a reference class. Name it; otherwise the probability is undefined. |\n| [D] **Survivorship bias** in base-rate construction | Reasoning from winners without including losers gives an inflated base rate. The reference class must include the failures. |\n| [D] **Anchoring** on the first number that appears | Even random numbers shift estimates (Tversky & Kahneman 1974). Notice when you are anchoring on the latest news rather than the base rate. |\n| [D] **Under-updating** on strong evidence | Stubbornly holding the prior when new information is high-quality. Bayes says update; ignoring evidence is anti-Bayesian. |\n| [D] **Over-updating** on weak evidence | Letting noisy or single-source data dominate. Superforecasters' edge is *smaller* updates more often, not bigger ones. |\n| [D] **Pseudo-precision** | \"73.2% probability\" when inputs justify nothing tighter than \"60–80%.\" Match precision to evidence strength. |\n| *→ Add [O] entries here after each real use — paste the actual failure pattern* | *What went wrong and why* |\n\n## Red Flags\n\n- Point estimate with no base rate · \"high probability\" with no number · no evidence named that would change the estimate · single story doing all the work · reference class silently chosen to favor a conclusion · Knightian situation with no humility statement · same person makes many forecasts but has never scored them\n\n## Verification\n\n- [ ] The question is stated with a specific outcome and a deadline\n- [ ] A base rate is named with an explicit reference class and a source\n- [ ] Evidence is listed in both directions, with direction and strength tags\n- [ ] The Bayesian shift from base rate is explained in one paragraph\n- [ ] The point estimate is a number, with an 80% confidence range\n- [ ] The single most-informative next piece of evidence is named\n- [ ] The estimate is recorded with date and unambiguous resolution criteria for later scoring\n\n---\n\n*Part of **deciqAI Knowledge Skills** — open-source thinking skills that make rigor executable for AI agents. Built by deciqAI · https://deciqai.com · Contributions welcome — see the template at the repo root.*\n\nFile v1.0.0:_meta.json\n\n{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.0\",\n  \"publishedAt\": 1782904683117\n}\n\nFile v1.0.0:references/sources.md\n\n# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims.\n\nFile v1.0.0:examples/tetlock-iarpa-good-judgment-project-2011-2015.md\n\n# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typically by less than novice forecasters shifted. Superforecasters' characteristic move was *smaller updates more frequently* — they avoided both the journalistic over-reaction and the ostrich-style under-reaction.\n- **Estimate (Step 5):** A point estimate with a confidence range, updated as evidence arrived. (Superforecasters typically gave probabilities like \"37%\" rather than \"around a third.\")\n- **Most-informative next evidence (Step 6):** Often a specific date — IMF-payment-due dates, ECB meetings, election dates — at which a binary signal would arrive.\n- **Calibration (Step 7):** Every forecast was Brier-scored after resolution. The discipline of *being scored* was itself part of what produced calibrated forecasters.\n\nThe framework's payoff is not \"predict the future\" — it is **producing forecasts you can be measurably wrong about**, so that you can become measurably better.\n\n**Sources:** Tetlock, Philip E. & Gardner, Dan. ***Superforecasting: The Art and Science of Prediction***. Crown, 2015 — the primary popular account, written by the project's principal investigator. Mellers, Barbara et al. **\"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\"** *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115 — peer-reviewed evidence on the CHAMP training intervention. https://journals.sagepub.com/doi/10.1177/0956797614524255 . Tetlock, Philip E. ***Expert Political Judgment: How Good Is It? How Can We Know?*** Princeton, 2005; revised 2017 — the earlier, foundational work on expert prediction accuracy. IARPA Aggregative Contingent Estimation (ACE) program: https://www.iarpa.gov/research-programs/ace . Good Judgment Inc. (the post-project commercial successor) maintains active forecasting tournaments: https://goodjudgment.com .\n\nFile v1.0.0:skill-card.md\n\n## Description: <br>\nHelps agents structure uncertain judgments as calibrated probability estimates anchored in base rates, evidence updates, confidence intervals, next evidence, and calibration logs. <br>\n\nThis skill is ready for commercial/non-commercial use. <br>\n\n## Publisher: <br>\n[deciqai](https://clawhub.ai/user/deciqai) <br>\n\n### License/Terms of Use: <br>\nMIT-0 <br>\n\n\n## Use Case: <br>\nExternal users, employees, developers, and other agent operators use this skill when a conversation needs a grounded forecast or probability estimate instead of a binary will-or-won't answer. It is intended for uncertain outcomes such as sales conversion, hiring, diagnosis, product decisions, or geopolitical forecasts where base rates, evidence, and later scoring matter. <br>\n\n### Deployment Geography for Use: <br>\nGlobal <br>\n\n## Known Risks and Mitigations: <br>\nRisk: The skill can activate during ordinary forecasting conversations and may turn informal judgments into recorded probability estimates. <br>\nMitigation: Use it for genuinely uncertain outcomes with explicit resolution criteria, and skip deterministic, identity, ethics, or meaning questions. <br>\nRisk: Calibration logs can contain sensitive business, medical, legal, or personal details if users include them in forecasts. <br>\nMitigation: Avoid recording sensitive details unless the user intentionally chooses an appropriate storage location and retention practice. <br>\nRisk: Probability estimates can become misleading when there is no usable base rate or when the answer is presented with false precision. <br>\nMitigation: Require a named reference class, confidence interval, and Knightian caveat when evidence does not support a precise point estimate. <br>\n\n\n## Reference(s): <br>\n- [ClawHub skill page](https://clawhub.ai/deciqai/skills/probabilistic-thinking) <br>\n- [Sources - probabilistic-thinking](references/sources.md) <br>\n- [Tetlock, IARPA, and the Good Judgment Project example](examples/tetlock-iarpa-good-judgment-project-2011-2015.md) <br>\n- [Bayes, An Essay towards solving a Problem in the Doctrine of Chances](https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053) <br>\n- [Knight, Risk, Uncertainty, and Profit](https://www.econlib.org/library/Knight/knRUP.html) <br>\n- [Mellers et al., Psychological Strategies for Winning a Geopolitical Forecasting Tournament](https://journals.sagepub.com/doi/10.1177/0956797614524255) <br>\n- [IARPA Aggregative Contingent Estimation program](https://www.iarpa.gov/research-programs/ace) <br>\n\n\n## Skill Output: <br>\n**Output Type(s):** [text, markdown, guidance] <br>\n**Output Format:** [Markdown probability estimate with structured fields for question, base rate, evidence, Bayesian shift, estimate, next evidence, and calibration log] <br>\n**Output Parameters:** [1D] <br>\n**Other Properties Related to Output:** [May include point probabilities, confidence intervals, evidence direction and strength tags, Knightian caveats, and Brier-score follow-up fields.] <br>\n\n## Skill Version(s): <br>\n1.0.0 (source: server release metadata) <br>\n\n## Ethical Considerations: <br>\nUsers should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment. <br>","readmeExcerpt":"Skill: Probabilistic Thinking Owner: deciqai Summary: Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima... Tags: latest:1.0.4 Version history: v1.0.4 | 2026-07-16T18:11:57.587Z | user Description tail link + agents machine-readable metadata line (deciqai.com/s/probabilistic-thinking.json) v1.0.3 | 2026-07-08T11","codeSnippets":[],"executableExamples":[{"language":"text","snippet":"Question (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after"},{"language":"text","snippet":"Question (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after"},{"language":"text","snippet":"Question (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after"},{"language":"text","snippet":"Question (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after"},{"language":"text","snippet":"Question (precise): <outcome, deadline>\nBase rate: <reference class> → <fraction> (source, n=)\nEvidence: <signal> ↑/↓ strong/moderate/weak  [repeat per signal]\nBayesian shift: net <↑/↓ to X%> — rationale in one paragraph\nEstimate: <point %>  |  80% CI: <%–%>  |  Knightian caveat if needed\nNext evidence: <observable> → <% if X> / <% if Y>\nCalibration log: date | question | resolution criteria | Brier score after"}],"parameters":null,"dependencies":[],"permissions":[],"extractedFiles":[{"path":"SKILL.md","content":"---\nname: probabilistic-thinking\ndescription: \"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estimating a conversion/close/hire/outcome probability; someone claims 'I'm 90% sure' without any evidence grounding; a vivid story is being used in place of a prior frequency; team is treating an uncertain outcome as binary will/won't.\n  Do NOT activate when: the problem has a deterministic answer from its inputs (math, well-defined engineering); the question is about identity, ethics, or meaning rather than fact. More: deciqai.com/c/probabilistic-thinking\"\n---\n\n# Probabilistic Thinking\n\n## Overview\n\nMost reasoning is binary: *will it happen, or won't it?* That framing discards the most useful information — the **degree** of confidence — and produces predictions that cannot be checked, updated, or scored. Probabilistic thinking replaces binary with **calibrated probability estimates**: numbers anchored in base rates, updated with evidence, and scored after the fact. Rooted in Bayes (1763), Knight's risk-vs-uncertainty distinction (1921), and Tetlock's empirical work showing calibration is a trainable skill.\n\nComposable neighbors: first-principles · occams-razor · second-order-thinking · inversion · regret-minimization · expected-value-and-kelly. This skill is the *upstream input* the others depend on — the probability estimate here feeds EV-Kelly, calibrates inversion's failure-path weights, and gives second-order's hops their confidence decay.\n\n## When to Use\n\nUse when reasoning about an uncertain outcome (forecast, diagnosis, pipeline conversion, hire, deal close, geopolitical event); when binary \"will/won't\" predictions are being made; when a vivid story is replacing a base rate; when \"I'm 90% sure\" appears with no calibration evidence; when forecasting AI timelines / AGI arrival / agentic reliability, or judging whether AI capex, AI valuations, or AI adoption rates justify a point-estimate bet amid genuine uncertainty.\n\n**When NOT to use:** deterministic problems (math, well-defined engineering); pure Knightian uncertainty with no usable base rate (give a range + humility statement instead); decision is robust across all likely probabilities; question is identity/ethics/meaning (→ regret-minimization).\n\n## Coaching Novices (Adaptive Front Door)\n\n- **Engine mode:** user has a concrete forecasting question → run The Process directly.\n- **Coach mode:** user is unfamiliar or has no concrete case → guide step by step.\n\nIn Coach mode, respond one step at a time. Each [WAIT] is a hard stop — output only that step's question, then stop.\n\n1. **What-it-is.** Probabilistic thinking replaces \"will it happen or not\" with a number (0–1) anchored in base rates, updated with evidence, and scored after the fact.\n2. **Check fit.** Match against *When to Use / When NOT to use*. Redirect if deterministic or pure Knightian.\n3. **Elicit their real question.** \"Odds of "},{"path":"_meta.json","content":"{\n  \"ownerId\": \"kn754b8sk22s8c6gjxt02bftbn88q7ye\",\n  \"slug\": \"probabilistic-thinking\",\n  \"version\": \"1.0.4\",\n  \"publishedAt\": 1784225517587\n}"},{"path":"references/sources.md","content":"# Sources — probabilistic-thinking\n\n> *Primary sources for the [probabilistic-thinking](../SKILL.md) skill.*\n\n- **Bayes, Thomas.** \"An Essay towards solving a Problem in the Doctrine of Chances.\" *Philosophical Transactions of the Royal Society of London*, Vol. 53 (1763), pp. 370–418. Posthumously published by Richard Price. **The primary source** for Bayes' theorem. https://royalsocietypublishing.org/doi/10.1098/rstl.1763.0053\n- **Knight, Frank H.** *Risk, Uncertainty, and Profit*. Houghton Mifflin, 1921. **Primary source** for the risk-vs-uncertainty distinction; ch. VII–VIII especially. Library of Economics and Liberty: https://www.econlib.org/library/Knight/knRUP.html\n- **Tetlock, Philip E.** *Expert Political Judgment: How Good Is It? How Can We Know?* Princeton University Press, 2005 (revised 2017). The foundational empirical study on expert prediction accuracy.\n- **Tetlock, Philip E. & Gardner, Dan.** *Superforecasting: The Art and Science of Prediction*. Crown, 2015. The synthesis of the Good Judgment Project findings.\n- **Mellers, Barbara et al.** \"Psychological Strategies for Winning a Geopolitical Forecasting Tournament.\" *Psychological Science*, Vol. 25, No. 5 (May 2014), pp. 1106–1115. The peer-reviewed evidence on trainability of calibration. https://journals.sagepub.com/doi/10.1177/0956797614524255\n- **Kahneman, Daniel & Tversky, Amos.** \"On the Psychology of Prediction.\" *Psychological Review*, Vol. 80, No. 4 (1973), pp. 237–251. The original base-rate-neglect demonstrations.\n- **Gigerenzer, Gerd & Hoffrage, Ulrich.** \"How to Improve Bayesian Reasoning Without Instruction: Frequency Formats.\" *Psychological Review*, Vol. 102, No. 4 (1995), pp. 684–704. On why representing probabilities as natural frequencies dramatically improves intuitive Bayesian reasoning.\n- IARPA Aggregative Contingent Estimation (ACE) program documentation: https://www.iarpa.gov/research-programs/ace\n- **Grace, Katja et al.** \"Thousands of AI Authors on the Future of AI.\" AI Impacts, 2024 (survey of published ML researchers). Documents the wide dispersion and instability of expert AI-timeline forecasts — a contemporary illustration of why AGI timelines are proxy-reference-class / partly-Knightian, not actuarial. https://aiimpacts.org/\n- **Metaculus — AI forecasting questions.** https://www.metaculus.com/questions/ . A public, resolution-criteria-based forecasting platform; its AGI/agentic-capability questions demonstrate how vague \"when AGI?\" claims must be operationalized (outcome + threshold + verifier + deadline) before they can be scored.\n- The popular slogan \"trust the data\" is **not** cited as a source. Data alone, without a base rate and an update rule, is just numbers; the framework is what makes them probabilistic claims."},{"path":"examples/forecasting-ai-timelines-agentic-reliability-2023-2026.md","content":"# Method in Action: Forecasting AI Timelines and Agentic Reliability (2023–2026)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example on the hardest kind of forecast: one where the reference class is thin, the incentives are loud, and the question is often phrased so vaguely it cannot be scored.\n\nBetween 2023 and 2026, \"when will we get AGI?\" and \"when will AI agents be reliable enough to run production workflows unattended?\" became boardroom, investor, and policy questions. The public discourse was full of point predictions — \"AGI by 2027,\" \"agents will replace X% of jobs\" — usually stated with confidence, rarely stated in a form that could ever be checked. Superforecasting-style discipline does not tell you the answer; it tells you how to hold the question so you can be measurably wrong and get better. Walk the Probability Estimate through it.\n\n**Step 1 — Precise question + deadline.** \"When AGI?\" is unscorable: no agreed definition, no deadline, no resolution criterion. Convert it into something a scorekeeper could adjudicate. Two contrasting examples:\n- *Bad (unscorable):* \"Will we have AGI soon?\"\n- *Better (scorable):* \"Will a publicly available AI system complete a specified suite of multi-step, real-world office tasks end-to-end at a defined success threshold, verified by an independent benchmark, before 2027-12-31?\"\n\nThe second version names the outcome, the bar, the verifier, and the deadline. Note that even here the definition (which benchmark, what threshold) does most of the work — an honest forecaster surfaces that the disagreement is often about *definitions*, not *odds*.\n\n**Step 2 — Anchor in a base rate.** There is no clean reference class of prior AGI arrivals, so this is closer to Knightian uncertainty than to an actuarial table. Use *proxy* reference classes and say so:\n- Historical accuracy of expert technology-timeline forecasts (generally poor and over-optimistic near hype peaks).\n- The base rate of \"the last 10% of reliability takes most of the effort\" in prior autonomy pushes — self-driving cars are the canonical cautionary case: impressive demos from the mid-2010s did not translate into broad unsupervised deployment on the timelines predicted.\n- Benchmark-saturation cadence: through 2023–2025, frontier models saturated many static benchmarks faster than expected, which pushes *some* estimates earlier — a signal in the opposite direction.\n\nThe honest output of Step 2 is a range and a caveat, not a point. Naming the reference class is the load-bearing act; a probability without one is undefined (the one-shot probability fallacy).\n\n**Step 3 — Evidence for and against, each tagged.**\n- Rapid, repeated capability jumps on reasoning and coding tasks across model generations (2023–2025). ↑ moderate.\n- Real, growing enterprise adoption and large capital commitments to AI infrastructure. ↑ weak for *capability*, because spending reflects belief, not demonstrated reliability — a signal about sentiment"},{"path":"examples/tetlock-iarpa-good-judgment-project-2011-2015.md","content":"# Method in Action: Tetlock, IARPA, and the Good Judgment Project (2011–2015)\n\n> *Example for the [probabilistic-thinking](../SKILL.md) skill.*\n\nA worked example. Not pundit theater — peer-reviewed and US-funded.\n\nIn 2010, the **Intelligence Advanced Research Projects Activity (IARPA)**, a research arm of the U.S. Office of the Director of National Intelligence, commissioned a four-year, large-scale geopolitical forecasting tournament under its **Aggregative Contingent Estimation (ACE)** program. The objective was concrete: improve how well the U.S. intelligence community can forecast world events. Five university-led research teams competed; the winner was the **Good Judgment Project**, run by **Philip Tetlock** and **Barbara Mellers** at the University of Pennsylvania.\n\nThe project recruited several thousand volunteer forecasters and asked them to assign numerical probabilities to hundreds of specific, time-bounded geopolitical questions (\"Will Greece exit the eurozone by date X?\"; \"Will Bashar al-Assad remain in power as of date Y?\"). Forecasters could update their estimates as evidence emerged. Each forecast was scored after resolution against a strict accuracy metric (the Brier score).\n\nThe findings — published in peer-reviewed papers and synthesized in Tetlock & Gardner's 2015 book *Superforecasting* — were stark:\n\n- **Top 2% of forecasters (\"superforecasters\")** beat the average forecaster by ~60% on Brier accuracy, and **outperformed intelligence community analysts with classified access by approximately 30%**, using only public information (Tetlock & Mellers, multiple papers; *Superforecasting* ch. 4).\n- Superforecasters were not domain experts. They were **ordinary people** — a retired pharmacist, a software engineer, a homemaker — who had been trained on probabilistic reasoning skills.\n- **The skills that distinguished them were teachable**: anchoring on base rates, decomposing questions, updating incrementally with new evidence, *avoiding* both over- and under-reaction to news, taking outside views seriously.\n- A short training intervention (≈ 60 minutes) on the CHAMP method (Compare, Historical perspective, Average, Mathematics, Probability) **improved accuracy by 10%** in the average forecaster (Mellers et al., *Psychological Science*, 2014).\n\nWalk through the Probability Estimate applied to one Good Judgment question — *\"Will the Greek government default on the IMF before 2016-06-30?\"* — as superforecasters approached it:\n\n- **Question (Step 1):** Specific, time-bounded, resolution criteria unambiguous. ✓\n- **Base rate (Step 2):** Historical Greek default frequencies on IMF over the prior decade; Eurozone sovereign-default frequency. Anchored around a small but non-zero number.\n- **Evidence (Step 3):** Bailout-talks status, deposit flight from Greek banks, ECB liquidity stance, German political signals — each a signal in a direction with a strength.\n- **Bayesian update (Step 4):** Each piece of news shifted the estimate, but typica"}],"languages":[],"docsSourceLabel":"CLAWHUB","editorialOverview":"Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima... Skill: Probabilistic Thinking Owner: deciqai Summary: Activate when: user says 'what are the odds,' 'base rate,' 'Bayesian update,' 'calibration,' or 'what's the probability'; user is making a forecast or estima... Tags: latest:1.0.4 Version history: v1.0.4 | 2026-07-16T18:11:57.587Z | user Description tail link + agents machine-readable metadata line (deciqai.com/s/probabilistic-thinking.json) v1.0.3 | 2026-07-08T11","editorialQuality":{"score":100,"threshold":65,"status":"ready","wordCount":2040,"uniquenessScore":53,"reasons":[]}},"media":{"evidence":{"source":"no-media","verified":false,"confidence":"low","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":"No screenshots, media assets, or demo links are available."},"primaryImageUrl":null,"mediaAssetCount":0,"assets":[],"demoUrl":null},"ownerResources":{"evidence":{"source":"unclaimed","verified":false,"confidence":"low","updatedAt":"2026-10-12T00:17:08.312Z","emptyReason":"This page has not been claimed by the agent owner."},"hasCustomPage":false,"customPageUpdatedAt":null,"customLinks":[],"structuredLinks":{"docsUrl":null,"demoUrl":null,"supportUrl":null,"pricingUrl":null,"statusUrl":null},"customPage":null},"relatedAgents":{"evidence":{"source":"protocol-neighbors","verified":false,"confidence":"medium","updatedAt":"2026-10-12T14:48:42.152Z","emptyReason":null},"items":[{"id":"8ebccd8e-3863-4187-8355-c3f14e1f9edf","entityType":"agent","canonicalPath":"/agent/iofficeai-aionui","slug":"iofficeai-aionui","name":"AionUi","description":"Free, local, open-source 24/7 Cowork app and OpenClaw for Gemini CLI, Claude Code, Codex, OpenCode, Qwen Code, Goose CLI, Auggie, and more | 🌟 Star if you like it!","url":"https://github.com/iOfficeAI/AionUi","homepage":"https://www.aionui.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-10-09T19:11:12.944Z","createdAt":"2026-02-25T03:38:16.584Z","downloads":null},{"id":"b917f68a-ebff-438e-84f8-3f4b2494c0bc","entityType":"agent","canonicalPath":"/agent/activepieces-activepieces","slug":"activepieces-activepieces","name":"activepieces","description":"AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents","url":"https://github.com/activepieces/activepieces","homepage":"https://www.activepieces.com","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-15T02:22:12.426Z","createdAt":"2026-02-25T03:38:12.412Z","downloads":null},{"id":"5cb26759-3a39-483f-94cf-276a98c13bb8","entityType":"agent","canonicalPath":"/agent/cherryhq-cherry-studio","slug":"cherryhq-cherry-studio","name":"cherry-studio","description":"AI productivity studio with smart chat, autonomous agents, and 300+ assistants. Unified access to frontier LLMs","url":"https://github.com/CherryHQ/cherry-studio","homepage":"https://cherry-ai.com","source":"GITHUB_REPOS","protocols":["MCP","OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-04-11T14:38:40.986Z","createdAt":"2026-02-25T03:38:19.379Z","downloads":null},{"id":"6f6582d0-5d76-4f0f-b81d-86520247950b","entityType":"agent","canonicalPath":"/agent/copilotkit-copilotkit","slug":"copilotkit-copilotkit","name":"CopilotKit","description":"The Frontend for Agents & Generative UI. React + Angular","url":"https://github.com/CopilotKit/CopilotKit","homepage":"https://docs.copilotkit.ai","source":"GITHUB_REPOS","protocols":["OPENCLAW"],"capabilities":[],"safetyScore":100,"overallRank":70,"updatedAt":"2026-03-25T09:50:57.846Z","createdAt":"2026-02-25T03:39:14.617Z","downloads":null}],"links":{"hub":"/agent","source":"/agent/source/clawhub","protocols":[{"label":"OpenClaw","href":"/agent/protocol/openclew"}]}}}