"""Wasserman (2004), Chapter 3: Expectation.

Code for Chapter 3 of L. Wasserman, All of Statistics: A Concise Course in Statistical Inference
(Springer, 2004), pp. 47-61, written from scratch with numpy, scipy and matplotlib, to be read next to
the lecture notes (../ch03-expectation.html).

The chapter itself has no figures: it is a chapter of definitions and identities. Its three
computer experiments (Exercises 3.9, 3.11 and 3.19) and its main identities are turned here into
six supplementary figures, numbered 3.S1 to 3.S6:

* 3.S1  Exercise 3.9 and Example 3.5: running means of N(0, 1) and Cauchy draws, and the truncated
        integral of |x| under the Cauchy density, which grows like log(1 + M^2) / pi.
* 3.S2  Exercise 3.11: four simulated random walks ("stock prices") with the bands +-sqrt(n) and
        +-2 sqrt(n) given by V(X_n) = n.
* 3.S3  Exercise 3.19: the sampling distribution of the mean of n Uniform(0, 1) draws,
        E(Xbar_n) = 1/2 and V(Xbar_n) = 1/(12 n), for n = 1, 5, 25, 100.
* 3.S4  Theorem 3.17: the sample variance with divisor n - 1 is unbiased; the divisor n is biased by
        the factor (n - 1)/n; the mean squared error of the divisors n - 1, n and n + 1; the divisor
        that minimises the mean squared error, n + 1 + kappa (n - 1)/n (kappa = excess kurtosis).
* 3.S5  Section 3.5: conditional expectation in the two-stage experiments of Examples 3.23 and 3.28,
        and the law of total variance V(X) = n/6 + n^2/12 of Example 3.28.
* 3.S6  Section 3.6: the moment generating function of Exp(1) and its Taylor polynomial (Example
        3.30), the sum of five Exp(1) draws against Gamma(5, 1) (Exercise 3.24), and the sum of
        independent Poisson(1) and Poisson(2) draws against Poisson(3) (Example 3.35).

No figure of the book is skipped, because the chapter has none.

The functions behind the browser demonstrations use JSRandom, a port of the seeded generator of the
demos (DK.rng in courses/demo-kit.js: mulberry32 uniforms, Box-Muller normals), and take their draws
in the same order as the JavaScript:

* sample_variance_study  (sample-variance-demo.html): the divisors n - 1, n and n + 1;
* two_stage_study         (total-variance-demo.html): conditional expectation and the law of total
                          variance for the beta-binomial (Example 3.28), the broken stick (Examples
                          3.23 and 3.25) and the coin-then-uniform experiment (Exercise 3.13);
* mgf_sum_study           (mgf-sums-demo.html): sums of independent variables, their moment
                          generating functions and the distribution the mgf identifies.

export_demo_data() writes the nerve-pulse waiting times (the book's nerve.dat, 799 values) to
../data/ch03-nerve.json for the sample-variance demo.

Run `python3 ch03_expectation.py --list`, `python3 ch03_expectation.py` (all six figures, about
twenty seconds) or `python3 ch03_expectation.py --fig 3.S4`. Files go to ../img/ch03/.
"""
import json
import math
import os

import numpy as np
from scipy import integrate, special, stats

import common
from common import savefig, panel_label, RED, BLUE, GREEN, ORANGE, PURPLE, GREY
import matplotlib.pyplot as plt

CH = 3
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))


# ============================================================================= seeded generator
M32 = 0xFFFFFFFF


def _imul(a, b):
    """Math.imul: 32-bit integer multiplication, result as an unsigned 32-bit int."""
    return ((a & M32) * (b & M32)) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    Uniforms come from mulberry32 and normals from the Box-Muller transform, the second normal of
    each pair being kept for the next call, exactly as in the JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def pos(self):
        x = 0.0
        while x == 0.0:
            x = self.u()
        return x

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        r = math.sqrt(-2.0 * math.log(self.pos()))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)

    def exp(self, mean):
        return -mean * math.log(self.pos())

    def gamma(self, a):
        """Gamma(a, 1) by Marsaglia and Tsang (2000); for a < 1 the boost G(a + 1) U^(1/a)."""
        if a < 1.0:
            g = self.gamma(a + 1.0)
            return g * math.pow(self.pos(), 1.0 / a)
        d = a - 1.0 / 3.0
        c = 1.0 / math.sqrt(9.0 * d)
        while True:
            x = self.norm()
            v = 1.0 + c * x
            if v <= 0.0:
                continue
            v = v * v * v
            u = self.pos()
            if math.log(u) < 0.5 * x * x + d - d * v + d * math.log(v):
                return d * v

    def beta(self, a, b):
        if a == 1.0 and b == 1.0:
            return self.u()
        g1 = self.gamma(a)
        g2 = self.gamma(b)
        return g1 / (g1 + g2)

    def poisson(self, lam):
        """Knuth's multiplication method (fine for the small means used in the demos)."""
        L = math.exp(-lam)
        k = 0
        p = 1.0
        while True:
            k += 1
            p *= self.u()
            if p <= L:
                return k - 1


# ============================================================================= data
def nerve():
    """The 799 waiting times (seconds) between successive nerve pulses of the book's nerve.dat."""
    with open(common.aos_file("nerve.dat")) as f:
        return np.array(f.read().split(), dtype=float)


def export_demo_data():
    os.makedirs(DATA_OUT, exist_ok=True)
    x = nerve()
    path = os.path.join(DATA_OUT, "ch03-nerve.json")
    with open(path, "w") as f:
        json.dump({"source": "nerve.dat, the book's website (Wasserman 2004)",
                   "units": "seconds between successive nerve pulses", "x": [float(v) for v in x]}, f)
    print("wrote", os.path.relpath(path, os.getcwd()))


# ============================================================================= Section 3.3: the sample variance
def population_moments(dist, p=0.5, data=None):
    """Mean, variance and excess kurtosis kappa = E(X - mu)^4 / sigma^4 - 3 of the population."""
    if dist == "normal":
        return 0.0, 1.0, 0.0
    if dist == "uniform":
        return 0.5, 1.0 / 12.0, -1.2
    if dist == "exponential":
        return 1.0, 1.0, 6.0
    if dist == "bernoulli":
        q = 1.0 - p
        return p, p * q, (1.0 - 6.0 * p * q) / (p * q)
    if dist == "nerve":
        x = np.asarray(data if data is not None else nerve())
        m = x.mean()
        v = ((x - m) ** 2).mean()
        return float(m), float(v), float(((x - m) ** 4).mean() / v ** 2 - 3.0)
    raise ValueError(dist)


def draw(r, dist, p=0.5, data=None):
    """One draw from the population, using the generator exactly as the demo does."""
    if dist == "normal":
        return r.norm()
    if dist == "uniform":
        return r.u()
    if dist == "exponential":
        return r.exp(1.0)
    if dist == "bernoulli":
        return 1.0 if r.u() < p else 0.0
    if dist == "nerve":
        return float(data[int(math.floor(r.u() * len(data)))])
    raise ValueError(dist)


def var_s2(n, sigma2, kappa):
    """V(S_n^2) = sigma^4 (2/(n - 1) + kappa/n) for iid data with excess kurtosis kappa."""
    return sigma2 ** 2 * (2.0 / (n - 1) + kappa / n)


def mse_divisor(d, n, sigma2, kappa):
    """Mean squared error of SS/d as an estimate of sigma^2, SS = sum (X_i - Xbar)^2."""
    ess = (n - 1) * sigma2
    vss = (n - 1) ** 2 * var_s2(n, sigma2, kappa)
    return (vss + (ess - d * sigma2) ** 2) / d ** 2


def best_divisor(n, kappa):
    """The divisor d minimising the mean squared error of SS/d: n + 1 + kappa (n - 1)/n."""
    return n + 1.0 + kappa * (n - 1.0) / n


def sample_variance_study(dist="normal", n=5, R=4000, seed=1, p=0.5, data=None):
    """R samples of size n; the estimates SS/(n - 1), SS/n and SS/(n + 1) of each.

    Written with plain loops, in the order used by sample-variance-demo.html."""
    if dist == "nerve" and data is None:
        data = nerve()
    r = JSRandom(seed)
    mu, s2, kappa = population_moments(dist, p, data)
    est = {"n-1": [], "n": [], "n+1": []}
    for _ in range(R):
        xs = [draw(r, dist, p, data) for _ in range(n)]
        m = 0.0
        for v in xs:
            m += v
        m /= n
        ss = 0.0
        for v in xs:
            ss += (v - m) * (v - m)
        est["n-1"].append(ss / (n - 1))
        est["n"].append(ss / n)
        est["n+1"].append(ss / (n + 1))
    out = {"sigma2": s2, "kappa": kappa, "mu": mu, "est": est}
    for k, vals in est.items():
        a = 0.0
        e = 0.0
        for v in vals:
            a += v
            e += (v - s2) * (v - s2)
        d = {"n-1": n - 1, "n": n, "n+1": n + 1}[k]
        out[k] = {"mean": a / R, "mse": e / R, "mean_theory": (n - 1) * s2 / d,
                  "mse_theory": mse_divisor(d, n, s2, kappa)}
    return out


# ============================================================================= Section 3.5: two-stage experiments
def two_stage_theory(model, n=10, a=1.0, b=1.0, d=3.0):
    """E(Y), E V(Y | X), V E(Y | X) and V(Y) for the three two-stage experiments of the demo."""
    if model == "binomial":                      # Q ~ Beta(a, b), X | Q ~ Binomial(n, Q) (Example 3.28)
        mq = a / (a + b)
        vq = a * b / ((a + b) ** 2 * (a + b + 1.0))
        ev = n * (mq - (vq + mq * mq))
        ve = n * n * vq
        mean = n * mq
    elif model == "stick":                       # X ~ U(0, 1), Y | X ~ U(X, 1) (Examples 3.23, 3.25)
        mean, ev, ve = 0.75, 1.0 / 36.0, 1.0 / 48.0
    elif model == "coin":                        # fair coin, then U(0, 1) or U(d, d + 1) (Exercise 3.13)
        mean, ev, ve = (d + 1.0) / 2.0, 1.0 / 12.0, d * d / 4.0
    else:
        raise ValueError(model)
    return {"mean": mean, "ev": ev, "ve": ve, "var": ev + ve}


def two_stage_study(model="binomial", R=3000, seed=1, n=10, a=1.0, b=1.0, d=3.0):
    """R draws of (first stage, second stage) and the Monte Carlo version of each term of
    V(Y) = E V(Y | X) + V E(Y | X): the average of the exact conditional variances at the simulated
    first stages, and the variance (divisor R) of the exact conditional means.

    Written in the order used by total-variance-demo.html."""
    r = JSRandom(seed)
    s1, s2, cm, cv = [], [], [], []
    for _ in range(R):
        if model == "binomial":
            q = r.beta(a, b)
            x = 0
            for _ in range(n):
                if r.u() < q:
                    x += 1
            s1.append(q); s2.append(float(x)); cm.append(n * q); cv.append(n * q * (1.0 - q))
        elif model == "stick":
            x = r.u()
            y = x + (1.0 - x) * r.u()
            s1.append(x); s2.append(y); cm.append((1.0 + x) / 2.0); cv.append((1.0 - x) * (1.0 - x) / 12.0)
        elif model == "coin":
            h = 1.0 if r.u() < 0.5 else 0.0
            y = r.u() if h else d + r.u()
            s1.append(h); s2.append(y); cm.append(0.5 if h else d + 0.5); cv.append(1.0 / 12.0)
        else:
            raise ValueError(model)

    def mean(v):
        s = 0.0
        for x in v:
            s += x
        return s / len(v)

    def var(v):
        m = mean(v)
        s = 0.0
        for x in v:
            s += (x - m) * (x - m)
        return s / len(v)

    return {"s1": s1, "s2": s2, "mean": mean(s2), "var": var(s2), "ev": mean(cv), "ve": var(cm),
            "theory": two_stage_theory(model, n, a, b, d)}


# ============================================================================= Section 3.6: moment generating functions
def mgf(dist, t, p=0.3, lam=1.5, beta=1.0, mu=0.0, sigma=1.0):
    """psi_X(t) = E exp(tX) for one summand (the table of Section 3.6, plus the uniform)."""
    t = float(t)
    if dist == "bernoulli":
        return 1.0 - p + p * math.exp(t)
    if dist == "poisson":
        return math.exp(lam * math.expm1(t))
    if dist == "exponential":
        return 1.0 / (1.0 - beta * t) if beta * t < 1.0 else math.inf
    if dist == "uniform":
        return 1.0 if t == 0.0 else math.expm1(t) / t
    if dist == "normal":
        return math.exp(mu * t + 0.5 * sigma * sigma * t * t)
    raise ValueError(dist)


def summand_moments(dist, p=0.3, lam=1.5, beta=1.0, mu=0.0, sigma=1.0):
    return {"bernoulli": (p, p * (1 - p)), "poisson": (lam, lam), "exponential": (beta, beta * beta),
            "uniform": (0.5, 1.0 / 12.0), "normal": (mu, sigma * sigma)}[dist]


def mgf_moments(dist, n, h=1e-4, **par):
    """E(Y) and V(Y) of Y = X_1 + ... + X_n from the derivatives of psi_X at 0 (central differences):
    psi_Y = psi_X^n gives psi_Y'(0) = n psi'(0), psi_Y''(0) = n(n - 1) psi'(0)^2 + n psi''(0)."""
    fp = mgf(dist, h, **par)
    fm = mgf(dist, -h, **par)
    d1 = (fp - fm) / (2.0 * h)
    d2 = (fp - 2.0 + fm) / (h * h)
    m1 = n * d1
    m2 = n * (n - 1) * d1 * d1 + n * d2
    return m1, m2 - m1 * m1


def irwin_hall_pdf(x, n):
    """Density of the sum of n independent Uniform(0, 1) variables."""
    x = np.asarray(x, dtype=float)
    out = np.zeros_like(x)
    for k in range(n + 1):
        out += (-1) ** k * special.comb(n, k) * np.where(x > k, (x - k), 0.0) ** (n - 1)
    out /= math.factorial(n - 1)
    return np.where((x >= 0) & (x <= n), out, 0.0)


def sum_law(dist, n, p=0.3, lam=1.5, beta=1.0, mu=0.0, sigma=1.0):
    """The distribution the mgf psi_X^n identifies (Theorem 3.33), as a scipy frozen law or a pdf."""
    if dist == "bernoulli":
        return stats.binom(n, p)
    if dist == "poisson":
        return stats.poisson(n * lam)
    if dist == "exponential":
        return stats.gamma(a=n, scale=beta)
    if dist == "normal":
        return stats.norm(n * mu, sigma * math.sqrt(n))
    if dist == "uniform":
        return None          # Irwin-Hall: irwin_hall_pdf
    raise ValueError(dist)


def mgf_sum_study(dist="bernoulli", n=10, R=4000, seed=1, p=0.3, lam=1.5, beta=1.0, mu=0.0, sigma=1.0):
    """R sums of n independent draws, in the order used by mgf-sums-demo.html."""
    r = JSRandom(seed)
    sums = []
    for _ in range(R):
        s = 0.0
        for _ in range(n):
            if dist == "bernoulli":
                s += 1.0 if r.u() < p else 0.0
            elif dist == "poisson":
                s += r.poisson(lam)
            elif dist == "exponential":
                s += r.exp(beta)
            elif dist == "uniform":
                s += r.u()
            elif dist == "normal":
                s += mu + sigma * r.norm()
            else:
                raise ValueError(dist)
        sums.append(s)
    m = 0.0
    for v in sums:
        m += v
    m /= R
    q = 0.0
    for v in sums:
        q += (v - m) * (v - m)
    q /= R - 1
    mx, vx = summand_moments(dist, p, lam, beta, mu, sigma)
    par = dict(p=p, lam=lam, beta=beta, mu=mu, sigma=sigma)
    mm, vm = mgf_moments(dist, n, **par)
    return {"sums": sums, "mean": m, "var": q, "mean_exact": n * mx, "var_exact": n * vx,
            "mean_mgf": mm, "var_mgf": vm}


def empirical_log_mgf(sums, t):
    """log of (1/R) sum_r exp(t Y_r), the plug-in estimate of log psi_Y(t)."""
    s = 0.0
    for v in sums:
        s += math.exp(t * v)
    return math.log(s / len(sums))


# ============================================================================= exercise helpers
def running_mean(x):
    return np.cumsum(x) / np.arange(1, len(x) + 1)


def exercise_values():
    """Closed-form answers of the computational exercises, used in the notes."""
    out = {}
    out["3.1 (n=10, c=1)"] = 1.25 ** 10
    out["3.3 (n=4)"] = 4 / 5
    out["3.10 E"] = math.exp(0.5)
    out["3.10 V"] = math.exp(2) - math.exp(1)
    out["3.13 mean"] = 2.0
    out["3.13 sd"] = math.sqrt(7.0 / 3.0)
    f = lambda y, x: (x + y) / 3.0
    E = lambda g: integrate.dblquad(lambda y, x: g(x, y) * f(y, x), 0, 1, 0, 2)[0]
    ex, ey = E(lambda x, y: x), E(lambda x, y: y)
    vx = E(lambda x, y: x * x) - ex ** 2
    vy = E(lambda x, y: y * y) - ey ** 2
    cxy = E(lambda x, y: x * y) - ex * ey
    out["3.15 V(2X-3Y+8)"] = 4 * vx + 9 * vy - 12 * cxy
    return out


# ============================================================================= figures
def fig_3_s1(out_dir=None):
    """Exercise 3.9 and Example 3.5: running means of normal and Cauchy draws."""
    n = 10_000
    k = np.arange(1, n + 1)
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.7))
    cols = (BLUE, RED, GREEN)
    for j, (name, sampler) in enumerate((("N(0, 1)", lambda g: g.standard_normal(n)),
                                         ("Cauchy", lambda g: g.standard_cauchy(n)))):
        ax = axes[0, j]
        for s, c in zip((1, 2, 3), cols):
            g = np.random.default_rng(309 + 10 * j + s)
            ax.plot(k, running_mean(sampler(g)), color=c, lw=1.0)
        ax.axhline(0, color=GREY, lw=0.8, ls="--")
        ax.set_xscale("log")
        ax.set_xlabel("n")
        ax.set_ylabel(r"$\bar X_n$")
        ax.set_title(r"running mean, %s draws" % name)
        if j == 0:
            ax.set_ylim(-1.5, 1.5)
        panel_label(ax, "ab"[j], "upper right")
    ax = axes[0, 2]
    M = np.logspace(0, 6, 200)
    trunc = np.log1p(M ** 2) / np.pi          # integral of |x| / (pi (1 + x^2)) over [-M, M]
    ax.plot(M, trunc, color=PURPLE)
    ax.set_xscale("log")
    ax.set_xlabel("M")
    ax.set_ylabel(r"$\int_{-M}^{M}|x|\,f(x)\,dx$")
    ax.set_title("Cauchy: truncated mean of |X|")
    panel_label(ax, "c", "upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig3-S1", out_dir)


def fig_3_s2(out_dir=None):
    """Exercise 3.11: four random walks with the +-sqrt(n) and +-2 sqrt(n) bands."""
    n = 10_000
    k = np.arange(1, n + 1)
    fig, ax = plt.subplots(figsize=(6.4, 3.4))
    for s, c in zip(range(4), (BLUE, RED, GREEN, ORANGE)):
        g = np.random.default_rng(311 + s)
        y = np.where(g.random(n) < 0.5, 1, -1)
        ax.plot(k, np.cumsum(y), color=c, lw=0.9, label="run %d" % (s + 1))
    for m, ls in ((1, "--"), (2, ":")):
        ax.plot(k, m * np.sqrt(k), color="k", lw=0.9, ls=ls)
        ax.plot(k, -m * np.sqrt(k), color="k", lw=0.9, ls=ls)
    ax.axhline(0, color=GREY, lw=0.6)
    ax.set_xlabel("day n")
    ax.set_ylabel(r"$X_n$")
    ax.set_title(r"Random walks: $E(X_n)=0$, $\mathrm{sd}(X_n)=\sqrt{n}$ (dashed $\pm\sqrt{n}$, dotted $\pm2\sqrt{n}$)")
    ax.legend(loc="upper left", ncol=4, fontsize=8, frameon=False)
    fig.tight_layout()
    savefig(fig, CH, "fig3-S2", out_dir)


def fig_3_s3(out_dir=None):
    """Exercise 3.19: the sampling distribution of the mean of n Uniform(0, 1) draws."""
    fig = plt.figure(figsize=(9.2, 5.0))
    gs = fig.add_gridspec(2, 4)
    ax = fig.add_subplot(gs[0, 0:2])
    xx = np.linspace(-0.25, 1.25, 400)
    ax.plot(xx, ((xx > 0) & (xx < 1)).astype(float), color=BLUE)
    ax.set_title(r"(a) the data: $f_X$ of Uniform(0, 1)")
    ax.set_xlabel("x")
    ax.set_ylim(0, 1.3)
    ax = fig.add_subplot(gs[0, 2:4])
    nn = np.arange(1, 101)
    ax.plot(nn, np.full(nn.shape, 0.5), color=BLUE, label=r"$E(\bar X_n)=1/2$")
    ax.plot(nn, 1 / (12 * nn), color=RED, label=r"$V(\bar X_n)=1/(12n)$")
    ax.set_yscale("log")
    ax.set_xlabel("n")
    ax.set_title("(b) mean and variance of the statistic")
    ax.legend(frameon=False, fontsize=8)
    g = np.random.default_rng(319)
    reps = 10_000
    for j, n in enumerate((1, 5, 25, 100)):
        ax = fig.add_subplot(gs[1, j])
        xb = g.random((reps, n)).mean(axis=1)
        ax.hist(xb, bins=40, range=(0, 1), density=True, color="#9ec3ea", edgecolor="white", lw=0.3)
        sd = math.sqrt(1 / (12 * n))
        ax.plot(xx, stats.norm.pdf(xx, 0.5, sd), color=RED, lw=1.1)
        ax.set_xlim(0, 1)
        ax.set_title("(%s) n = %d" % ("cdef"[j], n))
        ax.text(0.03, 0.97, "mean %.4f\nvar %.5f\n(1/(12n) = %.5f)" % (xb.mean(), xb.var(ddof=1), 1 / (12 * n)),
                transform=ax.transAxes, va="top", fontsize=7)
        ax.set_xlabel(r"$\bar x_n$")
    fig.tight_layout()
    savefig(fig, CH, "fig3-S3", out_dir)


def fig_3_s4(out_dir=None):
    """Theorem 3.17: bias and mean squared error of the sample variance with divisors n - 1, n, n + 1."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.8))
    g = np.random.default_rng(317)
    nn = np.arange(2, 31)
    ax = axes[0, 0]
    reps = 20_000
    sim1, simn = [], []
    for n in nn:
        x = g.standard_normal((reps, n))
        ss = ((x - x.mean(1, keepdims=True)) ** 2).sum(1)
        sim1.append((ss / (n - 1)).mean())
        simn.append((ss / n).mean())
    ax.plot(nn, np.ones_like(nn, dtype=float), color=BLUE, label=r"$E(S_n^2)/\sigma^2=1$")
    ax.plot(nn, (nn - 1) / nn, color=RED, label=r"$E(\hat\sigma_n^2)/\sigma^2=(n-1)/n$")
    ax.plot(nn, sim1, "o", ms=2.5, color=BLUE)
    ax.plot(nn, simn, "o", ms=2.5, color=RED)
    ax.set_xlabel("n")
    ax.set_title("(a) bias, N(0, 1) data")
    ax.legend(frameon=False, fontsize=7.5, loc="lower right")
    ax = axes[0, 1]
    for d_off, c, lab in ((-1, BLUE, "n - 1"), (0, RED, "n"), (1, GREEN, "n + 1")):
        ax.plot(nn, [mse_divisor(n + d_off, n, 1.0, 0.0) for n in nn], color=c, label="divisor " + lab)
    ax.set_yscale("log")
    ax.set_xlabel("n")
    ax.set_title(r"(b) MSE$/\sigma^4$, normal data")
    ax.legend(frameon=False, fontsize=7.5)
    ax = axes[0, 2]
    n = 10
    dd = np.linspace(6, 22, 300)
    for kappa, c, lab in ((-1.2, PURPLE, "uniform"), (0.0, BLUE, "normal"), (6.0, ORANGE, "exponential")):
        m = mse_divisor(dd, n, 1.0, kappa)
        ax.plot(dd, m, color=c, label=r"%s ($\kappa=%g$)" % (lab, kappa))
        db = best_divisor(n, kappa)
        ax.plot([db], [mse_divisor(db, n, 1.0, kappa)], "o", color=c, ms=4)
    ax.axvline(n - 1, color=GREY, lw=0.7, ls="--")
    ax.set_xlabel("divisor d")
    ax.set_title(r"(c) MSE$/\sigma^4$ of SS$/d$, n = 10")
    ax.legend(frameon=False, fontsize=7.5, loc="upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig3-S4", out_dir)


def fig_3_s5(out_dir=None):
    """Section 3.5: conditional expectation in two-stage experiments and the law of total variance."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.8))
    g = np.random.default_rng(323)
    ax = axes[0, 0]
    x = g.random(1500)
    y = x + (1 - x) * g.random(1500)
    ax.plot(x, y, ".", ms=2, color="#9ec3ea")
    bins = np.linspace(0, 1, 11)
    idx = np.digitize(x, bins) - 1
    mids = (bins[:-1] + bins[1:]) / 2
    ax.plot(mids, [y[idx == i].mean() for i in range(10)], "o", color=RED, ms=4, label="binned means of Y")
    ax.plot([0, 1], [0.5, 1], color=BLUE, label=r"$E(Y\mid X=x)=(1+x)/2$")
    ax.set_xlabel("x")
    ax.set_ylabel("y")
    ax.set_title(r"(a) $Y\mid X=x\sim U(x,1)$ (Ex. 3.23)")
    ax.legend(frameon=False, fontsize=7, loc="lower right")
    ax = axes[0, 1]
    n = 10
    reps = 20_000
    q = g.random(reps)
    xs = g.binomial(n, q)
    cnt = np.bincount(xs, minlength=n + 1) / reps
    ax.bar(np.arange(n + 1), cnt, color="#9ec3ea", edgecolor="white")
    ax.axhline(1 / (n + 1), color=RED, lw=1.1, label=r"$1/(n+1)$")
    ax.set_xlabel("x")
    ax.set_title("(b) Ex. 3.28: X is uniform on {0,...,10}")
    ax.legend(frameon=False, fontsize=7.5, loc="lower right")
    ax.set_ylim(0, 0.13)
    ax = axes[0, 2]
    nn = np.arange(1, 41)
    ax.fill_between(nn, 0, nn / 6, color="#f2b8a2", label=r"$E\,V(X\mid Q)=n/6$")
    ax.fill_between(nn, nn / 6, nn / 6 + nn ** 2 / 12, color="#9ec3ea", label=r"$V\,E(X\mid Q)=n^2/12$")
    sims = []
    for n in nn[::3]:
        qq = g.random(reps)
        sims.append(g.binomial(n, qq).var(ddof=1))
    ax.plot(nn[::3], sims, "o", color="k", ms=3, label="simulated V(X)")
    ax.set_xlabel("n")
    ax.set_title("(c) law of total variance")
    ax.legend(frameon=False, fontsize=7, loc="upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig3-S5", out_dir)


def fig_3_s6(out_dir=None):
    """Section 3.6: an mgf and its Taylor polynomial; sums identified by their mgfs."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.8))
    ax = axes[0, 0]
    t = np.linspace(-1.5, 0.8, 300)
    ax.plot(t, 1 / (1 - t), color=BLUE, label=r"$\psi(t)=1/(1-t)$")
    ax.plot(t, 1 + t, color=RED, ls="--", label=r"$1+E(X)\,t$")
    ax.plot(t, 1 + t + t ** 2, color=GREEN, ls=":", label=r"$1+E(X)t+E(X^2)t^2/2$")
    g = np.random.default_rng(330)
    xs = g.exponential(1.0, 2000)
    te = np.linspace(-1.5, 0.45, 40)
    ax.plot(te, [np.exp(s * xs).mean() for s in te], "o", ms=2.5, color="k", label="empirical, 2000 draws")
    ax.set_ylim(0, 5)
    ax.set_xlabel("t")
    ax.set_title("(a) Exp(1): mgf near t = 0")
    ax.legend(frameon=False, fontsize=6.8, loc="upper left")
    ax = axes[0, 1]
    reps = 20_000
    s = g.exponential(1.0, (reps, 5)).sum(1)
    ax.hist(s, bins=60, range=(0, 18), density=True, color="#9ec3ea", edgecolor="white", lw=0.3)
    yy = np.linspace(0, 18, 300)
    ax.plot(yy, stats.gamma.pdf(yy, a=5), color=RED, label="Gamma(5, 1)")
    ax.set_xlabel("y")
    ax.set_title(r"(b) $X_1+\cdots+X_5$, $X_i\sim$ Exp(1)")
    ax.legend(frameon=False, fontsize=7.5)
    ax = axes[0, 2]
    y = g.poisson(1.0, reps) + g.poisson(2.0, reps)
    k = np.arange(0, 13)
    ax.bar(k, np.bincount(y, minlength=13)[:13] / reps, color="#9ec3ea", edgecolor="white", label="simulated")
    ax.plot(k, stats.poisson.pmf(k, 3.0), "o", color=RED, ms=4, label="Poisson(3) pmf")
    ax.set_xlabel("y")
    ax.set_title("(c) Poisson(1) + Poisson(2)")
    ax.legend(frameon=False, fontsize=7.5)
    fig.tight_layout()
    savefig(fig, CH, "fig3-S6", out_dir)


FIGURES = {"3.S1": fig_3_s1, "3.S2": fig_3_s2, "3.S3": fig_3_s3, "3.S4": fig_3_s4, "3.S5": fig_3_s5,
           "3.S6": fig_3_s6}

if __name__ == "__main__":
    import sys
    common.run(FIGURES, CH)
    if "--list" not in sys.argv:
        export_demo_data()
