"""Chapter 2 (Probabilities) of Bishop and Bishop (2024), Deep Learning: Foundations and Concepts.

Reproduces every figure of the chapter that is computed from data, a model or a simulation:

    2.1   the two-dimensional sine regression problem: hidden x2 makes the data noisy
    2.3   the screening test drawn as two grids of 100 people (redrawn from the stated rates)
    2.5   a sample of 60 points from a joint distribution over X (9 values) and Y (2 values),
          with the histogram estimates of p(X), p(Y) and p(X | Y = 1)
    2.6   a density p(x), its cumulative distribution P(x) and a slice of width delta-x
    2.7   the uniform, exponential and Laplace densities
    2.8   the Gaussian density with its mean and standard deviation marked
    2.9   the Gaussian likelihood as a product of densities at the data points
    2.10  bias of the maximum likelihood variance: fits to three data sets of two points
    2.11  the Gaussian conditional p(t | x, w, sigma^2) around a fitted polynomial
    2.12  a density transforms with a Jacobian, so its mode moves (50,000 samples)
    2.13  a nonlinear change of variables in two dimensions: grid, density and samples
    2.14  two distributions over 30 bins with entropies 1.77 and 3.09 nats
    2.15  a convex function lies below every chord
    2.16  the four non-transitive (Efron) dice, with the computed winning probabilities

Skipped, with reasons:
    2.2   a photograph of a bent coin (no computation behind it);
    2.4   a hand-drawn schematic of the cells n_ij, column counts c_i and row counts r_j
          (Figure 2.5 shows the same construction on simulated data).

The module also holds the small helpers used by the browser demos:
the screening posterior, the Efron dice, the mode-shift example of Section 2.4, the
two-dimensional transformation (2.78)-(2.79) with its Jacobian and inverse, entropies and
KL divergences, the bent-coin posterior of Exercise 2.40 and a Python copy of the demos'
seeded random number generator (JSRandom), so that the demos can be checked number for number.

Usage: python3 ch02_probabilities.py [--fig 2.12 2.13] [--list]
"""
import math
import os

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, FancyArrowPatch
from scipy import optimize, special

import common
from common import BLUE, RED, GREEN, MAGENTA, GREY, ORANGE, savefig

CH = 2
HERE = os.path.dirname(os.path.abspath(__file__))
SQ2PI = math.sqrt(2.0 * math.pi)


# =========================================================================== seeded generator of the demos
M32 = 0xFFFFFFFF


def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    mulberry32 uniforms and Box-Muller normals (the second normal of each pair is cached),
    exactly as in JavaScript, so Python and the browser draw identical numbers from a seed.
    """

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def _pos(self):
        x = 0.0
        while x == 0.0:
            x = self.u()
        return x

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        r = math.sqrt(-2.0 * math.log(self._pos()))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


# =========================================================================== Section 2.1: screening
def screening(prior, sens, fpr):
    """Sum and product rules for the screening test (2.20)-(2.22).

    prior = p(C=1), sens = p(T=1 | C=1), fpr = p(T=1 | C=0).
    Returns p(T=1) and the posteriors p(C=1 | T=1), p(C=1 | T=0).
    """
    pt1 = sens * prior + fpr * (1 - prior)                      # sum rule + product rule (2.20)
    post_pos = sens * prior / pt1                               # Bayes (2.21)
    post_neg = (1 - sens) * prior / (1 - pt1)
    return {"pT1": pt1, "pT0": 1 - pt1, "post_pos": post_pos, "post_neg": post_neg}


def screening_sequence(prior, sens, fpr, results):
    """Posterior p(C=1 | T_1..T_k) after a sequence of test results, assuming the results are
    conditionally independent given C. Each posterior becomes the prior for the next test."""
    out = [prior]
    p = prior
    for r in results:
        lik1 = sens if r else 1 - sens
        lik0 = fpr if r else 1 - fpr
        p = lik1 * p / (lik1 * p + lik0 * (1 - p))
        out.append(p)
    return out


# Efron dice of Figure 2.16 / Exercise 2.2, in the cyclic order yellow -> blue -> green -> red -> yellow
EFRON = {"yellow": [3, 3, 3, 3, 3, 3], "blue": [4, 4, 4, 4, 0, 0],
         "green": [5, 5, 5, 1, 1, 1], "red": [2, 2, 2, 2, 6, 6]}
EFRON_CYCLE = ["yellow", "blue", "green", "red"]


def p_beats(a, b):
    """Exact probability (a Fraction) that die a shows a higher number than die b."""
    from fractions import Fraction
    wins = sum(1 for x in EFRON[a] for y in EFRON[b] if x > y)
    return Fraction(wins, 36)


# =========================================================================== Section 2.3: Gaussian
def gauss(x, mu=0.0, var=1.0):
    return np.exp(-0.5 * (np.asarray(x, float) - mu) ** 2 / var) / np.sqrt(2 * np.pi * var)


def gauss_ml(x):
    """Maximum likelihood mean (2.57) and variance (2.58)."""
    x = np.asarray(x, float)
    mu = x.mean()
    return mu, np.mean((x - mu) ** 2)


def gauss_loglik(x, mu, var):
    """Log likelihood (2.56)."""
    x = np.asarray(x, float)
    N = x.size
    return -0.5 / var * np.sum((x - mu) ** 2) - 0.5 * N * np.log(var) - 0.5 * N * np.log(2 * np.pi)


def ml_bias_js(N, K, mu=0.0, sigma=1.0, seed=11):
    """The bias experiment of the Gaussian ML demo: K data sets of size N drawn with JSRandom(seed).

    Returns the averages over data sets of mu_ML, sigma^2_ML (2.58), the unbiased estimator (2.63)
    and the known-mean estimator (2.61), in that order, drawing the points exactly as the demo does.
    """
    r = JSRandom(seed)
    s_mu = s_ml = s_unb = s_known = 0.0
    for _ in range(K):
        x = [mu + sigma * r.norm() for _ in range(N)]
        m = sum(x) / N
        ss = sum((v - m) ** 2 for v in x)
        s_mu += m
        s_ml += ss / N
        s_unb += ss / (N - 1) if N > 1 else 0.0
        s_known += sum((v - mu) ** 2 for v in x) / N
    return s_mu / K, s_ml / K, s_unb / K, s_known / K


def likelihood_data_js(N=8, mu=1.0, sigma=0.8, seed=3):
    """The data of the likelihood view of the Gaussian ML demo (JSRandom(seed))."""
    r = JSRandom(seed)
    return np.array([mu + sigma * r.norm() for _ in range(N)])


def poly_design(x, M):
    return np.vander(np.asarray(x, float), M + 1, increasing=True)


def fit_poly_ml(x, t, M):
    """Least squares (2.67) = maximum likelihood under Gaussian noise; sigma^2_ML from (2.68)."""
    Phi = poly_design(x, M)
    w, *_ = np.linalg.lstsq(Phi, t, rcond=None)
    var = np.mean((Phi @ w - t) ** 2)
    return w, var


def fit_poly_map(x, t, M, sigma2, s2):
    """MAP weights: minimiser of (2.117), i.e. ridge regression with lambda = sigma^2 / s^2."""
    Phi = poly_design(x, M)
    lam = sigma2 / s2
    return np.linalg.solve(Phi.T @ Phi + lam * np.eye(M + 1), Phi.T @ t)


# =========================================================================== Section 2.4: change of variables
MS_MU, MS_SIGMA, MS_SHIFT = 6.0, 1.0, 5.0      # p_x = N(6, 1); x = g(y) = ln y - ln(1 - y) + 5  (2.74)


def ms_g(y, c=MS_SHIFT):
    """x = g(y) = ln(y) - ln(1 - y) + c  (2.74)."""
    y = np.asarray(y, float)
    return np.log(y) - np.log1p(-y) + c


def ms_ginv(x, c=MS_SHIFT):
    """y = g^{-1}(x) = 1 / (1 + exp(-x + c))  (2.75), a logistic sigmoid."""
    return common.sigmoid(np.asarray(x, float) - c)


def ms_py(y, mu=MS_MU, sigma=MS_SIGMA, c=MS_SHIFT):
    """Density of y by (2.71): p_x(g(y)) |g'(y)| with g'(y) = 1 / (y (1 - y))."""
    y = np.asarray(y, float)
    return gauss(ms_g(y, c), mu, sigma ** 2) / (y * (1 - y))


def ms_modes(mu=MS_MU, sigma=MS_SIGMA, c=MS_SHIFT):
    """Return (sigmoid of the x-mode, mode of p_y in y, the x value it maps back to).

    The first is the mode of the 'function' p_x(g(y)); the second solves the stationarity
    condition derived in the notes, (x - mu) / sigma^2 = 2 y - 1 with y = sigmoid(x - c)."""
    y_naive = float(ms_ginv(mu, c))
    h = lambda x: (x - mu) / sigma ** 2 - (2 * common.sigmoid(x - c) - 1)
    # h is increasing when 1/sigma^2 > 2 max sigmoid' = 1/2, i.e. sigma < sqrt(2): one root
    xm = optimize.brentq(h, mu - 10 * sigma - 10, mu + 10 * sigma + 10, xtol=1e-14)
    return y_naive, float(ms_ginv(xm, c)), xm


def ms_samples_js(n=50000, seed=1, mu=MS_MU, sigma=MS_SIGMA):
    r = JSRandom(seed)
    return np.array([mu + sigma * r.norm() for _ in range(n)])


# two-dimensional transformation (2.78)-(2.79)
def t2d(x1, x2, a=5.0, b=1.0 / 3.0):
    """y1 = x1 + tanh(a x1), y2 = x2 + tanh(a x2) + b x1^3  (book: a = 5, b = 1/3)."""
    return x1 + np.tanh(a * x1), x2 + np.tanh(a * x2) + b * x1 ** 3


def t2d_jac(x1, x2, a=5.0, b=1.0 / 3.0):
    """Jacobian dy/dx of (2.78)-(2.79) (Exercise 2.20). Lower triangular."""
    s1 = 1.0 / np.cosh(a * x1) ** 2
    s2 = 1.0 / np.cosh(a * x2) ** 2
    return np.array([[1 + a * s1, 0.0 * x1], [3 * b * x1 ** 2, 1 + a * s2]])


def _solve_inc(target, a):
    """Solve u + tanh(a u) = target for u (the left side is strictly increasing when a >= 0)."""
    target = np.asarray(target, float)
    u = target - np.sign(target) * np.minimum(1.0, np.abs(target))   # start inside the root bracket
    for _ in range(60):                                             # safeguarded Newton
        f = u + np.tanh(a * u) - target
        d = 1 + a / np.cosh(a * u) ** 2
        u = u - f / d
    return u


def t2d_inv(y1, y2, a=5.0, b=1.0 / 3.0):
    """Inverse of (2.78)-(2.79): first x1 from y1, then x2 from y2 - b x1^3."""
    x1 = _solve_inc(y1, a)
    x2 = _solve_inc(np.asarray(y2, float) - b * x1 ** 3, a)
    return x1, x2


def t2d_density(y1, y2, s=0.5, a=5.0, b=1.0 / 3.0):
    """p_y(y) = p_x(x) / |det(dy/dx)| at x = g(y), for p_x = N(0, s^2 I) (2.76)."""
    x1, x2 = t2d_inv(y1, y2, a, b)
    px = np.exp(-0.5 * (x1 ** 2 + x2 ** 2) / s ** 2) / (2 * np.pi * s ** 2)
    det = (1 + a / np.cosh(a * x1) ** 2) * (1 + a / np.cosh(a * x2) ** 2)
    return px / det


# =========================================================================== Section 2.5: information theory
def entropy(p, base=np.e):
    p = np.asarray(p, float)
    nz = p[p > 0]
    return float(-np.sum(nz * np.log(nz)) / np.log(base))


def kl(p, q):
    p, q = np.asarray(p, float), np.asarray(q, float)
    m = p > 0
    return float(np.sum(p[m] * np.log(p[m] / q[m])))


def kl_gauss(mu, var, m, s2):
    """KL(N(mu, var) || N(m, s2)) (Exercise 2.27)."""
    return 0.5 * (np.log(s2 / var) + (var + (mu - m) ** 2) / s2 - 1.0)


def binned_gauss(width, M=30, centre=None):
    """Probabilities of a Gaussian of standard deviation `width` (in bins) over M unit bins."""
    centre = (M - 1) / 2.0 if centre is None else centre
    edges = np.arange(M + 1) - 0.5
    c = special.ndtr((edges - centre) / width)
    p = np.diff(c)
    return p / p.sum()


def width_for_entropy(H, M=30):
    """Bin width parameter of binned_gauss with entropy H (nats)."""
    return optimize.brentq(lambda w: entropy(binned_gauss(w, M)) - H, 0.05, 200.0, xtol=1e-12)


def ln_multiplicity(n):
    """ln W for occupation numbers n (2.82), exactly, via log-gamma."""
    n = np.asarray(n, float)
    return special.gammaln(n.sum() + 1) - special.gammaln(n + 1).sum()


def gauss_entropy(var):
    """Differential entropy of a univariate Gaussian (2.99)."""
    return 0.5 * (1 + np.log(2 * np.pi * var))


def discrete_mi(P):
    """Entropies and mutual information of a joint table P[x, y] (nats)."""
    P = np.asarray(P, float)
    px, py = P.sum(1), P.sum(0)
    Hxy, Hx, Hy = entropy(P.ravel()), entropy(px), entropy(py)
    return {"Hx": Hx, "Hy": Hy, "Hxy": Hxy, "Hy_x": Hxy - Hx, "Hx_y": Hxy - Hy,
            "I": kl(P.ravel(), np.outer(px, py).ravel())}


# Exercise 2.36 joint table p(x, y): rows x = 0, 1; columns y = 0, 1
EX236 = np.array([[1 / 3, 1 / 3], [0.0, 1 / 3]])


def bimodal(x, w=0.5, m1=-3.0, m2=3.0, s=0.8):
    """The bimodal target used for forward and reverse KL fits in the entropy demo."""
    return w * gauss(x, m1, s * s) + (1 - w) * gauss(x, m2, s * s)


def fit_forward_kl(w=0.5, m1=-3.0, m2=3.0, s=0.8):
    """argmin over Gaussians q of KL(p || q): moment matching (Exercise 2.26)."""
    mean = w * m1 + (1 - w) * m2
    var = s * s + w * m1 ** 2 + (1 - w) * m2 ** 2 - mean ** 2
    return mean, var


def reverse_kl(mu, logsd, w=0.5, m1=-3.0, m2=3.0, s=0.8, lo=-12.0, hi=12.0, n=4801):
    """KL(q || p) for q = N(mu, exp(logsd)^2), by the trapezoidal rule on a fixed grid."""
    x = np.linspace(lo, hi, n)
    sd = np.exp(logsd)
    q = gauss(x, mu, sd * sd)
    p = bimodal(x, w, m1, m2, s)
    f = np.where(q > 1e-300, q * (np.log(np.maximum(q, 1e-300)) - np.log(np.maximum(p, 1e-300))), 0.0)
    return float(np.trapz(f, x))


def fit_reverse_kl(mu0=2.0, w=0.5, m1=-3.0, m2=3.0, s=0.8):
    res = optimize.minimize(lambda v: reverse_kl(v[0], v[1], w, m1, m2, s), [mu0, 0.0],
                            method="Nelder-Mead", options={"xatol": 1e-10, "fatol": 1e-14, "maxiter": 4000})
    return res.x[0], math.exp(res.x[1]), res.fun


# =========================================================================== Section 2.6: the bent coin
def coin_posterior(prior_q, heads, tails, p_concave=0.6):
    """Exercise 2.40. Q = 1 means 'the concave side is heads'; the coin lands concave side up with
    probability p_concave. Flips are independent GIVEN Q. Returns p(Q=1 | D) and p(next = H | D)."""
    lik1 = p_concave ** heads * (1 - p_concave) ** tails        # Q = 1: heads <=> concave up
    lik0 = (1 - p_concave) ** heads * p_concave ** tails        # Q = 0: heads <=> convex up
    post = lik1 * prior_q / (lik1 * prior_q + lik0 * (1 - prior_q))
    pred = p_concave * post + (1 - p_concave) * (1 - post)
    return post, pred


# =========================================================================== figures
def fig_2_1(out=None):
    """Two-dimensional sine regression: unobserved x2 looks like noise."""
    rng = np.random.default_rng(1)
    f = lambda a, b: np.sin(2 * np.pi * a) * np.sin(2 * np.pi * b)
    fig = plt.figure(figsize=(10.2, 3.2))
    ax = fig.add_subplot(1, 3, 1, projection="3d")
    g = np.linspace(0, 1, 60)
    A, B = np.meshgrid(g, g)
    ax.plot_surface(A, B, f(A, B), cmap="Reds", linewidth=0, antialiased=True, alpha=0.95,
                    vmin=-1.6, vmax=1.0)
    ax.set_xlabel("$x_1$", labelpad=-8)
    ax.set_ylabel("$x_2$", labelpad=-8)
    ax.set_zlabel("$y$", labelpad=-8)
    ax.set_xticklabels([]); ax.set_yticklabels([]); ax.set_zticklabels([])
    ax.view_init(elev=24, azim=-60)
    ax.set_title("(a)  $y=\\sin(2\\pi x_1)\\sin(2\\pi x_2)$", fontsize=9.5)
    noise = 0.1
    x1 = rng.random(100)
    x2 = rng.random(100)                       # x2 drawn but not observed
    y_b = f(x1, x2) + noise * rng.normal(size=100)
    x1c = rng.random(100)
    y_c = f(x1c, np.pi / 2) + noise * rng.normal(size=100)
    for k, (xx, yy, lab) in enumerate([(x1, y_b, "(b)  $x_2$ unobserved"),
                                       (x1c, y_c, "(c)  $x_2=\\pi/2$ observed")]):
        a = fig.add_subplot(1, 3, k + 2)
        a.plot(xx, yy, "o", color=RED, ms=3.2)
        a.set_xlim(0, 1); a.set_ylim(-1.25, 1.25)
        a.set_xlabel("$x_1$"); a.set_ylabel("$y$")
        a.set_title(lab, fontsize=9.5)
    fig.subplots_adjust(left=0.01, right=0.99, bottom=0.16, top=0.9, wspace=0.3)
    savefig(fig, CH, "fig2-1", out)


def fig_2_3(out=None):
    """The screening test as two grids of 100 people."""
    fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.6))
    for ax, npos, title in [(axes[0], 3, "No cancer: 3 in 100 test positive"),
                            (axes[1], 90, "Cancer: 90 in 100 test positive")]:
        k = 0
        for r in range(10):
            for c in range(10):
                pos = k < npos
                ax.add_patch(plt.Circle((c, 9 - r), 0.36, color=RED if pos else "#cfd8e3", ec="#43546a", lw=0.5))
                k += 1
        ax.set_xlim(-0.7, 9.7); ax.set_ylim(-0.7, 9.7)
        ax.set_aspect("equal"); ax.axis("off")
        ax.set_title(title, fontsize=10)
    fig.tight_layout()
    savefig(fig, CH, "fig2-3", out)


def joint_xy_sample(N=60, seed=4):
    """Sample of Figure 2.5: X in {1..9}, Y in {1, 2}."""
    rng = np.random.default_rng(seed)
    xs = np.arange(1, 10)
    pY = np.array([0.55, 0.45])
    pX_Y = np.array([binned_gauss(1.6, 9, 3.3), binned_gauss(1.4, 9, 5.8)])
    Y = rng.choice([1, 2], size=N, p=pY)
    X = np.array([rng.choice(xs, p=pX_Y[y - 1]) for y in Y])
    return X, Y


def fig_2_5(out=None):
    """Joint, marginal and conditional histograms from 60 samples."""
    X, Y = joint_xy_sample()
    rng = np.random.default_rng(5)
    fig, axes = plt.subplots(2, 2, figsize=(8.4, 5.6))
    ax = axes[0, 0]
    for i in range(10):
        ax.plot([i + 0.5, i + 0.5], [0.5, 2.5], color=RED, lw=1.6)
    for j in range(3):
        ax.plot([0.5, 9.5], [j + 0.5, j + 0.5], color=RED, lw=1.6)
    ax.plot(X + rng.uniform(-0.38, 0.38, X.size), Y + rng.uniform(-0.38, 0.38, Y.size), "o", color=BLUE, ms=4.5)
    ax.set_xlim(0.4, 9.6); ax.set_ylim(0.4, 2.6)
    ax.set_yticks([1, 2]); ax.set_yticklabels(["$Y=1$", "$Y=2$"]); ax.set_xticks([])
    ax.set_xlabel("$X$"); ax.set_title("$p(X,Y)$: 60 samples")
    ax = axes[0, 1]
    pY = np.array([(Y == 1).mean(), (Y == 2).mean()])
    ax.barh([1, 2], pY, color="#b3b3ff", edgecolor="k", height=0.7)
    ax.set_yticks([1, 2]); ax.set_yticklabels(["$Y=1$", "$Y=2$"]); ax.set_xlim(0, 0.7)
    ax.set_title("$p(Y)$")
    ax = axes[1, 0]
    pX = np.array([(X == i).mean() for i in range(1, 10)])
    ax.bar(np.arange(1, 10), pX, color="#b3b3ff", edgecolor="k", width=0.8)
    ax.set_xticks(range(1, 10)); ax.set_xlabel("$X$"); ax.set_title("$p(X)$")
    ax = axes[1, 1]
    pX1 = np.array([((X == i) & (Y == 1)).sum() for i in range(1, 10)]) / (Y == 1).sum()
    ax.bar(np.arange(1, 10), pX1, color="#b3b3ff", edgecolor="k", width=0.8)
    ax.set_xticks(range(1, 10)); ax.set_xlabel("$X$"); ax.set_title("$p(X\\,|\\,Y=1)$")
    fig.tight_layout()
    savefig(fig, CH, "fig2-5", out)


def fig_2_6(out=None):
    """A density p(x), its CDF P(x) = integral of p, and the probability p(x) dx of a thin slice."""
    x = np.linspace(-3, 4, 800)
    p = 0.65 * gauss(x, 0.0, 0.55) + 0.35 * gauss(x, 2.0, 0.35)
    P = 0.65 * special.ndtr(x / math.sqrt(0.55)) + 0.35 * special.ndtr((x - 2.0) / math.sqrt(0.35))
    fig, ax = plt.subplots(figsize=(5.4, 3.4))
    ax.plot(x, p, color=RED, label="$p(x)$")
    ax.plot(x, P, color=BLUE, label="$P(x)$")
    a, dx = 0.55, 0.22
    xs = np.linspace(a, a + dx, 20)
    ps = 0.65 * gauss(xs, 0.0, 0.55) + 0.35 * gauss(xs, 2.0, 0.35)
    ax.fill_between(xs, 0, ps, color=GREEN, alpha=0.6, label="area $\\approx p(x)\\,\\delta x$")
    ax.annotate("", xy=(a + dx, -0.06), xytext=(a, -0.06), arrowprops=dict(arrowstyle="<->", lw=1))
    ax.text(a + dx / 2, -0.13, "$\\delta x$", ha="center", va="top")
    ax.set_ylim(-0.22, 1.08); ax.set_xlim(-3, 4)
    ax.set_xlabel("$x$"); ax.legend(loc="upper left", frameon=False)
    savefig(fig, CH, "fig2-6", out)


def fig_2_7(out=None):
    """Uniform on (-1, 1), exponential with lambda = 1 and Laplace with mu = 1, gamma = 1."""
    x = np.linspace(-2, 4, 1201)
    uni = np.where((x > -1) & (x < 1), 0.5, 0.0)
    ex = np.where(x >= 0, np.exp(-np.maximum(x, 0)), 0.0)
    lap = 0.5 * np.exp(-np.abs(x - 1))
    fig, ax = plt.subplots(figsize=(5.2, 3.2))
    ax.plot(x, uni, color=RED, label="uniform $(-1,1)$")
    ax.plot(x, ex, color=BLUE, label="exponential, $\\lambda=1$")
    ax.plot(x, lap, color=GREEN, label="Laplace, $\\mu=1,\\ \\gamma=1$")
    ax.set_xlim(-2, 4); ax.set_ylim(0, 1.1)
    ax.set_xlabel("$x$"); ax.set_ylabel("$p(x)$"); ax.legend(frameon=False)
    savefig(fig, CH, "fig2-7", out)


def fig_2_8(out=None):
    """The Gaussian density with mean mu and width 2 sigma."""
    mu, s = 0.0, 1.0
    x = np.linspace(-4, 4, 600)
    fig, ax = plt.subplots(figsize=(5.0, 3.0))
    ax.plot(x, gauss(x, mu, s * s), color=RED)
    h = gauss(mu + s, mu, s * s)
    ax.annotate("", xy=(mu + s, h), xytext=(mu - s, h), arrowprops=dict(arrowstyle="<->", lw=1.1))
    ax.text(mu, h + 0.015, "$2\\sigma$", ha="center", va="bottom")
    ax.axvline(mu, color=GREY, ls="--", lw=1)
    ax.set_xticks([mu]); ax.set_xticklabels(["$\\mu$"]); ax.set_yticks([])
    ax.set_xlabel("$x$"); ax.set_ylabel("$\\mathcal{N}(x\\,|\\,\\mu,\\sigma^2)$")
    ax.set_ylim(0, 0.46)
    savefig(fig, CH, "fig2-8", out)


def fig_2_9(out=None):
    """The likelihood is the product of the densities at the data points."""
    x = likelihood_data_js()
    mu, var = gauss_ml(x)
    g = np.linspace(mu - 3.5, mu + 3.5, 600)
    fig, ax = plt.subplots(figsize=(5.4, 3.2))
    ax.plot(g, gauss(g, mu, var), color=RED, label="$\\mathcal{N}(x\\,|\\,\\mu_{\\rm ML},\\sigma^2_{\\rm ML})$")
    for xn in x:
        ax.plot([xn, xn], [0, gauss(xn, mu, var)], color=GREEN, lw=0.9)
    ax.plot(x, np.zeros_like(x), "o", color=GREY, ms=6, clip_on=False, zorder=3, label="data $x_n$")
    ax.plot(x, gauss(x, mu, var), "o", color=BLUE, ms=6, zorder=3, label="$\\mathcal{N}(x_n\\,|\\,\\mu,\\sigma^2)$")
    ax.set_ylim(0, None); ax.set_xlabel("$x$"); ax.set_ylabel("$p(x)$")
    ax.legend(frameon=False, fontsize=8.5, loc="upper right")
    ax.set_title("$\\ln$ likelihood $= %.3f$ (N = %d)" % (gauss_loglik(x, mu, var), x.size), fontsize=9.5)
    savefig(fig, CH, "fig2-9", out)


def fig_2_10(out=None):
    """Three data sets of two points: the ML fits are centred correctly on average but too narrow."""
    rng = np.random.default_rng(8)
    sets = [np.sort(rng.normal(size=2)) for _ in range(3)]
    sets.sort(key=lambda s: s.mean())
    x = np.linspace(-3.2, 3.2, 600)
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 2.4), sharey=True)
    top = 1.08 * max(gauss(0.0, 0.0, gauss_ml(s)[1]) for s in sets)
    for ax, s in zip(axes, sets):
        m, v = gauss_ml(s)
        ax.plot(x, gauss(x), color=RED)
        ax.plot(x, gauss(x, m, v), color=BLUE)
        ax.plot(s, [0, 0], "o", color=GREEN, ms=7, clip_on=False, zorder=3)
        ax.axvline(0, color=GREY, ls="--", lw=1)
        ax.set_xticks([0]); ax.set_xticklabels(["$\\mu$"]); ax.set_yticks([])
        ax.set_ylim(0, top)
        ax.set_title("$\\sigma^2_{\\rm ML}=%.2f$" % v, fontsize=9.5)
        for sp in ("top", "right", "left"):
            ax.spines[sp].set_visible(False)
    fig.tight_layout()
    savefig(fig, CH, "fig2-10", out)


def sine_data(N=10, seed=2, noise=0.3):
    rng = np.random.default_rng(seed)
    x = np.linspace(0, 1, N)
    return x, np.sin(2 * np.pi * x) + noise * rng.normal(size=N)


def fig_2_11(out=None):
    """p(t | x0, w, sigma^2) around a fitted cubic."""
    x, t = sine_data()
    w, var = fit_poly_ml(x, t, 3)
    s = math.sqrt(var)
    g = np.linspace(0, 1, 400)
    y = poly_design(g, 3) @ w
    fig, ax = plt.subplots(figsize=(5.4, 3.4))
    ax.plot(g, y, color=RED, label="$y(x,\\mathbf{w}_{\\rm ML})$")
    ax.fill_between(g, y - 2 * s, y + 2 * s, color=RED, alpha=0.1, lw=0, label="$\\pm 2\\sigma_{\\rm ML}$")
    ax.plot(x, t, "o", mfc="none", mec=BLUE, ms=6, label="data")
    x0 = 0.62
    y0 = (poly_design([x0], 3) @ w)[0]
    tt = np.linspace(y0 - 3.2 * s, y0 + 3.2 * s, 200)
    dens = gauss(tt, y0, var)
    ax.plot(x0 + 0.17 * dens / dens.max(), tt, color=BLUE)
    ax.plot([x0, x0], [tt[0], tt[-1]], color=GREY, lw=0.9, ls="--")
    ax.text(x0 + 0.19, y0 + 0.35, "$p(t\\,|\\,x_0,\\mathbf{w},\\sigma^2)$", color=BLUE, fontsize=9)
    ax.set_xticks([0, x0, 1]); ax.set_xticklabels(["0", "$x_0$", "1"])
    ax.set_xlabel("$x$"); ax.set_ylabel("$t$")
    ax.legend(frameon=False, fontsize=8.5, loc="lower left")
    savefig(fig, CH, "fig2-11", out)


def fig_2_12(out=None):
    """The mode of a density is not preserved by a nonlinear change of variables."""
    xs = ms_samples_js(50000, seed=1)
    ys = ms_ginv(xs)
    y_naive, y_mode, _ = ms_modes()
    fig, ax = plt.subplots(figsize=(6.0, 4.2))
    x = np.linspace(0, 10, 800)
    ax.plot(x, ms_ginv(x), color=BLUE, label="$y=g^{-1}(x)$")
    # p_x and its histogram on the x axis (scaled to fit), p_y on the y axis
    sx = 0.35 / gauss(MS_MU, MS_MU, 1.0)
    cnt, edges = np.histogram(xs, bins=np.linspace(0, 10, 61), density=True)
    ax.bar(edges[:-1], cnt * sx, width=np.diff(edges), align="edge", color="#b3b3ff", edgecolor="#6a6ad0", lw=0.4)
    ax.plot(x, gauss(x, MS_MU, MS_SIGMA ** 2) * sx, color=RED, label="$p_x(x)$")
    yy = np.linspace(1e-4, 1 - 1e-4, 2000)
    py = ms_py(yy)
    sy = 3.6 / py.max()
    cy, ey = np.histogram(ys, bins=np.linspace(0, 1, 61), density=True)
    ax.barh(ey[:-1], cy * sy, height=np.diff(ey), align="edge", color="#f2c9f2", edgecolor="#c060c0", lw=0.4)
    ax.plot(py * sy, yy, color=MAGENTA, label="$p_y(y)$, by (2.71)")
    naive = gauss(ms_g(yy), MS_MU, MS_SIGMA ** 2)
    ax.plot(naive * 3.6 / naive.max(), yy, color=GREEN, label="$p_x(g(y))$ (no Jacobian)")
    ax.plot([0, MS_MU], [y_naive, y_naive], color=GREEN, ls="--", lw=1)
    ax.plot([MS_MU, MS_MU], [0, y_naive], color=GREEN, ls="--", lw=1)
    ax.plot([0, 3.6], [y_mode, y_mode], color=MAGENTA, ls=":", lw=1.2)
    ax.set_xlim(0, 10); ax.set_ylim(0, 1)
    ax.set_xlabel("$x$"); ax.set_ylabel("$y$")
    ax.legend(loc="center right", frameon=False, fontsize=8.5)
    ax.set_title("mode of $p_x(g(y))$: $y=%.3f$;  mode of $p_y$: $y=%.3f$" % (y_naive, y_mode), fontsize=9.5)
    savefig(fig, CH, "fig2-12", out)


def fig_2_13(out=None):
    """Change of variables (2.78)-(2.79) applied to a grid, a Gaussian density and samples."""
    s, L = 0.5, 1.5
    fig, axes = plt.subplots(2, 3, figsize=(8.4, 5.8))
    g = np.linspace(-L, L, 21)
    t = np.linspace(-L, L, 300)
    # column 1: grid
    for v in g:
        axes[0, 0].plot([v, v], [-L, L], color=GREY, lw=0.5)
        axes[0, 0].plot([-L, L], [v, v], color=GREY, lw=0.5)
        y1, y2 = t2d(np.full_like(t, v), t)
        axes[1, 0].plot(y1, y2, color=GREY, lw=0.5)
        y1, y2 = t2d(t, np.full_like(t, v))
        axes[1, 0].plot(y1, y2, color=GREY, lw=0.5)
    # column 2: densities
    n = 400
    u = np.linspace(-L, L, n)
    U1, U2 = np.meshgrid(u, u)
    px = np.exp(-0.5 * (U1 ** 2 + U2 ** 2) / s ** 2) / (2 * np.pi * s * s)
    axes[0, 1].imshow(px, extent=[-L, L, -L, L], origin="lower", cmap="afmhot", aspect="auto")
    YL1 = L + math.tanh(5 * L)
    YL2 = L + math.tanh(5 * L) + L ** 3 / 3
    v1 = np.linspace(-YL1, YL1, n)
    v2 = np.linspace(-YL2, YL2, n)
    V1, V2 = np.meshgrid(v1, v2)
    py = t2d_density(V1, V2, s)
    axes[1, 1].imshow(np.sqrt(py), extent=[-YL1, YL1, -YL2, YL2], origin="lower", cmap="afmhot", aspect="auto")
    # column 3: samples
    rng = np.random.default_rng(3)
    X = s * rng.normal(size=(500, 2))
    axes[0, 2].plot(X[:, 0], X[:, 1], ".", color=RED, ms=2.5)
    Y1, Y2 = t2d(X[:, 0], X[:, 1])
    axes[1, 2].plot(Y1, Y2, ".", color=RED, ms=2.5)
    for j in range(3):
        axes[0, j].set_xlim(-L, L); axes[0, j].set_ylim(-L, L)
        axes[1, j].set_xlim(-YL1, YL1); axes[1, j].set_ylim(-YL2, YL2)
        axes[0, j].set_xlabel("$x_1$"); axes[0, j].set_ylabel("$x_2$")
        axes[1, j].set_xlabel("$y_1$"); axes[1, j].set_ylabel("$y_2$")
        for i in range(2):
            axes[i, j].set_xticks([]); axes[i, j].set_yticks([])
    axes[0, 0].set_title("grid in $\\mathbf{x}$, mapped to $\\mathbf{y}$", fontsize=9.5)
    axes[0, 1].set_title("$p_{\\mathbf{x}}$ and $p_{\\mathbf{y}}$ (square root)", fontsize=9.5)
    axes[0, 2].set_title("500 samples, mapped", fontsize=9.5)
    fig.tight_layout(h_pad=2.2)
    for j in range(3):
        b0, b1 = axes[0, j].get_position(), axes[1, j].get_position()
        xm = (b0.x0 + b0.x1) / 2
        fig.add_artist(FancyArrowPatch((xm, b0.y0 - 0.045), (xm, b1.y1 + 0.012), transform=fig.transFigure,
                                       arrowstyle="-|>", mutation_scale=12, color="k", lw=1))
    savefig(fig, CH, "fig2-13", out)


def fig_2_14(out=None):
    """Entropy of a narrow and a broad distribution over 30 bins."""
    fig, axes = plt.subplots(1, 2, figsize=(8.0, 3.0), sharey=True)
    for ax, H in zip(axes, (1.77, 3.09)):
        p = binned_gauss(width_for_entropy(H))
        ax.bar(np.arange(30), p, width=1.0, color="#7b7bea", edgecolor="k", lw=0.6)
        ax.set_ylim(0, 0.5); ax.set_yticks([0, 0.25, 0.5]); ax.set_xticks([])
        ax.text(15, 0.4, "H = %.2f" % entropy(p), ha="center")
    axes[0].set_ylabel("probabilities")
    fig.tight_layout()
    savefig(fig, CH, "fig2-14", out)


def fig_2_15(out=None):
    """A convex function and one of its chords."""
    f = lambda x: 0.25 + 1.2 * (x - 0.45) ** 2 + 0.25 * (x - 0.45) ** 4
    a, b, lam = 0.0, 1.25, 0.45
    xl = lam * a + (1 - lam) * b
    x = np.linspace(-0.15, 1.4, 400)
    fig, ax = plt.subplots(figsize=(4.6, 3.2))
    ax.plot(x, f(x), color=RED, label="$f(x)$")
    ax.plot([a, b], [f(a), f(b)], color=BLUE, label="chord")
    ax.plot([xl, xl], [0, lam * f(a) + (1 - lam) * f(b)], color=GREY, ls="--", lw=1)
    ax.plot([xl], [f(xl)], "o", color=RED, ms=4)
    ax.plot([xl], [lam * f(a) + (1 - lam) * f(b)], "o", color=BLUE, ms=4)
    ax.set_xticks([a, xl, b]); ax.set_xticklabels(["$a$", "$x_\\lambda$", "$b$"]); ax.set_yticks([])
    ax.set_ylim(0, f(b) * 1.15); ax.set_xlabel("$x$")
    ax.legend(frameon=False, loc="upper left")
    savefig(fig, CH, "fig2-15", out)


def fig_2_16(out=None):
    """The four Efron dice; each beats the one before it in the cycle with probability 2/3."""
    cols = {"yellow": "#f2b705", "blue": "#2a9fd6", "green": "#76b82a", "red": "#e85a2a"}
    pos = {"yellow": (0.0, 1.0), "blue": (1.0, 0.0), "green": (0.0, -1.0), "red": (-1.0, 0.0)}
    fig, ax = plt.subplots(figsize=(5.4, 5.0))
    sz = 0.17
    for name, faces in EFRON.items():
        cx, cy = pos[name]
        for k, v in enumerate(faces):
            r, c = divmod(k, 3)
            x0 = cx + (c - 1.5) * sz
            y0 = cy + (0.5 - r) * sz
            ax.add_patch(Rectangle((x0, y0), sz * 0.94, sz * 0.94, color=cols[name]))
            ax.text(x0 + sz * 0.47, y0 + sz * 0.47, str(v), ha="center", va="center", color="white",
                    fontsize=11, weight="bold")
    for i, name in enumerate(EFRON_CYCLE):
        nxt = EFRON_CYCLE[(i + 1) % 4]
        (x0, y0), (x1, y1) = pos[name], pos[nxt]
        ax.add_patch(FancyArrowPatch((x0 + 0.42 * (x1 - x0), y0 + 0.42 * (y1 - y0)),
                                     (x0 + 0.62 * (x1 - x0), y0 + 0.62 * (y1 - y0)),
                                     arrowstyle="-|>", mutation_scale=14, color="k", lw=1.2))
        p = p_beats(nxt, name)
        ax.text(x0 + 0.52 * (x1 - x0) + 0.16 * np.sign(x0 + x1), y0 + 0.52 * (y1 - y0) + 0.16 * np.sign(y0 + y1),
                "%d/%d" % (p.numerator, p.denominator), ha="center", va="center", fontsize=10)
    ax.set_xlim(-1.5, 1.5); ax.set_ylim(-1.4, 1.4); ax.set_aspect("equal"); ax.axis("off")
    ax.set_title("each die beats the previous one with probability 2/3", fontsize=9.5)
    savefig(fig, CH, "fig2-16", out)


FIGURES = {"2.1": fig_2_1, "2.3": fig_2_3, "2.5": fig_2_5, "2.6": fig_2_6, "2.7": fig_2_7,
           "2.8": fig_2_8, "2.9": fig_2_9, "2.10": fig_2_10, "2.11": fig_2_11, "2.12": fig_2_12,
           "2.13": fig_2_13, "2.14": fig_2_14, "2.15": fig_2_15, "2.16": fig_2_16}

if __name__ == "__main__":
    common.run(FIGURES, CH)
