"""Chapter 3, "Probability: Multivariate Models", of the course "Probabilistic Machine Learning: An
Introduction", following Murphy (2022), Chapter 3.

Reproduces Figures 3.1 to 3.13 (every figure of the chapter that is computed from data, a model or a
simulation), plus the numbers used by the interactive demonstrations (exported as JSON to ../data).

Run:  python ch03_probability_multivariate.py [--fig 3.8 3.9 ...] [--list]

Skipped (hand-drawn diagrams or third-party material with nothing to compute):
  Figure 3.14  the water sprinkler network with its tables: a diagram. The inference it supports
               (p(R=1|W=1) = 0.7079, p(R=1|W=1,S=1) = 0.3204) is computed by `sprinkler_posterior` below.
  Figures 3.15, 3.16, 3.17  graphical-model diagrams (Markov chains, plate notation, the GMM as a PGM).
Stand-ins (stated in each caption):
  Figure 3.2   simulated monthly series with a common seasonal cause (the book shows real data).
  Figure 3.4   hypothetical case fatality rates and age mixes (the book shows reported COVID-19 counts).
  Figure 3.12  simulated two-dimensional clusters (the book's data are not available offline).
  Figure 3.13  a random subset of 20 000 MNIST training images (binarised at 0.5).
"""
import json
import os

import numpy as np
from scipy import stats

import common
from common import (BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, ellipse_xy, gauss_logpdf,
                    logsumexp, panel_grid, panel_label, plt, savefig)

HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
CH = 3


# =============================================================================== core mathematics
def gauss_condition(mu, S, idx1, idx2, y2):
    """Mean and covariance of p(y1 | y2) for y ~ N(mu, S), eq. (3.28)."""
    mu = np.asarray(mu, float)
    S = np.asarray(S, float)
    S11, S12, S22 = S[np.ix_(idx1, idx1)], S[np.ix_(idx1, idx2)], S[np.ix_(idx2, idx2)]
    K = np.linalg.solve(S22, S12.T).T          # S12 S22^{-1}
    return mu[idx1] + K @ (np.asarray(y2, float) - mu[idx2]), S11 - K @ S12.T


def bayes_rule_gauss(mu_z, S_z, W, b, S_y, y):
    """Posterior N(z | m, V) for p(z)=N(mu_z,S_z), p(y|z)=N(Wz+b,S_y), eq. (3.37)."""
    Pz, Py = np.linalg.inv(S_z), np.linalg.inv(S_y)
    V = np.linalg.inv(Pz + W.T @ Py @ W)
    m = V @ (W.T @ Py @ (np.asarray(y, float) - b) + Pz @ mu_z)
    return m, V


def scalar_posterior(mu0, tau0sq, sigsq, ybar, n):
    """Posterior mean and variance of an unknown scalar, eqs. (3.53)-(3.57)."""
    lam_n = 1.0 / tau0sq + n / sigsq
    return (n * ybar / sigsq + mu0 / tau0sq) / lam_n, 1.0 / lam_n


SPRINKLER = {  # Figure 3.14: CPTs as the probability that the node is true
    "C": 0.5,
    "S|C": {0: 0.5, 1: 0.1},
    "R|C": {0: 0.2, 1: 0.8},
    "W|SR": {(0, 0): 0.0, (1, 0): 0.9, (0, 1): 0.9, (1, 1): 0.99},
}


def sprinkler_joint(cpt=SPRINKLER):
    """Joint table p(C,S,R,W) as an array indexed [c, s, r, w] (eq. 3.105)."""
    J = np.zeros((2, 2, 2, 2))
    for c in (0, 1):
        for s in (0, 1):
            for r in (0, 1):
                for w in (0, 1):
                    pc = cpt["C"] if c else 1 - cpt["C"]
                    ps = cpt["S|C"][c] if s else 1 - cpt["S|C"][c]
                    pr = cpt["R|C"][c] if r else 1 - cpt["R|C"][c]
                    pw = cpt["W|SR"][(s, r)] if w else 1 - cpt["W|SR"][(s, r)]
                    J[c, s, r, w] = pc * ps * pr * pw
    return J


def sprinkler_posterior(query, evidence, cpt=SPRINKLER):
    """p(query = 1 | evidence) by enumeration; query and evidence use the names 'C','S','R','W'."""
    J = sprinkler_joint(cpt)
    names = "CSRW"
    num = den = 0.0
    for idx in np.ndindex(2, 2, 2, 2):
        if all(idx[names.index(k)] == v for k, v in evidence.items()):
            den += J[idx]
            if idx[names.index(query)] == 1:
                num += J[idx]
    return num / den


def gmm_em(X, K, n_iter=200, seed=0, tol=1e-9, reg=1e-6):
    """EM for a Gaussian mixture with full covariances (Murphy 2022, Section 3.5.1; Bishop 2006, Section 9.2).
    Returns (pi, mu, Sigma, loglik history)."""
    rng = np.random.default_rng(seed)
    N, D = X.shape
    mu = X[rng.choice(N, K, replace=False)].copy()
    Sig = np.stack([np.cov(X.T) + reg * np.eye(D)] * K)
    pi = np.full(K, 1.0 / K)
    hist = []
    for _ in range(n_iter):
        lp = np.stack([np.log(pi[k]) + gauss_logpdf(X, mu[k], Sig[k]) for k in range(K)], 1)
        ll = logsumexp(lp, axis=1)
        hist.append(ll.sum())
        R = np.exp(lp - ll[:, None])
        Nk = R.sum(0)
        pi = Nk / N
        mu = (R.T @ X) / Nk[:, None]
        for k in range(K):
            d = X - mu[k]
            Sig[k] = (R[:, k, None] * d).T @ d / Nk[k] + reg * np.eye(D)
        if len(hist) > 1 and abs(hist[-1] - hist[-2]) < tol * abs(hist[-1]):
            break
    return pi, mu, Sig, np.array(hist)


def gmm_responsibilities(X, pi, mu, Sig):
    lp = np.stack([np.log(pi[k]) + gauss_logpdf(X, mu[k], Sig[k]) for k in range(len(pi))], 1)
    return np.exp(lp - logsumexp(lp, axis=1)[:, None])


def bmm_em(X, K, n_iter=40, seed=0, eps=1e-3):
    """EM for a mixture of Bernoulli distributions, eq. (3.102). Returns (pi, mu, loglik history)."""
    rng = np.random.default_rng(seed)
    N, D = X.shape
    mu = np.clip(X[rng.choice(N, K, replace=False)] * 0.5 + 0.25 + 0.1 * rng.random((K, D)), eps, 1 - eps)
    pi = np.full(K, 1.0 / K)
    hist = []
    for _ in range(n_iter):
        lp = X @ np.log(mu).T + (1 - X) @ np.log(1 - mu).T + np.log(pi)
        ll = logsumexp(lp, axis=1)
        hist.append(ll.sum())
        R = np.exp(lp - ll[:, None])
        Nk = R.sum(0)
        pi = Nk / N
        mu = np.clip((R.T @ X) / Nk[:, None], eps, 1 - eps)
    return pi, mu, np.array(hist)


def standardized_pair(rho, n, rng):
    """n points of a standard bivariate sample whose sample correlation is exactly rho."""
    z = rng.standard_normal((n, 2))
    z -= z.mean(0)
    a = z[:, 0] / z[:, 0].std()
    b = z[:, 1] - (z[:, 1] @ a) / (a @ a) * a
    b /= b.std()
    return a, rho * a + np.sqrt(1 - rho ** 2) * b


def impute_experiment(seed=7, N=10, D=8, frac=0.5):
    """Figure 3.7: sample N vectors from a D-variate Gaussian, hide a fraction, impute with the exact conditional mean."""
    rng = np.random.default_rng(seed)
    A = rng.standard_normal((D, 3))
    S = A @ A.T + 0.3 * np.eye(D)
    mu = np.zeros(D)
    X = rng.multivariate_normal(mu, S, N)
    mask = rng.random((N, D)) < frac            # True = hidden
    Xhat = X.copy()
    for n in range(N):
        h, v = np.where(mask[n])[0], np.where(~mask[n])[0]
        if len(h) and len(v):
            Xhat[n, h], _ = gauss_condition(mu, S, h, v, X[n, v])
        elif len(h):
            Xhat[n, h] = mu[h]
    return X, mask, Xhat, S


# =============================================================================== figures
def _slope_set(ax, slope, rng):
    x = np.linspace(-1, 1, 40)
    ax.plot(x, slope * x, "o", ms=2.5, color=BLUE)


def fig_3_1(out=None):
    """Correlation coefficients of several point sets (own simulations)."""
    rng = np.random.default_rng(3)
    fig, axes = panel_grid(3, 7, size=(1.5, 1.45))
    for j, rho in enumerate([1.0, 0.8, 0.4, 0.0, -0.4, -0.8, -1.0]):
        a, b = standardized_pair(rho, 120, rng)
        axes[0, j].plot(a, b, "o", ms=2, color=BLUE)
        axes[0, j].set_title("%.1f" % np.corrcoef(a, b)[0, 1], fontsize=8.5)
    x = np.linspace(-1, 1, 40)
    for j, (sl, lab) in enumerate([(0.3, None), (1.0, None), (3.0, None), (0.0, "undefined"), (-3.0, None), (-1.0, None), (-0.3, None)]):
        ax = axes[1, j]
        ax.plot(x, sl * x, "o", ms=2, color=BLUE)
        ax.set_ylim(-3.2, 3.2) if sl == 0 else None
        ax.set_title(lab if lab else "%.0f" % np.sign(sl), fontsize=8.5)
    t = np.linspace(0, 2 * np.pi, 60, endpoint=False)
    xs = rng.uniform(-1, 1, 70)
    shapes = [
        (np.cos(t), np.sin(t)),                                       # circle
        (np.r_[xs, xs], np.r_[xs, -xs]),                              # X
        (xs, xs ** 2),                                                # parabola (Exercise 3.1)
        (np.linspace(-np.pi, np.pi, 70), np.sin(np.linspace(-np.pi, np.pi, 70) * 2)),  # one full wave
        (np.r_[np.cos(t[:30]), np.cos(t[:30]) / 2], np.r_[np.sin(t[:30]), np.sin(t[:30]) / 2 + 0.0]),
        (np.r_[rng.normal(-1, .15, 40), rng.normal(1, .15, 40)], np.r_[rng.normal(1, .15, 40), rng.normal(1, .15, 40)]),
        (np.r_[np.linspace(-1, 1, 20), np.linspace(-1, 1, 20), np.full(20, -1.0), np.full(20, 1.0)],
         np.r_[np.full(20, -1.0), np.full(20, 1.0), np.linspace(-1, 1, 20), np.linspace(-1, 1, 20)]),  # square outline
    ]
    for j, (a, b) in enumerate(shapes):
        axes[2, j].plot(a, b, "o", ms=2, color=BLUE)
        r = np.corrcoef(a, b)[0, 1]
        axes[2, j].set_title("%.1f" % (0.0 if abs(r) < 0.05 else r), fontsize=8.5)
    for ax in axes.ravel():
        ax.set_xticks([])
        ax.set_yticks([])
    fig.tight_layout(pad=0.4)
    return savefig(fig, CH, "fig3-1", out)


def fig_3_2(out=None):
    """Spurious correlation: two series driven by one hidden cause (simulated)."""
    rng = np.random.default_rng(11)
    T = 60
    t = np.arange(T)
    temp = 15 + 10 * np.sin(2 * np.pi * (t - 3) / 12) + 1.5 * rng.standard_normal(T)
    ice = 100 + 6 * temp + 12 * rng.standard_normal(T)
    crime = 20 + 0.9 * temp + 2.5 * rng.standard_normal(T)
    r = np.corrcoef(ice, crime)[0, 1]
    # partial correlation given temperature
    ri = ice - np.polyval(np.polyfit(temp, ice, 1), temp)
    rc = crime - np.polyval(np.polyfit(temp, crime, 1), temp)
    rp = np.corrcoef(ri, rc)[0, 1]
    z = lambda v: (v - v.mean()) / v.std()
    fig, axes = panel_grid(1, 3, size=(3.3, 2.5))
    axes[0, 0].plot(t, z(ice), color=RED, label="ice cream sales")
    axes[0, 0].plot(t, z(crime), color=ORANGE, label="violent crime rate")
    axes[0, 0].set_xlabel("month")
    axes[0, 0].set_ylabel("standardised")
    axes[0, 0].legend(loc="upper right", fontsize=7, ncol=1)
    panel_label(axes[0, 0], "a", "lower left")
    axes[0, 1].plot(ice, crime, "o", ms=3.5, color=BLUE)
    axes[0, 1].set_xlabel("ice cream sales")
    axes[0, 1].set_ylabel("crime rate")
    axes[0, 1].set_title("corr = %.2f" % r)
    panel_label(axes[0, 1], "b", "lower right")
    axes[0, 2].plot(ri, rc, "o", ms=3.5, color=GREEN)
    axes[0, 2].set_xlabel("ice cream, temperature removed")
    axes[0, 2].set_ylabel("crime, temperature removed")
    axes[0, 2].set_title("partial corr = %.2f" % rp)
    panel_label(axes[0, 2], "c", "lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-2", out)


def iris_simpson():
    from sklearn.datasets import load_iris
    d = load_iris()
    X = d.data[:, :2]
    return X, d.target, d.target_names


def fig_3_3(out=None):
    """Simpson's paradox on the iris data: sepal width against sepal length."""
    X, y, names = iris_simpson()
    cols = [RED, BLUE, GREEN]
    fig, axes = panel_grid(1, 2, size=(3.6, 2.9))
    ax = axes[0, 0]
    ax.plot(X[:, 0], X[:, 1], "o", ms=3, color="k", alpha=0.7)
    p = np.polyfit(X[:, 0], X[:, 1], 1)
    xx = np.array([4.2, 8.0])
    ax.plot(xx, np.polyval(p, xx), "k-")
    ax.set_title("all species: R = %.2f" % np.corrcoef(X.T)[0, 1])
    ax2 = axes[0, 1]
    for k in range(3):
        m = y == k
        ax2.plot(X[m, 0], X[m, 1], "o", ms=3, color=cols[k], label="%s: R = %.2f" % (names[k], np.corrcoef(X[m].T)[0, 1]))
        pk = np.polyfit(X[m, 0], X[m, 1], 1)
        xk = np.array([X[m, 0].min(), X[m, 0].max()])
        ax2.plot(xk, np.polyval(pk, xk), "-", color=cols[k])
    ax2.legend(fontsize=7.5, loc="lower right")
    for a in (ax, ax2):
        a.set_xlabel("sepal length")
        a.set_ylabel("sepal width")
        a.set_xlim(4, 8.2)
        a.set_ylim(1.9, 4.6)
    fig.tight_layout()
    return savefig(fig, CH, "fig3-3", out)


# Hypothetical case fatality rates (per cent) and age mixes (per cent of cases), nine age bands
AGE = ["0-9", "10-19", "20-29", "30-39", "40-49", "50-59", "60-69", "70-79", "80+"]
CFR_A = np.array([0.0, 0.2, 0.2, 0.3, 0.5, 1.6, 5.0, 11.0, 20.0])
CFR_B = CFR_A * 0.8
MIX_A = np.array([1, 4, 12, 17, 20, 20, 14, 8, 4], float)
MIX_B = np.array([1, 1, 4, 6, 9, 16, 22, 23, 18], float)


def simpson_table():
    MIX_A_ = MIX_A / MIX_A.sum()
    MIX_B_ = MIX_B / MIX_B.sum()
    return float(CFR_A @ MIX_A_), float(CFR_B @ MIX_B_)


def fig_3_4(out=None):
    """Simpson's paradox with a table of hypothetical age-specific fatality rates and age mixes."""
    oa, ob = simpson_table()
    fig, axes = panel_grid(1, 2, size=(4.3, 2.7))
    x = np.arange(len(AGE) + 1)
    w = 0.38
    axes[0, 0].bar(x[:-1] - w / 2, CFR_A, w, color=BLUE, label="country A")
    axes[0, 0].bar(x[:-1] + w / 2, CFR_B, w, color=RED, label="country B")
    axes[0, 0].bar(x[-1] - w / 2, oa, w, color=BLUE)
    axes[0, 0].bar(x[-1] + w / 2, ob, w, color=RED)
    axes[0, 0].set_xticks(x)
    axes[0, 0].set_xticklabels(AGE + ["total"], rotation=45, ha="right", fontsize=7.5)
    axes[0, 0].set_ylabel("case fatality rate (%)")
    axes[0, 0].legend(fontsize=8)
    panel_label(axes[0, 0], "a")
    axes[0, 1].bar(x[:-1] - w / 2, 100 * MIX_A / MIX_A.sum(), w, color=BLUE, label="country A")
    axes[0, 1].bar(x[:-1] + w / 2, 100 * MIX_B / MIX_B.sum(), w, color=RED, label="country B")
    axes[0, 1].set_xticks(x[:-1])
    axes[0, 1].set_xticklabels(AGE, rotation=45, ha="right", fontsize=7.5)
    axes[0, 1].set_ylabel("share of confirmed cases (%)")
    panel_label(axes[0, 1], "b")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-4", out)


COV_FULL = np.array([[8.0, 6.0], [6.0, 6.0]]) * 1.0
COV_DIAG = np.diag([1.5, 6.0])
COV_SPH = 4.0 * np.eye(2)


def fig_3_5(out=None):
    """Surface plots of three 2-d Gaussian densities: full, diagonal, spherical covariance."""
    g = np.linspace(-9, 9, 90)
    X, Y = np.meshgrid(g, g)
    P = np.stack([X.ravel(), Y.ravel()], 1)
    fig = plt.figure(figsize=(8.6, 2.7))
    for j, (S, name) in enumerate([(COV_FULL, "full"), (COV_DIAG, "diagonal"), (COV_SPH, "spherical")]):
        ax = fig.add_subplot(1, 3, j + 1, projection="3d")
        Z = np.exp(gauss_logpdf(P, np.zeros(2), S)).reshape(X.shape)
        ax.plot_surface(X, Y, Z, cmap="viridis", linewidth=0, antialiased=True, rcount=60, ccount=60)
        ax.set_zlim(0, 0.2)
        ax.set_title(name + "  (" + "abc"[j] + ")")
        ax.tick_params(labelsize=6)
        ax.view_init(28, -55)
    fig.tight_layout(pad=0.5)
    return savefig(fig, CH, "fig3-5", out)


def fig_3_6(out=None):
    """Level sets of the three densities of Figure 3.5."""
    g = np.linspace(-9, 9, 200)
    X, Y = np.meshgrid(g, g)
    P = np.stack([X.ravel(), Y.ravel()], 1)
    fig, axes = panel_grid(1, 3, size=(2.9, 2.8))
    for j, (S, name) in enumerate([(COV_FULL, "full"), (COV_DIAG, "diagonal"), (COV_SPH, "spherical")]):
        Z = np.exp(gauss_logpdf(P, np.zeros(2), S)).reshape(X.shape)
        axes[0, j].contour(X, Y, Z, 8, cmap="viridis")
        axes[0, j].set_aspect("equal")
        axes[0, j].set_title(name)
        panel_label(axes[0, j], "abc"[j], "lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-6", out)


def _hinton(ax, M, mask=None, title=""):
    N, D = M.shape   # rows are features (D), columns are samples
    for i in range(D):
        for n in range(N):
            if mask is not None and mask[n, i]:
                continue
            v = M[n, i]
            s = min(np.sqrt(abs(v)) * 0.55, 0.95)
            ax.add_patch(plt.Rectangle((n - s / 2, i - s / 2), s, s, color=BLUE if v > 0 else GREEN))
    ax.set_xlim(-0.6, N - 0.4)
    ax.set_ylim(D - 0.4, -0.6)
    ax.set_xticks(range(N))
    ax.set_yticks(range(D))
    ax.set_aspect("equal")
    ax.set_title(title, fontsize=9)
    ax.grid(color="0.85", lw=0.5)


def fig_3_7(out=None):
    """Imputation of missing entries with a multivariate Gaussian and the true parameters."""
    X, mask, Xhat, S = impute_experiment()
    fig, axes = panel_grid(1, 3, size=(3.1, 2.6))
    _hinton(axes[0, 0], X, mask, "observed")
    _hinton(axes[0, 1], X, None, "hidden truth")
    _hinton(axes[0, 2], Xhat, None, "imputed (true parameters)")
    for j in range(3):
        panel_label(axes[0, j], "abc"[j], "lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-7", out)


def fig_3_8(out=None):
    """Inference about z from one noisy observation y = 3: strong and weak prior."""
    y, sig2 = 3.0, 1.0
    zs = np.linspace(-5, 8, 400)
    fig, axes = panel_grid(1, 2, size=(3.9, 2.7))
    for j, tau2 in enumerate([1.0, 5.0]):
        m, v = scalar_posterior(0.0, tau2, sig2, y, 1)
        ax = axes[0, j]
        ax.plot(zs, stats.norm.pdf(zs, 0, np.sqrt(tau2)), color=BLUE, label="prior")
        ax.plot(zs, stats.norm.pdf(zs, y, np.sqrt(sig2)), color=RED, ls="--", label="likelihood")
        ax.plot(zs, stats.norm.pdf(zs, m, np.sqrt(v)), color=GREEN, lw=2, label="posterior")
        ax.set_xlabel("z")
        ax.set_title("prior variance %g: posterior mean %.2f" % (tau2, m), fontsize=9)
        ax.legend(fontsize=8)
        panel_label(ax, "ab"[j], "upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-8", out)


def fig_3_9(out=None):
    """Bayesian inference for a 2-d Gaussian mean from 10 noisy points."""
    rng = np.random.default_rng(5)
    z = np.array([0.5, 0.5])
    Sy = 0.1 * np.array([[2.0, 1.0], [1.0, 1.0]])
    Y = rng.multivariate_normal(z, Sy, 10)
    S0 = 0.1 * np.eye(2)
    m, V = bayes_rule_gauss(np.zeros(2), S0, np.eye(2), np.zeros(2), Sy / len(Y), Y.mean(0))
    g = np.linspace(-1, 1, 160)
    X, Yg = np.meshgrid(g, g)
    P = np.stack([X.ravel(), Yg.ravel()], 1)
    fig, axes = panel_grid(1, 3, size=(2.9, 2.7))
    axes[0, 0].plot(Y[:, 0], Y[:, 1], "o", color=BLUE, ms=5)
    axes[0, 0].plot(*z, "kx", ms=9, mew=2)
    axes[0, 0].set_xlim(-0.5, 1.5)
    axes[0, 0].set_ylim(-1, 1.1)
    axes[0, 0].set_title("data")
    for j, (mu, S, t) in enumerate([(np.zeros(2), S0, "prior"), (m, V, "posterior after 10 points")]):
        ax = axes[0, j + 1]
        D = np.exp(gauss_logpdf(P, mu, S)).reshape(X.shape)
        im = ax.contourf(X, Yg, D, 12, cmap="viridis")
        fig.colorbar(im, ax=ax, fraction=0.045)
        ax.set_aspect("equal")
        ax.set_title(t, fontsize=9)
        ax.set_xlabel("$z_1$")
    axes[0, 1].set_ylabel("$z_2$")
    for j in range(3):
        panel_label(axes[0, j], "abc"[j], "lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-9", out)


def sensor_fusion(y1, y2, S1, S2):
    """Posterior (flat prior) of z from two sensors, Section 3.3.5."""
    P = np.linalg.inv(S1) + np.linalg.inv(S2)
    V = np.linalg.inv(P)
    return V @ (np.linalg.solve(S1, y1) + np.linalg.solve(S2, y2)), V


FUSION_CASES = [
    (0.01 * np.eye(2), 0.01 * np.eye(2)),
    (0.05 * np.eye(2), 0.01 * np.eye(2)),
    (0.01 * np.array([[10.0, 1], [1, 1]]), 0.01 * np.array([[1.0, 1], [1, 10]])),
]


def fig_3_10(out=None):
    """Sensor fusion with two sensors in 2-d."""
    y1, y2 = np.array([0.0, -1.0]), np.array([1.0, 0.0])
    fig, axes = panel_grid(1, 3, size=(2.9, 2.8))
    for j, (S1, S2) in enumerate(FUSION_CASES):
        m, V = sensor_fusion(y1, y2, S1, S2)
        ax = axes[0, j]
        ax.plot(*y1, "r+", ms=10, mew=2, label="$y_1$")
        ax.plot(*y2, "g+", ms=10, mew=2, label="$y_2$")
        for S, c in ((S1, RED), (S2, GREEN)):
            ax.plot(*ellipse_xy(y1 if c == RED else y2, S, 1.0), color=c, lw=1)
        ax.plot(*ellipse_xy(m, V, 1.0), "k-", lw=1.4)
        ax.plot(*m, "k*", ms=9)
        ax.set_aspect("equal")
        ax.set_xlim(-0.5, 1.5)
        ax.set_ylim(-1.5, 0.5)
        panel_label(ax, "abc"[j], "lower left")
        if j == 2:
            ax.legend(fontsize=8, loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-10", out)


GMM3 = dict(pi=np.array([0.35, 0.4, 0.25]),
            mu=np.array([[0.22, 0.45], [0.5, 0.5], [0.77, 0.55]]),
            Sig=np.array([[[0.011, -0.01], [-0.01, 0.04]], [[0.018, -0.01], [-0.01, 0.03]], [[0.011, -0.008], [-0.008, 0.045]]]))


def fig_3_11(out=None):
    """A mixture of three Gaussians in 2-d: component contours and the total density surface."""
    g = np.linspace(0.0, 1.0, 120)
    X, Y = np.meshgrid(g, g)
    P = np.stack([X.ravel(), Y.ravel()], 1)
    fig = plt.figure(figsize=(7.4, 3.0))
    ax = fig.add_subplot(1, 2, 1)
    tot = np.zeros(len(P))
    for k, c in enumerate([RED, GREEN, BLUE]):
        d = np.exp(gauss_logpdf(P, GMM3["mu"][k], GMM3["Sig"][k]))
        ax.contour(X, Y, d.reshape(X.shape), 6, colors=c, linewidths=0.9)
        tot += GMM3["pi"][k] * d
    ax.set_aspect("equal")
    ax.set_title("components (a)")
    ax2 = fig.add_subplot(1, 2, 2, projection="3d")
    ax2.plot_surface(X, Y, tot.reshape(X.shape), cmap="copper", linewidth=0, rcount=70, ccount=70)
    ax2.set_title("mixture density (b)")
    ax2.tick_params(labelsize=6)
    ax2.view_init(32, -60)
    fig.tight_layout(pad=0.5)
    return savefig(fig, CH, "fig3-11", out)


def clusters_2d(seed=2, n=700):
    """Simulated 2-d data: five clusters, three round on the left and two elongated on the right."""
    rng = np.random.default_rng(seed)
    specs = [((-4.0, 3.0), [[0.15, 0], [0, 0.3]], 1), ((-4.0, 1.0), [[0.2, 0], [0, 0.3]], 1),
             ((-4.0, -2.0), [[0.2, 0], [0, 0.25]], 1), ((-1.2, 0.5), [[0.5, 0.9], [0.9, 2.0]], 1.6),
             ((0.8, -0.8), [[0.4, 0.8], [0.8, 1.9]], 1.4)]
    w = np.array([s[2] for s in specs])
    w = w / w.sum()
    lab = rng.choice(5, n, p=w)
    X = np.zeros((n, 2))
    for k, (m, S, _) in enumerate(specs):
        idx = lab == k
        X[idx] = rng.multivariate_normal(m, S, idx.sum())
    return X, lab


def fit_gmm_best(X, K, restarts=6):
    best = None
    for s in range(restarts):
        r = gmm_em(X, K, seed=s)
        if best is None or r[3][-1] > best[3][-1]:
            best = r
    return best


def fig_3_12(out=None):
    """Two-dimensional data and a K = 5 clustering from a Gaussian mixture fitted by EM."""
    X, _ = clusters_2d()
    pi, mu, Sig, hist = fit_gmm_best(X, 5)
    z = gmm_responsibilities(X, pi, mu, Sig).argmax(1)
    fig, axes = panel_grid(1, 2, size=(3.9, 2.8))
    axes[0, 0].plot(X[:, 0], X[:, 1], ".", ms=3, color=BLUE)
    cols = [GREEN, PURPLE, ORANGE, BLUE, RED]
    order = np.argsort(mu[:, 0])
    for r, k in enumerate(order):
        m = z == k
        axes[0, 1].plot(X[m, 0], X[m, 1], ".", ms=3, color=cols[r])
    for a in axes.ravel():
        a.set_xlim(-6.5, 3.5)
        a.set_ylim(-4, 5)
    panel_label(axes[0, 0], "a", "lower right")
    panel_label(axes[0, 1], "b", "lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig3-12", out)


def mnist_bmm(n=20000, K=20, seed=0, n_iter=40):
    cache = os.path.join(HERE, "cache")
    os.makedirs(cache, exist_ok=True)
    path = os.path.join(cache, "bmm_mnist_K%d_n%d.npz" % (K, n))
    if os.path.exists(path):
        z = np.load(path)
        return z["pi"], z["mu"], z["hist"]
    X, _ = common.mnist(train=True)
    rng = np.random.default_rng(seed)
    X = (X[rng.choice(len(X), n, replace=False)] > 0.5).astype(float)
    pi, mu, hist = bmm_em(X, K, n_iter=n_iter, seed=seed)
    np.savez(path, pi=pi, mu=mu, hist=hist)
    return pi, mu, hist


def fig_3_13(out=None):
    """A mixture of 20 Bernoulli distributions fitted to binarised MNIST digits."""
    pi, mu, hist = mnist_bmm()
    order = np.argsort(-pi)
    fig, axes = panel_grid(4, 5, size=(1.2, 1.35))
    for a, k in zip(axes.ravel(), order):
        a.imshow(mu[k].reshape(28, 28), cmap="gray_r", vmin=0, vmax=1)
        a.set_title("%.3f" % pi[k], fontsize=8, pad=2)
        a.axis("off")
    fig.tight_layout(pad=0.3)
    return savefig(fig, CH, "fig3-13", out)


# =============================================================================== demo data
def export_json(out=None):
    """Small data sets and reference numbers for the interactive demonstrations."""
    os.makedirs(DATA_OUT, exist_ok=True)
    X, y, names = iris_simpson()
    json.dump({"names": list(names), "sepal_length": X[:, 0].round(2).tolist(), "sepal_width": X[:, 1].round(2).tolist(),
               "species": y.tolist()}, open(os.path.join(DATA_OUT, "ch03-iris.json"), "w"))
    ref = {}
    m, v = gauss_condition([0, 0], [[1, .8], [.8, 1]], [0], [1], [1.0])
    ref["cond_rho08"] = [float(m[0]), float(v[0, 0])]
    y1, y2 = np.array([0.0, -1.0]), np.array([1.0, 0.0])
    ref["fusion"] = []
    for S1, S2 in FUSION_CASES:
        mm, VV = sensor_fusion(y1, y2, S1, S2)
        ref["fusion"].append({"mean": mm.tolist(), "cov": VV.tolist()})
    ref["sprinkler"] = {"R|W": sprinkler_posterior("R", {"W": 1}), "R|W,S": sprinkler_posterior("R", {"W": 1, "S": 1}),
                        "R": sprinkler_posterior("R", {})}
    ref["iris_pooled_r"] = float(np.corrcoef(X.T)[0, 1])
    ref["iris_group_r"] = [float(np.corrcoef(X[y == k].T)[0, 1]) for k in range(3)]
    mm, VV = bayes_rule_gauss(np.zeros(2), 0.1 * np.eye(2), np.eye(2), np.zeros(2), 0.1 * np.array([[2.0, 1], [1, 1]]) / 10, np.array([0.6, 0.4]))
    ref["bayes2d_example"] = {"mean": mm.tolist(), "cov": VV.tolist()}
    rng = np.random.default_rng(5)
    Sy = 0.1 * np.array([[2.0, 1.0], [1.0, 1.0]])
    pts = rng.multivariate_normal([0.5, 0.5], Sy, 50)
    json.dump({"z": [0.5, 0.5], "Sy": Sy.tolist(), "points": pts.round(4).tolist()}, open(os.path.join(DATA_OUT, "ch03-gauss-infer.json"), "w"))
    for n in (1, 2, 5, 10, 50):
        mm, VV = bayes_rule_gauss(np.zeros(2), 0.1 * np.eye(2), np.eye(2), np.zeros(2), Sy / n, pts[:n].round(4).mean(0))
        ref["infer_N%d" % n] = {"mean": mm.tolist(), "cov": VV.tolist()}
    json.dump(ref, open(os.path.join(DATA_OUT, "ch03-reference.json"), "w"), indent=1)
    print("wrote ch03 json")


FIGURES = {"3.1": fig_3_1, "3.2": fig_3_2, "3.3": fig_3_3, "3.4": fig_3_4, "3.5": fig_3_5, "3.6": fig_3_6,
           "3.7": fig_3_7, "3.8": fig_3_8, "3.9": fig_3_9, "3.10": fig_3_10, "3.11": fig_3_11, "3.12": fig_3_12,
           "3.13": fig_3_13}

if __name__ == "__main__":
    common.run(FIGURES, CH)
    export_json()
