"""Bishop and Bishop (2024), Chapter 5: Single-layer Networks: Classification.

Reproduces the figures of Chapter 5 of C. M. Bishop and H. Bishop, Deep Learning: Foundations
and Concepts (Springer, 2024) that are computed from data, a model or a formula, written from
scratch with numpy, scipy and matplotlib. Every algorithm is a small function meant to be read
next to the lecture notes (../ch05-single-layer-classification.html):

* Section 5.1, discriminant functions: least_squares_classifier ((5.13)-(5.16), the pseudo-inverse
  solution for 1-of-K targets) and one_hot ((5.11)).
* Section 5.2, decision theory: bayes_decision (the minimum-expected-loss rule (5.23)),
  reject_decision (the reject option, Section 5.2.3, with the loss lambda of Exercise 5.10),
  confusion_counts, classifier_scores ((5.28)-(5.33), (5.38)-(5.39)), roc_curve and auc_mann_whitney
  (Section 5.2.6), rebalance_posteriors (compensating for class priors, Section 5.2.4) and
  combine_posteriors ((5.27)).
* Section 5.3, generative classifiers: generative_fit (maximum likelihood (5.56)-(5.63), shared or
  separate covariances), gaussian_posteriors ((5.45)-(5.47)), lda_weights ((5.49)-(5.53)),
  naive_bayes_fit / naive_bayes_activations ((5.64), (5.65), Exercises 5.15-5.17).
* Section 5.4, discriminative classifiers: logistic_irls (Newton / IRLS for (5.74)),
  logistic_gd (batch gradient descent on the mean of (5.74), the optimiser of the browser demo),
  softmax_gd and softmax_fit ((5.76)-(5.81)), probit_fit ((5.83)-(5.88)), gaussian_basis
  (Figure 5.15) and the exponential-family / canonical-link helpers.
* decision_1d: the one-dimensional two-class problem of decision-roc-demo.html, computed on the
  same grid with the same trapezoidal rule as the page, so the two agree to rounding error.

Figures reproduced: 5.1, 5.2, 5.3, 5.4, 5.5, 5.7, 5.8, 5.10, 5.11, 5.12, 5.13, 5.14, 5.15, 5.17.
Skipped: Figure 5.6 (a 2 x 2 loss matrix, a table), Figure 5.9 (the confusion-matrix layout, a
table) and Figure 5.16 (a network schematic). The geometric sketches 5.1-5.3 are redrawn from
actual linear functions, so every line in them is computed.

Run `python3 ch05_single_layer_classification.py --list`, `python3 ch05_single_layer_classification.py`
(all figures, about 20 s) or `... --fig 5.4 5.14`. Files go to ../img/ch05/. The data sets used by
the browser demos are written to ../data/ch05-*.json by `--fig data` (also part of the default run).

Settings chosen where the book gives none (the book's synthetic data are not published):
* Figure 5.4: 50 points per class from N((-0.7, 1.1), C) (red crosses, t = 1) and N((0.9, -0.8), C)
  (blue circles, t = 0), C = [[0.9, 0.45], [0.45, 0.9]], seed 1; the right panel adds 10 blue points
  from N((7.4, -6.6), 0.35^2 I). Logistic regression is fitted by IRLS (maximum likelihood; the data
  are not linearly separable, so the maximum exists).
* Figures 5.5, 5.10: p(x, C1) = 0.42 N(x | -1.2, 0.55^2) + 0.23 N(x | 1.5, 0.6^2) and
  p(x, C2) = 0.35 N(x | 1.7, 0.7^2); x0 is their crossing point and x-hat = x0 + 0.75 in panel (a).
* Figure 5.7: p(x | C_k) = N(x | -+1, 1), equal priors, theta = 0.85.
* Figure 5.8: p(x | C1) = 0.4 N(0.2, 0.1^2) + 0.6 N(0.5, 0.12^2), p(x | C2) = N(0.7, 0.12^2), equal priors.
* Figure 5.11: the score of each classifier is Gaussian with unit variance in each class and class
  means 0 and d (d = 2.0 blue, d = 0.9 red), giving the binormal ROC curve TPR = Phi(d + Phi^-1(FPR)).
* Figures 5.13, 5.14: our own means and covariances (listed in fig_5_13 and fig_5_14).
* Figure 5.15: red class N(0, 0.25^2 I) (40 points), blue class half from N((-1, -1), 0.15^2 I) and
  half from N((0.75, 1.0), 0.2^2 I) (40 points), seed 2; basis functions
  phi_j(x) = exp(-||x - mu_j||^2 / (2 s^2)), mu_1 = (-1, -1), mu_2 = (0, 0), s^2 = 0.5. The
  classes are separable in feature space, so maximum likelihood would send ||w|| to infinity
  (Section 5.4.3); we add a small weight decay 0.01 ||w||^2 / 2 to obtain a finite boundary.
* Figure 5.17: p(theta) = 0.7 N(1.5, 0.4^2) + 0.3 N(3.1, 0.32^2), green line at a = 2.8.
"""
import json
import math
import os
import sys

import numpy as np
from scipy import optimize, special, stats

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import common  # noqa: E402
from common import plt, savefig, sigmoid, RED, BLUE, GREEN, MAGENTA, GREY, ORANGE  # noqa: E402

CH = 5
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
LAMBDA_PROBIT = math.sqrt(math.pi / 8.0)   # Figure 5.12 and Exercise 5.24


# =========================================================================== basic maps
def softmax(A, axis=-1):
    """Normalised exponential (5.45) / (5.76), computed stably (subtract the max)."""
    A = np.asarray(A, dtype=float)
    Z = A - A.max(axis=axis, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=axis, keepdims=True)


def softmax_jacobian(a):
    """dy_k / da_j = y_k (I_kj - y_j) (5.78), as the matrix diag(y) - y y^T."""
    y = softmax(a)
    return np.diag(y) - np.outer(y, y)


def logit(y):
    """Inverse of the logistic sigmoid (5.44)."""
    y = np.asarray(y, dtype=float)
    return np.log(y) - np.log1p(-y)


def probit(a):
    """Phi(a) (5.86), the standard normal cumulative distribution function."""
    return stats.norm.cdf(a)


def erf_book(a):
    """The function written as erf in (5.87): 2/sqrt(pi) int_0^a exp(-theta^2/2) d theta.

    It is NOT the erf of numerical libraries (which has exp(-theta^2)); the two are related by
    erf_book(a) = sqrt(2) erf(a / sqrt(2)). With this definition (5.88) holds as printed.
    """
    return math.sqrt(2.0) * special.erf(np.asarray(a, dtype=float) / math.sqrt(2.0))


def one_hot(labels, K):
    """1-of-K (one-hot) coding (5.11): row n has a 1 in column labels[n]."""
    T = np.zeros((len(labels), K))
    T[np.arange(len(labels)), np.asarray(labels, dtype=int)] = 1.0
    return T


def augment(X):
    """Prepend the dummy input x0 = 1 (5.6): rows (1, x^T)."""
    X = np.atleast_2d(X)
    return np.hstack([np.ones((X.shape[0], 1)), X])


# =========================================================================== 5.1 least squares
def least_squares_classifier(X, T):
    """W-tilde = X-tilde^dagger T (5.15); predictions y(x) = W-tilde^T x-tilde (5.16).

    X is N x D, T is N x K (1-of-K rows, or a single column of 0/1 targets). Returns the
    (D+1) x K matrix whose column k is (w_k0, w_k^T)^T.
    """
    return np.linalg.pinv(augment(X)) @ T


def sum_of_squares_classification(Wt, X, T):
    """E_D(W-tilde) = 1/2 Tr{(X W - T)^T (X W - T)} (5.14)."""
    R = augment(X) @ Wt - T
    return 0.5 * np.trace(R.T @ R)


# =========================================================================== 5.2 decision theory
def bayes_decision(post, L=None):
    """Assign each row of post (N x K posteriors) to the class j minimising sum_k L_kj p(C_k | x) (5.23).

    L[k, j] is the loss for true class k and decision j; L = None means 1 - I (minimum
    misclassification: the largest posterior wins, Exercise 5.7).
    """
    post = np.atleast_2d(post)
    K = post.shape[1]
    if L is None:
        L = 1.0 - np.eye(K)
    return np.argmin(post @ np.asarray(L, dtype=float), axis=1)


def reject_decision(post, theta=None, L=None, lam=None):
    """Decisions with a reject option; -1 means 'reject'.

    Either the threshold rule of Section 5.2.3 (reject if max_k p(C_k|x) <= theta), or, if lam is
    given, the minimum-expected-loss rule of Exercise 5.10: reject when the smallest expected loss
    of a decision, min_j sum_k L_kj p(C_k|x), exceeds the loss lam of rejecting.
    """
    post = np.atleast_2d(post)
    K = post.shape[1]
    if lam is not None:
        if L is None:
            L = 1.0 - np.eye(K)
        risk = post @ np.asarray(L, dtype=float)
        j = np.argmin(risk, axis=1)
        return np.where(risk[np.arange(len(j)), j] < lam, j, -1)
    j = np.argmax(post, axis=1)
    return np.where(post.max(axis=1) > theta, j, -1)


def confusion_counts(t, pred):
    """N_TP, N_FP, N_TN, N_FN (5.28) for 0/1 labels (1 = positive, e.g. cancer)."""
    t = np.asarray(t).astype(int)
    pred = np.asarray(pred).astype(int)
    tp = int(np.sum((t == 1) & (pred == 1)))
    fp = int(np.sum((t == 0) & (pred == 1)))
    tn = int(np.sum((t == 0) & (pred == 0)))
    fn = int(np.sum((t == 1) & (pred == 0)))
    return tp, fp, tn, fn


def classifier_scores(tp, fp, tn, fn):
    """Accuracy (5.29), precision (5.30), recall (5.31), false positive rate (5.32), false
    discovery rate (5.33) and the F-score in both forms (5.38), (5.39)."""
    prec = tp / (tp + fp) if tp + fp else float("nan")
    rec = tp / (tp + fn) if tp + fn else float("nan")
    return {
        "accuracy": (tp + tn) / (tp + fp + tn + fn),
        "precision": prec,
        "recall": rec,
        "fpr": fp / (fp + tn) if fp + tn else float("nan"),
        "fdr": fp / (fp + tp) if fp + tp else float("nan"),
        "F_harmonic": 2 * prec * rec / (prec + rec) if prec + rec else float("nan"),
        "F_counts": 2 * tp / (2 * tp + fp + fn) if tp + fp + fn else float("nan"),
    }


def roc_curve(scores, t):
    """Empirical ROC curve: (FPR, TPR) as the threshold sweeps from +inf down through all scores.

    Ties are handled by moving diagonally (all tied points cross the threshold together).
    """
    scores = np.asarray(scores, dtype=float)
    t = np.asarray(t).astype(int)
    order = np.argsort(-scores, kind="mergesort")
    s, y = scores[order], t[order]
    P, Nn = y.sum(), len(y) - y.sum()
    tps = np.cumsum(y)
    fps = np.cumsum(1 - y)
    keep = np.r_[np.where(np.diff(s) != 0)[0], len(s) - 1]
    fpr = np.r_[0.0, fps[keep] / Nn]
    tpr = np.r_[0.0, tps[keep] / P]
    return fpr, tpr


def auc_trapezoid(fpr, tpr):
    return float(np.trapz(tpr, fpr))


def auc_mann_whitney(scores, t):
    """AUC = P(score of a random positive > score of a random negative) + 1/2 P(tie)."""
    scores = np.asarray(scores, dtype=float)
    t = np.asarray(t).astype(int)
    pos, neg = scores[t == 1], scores[t == 0]
    d = pos[:, None] - neg[None, :]
    return float(np.mean((d > 0) + 0.5 * (d == 0)))


def binormal_roc(fpr, d):
    """ROC of a score that is N(0, 1) in the negative and N(d, 1) in the positive class."""
    return stats.norm.cdf(d + stats.norm.ppf(fpr))


def rebalance_posteriors(post, train_priors, new_priors):
    """Section 5.2.4: divide by the training class fractions, multiply by the new priors, normalise."""
    w = np.asarray(post) * (np.asarray(new_priors) / np.asarray(train_priors))
    return w / w.sum(axis=-1, keepdims=True)


def combine_posteriors(post_a, post_b, priors):
    """(5.27): p(C_k | x_A, x_B) proportional to p(C_k|x_A) p(C_k|x_B) / p(C_k), normalised."""
    w = np.asarray(post_a) * np.asarray(post_b) / np.asarray(priors)
    return w / w.sum(axis=-1, keepdims=True)


# =========================================================================== the 1-D demo problem
def npdf(x, m, s):
    return np.exp(-0.5 * ((x - m) / s) ** 2) / (s * math.sqrt(2 * math.pi))


DEMO_GRID = np.linspace(-6.0, 8.0, 2801)   # the grid of decision-roc-demo.html


def decision_1d(d=2.0, prior2=0.5, bimodal=False, tau=0.5, L12=1.0, L21=1.0, theta=None, grid=DEMO_GRID):
    """The two-class problem of decision-roc-demo.html.

    Class C1 ('normal', negative): N(0, 1), or, with bimodal=True, 0.35 N(-3, 0.6^2) + 0.65 N(0, 1)
    (an extra mode far from the other class, as in Figure 5.8). Class C2 ('cancer', positive):
    N(d, 1). Prior p(C2) = prior2. Decide C2 where p(C2|x) >= tau. L12 is the loss of a false
    positive (true C1, decided C2), L21 of a false negative. With theta, points whose largest
    posterior is <= theta are rejected. All integrals by the trapezoidal rule on `grid`.
    """
    x = grid
    p1 = (0.35 * npdf(x, -3.0, 0.6) + 0.65 * npdf(x, 0.0, 1.0)) if bimodal else npdf(x, 0.0, 1.0)
    p2 = npdf(x, d, 1.0)
    j1, j2 = (1 - prior2) * p1, prior2 * p2
    post2 = j2 / (j1 + j2)
    dec2 = post2 >= tau
    acc = np.ones_like(x, dtype=bool) if theta is None else (np.maximum(post2, 1 - post2) > theta)
    I = lambda f: float(np.trapz(f, x))  # noqa: E731
    fp = I(j1 * (dec2 & acc))
    tp = I(j2 * (dec2 & acc))
    fn = I(j2 * (~dec2 & acc))
    tn = I(j1 * (~dec2 & acc))
    rej = I((j1 + j2) * (~acc))
    # ROC: sweep tau down over the posterior; TPR(tau) = P(post2 >= tau | C2), FPR likewise
    # (theta ignored). Grid points enter in order of decreasing posterior, each carrying its
    # trapezoid weight; the curve is the cumulative sums.
    order = np.argsort(-post2, kind="mergesort")
    tw = np.full_like(x, x[1] - x[0])
    tw[0] = tw[-1] = 0.5 * (x[1] - x[0])
    w1, w2 = p1 * tw, p2 * tw
    tpr = np.r_[0.0, np.cumsum(w2[order])] / w2.sum()
    fpr = np.r_[0.0, np.cumsum(w1[order])] / w1.sum()
    auc = float(np.trapz(tpr, fpr))
    loss = L12 * fp + L21 * fn
    return {"fp": fp, "tp": tp, "fn": fn, "tn": tn, "reject": rej, "error": fp + fn, "loss": loss,
            "tpr": tp / (tp + fn) if tp + fn else float("nan"), "fpr": fp / (fp + tn) if fp + tn else float("nan"),
            "auc": auc, "roc_fpr": fpr, "roc_tpr": tpr, "tau_opt": L12 / (L12 + L21), "x": x, "post2": post2,
            "j1": j1, "j2": j2}


# =========================================================================== 5.3 generative classifiers
def generative_fit(X, labels, K, shared=True):
    """Maximum likelihood for Gaussian class-conditionals (Section 5.3.2, Exercises 5.13, 5.14).

    Returns priors pi_k = N_k / N (5.56), (5.101), means (5.58), (5.103) and covariances: the pooled
    Sigma = sum_k (N_k/N) S_k (5.61), (5.104) if shared, else the per-class S_k (5.62), (5.105).
    """
    X = np.asarray(X, dtype=float)
    labels = np.asarray(labels, dtype=int)
    N, D = X.shape
    pri, mus, Ss = [], [], []
    for k in range(K):
        Xk = X[labels == k]
        Nk = len(Xk)
        mu = Xk.mean(0)
        S = (Xk - mu).T @ (Xk - mu) / Nk
        pri.append(Nk / N)
        mus.append(mu)
        Ss.append(S)
    pri, mus = np.array(pri), np.array(mus)
    if shared:
        Sig = sum(p * S for p, S in zip(pri, Ss))
        covs = np.array([Sig] * K)
    else:
        covs = np.array(Ss)
    return pri, mus, covs


def gaussian_activations(X, priors, means, covs):
    """a_k(x) = ln p(x|C_k) + ln p(C_k) (5.46) for Gaussian class densities (5.47)."""
    return np.stack([common.gauss_logpdf(X, m, S) + math.log(p) for p, m, S in zip(priors, means, covs)], axis=1)


def gaussian_posteriors(X, priors, means, covs):
    """p(C_k | x) = softmax_k(a_k) (5.45)."""
    return softmax(gaussian_activations(np.atleast_2d(X), priors, means, covs), axis=1)


def lda_weights(priors, means, Sigma):
    """w_k = Sigma^-1 mu_k (5.52), w_k0 = -1/2 mu_k^T Sigma^-1 mu_k + ln p(C_k) (5.53)."""
    Si = np.linalg.inv(Sigma)
    W = np.array([Si @ m for m in means])
    w0 = np.array([-0.5 * m @ Si @ m + math.log(p) for p, m in zip(priors, means)])
    return W, w0


def two_class_lda(prior1, mu1, mu2, Sigma):
    """w = Sigma^-1 (mu1 - mu2) (5.49), w0 = -1/2 mu1^T Sigma^-1 mu1 + 1/2 mu2^T Sigma^-1 mu2 + ln(p1/p2) (5.50)."""
    Si = np.linalg.inv(Sigma)
    w = Si @ (mu1 - mu2)
    w0 = -0.5 * mu1 @ Si @ mu1 + 0.5 * mu2 @ Si @ mu2 + math.log(prior1 / (1 - prior1))
    return w, w0


def naive_bayes_fit(X, labels, K, L=None, smooth=0.0):
    """Maximum likelihood for naive Bayes with binary features (Exercise 5.15) or, if L is given,
    features in L states coded 1-of-L (X is N x M with integer states; Exercise 5.17).

    Binary: mu[k, i] = mean of x_i over the points of class k. L states: mu[k, m, l] = fraction of the
    points of class k whose feature m is in state l. `smooth` adds pseudo-counts (0 = pure ML).
    """
    X = np.asarray(X)
    labels = np.asarray(labels, dtype=int)
    pri = np.array([np.mean(labels == k) for k in range(K)])
    if L is None:
        mu = np.array([(X[labels == k].sum(0) + smooth) / (np.sum(labels == k) + 2 * smooth) for k in range(K)])
    else:
        M = X.shape[1]
        mu = np.zeros((K, M, L))
        for k in range(K):
            Xk = X[labels == k]
            for m in range(M):
                c = np.bincount(Xk[:, m], minlength=L) + smooth
                mu[k, m] = c / c.sum()
    return pri, mu


def naive_bayes_activations(X, pri, mu):
    """a_k(x) = sum_i {x_i ln mu_ki + (1 - x_i) ln(1 - mu_ki)} + ln p(C_k) (5.65)."""
    X = np.atleast_2d(X).astype(float)
    return X @ np.log(mu).T + (1 - X) @ np.log1p(-mu).T + np.log(pri)


# =========================================================================== 5.4 discriminative
def cross_entropy(w, Phi, t, alpha=0.0):
    """E(w) (5.74) with y = sigma(Phi w), computed stably, plus alpha/2 ||w||^2."""
    a = Phi @ w
    # -[t ln sigma(a) + (1 - t) ln sigma(-a)] = softplus(a) - t a
    return float(np.sum(np.logaddexp(0.0, a) - t * a) + 0.5 * alpha * w @ w)


def cross_entropy_grad(w, Phi, t, alpha=0.0):
    """grad E = sum_n (y_n - t_n) phi_n (5.75) (+ alpha w)."""
    return Phi.T @ (sigmoid(Phi @ w) - t) + alpha * w


def logistic_hessian(w, Phi, alpha=0.0):
    """H = Phi^T R Phi with R = diag(y_n (1 - y_n)) (+ alpha I): positive semi-definite."""
    y = sigmoid(Phi @ w)
    return Phi.T @ (Phi * (y * (1 - y))[:, None]) + alpha * np.eye(Phi.shape[1])


def logistic_irls(Phi, t, alpha=0.0, iters=100, tol=1e-12):
    """Newton-Raphson (iterative reweighted least squares) for (5.74); returns w and the path."""
    w = np.zeros(Phi.shape[1])
    path = [w.copy()]
    for _ in range(iters):
        step = np.linalg.solve(logistic_hessian(w, Phi, alpha), cross_entropy_grad(w, Phi, t, alpha))
        w = w - step
        path.append(w.copy())
        if np.max(np.abs(step)) < tol:
            break
    return w, np.array(path)


def logistic_gd(Phi, t, lr, steps, alpha=0.0, w0=None):
    """Batch gradient descent on the MEAN cross-entropy E(w)/N + alpha/2 ||w||^2 (the demo's optimiser).

    Returns the final w and the list of (mean loss, ||w||) after 0, 1, ..., steps updates.
    """
    N = len(t)
    w = np.zeros(Phi.shape[1]) if w0 is None else np.array(w0, dtype=float)
    hist = []
    for k in range(steps + 1):
        a = Phi @ w
        y = sigmoid(a)
        loss = float(np.mean(np.logaddexp(0.0, a) - t * a) + 0.5 * alpha * w @ w)
        hist.append((loss, float(np.sqrt(w @ w))))
        if k == steps:
            break
        g = Phi.T @ (y - t) / N + alpha * w
        w = w - lr * g
    return w, hist


def softmax_cross_entropy(Wm, Phi, T):
    """E(w_1..w_K) = -sum_n sum_k t_nk ln y_nk (5.80); Wm is M x K."""
    A = Phi @ Wm
    return float(-np.sum(T * (A - common.logsumexp(A, axis=1, keepdims=True))))


def softmax_grad(Wm, Phi, T):
    """grad_{w_j} E = sum_n (y_nj - t_nj) phi_n (5.81), all j at once: Phi^T (Y - T)."""
    return Phi.T @ (softmax(Phi @ Wm, axis=1) - T)


def softmax_gd(Phi, T, lr, steps, alpha=0.0):
    """Batch gradient descent on the mean multiclass cross-entropy from W = 0 (the demo's optimiser)."""
    N = len(T)
    Wm = np.zeros((Phi.shape[1], T.shape[1]))
    hist = []
    for k in range(steps + 1):
        A = Phi @ Wm
        loss = float(-np.sum(T * (A - common.logsumexp(A, axis=1, keepdims=True))) / N + 0.5 * alpha * np.sum(Wm * Wm))
        hist.append((loss, float(np.sqrt(np.sum(Wm * Wm)))))
        if k == steps:
            break
        Wm = Wm - lr * (Phi.T @ (softmax(A, axis=1) - T) / N + alpha * Wm)
    return Wm, hist


def softmax_fit(Phi, T, alpha=1e-6):
    """Minimise (5.80) + alpha/2 ||W||^2 with L-BFGS (the tiny alpha removes the shift symmetry)."""
    M, K = Phi.shape[1], T.shape[1]

    def f(v):
        Wm = v.reshape(M, K)
        return softmax_cross_entropy(Wm, Phi, T) + 0.5 * alpha * v @ v, (softmax_grad(Wm, Phi, T)).ravel() + alpha * v
    r = optimize.minimize(f, np.zeros(M * K), jac=True, method="L-BFGS-B", options={"maxiter": 5000, "gtol": 1e-10})
    return r.x.reshape(M, K)


def probit_nll(w, Phi, t):
    a = Phi @ w
    return float(-np.sum(t * stats.norm.logcdf(a) + (1 - t) * stats.norm.logcdf(-a)))


def probit_grad(w, Phi, t):
    """grad = -sum_n [t_n phi(a)/Phi(a) - (1 - t_n) phi(a)/Phi(-a)] phi_n, using stable Mills ratios."""
    a = Phi @ w
    r_pos = np.exp(stats.norm.logpdf(a) - stats.norm.logcdf(a))
    r_neg = np.exp(stats.norm.logpdf(a) - stats.norm.logcdf(-a))
    return -Phi.T @ (t * r_pos - (1 - t) * r_neg)


def probit_fit(Phi, t, alpha=0.0):
    f = lambda w: (probit_nll(w, Phi, t) + 0.5 * alpha * w @ w, probit_grad(w, Phi, t) + alpha * w)  # noqa: E731
    r = optimize.minimize(f, np.zeros(Phi.shape[1]), jac=True, method="L-BFGS-B", options={"maxiter": 5000, "gtol": 1e-10})
    return r.x


def gaussian_basis(X, centres, s2):
    """phi_j(x) = exp(-||x - mu_j||^2 / (2 s^2)) (Figure 5.15)."""
    X = np.atleast_2d(X)
    return np.stack([np.exp(-np.sum((X - c) ** 2, axis=1) / (2 * s2)) for c in centres], axis=1)


# =========================================================================== data sets
def data_fig_5_4(seed=1):
    """Two classes as in Figure 5.4 (red crosses t = 1, blue circles t = 0) and 10 extra blue points."""
    rng = np.random.default_rng(seed)
    C = np.array([[0.9, 0.45], [0.45, 0.9]])
    Lc = np.linalg.cholesky(C)
    red = np.array([-0.7, 1.1]) + rng.normal(size=(50, 2)) @ Lc.T
    blue = np.array([0.9, -0.8]) + rng.normal(size=(50, 2)) @ Lc.T
    extra = np.array([7.4, -6.6]) + 0.35 * rng.normal(size=(10, 2))
    X = np.vstack([red, blue])
    t = np.r_[np.ones(50), np.zeros(50)]
    return X, t, extra


def data_fig_5_15(seed=2):
    rng = np.random.default_rng(seed)
    red = 0.25 * rng.normal(size=(40, 2))
    blue = np.vstack([np.array([-1.0, -1.0]) + 0.15 * rng.normal(size=(20, 2)),
                      np.array([0.75, 1.0]) + 0.2 * rng.normal(size=(20, 2))])
    X = np.vstack([red, blue])
    t = np.r_[np.ones(40), np.zeros(40)]
    return X, t


FIG15_CENTRES = np.array([[-1.0, -1.0], [0.0, 0.0]])
FIG15_S2 = 0.5
FIG15_ALPHA = 0.01


def data_three_class(seed=5, n=30):
    """Three Gaussian classes for the softmax demo."""
    rng = np.random.default_rng(seed)
    means = np.array([[-1.5, -0.8], [1.5, -0.6], [0.0, 1.6]])
    X = np.vstack([m + 0.75 * rng.normal(size=(n, 2)) for m in means])
    lab = np.repeat(np.arange(3), n)
    return X, lab


def data_separable(seed=6, n=15):
    rng = np.random.default_rng(seed)
    a = np.array([-1.2, 0.8]) + 0.45 * rng.normal(size=(n, 2))
    b = np.array([1.2, -0.8]) + 0.45 * rng.normal(size=(n, 2))
    return np.vstack([a, b]), np.r_[np.ones(n), np.zeros(n)]


def data_mislabel(seed=8, n=30):
    """Two overlapping classes plus one red point placed far on the blue side (a labelling error)."""
    rng = np.random.default_rng(seed)
    a = np.array([-1.0, 0.0]) + 0.8 * rng.normal(size=(n, 2))
    b = np.array([1.0, 0.0]) + 0.8 * rng.normal(size=(n, 2))
    X = np.vstack([a, b, [[10.0, -0.5]]])
    t = np.r_[np.ones(n), np.zeros(n), 1.0]
    return X, t


# Generative demo parameters (generative-classifier-demo.html, also used by Figures 5.13, 5.14)
GEN2 = {"means": np.array([[-0.7, 0.4], [0.7, -0.4]]), "Sigma": np.array([[1.0, 0.45], [0.45, 0.8]])}
GEN3 = {"means": np.array([[-1.3, -1.0], [-0.6, 1.6], [1.6, -0.2]]),
        "covs": np.array([[[0.55, 0.15], [0.15, 0.35]], [[0.55, 0.15], [0.15, 0.35]], [[0.25, -0.05], [-0.05, 0.55]]])}


def generative_sample(seed=9, n=20):
    """Samples for the 'fit by maximum likelihood' view of the generative demo: n points from each
    class of GEN3 (classes 0 and 1 share their covariance)."""
    rng = np.random.default_rng(seed)
    out, lab = [], []
    for k in range(3):
        Lc = np.linalg.cholesky(GEN3["covs"][k])
        out.append(GEN3["means"][k] + rng.normal(size=(n, 2)) @ Lc.T)
        lab += [k] * n
    return np.vstack(out), np.array(lab)


def export_data(out_dir=None):
    """Write the data sets of the browser demos to ../data/ch05-*.json."""
    out = out_dir or DATA_OUT
    os.makedirs(out, exist_ok=True)
    X4, t4, extra = data_fig_5_4()
    X15, t15 = data_fig_5_15()
    X3, l3 = data_three_class()
    Xs, ts = data_separable()
    Xm, tm = data_mislabel()
    r = lambda A: np.round(np.asarray(A, dtype=float), 6).tolist()  # noqa: E731
    d = {
        "note": "Data for logistic-softmax-demo.html, generated by ch05_single_layer_classification.py "
                "(Bishop and Bishop 2024, Chapter 5). Labels: t = 1 red, t = 0 blue; three-class labels 0, 1, 2.",
        "fig5_4": {"X": r(X4), "t": t4.astype(int).tolist(), "extra": r(extra)},
        "fig5_15": {"X": r(X15), "t": t15.astype(int).tolist(), "centres": FIG15_CENTRES.tolist(), "s2": FIG15_S2},
        "three": {"X": r(X3), "label": l3.tolist()},
        "separable": {"X": r(Xs), "t": ts.astype(int).tolist()},
        "mislabel": {"X": r(Xm), "t": tm.astype(int).tolist()},
    }
    p1 = os.path.join(out, "ch05-classifier-data.json")
    with open(p1, "w") as f:
        json.dump(d, f, separators=(",", ":"))
    Xg, lg = generative_sample()
    g = {"note": "Gaussian class-conditional parameters and samples for generative-classifier-demo.html "
                 "(Bishop and Bishop 2024, Section 5.3), generated by ch05_single_layer_classification.py.",
         "two": {"means": GEN2["means"].tolist(), "Sigma": GEN2["Sigma"].tolist()},
         "three": {"means": GEN3["means"].tolist(), "covs": GEN3["covs"].tolist()},
         "sample": {"X": r(Xg), "label": lg.tolist()}}
    p2 = os.path.join(out, "ch05-generative.json")
    with open(p2, "w") as f:
        json.dump(g, f, separators=(",", ":"))
    for p in (p1, p2):
        print("wrote", os.path.relpath(p, os.getcwd()), "(%.1f kB)" % (os.path.getsize(p) / 1024))


# =========================================================================== figures
def _square(ax, lim):
    ax.set_xlim(*lim[:2])
    ax.set_ylim(*lim[2:])
    ax.set_aspect("equal")


def fig_5_1(out=None):
    """Geometry of a two-class linear discriminant: w, the bias, the signed distance y(x)/||w||."""
    w = np.array([1.0, 1.6])
    w0 = -2.2
    nw = np.linalg.norm(w)
    u = w / nw
    fig, ax = plt.subplots(figsize=(4.6, 4.0))
    lim = (-0.5, 4.0, -0.5, 3.6)
    xx = np.linspace(lim[0], lim[1], 400)
    gx, gy = np.meshgrid(np.linspace(lim[0], lim[1], 200), np.linspace(lim[2], lim[3], 200))
    Y = w[0] * gx + w[1] * gy + w0
    ax.contourf(gx, gy, Y, levels=[-100, 0, 100], colors=["#dbe6f6", "#f8dcdc"], alpha=0.8)
    ax.plot(xx, -(w[0] * xx + w0) / w[1], color=RED, lw=2)
    foot = -w0 / nw * u            # closest point of the surface to the origin, (5.3)
    ax.annotate("", xy=foot, xytext=(0, 0), arrowprops=dict(arrowstyle="<->", color=GREY, lw=1.2))
    ax.text(foot[0] / 2 - 0.95, foot[1] / 2 + 0.2, r"$-w_0/\|\mathbf{w}\|$", fontsize=10)
    x = np.array([2.6, 2.4])
    y = w @ x + w0
    xp = x - y / nw * u            # orthogonal projection, (5.4)
    ax.annotate("", xy=xp + 0.9 * u, xytext=xp, arrowprops=dict(arrowstyle="-|>", color="k", lw=1.6))
    ax.text(*(xp + 0.9 * u + np.array([-0.3, 0.0])), r"$\mathbf{w}$", fontsize=11)
    ax.plot(*x, "ko", ms=4)
    ax.text(x[0] + 0.08, x[1] + 0.05, r"$\mathbf{x}$", fontsize=11)
    ax.plot(*xp, "ko", ms=4)
    ax.text(xp[0] + 0.1, xp[1] - 0.3, r"$\mathbf{x}_\perp$", fontsize=11)
    ax.plot([xp[0], x[0]], [xp[1], x[1]], "k--", lw=1)
    mid = (x + xp) / 2
    ax.text(mid[0] + 0.12, mid[1] - 0.3, r"$y(\mathbf{x})/\|\mathbf{w}\|$", fontsize=10)
    ax.text(0.2, 3.1, r"$\mathcal{R}_1$: $y>0$", fontsize=10)
    ax.text(-0.4, -0.35, r"$\mathcal{R}_2$: $y<0$", fontsize=10)
    ax.text(3.2, -0.2, r"$y=0$", color=RED, fontsize=10)
    ax.axhline(0, color=GREY, lw=0.6)
    ax.axvline(0, color=GREY, lw=0.6)
    _square(ax, lim)
    ax.set_xlabel(r"$x_1$")
    ax.set_ylabel(r"$x_2$")
    savefig(fig, CH, "fig5-1", out)


def _shade_labels(ax, gx, gy, lab, colours):
    for k, c in colours.items():
        ax.contourf(gx, gy, (lab == k).astype(float), levels=[0.5, 1.5], colors=[c], alpha=0.85)


def fig_5_2(out=None):
    """One-versus-the-rest (left) and one-versus-one (right) discriminants leave ambiguous regions."""
    fig, axes = plt.subplots(1, 2, figsize=(8.2, 3.9))
    gx, gy = np.meshgrid(np.linspace(-3, 3, 500), np.linspace(-3, 3, 500))
    # left: two one-vs-rest classifiers y1 = 'C1 vs not C1', y2 = 'C2 vs not C2'
    y1 = -gx + 0.3 * gy - 0.2       # > 0 means 'C1'
    y2 = gx + 0.5 * gy - 0.4        # > 0 means 'C2'
    lab = np.where((y1 > 0) & (y2 > 0), 3, np.where(y1 > 0, 0, np.where(y2 > 0, 1, 2)))
    ax = axes[0]
    _shade_labels(ax, gx, gy, lab, {0: "#f6d5d5", 1: "#d6e3f5", 2: "#e9e9e9", 3: "#a6dba0"})
    ax.contour(gx, gy, y1, levels=[0], colors="k", linewidths=1.4)
    ax.contour(gx, gy, y2, levels=[0], colors="k", linewidths=1.4)
    for (px, py, s) in [(-2.2, -1.6, r"$\mathcal{R}_1$"), (2.0, 1.6, r"$\mathcal{R}_2$"), (1.5, -2.3, r"$\mathcal{R}_3$"), (-0.6, 2.2, "?")]:
        ax.text(px, py, s, fontsize=13, ha="center")
    ax.text(-2.85, 2.55, r"$C_1$ | not $C_1$", fontsize=8.5)
    ax.text(0.75, -0.65, r"$C_2$ | not $C_2$", fontsize=8.5, rotation=-63)
    ax.set_title("one-versus-the-rest")
    # right: three one-vs-one classifiers, majority vote
    ang = np.deg2rad([90, 210, 330])
    cen = np.stack([np.cos(ang), np.sin(ang)], 1) * 1.6
    votes = np.zeros(gx.shape + (3,))
    lines = []
    # pairwise lines s_ij = n_ij . x - delta_ij |n_ij| with n_ij = c_i - c_j; positive means 'C_i'.
    # Offsets with a cyclic pattern (1 beats 0, 2 beats 1, 0 beats 2 near the origin) leave a
    # triangle in which every class receives exactly one vote.
    delta = {(0, 1): 0.45, (1, 2): 0.45, (0, 2): -0.45}
    for (i, j), dl in delta.items():
        n = cen[i] - cen[j]
        s = n[0] * gx + n[1] * gy - dl * np.linalg.norm(n)
        votes[..., i] += s > 0
        votes[..., j] += s <= 0
        lines.append(s)
    mx = votes.max(-1)
    tie = (votes == mx[..., None]).sum(-1) > 1
    lab2 = np.where(tie, 3, votes.argmax(-1))
    ax = axes[1]
    _shade_labels(ax, gx, gy, lab2, {0: "#f6d5d5", 1: "#d6e3f5", 2: "#e9e9e9", 3: "#a6dba0"})
    for s in lines:
        ax.contour(gx, gy, s, levels=[0], colors="k", linewidths=1.4)
    for k, s in enumerate([r"$\mathcal{R}_1$", r"$\mathcal{R}_2$", r"$\mathcal{R}_3$"]):
        ax.text(*(cen[k] * 1.25), s, fontsize=13, ha="center", va="center")
    ax.text(0, 0, "?", fontsize=13, ha="center", va="center")
    ax.set_title("one-versus-one (majority vote)")
    for ax in axes:
        _square(ax, (-3, 3, -3, 3))
        ax.set_xticks([])
        ax.set_yticks([])
    savefig(fig, CH, "fig5-2", out)


def fig_5_3(out=None):
    """A single K-class linear discriminant: convex decision regions, and the segment argument (5.9)-(5.10)."""
    W = np.array([[-1.0, 0.4], [1.0, 0.6], [0.1, -1.2]])
    b = np.array([0.0, 0.1, 0.2])
    gx, gy = np.meshgrid(np.linspace(-3, 3, 500), np.linspace(-3, 3, 500))
    Y = np.stack([W[k, 0] * gx + W[k, 1] * gy + b[k] for k in range(3)], -1)
    lab = Y.argmax(-1)
    fig, ax = plt.subplots(figsize=(4.0, 3.9))
    _shade_labels(ax, gx, gy, lab, {0: "#e9e9e9", 1: "#d6e3f5", 2: "#f6d5d5"})
    ax.contour(gx, gy, lab, levels=[0.5, 1.5], colors=RED, linewidths=1.6)
    xa, xb = np.array([0.4, -1.8]), np.array([2.2, -0.9])
    lam = 0.45
    xh = lam * xa + (1 - lam) * xb
    ax.plot([xa[0], xb[0]], [xa[1], xb[1]], "k-", lw=1.2)
    for p, s in [(xa, r"$\mathbf{x}_A$"), (xb, r"$\mathbf{x}_B$"), (xh, r"$\widehat{\mathbf{x}}$")]:
        ax.plot(*p, "ko", ms=4)
        ax.text(p[0] + 0.1, p[1] + 0.12, s, fontsize=11)
    ax.text(-2.0, 1.0, r"$\mathcal{R}_i$", fontsize=13)
    ax.text(1.8, 1.6, r"$\mathcal{R}_j$", fontsize=13)
    ax.text(0.3, -2.6, r"$\mathcal{R}_k$", fontsize=13)
    _square(ax, (-3, 3, -3, 3))
    ax.set_xticks([])
    ax.set_yticks([])
    savefig(fig, CH, "fig5-3", out)


def _boundary(ax, w, lim, **kw):
    """Plot w0 + w1 x1 + w2 x2 = 0 (w = (w0, w1, w2)) across the x1 range."""
    xx = np.linspace(lim[0], lim[1], 200)
    ax.plot(xx, -(w[0] + w[1] * xx) / w[2], **kw)


def fig_5_4(out=None):
    """Least squares (magenta) is pulled by distant, correctly classified points; logistic regression (green) is not."""
    X, t, extra = data_fig_5_4()
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 4.1))
    lim = (-4, 9, -9, 4.5)
    for ax, add in zip(axes, (False, True)):
        Xa = np.vstack([X, extra]) if add else X
        ta = np.r_[t, np.zeros(len(extra))] if add else t
        Wls = least_squares_classifier(Xa, one_hot(ta.astype(int), 2))
        wls = Wls[:, 1] - Wls[:, 0]       # boundary y_1 = y_0
        wlr, _ = logistic_irls(augment(Xa), ta)
        r = ta == 1
        ax.plot(Xa[r, 0], Xa[r, 1], "x", color=RED, ms=5, mew=1.3)
        ax.plot(Xa[~r, 0], Xa[~r, 1], "o", mfc="none", mec=BLUE, ms=5, mew=1.1)
        _boundary(ax, wls, lim, color=MAGENTA, lw=1.6, label="least squares")
        _boundary(ax, wlr, lim, color=GREEN, lw=1.6, label="logistic regression")
        ax.set_xlim(lim[0], lim[1])
        ax.set_ylim(lim[2], lim[3])
        ax.set_aspect("equal")
    axes[0].legend(loc="lower right", fontsize=8.5)
    savefig(fig, CH, "fig5-4", out)


# =========================================================================== 1-D decision-theory figures
def joints_5_5(x):
    j1 = 0.42 * npdf(x, -1.2, 0.55) + 0.23 * npdf(x, 1.5, 0.6)
    j2 = 0.35 * npdf(x, 1.7, 0.7)
    return j1, j2


def crossing_x0():
    f = lambda x: np.subtract(*joints_5_5(np.array([x]))).item()  # noqa: E731
    return optimize.brentq(f, 0.0, 1.6)


def _regions_panel(ax, xhat, x0, labels=False):
    x = np.linspace(-3.2, 4.6, 2000)
    j1, j2 = joints_5_5(x)
    lo = np.minimum(j1, j2)
    L = x < xhat
    ax.fill_between(x, 0, lo, where=L, color="#7cc47c", lw=0, interpolate=True)                     # green
    ax.fill_between(x, lo, j2, where=L & (j2 > j1), color="#f08080", lw=0, interpolate=True)        # red
    ax.fill_between(x, 0, j1, where=~L, color="#7f7ff0", lw=0, interpolate=True)                    # blue
    ax.plot(x, j1, "k", lw=1.2)
    ax.plot(x, j2, "k", lw=1.2)
    ax.axvline(xhat, color="k", lw=1.2)
    if abs(xhat - x0) > 1e-6:
        ax.axvline(x0, color="k", lw=1, ls="--")
        ax.annotate("", xy=(xhat + 0.35, 0.3), xytext=(xhat - 0.35, 0.3),
                    arrowprops=dict(arrowstyle="<->", color=RED, lw=1.6))
    ax.set_ylim(0, 0.34)
    ax.set_xlim(x[0], x[-1])
    ytop = 0.325
    ax.annotate("", xy=(x[0], ytop), xytext=(xhat, ytop), arrowprops=dict(arrowstyle="<->", lw=0.9))
    ax.annotate("", xy=(xhat, ytop), xytext=(x[-1], ytop), arrowprops=dict(arrowstyle="<->", lw=0.9))
    ax.text((x[0] + xhat) / 2, ytop + 0.006, r"$\mathcal{R}_1$", ha="center", fontsize=10)
    ax.text((xhat + x[-1]) / 2, ytop + 0.006, r"$\mathcal{R}_2$", ha="center", fontsize=10)
    ax.text(-1.9, 0.27, r"$p(x,\mathcal{C}_1)$", fontsize=10)
    ax.text(2.5, 0.2, r"$p(x,\mathcal{C}_2)$", fontsize=10)
    ticks, tl = [xhat], [r"$\widehat{x}$"]
    if abs(xhat - x0) > 1e-6:
        ticks, tl = [x0, xhat], [r"$x_0$", r"$\widehat{x}$"]
    ax.set_xticks(ticks)
    ax.set_xticklabels(tl)
    ax.set_yticks([])
    if labels:
        ax.text(-1.2, 0.12, "A", fontsize=11, ha="center")
        ax.text(0.5 * (x0 + xhat), 0.155, "B", fontsize=11, ha="center")
        ax.text(0.5 * (x0 + xhat) - 0.1, 0.05, "C", fontsize=11, ha="center")
        ax.text(xhat + 0.35, 0.17, "D", fontsize=11, ha="center")
        ax.text(xhat + 0.45, 0.04, "E", fontsize=11, ha="center")


def fig_5_5(out=None):
    """Joint densities p(x, C_k), a decision boundary x-hat and the three error regions; optimum at x0."""
    x0 = crossing_x0()
    fig, axes = plt.subplots(2, 1, figsize=(6.0, 5.6))
    _regions_panel(axes[0], x0 + 0.75, x0)
    _regions_panel(axes[1], x0, x0)
    common.panel_label(axes[0], "a", "upper right")
    common.panel_label(axes[1], "b", "upper right")
    fig.subplots_adjust(hspace=0.35)
    savefig(fig, CH, "fig5-5", out)


def fig_5_10(out=None):
    """As Figure 5.5(a) with the regions labelled A-E used in (5.34)-(5.37)."""
    x0 = crossing_x0()
    fig, ax = plt.subplots(figsize=(6.0, 2.9))
    _regions_panel(ax, x0 + 0.75, x0, labels=True)
    savefig(fig, CH, "fig5-10", out)


def fig_5_7(out=None):
    """The reject option: reject where the larger posterior is at most theta."""
    x = np.linspace(-4, 4, 800)
    p1 = sigmoid(-2 * x)       # N(-1,1) vs N(1,1), equal priors: a = ln N(x|-1)/N(x|1) = -2x
    theta = 0.85
    fig, ax = plt.subplots(figsize=(5.4, 3.0))
    ax.plot(x, p1, color=RED, lw=1.8)
    ax.plot(x, 1 - p1, color=BLUE, lw=1.8)
    ax.axhline(theta, color=GREY, lw=1, ls="--")
    xr = 0.5 * math.log(theta / (1 - theta))   # |x| < xr is rejected
    ax.axvspan(-xr, xr, color="#a6dba0", alpha=0.5, lw=0)
    ax.text(0, 0.5, "reject region", ha="center", fontsize=9.5)
    ax.text(-3.6, 1.04, r"$p(\mathcal{C}_1|x)$", color=RED, fontsize=10)
    ax.text(2.4, 1.04, r"$p(\mathcal{C}_2|x)$", color=BLUE, fontsize=10)
    ax.set_yticks([0, theta, 1])
    ax.set_yticklabels(["0.0", r"$\theta$", "1.0"])
    ax.set_ylim(-0.02, 1.15)
    ax.set_xlabel(r"$x$")
    savefig(fig, CH, "fig5-7", out)


def fig_5_8(out=None):
    """Class densities with a mode that does not influence the posterior (left) and the posteriors (right)."""
    x = np.linspace(0, 1, 800)
    c1 = 0.4 * npdf(x, 0.2, 0.1) + 0.6 * npdf(x, 0.5, 0.12)
    c2 = npdf(x, 0.7, 0.12)
    post1 = c1 / (c1 + c2)
    xb = optimize.brentq(lambda z: (0.4 * npdf(z, 0.2, 0.1) + 0.6 * npdf(z, 0.5, 0.12)) - npdf(z, 0.7, 0.12), 0.5, 0.7)
    fig, axes = plt.subplots(1, 2, figsize=(8.0, 3.2))
    ax = axes[0]
    ax.plot(x, c1, color=BLUE)
    ax.plot(x, c2, color=RED)
    ax.text(0.08, 1.45, r"$p(x|\mathcal{C}_1)$", color=BLUE, fontsize=10)
    ax.text(0.62, 3.5, r"$p(x|\mathcal{C}_2)$", color=RED, fontsize=10)
    ax.set_ylim(0, 5)
    ax.set_ylabel("class densities")
    ax = axes[1]
    ax.plot(x, post1, color=BLUE)
    ax.plot(x, 1 - post1, color=RED)
    ax.plot([xb, xb], [0, 1], color=GREEN, lw=1.5)
    ax.text(0.05, 1.05, r"$p(\mathcal{C}_1|x)$", color=BLUE, fontsize=10)
    ax.text(0.75, 1.05, r"$p(\mathcal{C}_2|x)$", color=RED, fontsize=10)
    ax.set_ylim(0, 1.2)
    for ax in axes:
        ax.set_xlim(0, 1)
        ax.set_xlabel(r"$x$")
    savefig(fig, CH, "fig5-8", out)


def fig_5_11(out=None):
    """ROC curves of two classifiers (binormal scores, d = 2.0 and 0.9) and of random guessing."""
    f = np.linspace(0, 1, 400)
    fig, ax = plt.subplots(figsize=(3.8, 3.7))
    ax.plot(f, binormal_roc(f, 2.0), color=BLUE, lw=1.8, label="AUC = %.3f" % stats.norm.cdf(2.0 / math.sqrt(2)))
    ax.plot(f, binormal_roc(f, 0.9), color=RED, lw=1.8, label="AUC = %.3f" % stats.norm.cdf(0.9 / math.sqrt(2)))
    ax.plot([0, 1], [0, 1], "k--", lw=1, label="random, AUC = 0.5")
    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)
    ax.set_xticks([0, 1])
    ax.set_yticks([0, 1])
    ax.set_aspect("equal")
    ax.set_xlabel("False positive rate")
    ax.set_ylabel("True positive rate")
    ax.legend(loc="lower right", fontsize=8)
    savefig(fig, CH, "fig5-11", out)


def fig_5_12(out=None):
    """The logistic sigmoid and the scaled probit Phi(lambda a), lambda^2 = pi/8."""
    a = np.linspace(-5, 5, 500)
    fig, ax = plt.subplots(figsize=(4.0, 2.9))
    ax.plot(a, sigmoid(a), color=RED, lw=1.8, label=r"$\sigma(a)$")
    ax.plot(a, probit(LAMBDA_PROBIT * a), color=BLUE, lw=1.8, ls="--", label=r"$\Phi(\lambda a)$, $\lambda^2=\pi/8$")
    ax.set_xlim(-5, 5)
    ax.set_yticks([0, 0.5, 1])
    ax.set_xlabel(r"$a$")
    ax.legend(loc="upper left", fontsize=8.5)
    savefig(fig, CH, "fig5-12", out)


def fig_5_13(out=None):
    """Two Gaussian class densities with a shared covariance (left) and the posterior p(C1|x) = sigma(w^T x + w0) (right)."""
    mu, S = GEN2["means"], GEN2["Sigma"]
    g = np.linspace(-3.2, 3.2, 46)
    gx, gy = np.meshgrid(g, g)
    P = np.stack([gx.ravel(), gy.ravel()], 1)
    d1 = np.exp(common.gauss_logpdf(P, mu[0], S)).reshape(gx.shape)
    d2 = np.exp(common.gauss_logpdf(P, mu[1], S)).reshape(gx.shape)
    w, w0 = two_class_lda(0.5, mu[0], mu[1], S)
    post = sigmoid(P @ w + w0).reshape(gx.shape)
    fig = plt.figure(figsize=(8.6, 3.8))
    ax = fig.add_subplot(1, 2, 1, projection="3d")
    ax.plot_surface(gx, gy, d1, color=RED, alpha=0.55, linewidth=0, rstride=1, cstride=1, shade=True)
    ax.plot_surface(gx, gy, d2, color=BLUE, alpha=0.55, linewidth=0, rstride=1, cstride=1, shade=True)
    ax2 = fig.add_subplot(1, 2, 2, projection="3d")
    cols = np.empty(post.shape + (4,))
    r, b = np.array(plt.matplotlib.colors.to_rgb(RED)), np.array(plt.matplotlib.colors.to_rgb(BLUE))
    cols[..., :3] = post[..., None] * r + (1 - post[..., None]) * b
    cols[..., 3] = 1.0
    ax2.plot_surface(gx, gy, post, facecolors=cols, linewidth=0, rstride=1, cstride=1, shade=False)
    ax2.set_zticks([0, 1])
    for a_ in (ax, ax2):
        a_.set_xlabel(r"$x_1$", labelpad=-10)
        a_.set_ylabel(r"$x_2$", labelpad=-10)
        a_.set_xticks([])
        a_.set_yticks([])
        a_.view_init(elev=24, azim=-115)
        a_.xaxis.pane.set_facecolor("#f2f2f2")
        a_.yaxis.pane.set_facecolor("#f2f2f2")
        a_.zaxis.pane.set_facecolor("#f2f2f2")
    ax.set_zticks([])
    savefig(fig, CH, "fig5-13", out, formats=("png", "svg", "pdf"))


def fig_5_14(out=None):
    """Three Gaussian classes; red and blue share a covariance (linear boundary), green has its own (quadratic)."""
    means, covs = GEN3["means"], GEN3["covs"]
    pri = np.ones(3) / 3
    g = np.linspace(-3.0, 3.2, 320)
    gx, gy = np.meshgrid(g, g)
    P = np.stack([gx.ravel(), gy.ravel()], 1)
    post = gaussian_posteriors(P, pri, means, covs).reshape(gx.shape + (3,))
    cols = np.array([plt.matplotlib.colors.to_rgb(c) for c in (RED, BLUE, GREEN)])
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 4.0))
    ax = axes[0]
    for k, c in enumerate((RED, BLUE, GREEN)):
        dk = np.exp(common.gauss_logpdf(P, means[k], covs[k])).reshape(gx.shape)
        m = dk.max()
        ax.contour(gx, gy, dk, levels=m * np.array([0.08, 0.2, 0.4, 0.6, 0.85]), colors=[c], linewidths=1.1)
    ax2 = axes[1]
    img = np.tensordot(post, cols, axes=([2], [0]))
    ax2.imshow(img, origin="lower", extent=(g[0], g[-1], g[0], g[-1]), interpolation="bilinear")
    lab = post.argmax(-1)
    ax2.contour(gx, gy, lab, levels=[0.5, 1.5], colors="white", linewidths=1.3)
    for a_ in axes:
        a_.set_xlim(g[0], g[-1])
        a_.set_ylim(g[0], g[-1])
        a_.set_aspect("equal")
        a_.set_xticks([])
        a_.set_yticks([])
        a_.set_xlabel(r"$x_1$")
        a_.set_ylabel(r"$x_2$")
    savefig(fig, CH, "fig5-14", out)


def fit_fig_5_15():
    X, t = data_fig_5_15()
    F = gaussian_basis(X, FIG15_CENTRES, FIG15_S2)
    w, _ = logistic_irls(augment(F), t, alpha=FIG15_ALPHA)
    return X, t, F, w


def fig_5_15(out=None):
    """Gaussian basis functions turn a non-linear problem in x into a linear one in (phi1, phi2)."""
    X, t, F, w = fit_fig_5_15()
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 4.0))
    ax = axes[0]
    r = t == 1
    ax.plot(X[r, 0], X[r, 1], ".", color=RED, ms=6)
    ax.plot(X[~r, 0], X[~r, 1], ".", color=BLUE, ms=6)
    th = np.linspace(0, 2 * np.pi, 300)
    for c in FIG15_CENTRES:
        ax.plot(c[0], c[1], "+", color=GREEN, ms=10, mew=2)
        for lev in (0.5, 0.1):   # phi = lev  <=>  radius sqrt(-2 s^2 ln lev)
            rad = math.sqrt(-2 * FIG15_S2 * math.log(lev))
            ax.plot(c[0] + rad * np.cos(th), c[1] + rad * np.sin(th), color=GREEN, lw=1)
    g = np.linspace(-1.5, 1.5, 300)
    gx, gy = np.meshgrid(g, g)
    Fg = gaussian_basis(np.stack([gx.ravel(), gy.ravel()], 1), FIG15_CENTRES, FIG15_S2)
    ag = (augment(Fg) @ w).reshape(gx.shape)
    ax.contour(gx, gy, ag, levels=[0], colors="k", linewidths=1.4)
    ax.set_xlim(-1.5, 1.5)
    ax.set_ylim(-1.5, 1.5)
    ax.set_aspect("equal")
    ax.set_xlabel(r"$x_1$")
    ax.set_ylabel(r"$x_2$")
    ax = axes[1]
    ax.plot(F[r, 0], F[r, 1], ".", color=RED, ms=6)
    ax.plot(F[~r, 0], F[~r, 1], ".", color=BLUE, ms=6)
    ff = np.linspace(0, 1, 50)
    ax.plot(ff, -(w[0] + w[1] * ff) / w[2], "k", lw=1.4)
    ax.set_xlim(-0.05, 1.05)
    ax.set_ylim(-0.05, 1.05)
    ax.set_aspect("equal")
    ax.set_xlabel(r"$\phi_1$")
    ax.set_ylabel(r"$\phi_2$")
    savefig(fig, CH, "fig5-15", out)


def fig_5_17(out=None):
    """A threshold density p(theta) (a two-Gaussian mixture) and its distribution function f(a)."""
    x = np.linspace(0, 4, 800)
    p = 0.7 * npdf(x, 1.5, 0.4) + 0.3 * npdf(x, 3.1, 0.32)
    F = 0.7 * stats.norm.cdf(x, 1.5, 0.4) + 0.3 * stats.norm.cdf(x, 3.1, 0.32)
    a = 2.8
    fig, ax = plt.subplots(figsize=(4.6, 3.0))
    ax.fill_between(x, 0, p, where=x <= a, color="#c8f0c8", lw=0)
    ax.plot(x, p, color=BLUE, lw=1.6, label=r"$p(\theta)$")
    ax.plot(x, F, color=RED, lw=1.6, label=r"$f(a)$")
    ax.axvline(a, color=GREEN, lw=1)
    ax.set_xlim(0, 4)
    ax.set_ylim(0, 1.02)
    ax.legend(loc="upper left", fontsize=8.5)
    savefig(fig, CH, "fig5-17", out)


def make_data(out=None):
    """Data sets for the browser demos (JSON)."""
    export_data()


FIGURES = {
    "5.1": fig_5_1, "5.2": fig_5_2, "5.3": fig_5_3, "5.4": fig_5_4, "5.5": fig_5_5, "5.7": fig_5_7,
    "5.8": fig_5_8, "5.10": fig_5_10, "5.11": fig_5_11, "5.12": fig_5_12, "5.13": fig_5_13,
    "5.14": fig_5_14, "5.15": fig_5_15, "5.17": fig_5_17, "data": make_data,
}

if __name__ == "__main__":
    common.run(FIGURES, CH)
