"""Murphy (2022), Chapter 10: Logistic Regression.

Companion code for Chapter 10 of K. P. Murphy, Probabilistic Machine Learning: An Introduction
(MIT Press, 2022; April 2025 draft), pp. 339-369, written from scratch with numpy, scipy and
matplotlib, to be read next to the lecture notes (../ch10-logistic-regression.html). scikit-learn is
only used to load the iris data and to confirm the implementations.

Methods
* Section 10.2, binary logistic regression: the NLL, gradient and Hessian (10.8)-(10.23)
  (nll_grad_hess), Newton's method / iteratively reweighted least squares with a step-halving
  safeguard (irls, Algorithm 10.1, (10.36)-(10.43)), batch and stochastic gradient descent (10.28)
  (gradient_descent, sgd), the perceptron (10.30) (perceptron), the bound-optimisation iteration
  with the fixed matrix B = -X'X/4 (10.82) (mm_binary), polynomial feature expansion
  (poly_features), l2-penalised fits (10.47)-(10.50) with an unpenalised bias.
* Section 10.3, multinomial logistic regression: softmax regression by Newton-free quasi-Newton
  steps (softmax_fit), the gradient (10.68) and the Hessian (10.69), the Bohning bound (10.79),
  the sum-to-zero property of the penalised optimum (10.89), the long-tail sampling rule (10.97).
* Section 10.4, robust logistic regression: the label-flip mixture likelihood (10.99) fitted by
  Metropolis sampling (robust_mh); tempered logarithm and exponential (10.103), (10.106), the
  tempered softmax with its normaliser by bisection and by Algorithm 10.2 (tempered_softmax,
  tempered_softmax_alg), the bi-tempered loss (10.102) and its minimisation (fit_bitempered).
* Section 10.5, Bayesian logistic regression: the Laplace approximation on a grid and by Newton's
  method (laplace_logreg), Monte Carlo predictive (10.110), the probit approximation (10.113).

Data. The iris measurements come from scikit-learn (Fisher, 1936). The two-dimensional
two-class data of Figures 10.4 and 10.6 and the separable data of Figures 10.13 and 10.14 are
simulated (seeds fixed in the code): the book's own point sets are not available, so those figures
are close stand-ins, not copies. The 3-class data of Figure 10.7 are simulated as well. The
contaminated one-dimensional iris data of Figure 10.10 follow the recipe in the text (six extra
versicolor flowers with abnormally low sepal length); the book used a hand-made version.

Figures. Reproduced here: 10.1b, 10.2, 10.3 (redrawn with computed points), 10.4, 10.5, 10.6, 10.7,
10.10, 10.11, 10.12, 10.13, 10.14. Skipped, with the reason:
  10.1a  a hand-drawn plane with its normal vector in 3-D (nothing is computed);
  10.8   the label hierarchy (a hand-drawn tree);
  10.9   the flat and hierarchical softmax diagram (a hand-drawn tree whose numbers are in the text);
  10.15  the data of Exercise 10.2 (a hand-drawn sketch; re-read from the book as Figure 10.S4 below)
         and the plot of w_k against c_k (a MATLAB screenshot of an unspecified model).
Supplementary figures for the gaps filled in the notes:
  10.S1 the class-sampling probabilities of (10.97) for a long-tailed class distribution;
  10.S2 the speed of gradient descent, stochastic gradient descent, Newton/IRLS and the bound
        optimisation iteration on the iris one-feature problem of Figure 10.5;
  10.S3 the quality of the probit approximation (10.111): sigmoid against the Gaussian cdf, and the
        exact, Monte Carlo and probit predictive probabilities;
  10.S4 the four fits of Exercise 10.2 (all weights, then w0, w1 or w2 heavily penalised).
Run `python3 ch10_logistic_regression.py --list`, `python3 ch10_logistic_regression.py` (all figures,
a few minutes) or `python3 ch10_logistic_regression.py --fig 10.4`. Files go to ../img/ch10/; the
data used by the browser demonstrations go to ../data/ch10-*.json.
"""
import itertools
import json
import os
import sys

os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")

import numpy as np
from scipy import optimize, special, stats

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common  # noqa: E402
from common import BLUE, CYAN, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, plt, savefig  # noqa: E402

CH = 10
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
sigmoid = common.sigmoid


# =========================================================================== data
def iris_binary(feature_idx, target=2):
    """Iris: the chosen feature columns and the label 1 for species `target` (default Virginica)."""
    from sklearn.datasets import load_iris
    d = load_iris()
    X = d.data[:, feature_idx]
    return X, (d.target == target).astype(float)


def iris_setosa_versicolor():
    from sklearn.datasets import load_iris
    d = load_iris()
    keep = d.target < 2
    return d.data[keep, 0], d.target[keep].astype(float)


def two_class_data(n_train=40, n_test=2000, seed=3):
    """Simulated 2-D two-class data for Figures 10.4 and 10.6: overlapping classes whose log-odds
    is a mild quadratic, so that low degrees underfit slightly and high degrees overfit a small
    training set."""
    rng = np.random.default_rng(seed)

    def draw(n):
        X = rng.multivariate_normal([0, 0], [[0.8, 0.35], [0.35, 1.0]], n)
        a = 1.6 * X[:, 0] - 0.9 * X[:, 1] + 0.8 * (X[:, 0] ** 2 - X[:, 1] ** 2) * 0.6 - 0.2
        y = (rng.random(n) < sigmoid(a)).astype(float)
        return X, y
    Xtr, ytr = draw(n_train)
    Xte, yte = draw(n_test)
    return Xtr, ytr, Xte, yte


def poly_features(X, degree):
    """All monomials x1^i x2^j with i + j <= degree (including the constant 1 first)."""
    X = np.atleast_2d(X)
    cols = []
    for tot in range(degree + 1):
        for i in range(tot, -1, -1):
            cols.append(X[:, 0] ** i * X[:, 1] ** (tot - i))
    return np.stack(cols, 1)


def add_bias(X):
    X = np.asarray(X, float)
    X = X.reshape(-1, 1) if X.ndim == 1 else X
    return np.hstack([np.ones((len(X), 1)), X])


# =========================================================================== Section 10.2
def nll_grad_hess(w, X, y, lam=0.0):
    """Mean NLL (10.8), its gradient (10.21) and Hessian (10.23) for X with a leading column of ones.

    lam is the l2 penalty strength in  NLL + (lam/2) ||w_{1:}||^2  (the bias w_0 is not penalised)."""
    N = len(y)
    a = X @ w
    mu = sigmoid(a)
    f = (np.logaddexp(0.0, a) - y * a).mean()
    g = X.T @ (mu - y) / N
    s = mu * (1 - mu)
    H = (X * s[:, None]).T @ X / N
    if lam:
        f += 0.5 * lam * np.sum(w[1:] ** 2)
        g = g + lam * np.r_[0.0, w[1:]]
        H = H + lam * np.diag(np.r_[0.0, np.ones(len(w) - 1)])
    return f, g, H


def irls(X, y, lam=0.0, w0=None, iters=100, tol=1e-12, trace=False):
    """Newton's method for logistic regression (Algorithm 10.1 in the form (10.36)), with step
    halving so that it also runs on almost-separable data. Returns w (and the NLL trace)."""
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    f, g, H = nll_grad_hess(w, X, y, lam)
    hist = [f]
    for _ in range(iters):
        try:
            step = np.linalg.solve(H + 1e-12 * np.eye(len(w)), g)
        except np.linalg.LinAlgError:
            step = np.linalg.lstsq(H, g, rcond=None)[0]
        t = 1.0
        while t > 1e-10:
            fn = nll_grad_hess(w - t * step, X, y, lam)[0]
            if fn <= f + 1e-4 * t * (-g @ step) or fn <= f:
                break
            t *= 0.5
        w = w - t * step
        f, g, H = nll_grad_hess(w, X, y, lam)
        hist.append(f)
        if np.linalg.norm(g) < tol:
            break
    return (w, np.array(hist)) if trace else w


def irls_wls(X, y, w0=None, iters=30):
    """The same iteration written as the sequence of weighted least-squares problems of
    Algorithm 10.1: w <- (X'SX)^{-1} X'S z with z = a + (y - mu)/s."""
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    for _ in range(iters):
        a = X @ w
        mu = sigmoid(a)
        s = mu * (1 - mu)
        z = a + (y - mu) / s
        w = np.linalg.solve((X * s[:, None]).T @ X, (X * s[:, None]).T @ z)
    return w


def gradient_descent(X, y, lr, iters, w0=None, lam=0.0):
    """Batch gradient descent on the mean NLL: w <- w - lr * grad. Returns the NLL after each step."""
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    hist = [nll_grad_hess(w, X, y, lam)[0]]
    for _ in range(iters):
        f, g, _ = nll_grad_hess(w, X, y, lam)
        w = w - lr * g
        hist.append(nll_grad_hess(w, X, y, lam)[0])
    return w, np.array(hist)


def sgd(X, y, lr0, epochs, seed=0, decay=0.0, w0=None):
    """Minibatch-of-one SGD (10.28) with step lr0 / (1 + decay * t); returns w and the full-data NLL
    after every epoch."""
    rng = np.random.default_rng(seed)
    N = len(y)
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    hist = [nll_grad_hess(w, X, y)[0]]
    t = 0
    for _ in range(epochs):
        for n in rng.permutation(N):
            w = w - lr0 / (1 + decay * t) * (sigmoid(X[n] @ w) - y[n]) * X[n]
            t += 1
        hist.append(nll_grad_hess(w, X, y)[0])
    return w, np.array(hist)


def mm_binary(X, y, iters, w0=None):
    """Bound optimisation (10.82): w <- w - 4 (X'X)^{-1} grad_sum, i.e. with the fixed matrix
    B = X'X/4 (the book's sign convention is for the log-likelihood; here for the NLL)."""
    N = len(y)
    B = X.T @ X / 4.0 / N
    Binv = np.linalg.inv(B)
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    hist = [nll_grad_hess(w, X, y)[0]]
    for _ in range(iters):
        g = nll_grad_hess(w, X, y)[1]
        w = w - Binv @ g
        hist.append(nll_grad_hess(w, X, y)[0])
    return w, np.array(hist)


def perceptron(X, y, epochs=100, seed=0):
    """Rosenblatt's perceptron (10.30) with labels in {0,1}; stops after an error-free epoch.
    Returns w and the number of mistakes made in total."""
    rng = np.random.default_rng(seed)
    w = np.zeros(X.shape[1])
    mistakes = 0
    for _ in range(epochs):
        err = 0
        for n in rng.permutation(len(y)):
            yhat = float(X[n] @ w > 0)
            if yhat != y[n]:
                w = w - (yhat - y[n]) * X[n]
                err += 1
        mistakes += err
        if err == 0:
            break
    return w, mistakes


def error_rate(X, y, w):
    return float(np.mean((X @ w > 0) != (y > 0.5)))


# =========================================================================== Section 10.3
def softmax(A):
    A = np.asarray(A, float)
    A = A - A.max(1, keepdims=True)
    E = np.exp(A)
    return E / E.sum(1, keepdims=True)


def softmax_nll_grad(W, X, Y, lam=0.0):
    """Mean NLL (10.58), gradient (10.68) (D x C), penalty (lam/2) sum ||w_c||^2 on all rows but the first."""
    N = len(X)
    A = X @ W
    lse = special.logsumexp(A, axis=1)
    f = (lse - (Y * A).sum(1)).mean()
    mu = softmax(A)
    G = X.T @ (mu - Y) / N
    if lam:
        pen = W.copy()
        pen[0] = 0
        f += 0.5 * lam * np.sum(pen ** 2)
        G = G + lam * pen
    return f, G


def softmax_hess(W, X, lam=0.0):
    """Hessian (10.69) of the mean NLL with respect to vec(W) laid out class by class (C blocks of D)."""
    N, D = X.shape
    C = W.shape[1]
    mu = softmax(X @ W)
    H = np.zeros((C * D, C * D))
    for c in range(C):
        for c2 in range(C):
            coef = mu[:, c] * ((c == c2) - mu[:, c2])
            H[c * D:(c + 1) * D, c2 * D:(c2 + 1) * D] = (X * coef[:, None]).T @ X / N
    if lam:
        P = np.diag(np.r_[0.0, np.ones(D - 1)])
        for c in range(C):
            H[c * D:(c + 1) * D, c * D:(c + 1) * D] += lam * P
    return H


def softmax_fit(X, y, C, lam=0.0, clamp=False, W0=None):
    """Multinomial logistic regression by L-BFGS on (10.58) (+ penalty). X has a leading column of
    ones. With clamp=True the last class has w_C = 0 (identifiable form (10.85))."""
    N, D = X.shape
    Y = np.eye(C)[y.astype(int)]
    free = C - 1 if clamp else C

    def unpack(v):
        W = np.zeros((D, C))
        W[:, :free] = v.reshape(D, free)
        return W

    def fun(v):
        f, G = softmax_nll_grad(unpack(v), X, Y, lam)
        return f, G[:, :free].ravel()
    v0 = np.zeros(D * free) if W0 is None else W0[:, :free].ravel()
    r = optimize.minimize(fun, v0, jac=True, method="L-BFGS-B", options=dict(maxiter=5000, ftol=1e-15, gtol=1e-10))
    return unpack(r.x)


def class_sampling_probs(counts, q):
    """(10.97): p_c proportional to N_c^q."""
    c = np.asarray(counts, float) ** q
    return c / c.sum()


def bohning_gap(mu):
    """Smallest eigenvalue of  1/2 (I - 11'/C) - (diag(mu) - mu mu')  (>= 0 by the Bohning bound)."""
    C = len(mu)
    M = 0.5 * (np.eye(C) - np.ones((C, C)) / C) - (np.diag(mu) - np.outer(mu, mu))
    return np.linalg.eigvalsh(M).min()


def multinomial_mm(X, y, C, iters, W0=None):
    """Bound optimisation for the clamped model (w_C = 0), using the Bohning matrix
    B = 1/2 (I - 11'/C)_{(C-1)x(C-1) block} kron (X'X) (10.79) for the NLL sum."""
    N, D = X.shape
    Y = np.eye(C)[y.astype(int)]
    Bc = 0.5 * (np.eye(C) - np.ones((C, C)) / C)[:C - 1, :C - 1]
    Bm = np.kron(Bc, X.T @ X)
    Binv = np.linalg.inv(Bm)
    W = np.zeros((D, C)) if W0 is None else W0.copy()
    hist = []
    for _ in range(iters):
        f, G = softmax_nll_grad(W, X, Y)
        hist.append(f)
        # vec in class-by-class layout of the first C-1 classes
        g = (G[:, :C - 1] * N).T.ravel()
        step = Binv @ g
        W[:, :C - 1] -= step.reshape(C - 1, D).T
    hist.append(softmax_nll_grad(W, X, Y)[0])
    return W, np.array(hist)


def three_class_data(n=50, seed=5):
    """Simulated 3-class 2-D data in [-1, 1]^2 (Figure 10.7): a blob at the origin and two half
    rings around it, so that a linear softmax cannot isolate the middle class but quadratic
    features can."""
    rng = np.random.default_rng(seed)
    X0 = rng.normal([0.0, 0.0], 0.17, (n, 2))
    r1 = rng.normal(0.68, 0.07, n)
    a1 = rng.uniform(np.radians(-20), np.radians(160), n)
    r2 = rng.normal(0.68, 0.07, n)
    a2 = rng.uniform(np.radians(200), np.radians(340), n)
    X1 = np.stack([r1 * np.cos(a1), r1 * np.sin(a1)], 1)
    X2 = np.stack([r2 * np.cos(a2), r2 * np.sin(a2)], 1)
    X = np.vstack([X0, X1, X2])
    y = np.repeat([0, 1, 2], n)
    return np.clip(X, -0.98, 0.98), y


# =========================================================================== Section 10.4
def log_t(x, t):
    """Tempered logarithm (10.103)."""
    x = np.asarray(x, float)
    if t == 1.0:
        return np.log(x)
    return (x ** (1.0 - t) - 1.0) / (1.0 - t)


def exp_t(x, t):
    """Tempered exponential (10.106); for t > 1 the argument must stay below 1/(t-1)."""
    x = np.asarray(x, float)
    if t == 1.0:
        return np.exp(x)
    base = 1.0 + (1.0 - t) * x
    with np.errstate(divide="ignore", invalid="ignore"):
        out = np.where(base > 0, np.power(np.maximum(base, 1e-300), 1.0 / (1.0 - t)), 0.0 if t < 1 else np.inf)
    return out


def tempered_softmax(A, t, method="newton", iters=60):
    """Tempered softmax (10.105) for each row of A: exp_t(a_c - lambda(a)) with lambda solving
    (10.107). method="bisection" brackets lambda in [max a, max a + (C^(t-1) - 1)/(t-1)]; the default
    "newton" starts at max a, where the sum is at least one, and uses that the sum is convex and
    decreasing in lambda for t > 1, so Newton's iterates increase monotonically to the root."""
    A = np.atleast_2d(np.asarray(A, float))
    if t == 1.0:
        return softmax(A)
    C = A.shape[1]
    m = A.max(1)
    if method == "bisection":
        lo = m.copy()
        hi = m + (C ** (t - 1.0) - 1.0) / (t - 1.0)
        for _ in range(100):
            mid = 0.5 * (lo + hi)
            s = exp_t(A - mid[:, None], t).sum(1)
            big = s > 1.0
            lo = np.where(big, mid, lo)
            hi = np.where(big, hi, mid)
        lam = 0.5 * (lo + hi)
    else:
        lam = m.copy()
        for _ in range(iters):
            E = exp_t(A - lam[:, None], t)
            f = E.sum(1) - 1.0
            fp = (E ** t).sum(1)
            lam = lam + f / fp
    P = exp_t(A - lam[:, None], t)
    return P / P.sum(1, keepdims=True)


def tempered_lambda_alg(a, t, iters=200):
    """Algorithm 10.2 for one logit vector a (t > 1): returns lambda(a)."""
    a = np.asarray(a, float)
    mu = a.max()
    at = a - mu
    for _ in range(iters):
        Z = exp_t(at, t).sum()
        new = Z ** (1.0 - t) * (a - mu)
        if np.max(np.abs(new - at)) < 1e-14:
            at = new
            break
        at = new
    Z = exp_t(at, t).sum()
    return -log_t(1.0 / Z, t) + mu


def tempered_softmax_alg(a, t):
    lam = tempered_lambda_alg(a, t)
    return exp_t(np.asarray(a, float) - lam, t)


def tempered_ce_onehot(yhat, c, t1):
    """(10.102) for a one-hot target c (rows of yhat)."""
    yhat = np.atleast_2d(yhat)
    yc = yhat[np.arange(len(yhat)), c]
    return -log_t(np.maximum(yc, 1e-300), t1) - (1.0 - (yhat ** (2.0 - t1)).sum(1)) / (2.0 - t1)


def bitempered_binary_loss(a, y, t1, t2):
    """Bi-tempered logistic loss for logit a (class 1 logit a, class 0 logit 0) and label y in {0,1}."""
    A = np.stack([np.zeros_like(a), a], 1)
    P = tempered_softmax(A, t2)
    return tempered_ce_onehot(P, y.astype(int), t1)


ROB_LAM = 0.01     # l2 penalty on the slope in the one-dimensional robust demonstration
BT_LAM = 0.02      # small l2 penalty on the weights (not the bias) used with the bi-tempered loss


def bitempered_objective(w, Xb, y, t1, t2, lam=BT_LAM):
    return bitempered_binary_loss(Xb @ w, y, t1, t2).mean() + 0.5 * lam * np.sum(w[1:] ** 2)


def fit_bitempered(X, y, t1, t2, lam=BT_LAM):
    """Minimise the mean bi-tempered loss plus (lam/2)||w||^2 over (b, w); X is N x 2. The loss is
    not convex for t1 < 1, so Nelder-Mead is started from the l2-penalised logistic fit and from two
    vertical boundaries and the best result is kept. Without the penalty the weights run off to
    infinity whenever the data are separable (the loss then tends to 0)."""
    Xb = add_bias(X)
    best = None
    for s in (irls(Xb, y, lam=lam), np.array([0, 1.0, 0]), np.array([0, 0.3, 0])):
        r = optimize.minimize(bitempered_objective, s, args=(Xb, y, t1, t2, lam), method="Nelder-Mead",
                              options=dict(xatol=1e-5, fatol=1e-10, maxiter=600))
        if best is None or r.fun < best.fun:
            best = r
    return best.x


def robust_loglik(w, pi, Xb, y):
    """Mixture likelihood (10.99): pi Ber(y|0.5) + (1-pi) Ber(y|sigmoid(w'x))."""
    mu = sigmoid(Xb @ w)
    py = np.where(y > 0.5, mu, 1 - mu)
    return np.log(pi * 0.5 + (1 - pi) * py)


def robust_mh(x, y, robust, n=30000, seed=0, prior_sd=10.0, step=0.25):
    """Random-walk Metropolis for (b0, b1[, logit pi]) on the centred feature, with N(0, prior_sd^2)
    priors on b0, b1 and a Beta(1, 9) prior on pi. Returns the draws after burn-in and the
    acceptance rate."""
    rng = np.random.default_rng(seed)
    Xb = add_bias(x)
    d = 3 if robust else 2
    lbeta = special.betaln(1, 9)

    def logpost(th):
        w = th[:2]
        if robust:
            pi = 1.0 / (1.0 + np.exp(-th[2]))
            mu = 1.0 / (1.0 + np.exp(-(Xb @ w)))
            py = np.where(y > 0.5, mu, 1 - mu)
            ll = np.log(pi * 0.5 + (1 - pi) * py).sum()
            lpi = 8 * np.log(1 - pi) - lbeta + np.log(pi * (1 - pi))       # Beta(1,9) density times the Jacobian
            ll += lpi
        else:
            a = Xb @ w
            ll = (y * a - np.logaddexp(0, a)).sum()
        return ll - 0.5 * (w @ w) / prior_sd ** 2
    th = np.zeros(d)
    if robust:
        th[2] = -2.0
    lp = logpost(th)
    out = np.zeros((n, d))
    Z = rng.standard_normal((n, d))
    U = np.log(rng.random(n))
    L = np.eye(d) * step
    acc = 0
    for i in range(n):
        prop = th + L @ Z[i]
        lpp = logpost(prop)
        if U[i] < lpp - lp:
            th, lp = prop, lpp
            acc += 1
        out[i] = th
        if i == 3000:       # adapt the proposal to the warm-up draws
            L = np.linalg.cholesky(np.cov(out[1000:3000].T) * (2.38 ** 2 / d) + 1e-8 * np.eye(d))
    return out[5000:], acc / n


# =========================================================================== Section 10.5
def laplace_logreg(X, y, prior_var):
    """MAP estimate and Laplace covariance (inverse Hessian of the penalised NLL sum) for
    p(w) = N(0, prior_var I); X has NO bias column here (as in Figure 10.13)."""
    N = len(y)
    P = 1.0 / prior_var

    def fun(w):
        a = X @ w
        f = (np.logaddexp(0, a) - y * a).sum() + 0.5 * P * w @ w
        g = X.T @ (sigmoid(a) - y) + P * w
        return f, g
    r = optimize.minimize(fun, np.zeros(X.shape[1]), jac=True, method="BFGS", options=dict(gtol=1e-10))
    w = r.x
    mu = sigmoid(X @ w)
    H = (X * (mu * (1 - mu))[:, None]).T @ X + P * np.eye(X.shape[1])
    return w, np.linalg.inv(H)


def predictive_mc(Xs, w, V, S=2000, seed=0):
    """(10.110): average of sigma(w_s' x) over draws from N(w, V)."""
    rng = np.random.default_rng(seed)
    ws = rng.multivariate_normal(w, V, S)
    return sigmoid(Xs @ ws.T).mean(1)


def kappa(v):
    """(10.112)."""
    return 1.0 / np.sqrt(1.0 + np.pi * v / 8.0)


def predictive_probit(Xs, w, V):
    """(10.113)-(10.115)."""
    m = Xs @ w
    v = np.einsum("ij,jk,ik->i", Xs, V, Xs)
    return sigmoid(kappa(v) * m)


def predictive_exact(m, v, n=6001):
    """E[sigmoid(a)] for a ~ N(m, v): trapezoid rule on the standard-normal variable z in [-9, 9]
    (spectrally accurate for this smooth integrand, also when v is large)."""
    z = np.linspace(-9, 9, n)
    wt = stats.norm.pdf(z) * (z[1] - z[0])
    m = np.atleast_1d(np.asarray(m, float))
    v = np.atleast_1d(np.asarray(v, float))
    return (sigmoid(m[:, None] + np.sqrt(v)[:, None] * z[None, :]) * wt[None, :]).sum(1)


def separable_data(seed=11, n=22):
    """Simulated linearly separable 2-D data (Figure 10.13): classes either side of a line through
    the origin with normal (2, 1)/sqrt(5) and a margin."""
    rng = np.random.default_rng(seed)
    w0 = np.array([2.0, 1.0]) / np.sqrt(5)
    pts = []
    while len(pts) < n:
        x = rng.uniform(-6, 6, 2)
        if abs(w0 @ x) > 0.9:
            pts.append(x)
    X = np.array(pts)
    y = (X @ w0 > 0).astype(float)
    return X, y


# =========================================================================== Exercise 10.2 data
def ex102_data():
    """The 13 points of Figure 10.15a, re-read from the page (x1, x2 on the grid of the sketch):
    six pluses (class 1) on the left and seven circles (class 0) on the right."""
    plus = np.array([[1.3, 5.9], [2.4, 5.9], [3.2, 6.6], [3.2, 5.6], [2.5, 4.6], [2.2, 0.5]])
    circ = np.array([[5.3, 4.6], [5.4, 2.5], [6.1, 2.1], [7.3, 2.4], [5.3, 1.1], [6.2, 1.1], [7.5, 1.3]])
    X = np.vstack([plus, circ])
    y = np.r_[np.ones(len(plus)), np.zeros(len(circ))]
    return X, y


def penalised_fit(X, y, pen, lam=1e4):
    """Logistic fit with penalty (lam/2) sum_j pen_j w_j^2 on (w0, w1, w2), by Newton with halving."""
    Xb = add_bias(X)
    pen = np.asarray(pen, float)

    def fun(w):
        a = Xb @ w
        return (np.logaddexp(0, a) - y * a).sum() + 0.5 * lam * np.sum(pen * w ** 2)

    def jac(w):
        return Xb.T @ (sigmoid(Xb @ w) - y) + lam * pen * w
    best = None
    for w0 in (np.zeros(3), np.array([-3.0, 1.0, 0.0]), np.array([-3.0, 0.0, 1.0]), np.array([3.0, -1, -1.0])):
        r = optimize.minimize(fun, w0, jac=jac, method="BFGS", options=dict(gtol=1e-9, maxiter=2000))
        if best is None or r.fun < best.fun:
            best = r
    return best.x


# =========================================================================== figure helpers
def _decision_image(ax, predict, xlim, ylim, n=220, contour=True):
    xs = np.linspace(*xlim, n)
    ys = np.linspace(*ylim, n)
    G = np.stack(np.meshgrid(xs, ys), -1).reshape(-1, 2)
    p = predict(G).reshape(n, n)
    ax.imshow(p, extent=[*xlim, *ylim], origin="lower", cmap="coolwarm_r", vmin=0, vmax=1, alpha=0.45,
              aspect="auto", interpolation="bilinear", rasterized=True)
    if contour:
        ax.contour(xs, ys, p, levels=[0.5], colors="k", linewidths=1.0)
    ax.set_xlim(*xlim)
    ax.set_ylim(*ylim)


def _scatter_classes(ax, X, y, s=16):
    ax.scatter(X[y > 0.5, 0], X[y > 0.5, 1], c=RED, marker="x", s=s, linewidths=1.2)
    ax.scatter(X[y < 0.5, 0], X[y < 0.5, 1], c=BLUE, marker="o", s=s, linewidths=0.6)


# =========================================================================== Figures
def fig_10_1(out=None):
    """Iris petal length and width: the optimal linear boundary for Virginica against the rest."""
    X, y = iris_binary([2, 3])
    w = irls(add_bias(X), y)
    fig, ax = plt.subplots(figsize=(4.6, 3.4))
    xs = np.linspace(0.8, 7.2, 200)
    ax.scatter(X[y < 0.5, 0], X[y < 0.5, 1], c=BLUE, s=14, label="Not Iris-Virginica")
    ax.scatter(X[y > 0.5, 0], X[y > 0.5, 1], c=GREEN, marker="s", s=14, label="Iris-Virginica")
    ax.plot(xs, -(w[0] + w[1] * xs) / w[2], "k", lw=1.5, label="boundary $P(y=1\\mid x)=1/2$")
    ax.set_xlim(0.8, 7.2)
    ax.set_ylim(0, 2.7)
    ax.set_xlabel("Petal length (cm)")
    ax.set_ylabel("Petal width (cm)")
    ax.legend(loc="lower right", fontsize=8)
    ax.set_title("Logistic-regression boundary, $N=150$")
    savefig(fig, CH, "fig10-1", out)


def fig_10_2(out=None):
    """sigma(w1 x1 + w2 x2) for nine weight vectors."""
    ws = [(-3, -3), (0, -3), (3, -3), (-3, 0), (0, 0.5), (3, 0), (-3, 3), (0, 3), (3, 3)]
    fig, axes = common.panel_grid(3, 3, size=(2.1, 2.1))
    g = np.linspace(-5, 5, 120)
    G = np.stack(np.meshgrid(g, g), -1)
    for ax, w in zip(axes.ravel(), ws):
        p = sigmoid(G[..., 0] * w[0] + G[..., 1] * w[1])
        ax.imshow(p, extent=[-5, 5, -5, 5], origin="lower", cmap="coolwarm_r", vmin=0, vmax=1, interpolation="bilinear", rasterized=True)
        ax.contour(g, g, p, levels=[0.1, 0.3, 0.5, 0.7, 0.9], colors="k", linewidths=0.5)
        if w != (0, 0):
            ax.annotate("", xy=(w[0] * 0.5, w[1] * 0.5), xytext=(0, 0), arrowprops=dict(arrowstyle="->", color="k", lw=1.2))
        ax.set_title("$w=(%g,%g)$" % w, fontsize=9)
        ax.set_xticks([-5, 0, 5])
        ax.set_yticks([-5, 0, 5])
    fig.tight_layout()
    savefig(fig, CH, "fig10-2", out)


def fig_10_3(out=None):
    """A circular boundary in (x1, x2) becomes a straight line in (x1^2, x2^2)."""
    rng = np.random.default_rng(2)
    X = rng.uniform(-2.2, 2.2, (160, 2))
    R = 1.5
    y = (X[:, 0] ** 2 + X[:, 1] ** 2 > R ** 2).astype(float)
    fig, axes = plt.subplots(1, 2, figsize=(7.4, 3.3))
    _scatter_classes(axes[0], X, y)
    t = np.linspace(0, 2 * np.pi, 200)
    axes[0].plot(R * np.cos(t), R * np.sin(t), "k", lw=1.2)
    axes[0].set_aspect("equal")
    axes[0].set_xlabel("$x_1$")
    axes[0].set_ylabel("$x_2$")
    axes[0].set_title("input space: $x_1^2+x_2^2=R^2$")
    Z = X ** 2
    _scatter_classes(axes[1], Z, y)
    u = np.linspace(0, R ** 2, 10)
    axes[1].plot(u, R ** 2 - u, "k", lw=1.2)
    axes[1].set_xlabel("$\\phi_1=x_1^2$")
    axes[1].set_ylabel("$\\phi_2=x_2^2$")
    axes[1].set_title("feature space: $\\phi_1+\\phi_2=R^2$")
    axes[1].set_xlim(0, 4.8)
    axes[1].set_ylim(0, 4.8)
    for a, l in zip(axes, "ab"):
        common.panel_label(a, l, "upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-3", out)


def _fit_poly(Xtr, ytr, deg, C):
    F = poly_features(Xtr, deg)
    return irls(F, ytr, lam=1.0 / (C * len(ytr)), iters=200)


def fig_10_4(out=None):
    """Polynomial expansion of a 2-D logistic regression: degrees 1, 2, 4 and train/test error."""
    Xtr, ytr, Xte, yte = two_class_data()
    C = 1e6
    fig, axes = common.panel_grid(2, 2, size=(3.4, 2.9))
    lim = (-3, 3)
    for ax, deg, lab in zip(axes.ravel()[:3], (1, 2, 4), "abc"):
        w = _fit_poly(Xtr, ytr, deg, C)
        _decision_image(ax, lambda G: sigmoid(poly_features(G, deg) @ w), lim, lim)
        _scatter_classes(ax, Xtr, ytr)
        ax.set_title("Degree %d" % deg)
        common.panel_label(ax, lab, "lower left")
    degs = [1, 2, 3, 4, 5, 6, 8, 10]
    tr, te = [], []
    for d in degs:
        w = _fit_poly(Xtr, ytr, d, C)
        tr.append(error_rate(poly_features(Xtr, d), ytr, w))
        te.append(error_rate(poly_features(Xte, d), yte, w))
    ax = axes[1, 1]
    ax.plot(degs, tr, "o-", color=BLUE, ms=4, label="train")
    ax.plot(degs, te, "s-", color=ORANGE, ms=4, label="test")
    ax.set_xlabel("polynomial degree")
    ax.set_ylabel("error rate")
    ax.legend()
    common.panel_label(ax, "d", "upper center")
    fig.tight_layout()
    savefig(fig, CH, "fig10-4", out)


def fig_10_5(out=None):
    """NLL surface of binary logistic regression on one iris feature (petal width) and a bias."""
    X, y = iris_binary([3])
    Xb = add_bias(X)
    w_hat = irls(Xb, y)
    bs = np.linspace(w_hat[0] - 4, w_hat[0] + 4, 90)
    ws = np.linspace(w_hat[1] - 3.2, w_hat[1] + 3.2, 90)
    Z = np.array([[nll_grad_hess(np.array([b, w]), Xb, y)[0] for w in ws] for b in bs])
    W, B = np.meshgrid(ws, bs)
    fig = plt.figure(figsize=(8.4, 3.5))
    ax = fig.add_subplot(1, 2, 1, projection="3d")
    ax.plot_surface(W, B, Z, cmap="viridis", linewidth=0, antialiased=True, alpha=0.9, rasterized=True)
    ax.set_xlabel("w")
    ax.set_ylabel("b")
    ax.set_zlabel("NLL")
    ax.set_title("Loss function surface")
    ax.view_init(28, -125)
    ax2 = fig.add_subplot(1, 2, 2)
    cs = ax2.contour(ws, bs, Z, levels=Z.min() + np.array([0.02, 0.05, 0.1, 0.2, 0.4, 0.8, 1.4, 2.2]), cmap="viridis")
    ax2.clabel(cs, fontsize=6, fmt="%.1f")
    ax2.plot(w_hat[1], w_hat[0], "r*", ms=11)
    ax2.set_xlabel("w")
    ax2.set_ylabel("b")
    ax2.set_title("contours; MLE $w=%.2f$, $b=%.2f$" % (w_hat[1], w_hat[0]))
    savefig(fig, CH, "fig10-5", out)


def fig_10_6(out=None):
    """Weight decay on a degree-4 polynomial: C = 1, 316, 100000, and train/test error against C."""
    Xtr, ytr, Xte, yte = two_class_data()
    lim = (-3, 3)
    fig, axes = common.panel_grid(2, 2, size=(3.4, 2.9))
    for ax, C, lab in zip(axes.ravel()[:3], (1.0, 316.0, 1e5), "abc"):
        w = _fit_poly(Xtr, ytr, 4, C)
        _decision_image(ax, lambda G: sigmoid(poly_features(G, 4) @ w), lim, lim)
        _scatter_classes(ax, Xtr, ytr)
        ax.set_title("$C=%s$" % (("%g" % C) if C < 1e4 else "10^5"))
        common.panel_label(ax, lab, "lower left")
    Cs = np.logspace(0, 5, 11)
    tr, te = [], []
    for C in Cs:
        w = _fit_poly(Xtr, ytr, 4, C)
        tr.append(error_rate(poly_features(Xtr, 4), ytr, w))
        te.append(error_rate(poly_features(Xte, 4), yte, w))
    ax = axes[1, 1]
    ax.semilogx(Cs, tr, "o-", color=BLUE, ms=4, label="train")
    ax.semilogx(Cs, te, "s-", color=ORANGE, ms=4, label="test")
    ax.set_xlabel("inverse regularisation $C$")
    ax.set_ylabel("error rate")
    ax.legend()
    common.panel_label(ax, "d", "lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-6", out)


def fig_10_7(out=None):
    """3-class logistic regression with linear and quadratic features."""
    X, y = three_class_data()
    cols = np.array([RED, BLUE, GREEN])
    fig, axes = plt.subplots(1, 2, figsize=(7.4, 3.4))
    feats = [lambda Z: add_bias(Z),
             lambda Z: np.stack([np.ones(len(Z)), Z[:, 0], Z[:, 1], Z[:, 0] ** 2, Z[:, 1] ** 2, Z[:, 0] * Z[:, 1]], 1)]
    names = ["Linear logistic regression", "Quadratic logistic regression"]
    g = np.linspace(-1, 1, 260)
    G = np.stack(np.meshgrid(g, g), -1).reshape(-1, 2)
    for ax, f, nm, l in zip(axes, feats, names, "ab"):
        F = f(X)
        W = softmax_fit(F, y, 3, lam=1e-4)
        pred = softmax(f(G) @ W).argmax(1).reshape(len(g), len(g))
        ax.imshow(pred, extent=[-1, 1, -1, 1], origin="lower", cmap=plt.matplotlib.colors.ListedColormap(["#f4b6b6", "#b6c8f4", "#b6e6b6"]),
                  vmin=0, vmax=2, aspect="auto", interpolation="nearest", rasterized=True)
        for c, m in zip(range(3), ["o", "s", "^"]):
            ax.scatter(X[y == c, 0], X[y == c, 1], c=cols[c], marker=m, s=14, edgecolor="k", linewidths=0.3)
        acc = np.mean(softmax(F @ W).argmax(1) == y)
        ax.set_title("%s (train accuracy %.2f)" % (nm, acc), fontsize=9)
        ax.set_xlim(-1, 1)
        ax.set_ylim(-1, 1)
        common.panel_label(ax, l, "lower left")
    fig.tight_layout()
    savefig(fig, CH, "fig10-7", out)


def contaminated_iris(seed=4):
    """Setosa vs versicolor sepal length plus six versicolor flowers with abnormally low sepal length."""
    x, y = iris_setosa_versicolor()
    rng = np.random.default_rng(seed)
    extra = np.round(rng.uniform(3.4, 4.2, 6), 1)
    return np.r_[x, extra], np.r_[y, np.ones(6)], extra


def robust_results():
    x0, y0 = iris_setosa_versicolor()
    xc, yc, extra = contaminated_iris()
    c = x0.mean()
    res = {}
    for key, (x, y, rob) in dict(clean=(x0, y0, False), std=(xc, yc, False), rob=(xc, yc, True)).items():
        draws, acc = robust_mh(x - c, y, rob)
        res[key] = (draws, acc)
    return res, c


def fig_10_10(out=None):
    """Logistic regression on iris sepal length with six outliers: standard and robust (mixture) model."""
    res, c = robust_results()
    xc, yc, extra = contaminated_iris()
    xs = np.linspace(3.2, 7.4, 220)
    rng = np.random.default_rng(0)
    fig, axes = plt.subplots(1, 2, figsize=(8.4, 3.2), sharey=True)
    jit = 0.04 * rng.standard_normal(len(yc))
    for ax, key, lab, ttl in zip(axes, ("std", "rob"), "ab", ("standard model", "robust (mixture) model")):
        draws = res[key][0]
        ax.scatter(xc[:-6], yc[:-6] + jit[:-6], c="k", s=8, alpha=0.6)
        ax.scatter(xc[-6:], yc[-6:] + jit[-6:], c=RED, marker="x", s=30, label="outliers")
        sub = draws[::25]
        P = sigmoid(sub[:, [0]] + sub[:, [1]] * (xs - c)[None, :])
        for p in P[:60]:
            ax.plot(xs, p, color=BLUE, alpha=0.08, lw=0.8)
        ax.plot(xs, P.mean(0), color=BLUE, lw=2)
        db = c - draws[:, 0] / draws[:, 1]
        q = np.percentile(db, [2.5, 97.5])
        ax.plot(q, [-0.07, -0.07], color="k", lw=5, solid_capstyle="butt")
        ax.axvline(np.median(db), color="k", lw=0.8)
        ax.set_title("%s: boundary %.2f (95%% interval %.2f to %.2f)" % (ttl, np.median(db), q[0], q[1]), fontsize=8.5)
        ax.set_xlabel("sepal length")
        common.panel_label(ax, lab, "upper left")
    axes[0].set_ylabel("$p(y=1)$")
    axes[0].legend(loc="center left", fontsize=8)
    fig.tight_layout()
    savefig(fig, CH, "fig10-10", out)


def fig_10_11(out=None):
    """Logistic loss and sigmoid against their tempered versions."""
    a = np.linspace(-6, 6, 400)
    fig, axes = plt.subplots(1, 2, figsize=(7.6, 3.0))
    ax = axes[0]
    ax.plot(a, bitempered_binary_loss(a, np.ones_like(a), 1.0, 1.0), color=BLUE, label="logistic loss ($t_1=1$)")
    ax.plot(a, bitempered_binary_loss(a, np.ones_like(a), 0.8, 1.0), color=RED, label="tempered, $t_1=0.8$")
    ax.set_xlabel("logit $a$ (true label $y=1$)")
    ax.set_ylabel("loss")
    ax.set_ylim(0, 6.2)
    ax.legend(fontsize=8)
    common.panel_label(ax, "a", "upper center")
    ax = axes[1]
    ax.plot(a, tempered_softmax(np.stack([np.zeros_like(a), a], 1), 1.0)[:, 1], color=BLUE, label="sigmoid ($t_2=1$)")
    ax.plot(a, tempered_softmax(np.stack([np.zeros_like(a), a], 1), 2.0)[:, 1], color=RED, label="tempered, $t_2=2$")
    ax.set_xlabel("logit $a$")
    ax.set_ylabel("$P(y=1)$")
    ax.legend(fontsize=8, loc="upper left")
    common.panel_label(ax, "b", "lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-11", out)


def noisy_blobs(kind, seed=7, n=60):
    """Two Gaussian blobs in 2-D with label noise: 'clean', 'near' (flips near the boundary),
    'far' (flips of points far from it) or 'both'."""
    rng = np.random.default_rng(seed)
    X = np.vstack([rng.normal([-1.6, 0.0], 0.9, (n, 2)), rng.normal([1.6, 0.0], 0.9, (n, 2))])
    y = np.r_[np.zeros(n), np.ones(n)]
    y0 = y.copy()
    if kind in ("near", "both"):
        near = np.argsort(np.abs(X[:, 0]))[:14]
        y[near[::2]] = 1 - y[near[::2]]
    if kind in ("far", "both"):
        far0 = np.where(y0 == 0)[0][np.argsort(X[y0 == 0, 0])[:5]]
        far1 = np.where(y0 == 1)[0][np.argsort(-X[y0 == 1, 0])[:5]]
        y[far0] = 1
        y[far1] = 0
    return X, y


def fig_10_12(out=None):
    """Standard (top) and bi-tempered (bottom) logistic regression on clean data and three kinds of label noise."""
    kinds = ["clean", "near", "far", "both"]
    temps = [(0.2, 4.0), (1.0, 4.0), (0.2, 1.0), (0.2, 4.0)]
    titles = ["clean data", "noise near the boundary", "noise far from the boundary", "both kinds of noise"]
    fig, axes = common.panel_grid(2, 4, size=(2.5, 2.5))
    lim = (-4.5, 4.5)
    for j, (k, (t1, t2), ttl) in enumerate(zip(kinds, temps, titles)):
        X, y = noisy_blobs(k)
        w_std = irls(add_bias(X), y, lam=BT_LAM)
        w_bt = fit_bitempered(X, y, t1, t2)
        for i, (w, pf) in enumerate([(w_std, lambda G, w=w_std: sigmoid(add_bias(G) @ w)),
                                     (w_bt, lambda G, w=w_bt: tempered_softmax(np.stack([np.zeros(len(G)), add_bias(G) @ w], 1), t2)[:, 1])]):
            ax = axes[i, j]
            _decision_image(ax, pf, lim, lim, n=140, contour=False)
            xl = np.array(lim)
            ax.plot(xl, -(w[0] + w[1] * xl) / w[2], "k", lw=1.0)       # the boundary is a = 0 for every temperature
            _scatter_classes(ax, X, y, s=10)
            ax.set_xticks([])
            ax.set_yticks([])
            if i == 0:
                ax.set_title(ttl, fontsize=8.5)
            else:
                ax.set_xlabel("$t_1=%g,\\ t_2=%g$" % (t1, t2), fontsize=8.5)
        axes[0, 0].set_ylabel("logistic")
        axes[1, 0].set_ylabel("bi-tempered")
    fig.tight_layout()
    savefig(fig, CH, "fig10-12", out)


def _laplace_setup(prior_var=100.0):
    X, y = separable_data()
    w_map, V = laplace_logreg(X, y, prior_var)
    return X, y, w_map, V


def fig_10_13(out=None):
    """Separable data, log-likelihood, unnormalised log posterior and the Laplace approximation."""
    X, y, w_map, V = _laplace_setup()
    g1 = np.linspace(-8, 14, 160)
    g2 = np.linspace(-8, 10, 160)
    W1, W2 = np.meshgrid(g1, g2)
    Wf = np.stack([W1.ravel(), W2.ravel()], 1)
    A = X @ Wf.T
    ll = (y[:, None] * A - np.logaddexp(0, A)).sum(0).reshape(W1.shape)
    lp = ll - 0.5 * (Wf ** 2).sum(1).reshape(W1.shape) / 100.0
    Vinv = np.linalg.inv(V)
    d = Wf - w_map
    lap = -0.5 * np.einsum("ij,jk,ik->i", d, Vinv, d).reshape(W1.shape)
    fig, axes = common.panel_grid(2, 2, size=(3.5, 3.1))
    ax = axes[0, 0]
    _scatter_classes(ax, X, y, s=16)
    for w in [(3, 1), (4, 2), (5, 3), (7, 3)]:
        t = np.array([-6, 6.0])
        ax.plot(t, -w[0] * t / w[1], lw=1, label="$w=(%d,%d)$" % w)
    ax.set_xlim(-6, 6)
    ax.set_ylim(-6, 6)
    ax.legend(fontsize=6.5, loc="lower left")
    ax.set_title("data and four separating lines", fontsize=9)
    for ax, Z, ttl, lev in [(axes[0, 1], ll, "log-likelihood", np.linspace(-14, -0.2, 12)),
                             (axes[1, 0], lp, "log unnormalised posterior", np.linspace(-18, lp.max(), 12)),
                             (axes[1, 1], lap, "Laplace approximation", np.linspace(-9, -0.5, 8))]:
        cs = ax.contour(g1, g2, Z, levels=lev, cmap="viridis", linewidths=0.9)
        ax.set_title(ttl, fontsize=9)
        ax.set_xlabel("$w_1$")
        ax.set_ylabel("$w_2$")
        if ttl != "log-likelihood":
            ax.plot(*w_map, "r.", ms=9)
    axes[0, 1].plot([0, 12], [0, 6], "k--", lw=0.8)
    for w in [(3, 1), (4, 2), (5, 3), (7, 3)]:
        axes[0, 1].plot(*w, "ko", ms=3)
    for ax, l in zip(axes.ravel(), "abcd"):
        common.panel_label(ax, l, "upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-13", out)


def fig_10_14(out=None):
    """Posterior predictive of the separable-data model: plug-in, sampled boundaries, Monte Carlo, probit."""
    X, y, w_map, V = _laplace_setup()
    lim = (-8, 8)
    g = np.linspace(*lim, 130)
    G = np.stack(np.meshgrid(g, g), -1).reshape(-1, 2)
    fig, axes = common.panel_grid(2, 2, size=(3.5, 3.1))
    lv = np.linspace(0.1, 0.9, 9)
    ax = axes[0, 0]
    p = sigmoid(G @ w_map).reshape(len(g), len(g))
    ax.contour(g, g, p, levels=lv, cmap="coolwarm_r")
    ax.set_title("plug-in $p(y=1\\mid x,\\hat w_{\\rm MAP})$", fontsize=9)
    ax = axes[0, 1]
    rng = np.random.default_rng(1)
    ws = rng.multivariate_normal(w_map, V, 10)
    t = np.array([-8, 8.0])
    for w in ws:
        ax.plot(t, -w[0] * t / w[1], color=BLUE, lw=0.9)
    ax.set_xlim(*lim)
    ax.set_ylim(*lim)
    ax.set_title("boundaries of 10 posterior draws", fontsize=9)
    ax = axes[1, 0]
    p = predictive_mc(G, w_map, V, S=1000, seed=0).reshape(len(g), len(g))
    ax.contour(g, g, p, levels=lv, cmap="coolwarm_r")
    ax.set_title("Monte Carlo average, $S=1000$", fontsize=9)
    ax = axes[1, 1]
    p2 = predictive_probit(G, w_map, V).reshape(len(g), len(g))
    ax.contour(g, g, p2, levels=lv, cmap="coolwarm_r")
    ax.set_title("moderated output (probit)", fontsize=9)
    for ax, l in zip(axes.ravel(), "abcd"):
        _scatter_classes(ax, X, y, s=10)
        ax.set_xlim(*lim)
        ax.set_ylim(*lim)
        common.panel_label(ax, l, "upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-14", out)


# ---- supplementary figures
def fig_10_S1(out=None):
    """Class sampling probabilities (10.97) for a long-tailed class distribution."""
    C = 20
    counts = np.round(5000 * np.arange(1, C + 1) ** -1.1).astype(int) + 5
    fig, ax = plt.subplots(figsize=(5.6, 3.0))
    for q, col, lab in [(1.0, BLUE, "$q=1$ instance-balanced"), (0.5, GREEN, "$q=1/2$ square-root"), (0.0, RED, "$q=0$ class-balanced")]:
        ax.plot(np.arange(1, C + 1), class_sampling_probs(counts, q), "o-", color=col, ms=3.5, label=lab)
    ax.set_xlabel("class rank $c$ (sorted by frequency)")
    ax.set_ylabel("sampling probability $p_c$")
    ax.set_yscale("log")
    ax.legend(fontsize=8)
    savefig(fig, CH, "fig10-S1", out)


def _petal_width_problem():
    X, y = iris_binary([3])
    return add_bias(X), y


def fig_10_S2(out=None):
    """Speed of GD, SGD, Newton/IRLS and the bound optimisation iteration (iris, petal width)."""
    Xb, y = _petal_width_problem()
    w_hat, hist_n = irls(Xb, y, trace=True, tol=1e-14)
    f_star = nll_grad_hess(w_hat, Xb, y)[0]
    _, h_gd = gradient_descent(Xb, y, 0.5, 400)
    _, h_mm = mm_binary(Xb, y, 400)
    _, h_sgd = sgd(Xb, y, 0.05, 40, seed=0)
    fig, ax = plt.subplots(figsize=(5.8, 3.4))
    for h, lab, col in [(h_gd, "gradient descent, step 0.5", BLUE), (h_mm, "bound optimisation (10.82)", GREEN),
                        (hist_n, "Newton / IRLS", RED)]:
        ax.semilogy(np.arange(len(h)), np.maximum(h - f_star, 1e-16), color=col, label=lab)
    ax.semilogy(np.arange(len(h_sgd)) * len(y) / 150.0, np.maximum(h_sgd - f_star, 1e-16), color=ORANGE, lw=1, label="SGD, one example per step (x-axis: epochs)")
    ax.set_xlim(0, 60)
    ax.set_xlabel("iteration (SGD: epoch)")
    ax.set_ylabel("NLL minus its minimum")
    ax.legend(fontsize=7.5)
    savefig(fig, CH, "fig10-S2", out)


def fig_10_S3(out=None):
    """Sigmoid against the Gaussian cdf and the quality of the probit approximation (10.111)."""
    lam = np.sqrt(np.pi / 8)
    a = np.linspace(-6, 6, 400)
    fig, axes = plt.subplots(1, 2, figsize=(7.6, 3.0))
    ax = axes[0]
    ax.plot(a, sigmoid(a), color=BLUE, label="$\\sigma(a)$")
    ax.plot(a, stats.norm.cdf(lam * a), "--", color=RED, label="$\\Phi(\\lambda a)$, $\\lambda^2=\\pi/8$")
    ax.set_xlabel("$a$")
    ax.legend(fontsize=8, loc="upper left")
    common.panel_label(ax, "a", "lower right")
    ax = axes[1]
    v = 4.0
    m = np.linspace(-6, 6, 300)
    ax.plot(m, predictive_exact(m, np.full_like(m, v)), color=BLUE, label="exact $E\\,\\sigma(a)$ (quadrature)")
    ax.plot(m, sigmoid(kappa(v) * m), "--", color=RED, label="probit $\\sigma(\\kappa m)$")
    ax.plot(m, sigmoid(m), ":", color=GREY, label="plug-in $\\sigma(m)$")
    ax.set_xlabel("mean $m$ of the logit ($v=4$)")
    ax.legend(fontsize=8, loc="upper left")
    common.panel_label(ax, "b", "lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig10-S3", out)


def fig_10_S4(out=None):
    """Exercise 10.2: all weights unpenalised, then w0, w1 or w2 heavily penalised."""
    X, y = ex102_data()
    cases = [("(a) unpenalised", [0, 0, 0], 0.0), ("(b) penalise $w_0$", [1, 0, 0], 1e4),
             ("(c) penalise $w_1$", [0, 1, 0], 1e4), ("(d) penalise $w_2$", [0, 0, 1], 1e4)]
    fig, axes = common.panel_grid(1, 4, size=(2.5, 2.5))
    for ax, (ttl, pen, lam) in zip(axes.ravel(), cases):
        w = penalised_fit(X, y, pen, lam) if lam else penalised_fit(X, y, [0, 0, 0], 1e-6)
        err = int(round(error_rate(add_bias(X), y, w) * len(y)))
        xs = np.array([0, 9.0])
        _scatter_classes(ax, X, y, s=18)
        with np.errstate(all="ignore"):
            if abs(w[2]) > 1e-9:
                ax.plot(xs, -(w[0] + w[1] * xs) / w[2], "k")
            else:
                ax.axvline(-w[0] / w[1], color="k")
        ax.set_xlim(0, 9)
        ax.set_ylim(0, 8)
        ax.set_title("%s: %d error%s" % (ttl, err, "" if err == 1 else "s"), fontsize=8.5)
        ax.set_xlabel("$x_1$")
    axes[0, 0].set_ylabel("$x_2$")
    fig.tight_layout()
    savefig(fig, CH, "fig10-S4", out)


FIGURES = {"10.1": fig_10_1, "10.2": fig_10_2, "10.3": fig_10_3, "10.4": fig_10_4, "10.5": fig_10_5,
           "10.6": fig_10_6, "10.7": fig_10_7, "10.10": fig_10_10, "10.11": fig_10_11, "10.12": fig_10_12,
           "10.13": fig_10_13, "10.14": fig_10_14, "10.S1": fig_10_S1, "10.S2": fig_10_S2,
           "10.S3": fig_10_S3, "10.S4": fig_10_S4}


# =========================================================================== JSON for the demonstrations
def export_json():
    os.makedirs(DATA_OUT, exist_ok=True)
    r = lambda a, k=6: [round(float(v), k) for v in np.ravel(a)]
    # 1. optimisers on the iris petal-width problem (Figure 10.5)
    Xb, y = _petal_width_problem()
    w_hat, hist_n = irls(Xb, y, trace=True, tol=1e-14)
    f_star = nll_grad_hess(w_hat, Xb, y)[0]
    _, h_gd = gradient_descent(Xb, y, 0.5, 60)
    _, h_gd2 = gradient_descent(Xb, y, 2.0, 60)
    wmm, h_mm = mm_binary(Xb, y, 60)
    d1 = dict(x=r(Xb[:, 1], 3), y=[int(v) for v in y], what=r(w_hat, 6), fstar=float(f_star),
              ref=dict(newton=r(hist_n[:8], 8), gd_lr0p5=r(h_gd[[0, 1, 2, 5, 10, 20, 40, 60]], 8), gd_lr2=r(h_gd2[[0, 1, 2, 5, 10, 20, 40, 60]], 8),
                       mm=r(h_mm[[0, 1, 2, 5, 10, 20, 40, 60]], 8), idx=[0, 1, 2, 5, 10, 20, 40, 60]))
    with open(os.path.join(DATA_OUT, "ch10-optimisers.json"), "w") as f:
        json.dump(d1, f, separators=(",", ":"))
    # 2. Laplace + predictive on the separable data
    X, ys, w_map, V = _laplace_setup(100.0)
    pts = np.array([[0.0, 0.0], [3.0, 3.0], [-4.0, 2.0], [6.0, -2.0], [-6.0, -6.0]])
    m = pts @ w_map
    v = np.einsum("ij,jk,ik->i", pts, V, pts)
    d2 = dict(X=[r(p, 4) for p in X], y=[int(v_) for v_ in ys],
              ref=dict(prior_var=100.0, wmap=r(w_map, 6), V=r(V, 8), pts=[r(p, 3) for p in pts], plugin=r(sigmoid(m), 8),
                       probit=r(predictive_probit(pts, w_map, V), 8), exact=r(predictive_exact(m, v), 8)))
    with open(os.path.join(DATA_OUT, "ch10-laplace.json"), "w") as f:
        json.dump(d2, f, separators=(",", ":"))
    # 3. robust logistic regression on the contaminated iris sepal length (centred), plus reference values
    xc, yc, extra = contaminated_iris()
    x0, y0 = iris_setosa_versicolor()
    cen = float(x0.mean())
    d3 = dict(x=r(xc, 2), y=[int(v_) for v_ in yc], centre=cen, nclean=int(len(x0)), lam=ROB_LAM, ref={})
    wfix = np.array([0.0, 2.0])
    Xc6 = add_bias(xc - cen)
    d3["ref"]["fixed_w"] = r(wfix, 3)
    d3["ref"]["bt_loss_fixed"] = {"%g_%g" % (t1, t2): float(bitempered_binary_loss(Xc6 @ wfix, yc, t1, t2).mean())
                                  for t1, t2 in [(1, 1), (0.2, 1), (1, 4), (0.2, 4)]}
    d3["ref"]["mix_nll_fixed_pi0.1"] = float(-robust_loglik(wfix, 0.1, Xc6, yc).mean())
    fits = {}
    for k_out in (0, 6):
        n_use = len(x0) + k_out
        Xk, yk = add_bias(xc[:n_use] - cen), yc[:n_use]
        for name, f in [("std", lambda w: (np.logaddexp(0, Xk @ w) - yk * (Xk @ w)).mean()),
                        ("mix0.1", lambda w: -robust_loglik(w, 0.1, Xk, yk).mean()),
                        ("bt1_4", lambda w: bitempered_binary_loss(Xk @ w, yk, 1.0, 4.0).mean()),
                        ("bt0.2_1", lambda w: bitempered_binary_loss(Xk @ w, yk, 0.2, 1.0).mean()),
                        ("bt0.2_4", lambda w: bitempered_binary_loss(Xk @ w, yk, 0.2, 4.0).mean())]:
            obj = lambda w, f=f: f(w) + 0.5 * ROB_LAM * w[1] ** 2
            best = None
            for s0 in ([0.0, 1.0], [0.0, 3.0], [0.0, 0.3], [0.5, 2.0], [-0.5, 2.0]):
                rr = optimize.minimize(obj, s0, method="Nelder-Mead", options=dict(xatol=1e-7, fatol=1e-12, maxiter=2000))
                if best is None or rr.fun < best.fun:
                    best = rr
            fits["%s_k%d" % (name, k_out)] = dict(w=r(best.x, 5), obj=float(best.fun), boundary=float(cen - best.x[0] / best.x[1]))
    d3["ref"]["fits"] = fits
    with open(os.path.join(DATA_OUT, "ch10-robust.json"), "w") as f:
        json.dump(d3, f, separators=(",", ":"))
    print("wrote ch10-*.json")


if __name__ == "__main__":
    if "--json" in sys.argv:
        export_json()
    else:
        common.run(FIGURES, CH)
        if "--fig" not in sys.argv and "--list" not in sys.argv:
            export_json()
