"""Bishop and Bishop (2024), Chapter 4: Single-layer Networks: Regression.

Reproduces the numerical figures of Chapter 4 of C. M. Bishop and H. Bishop, Deep Learning:
Foundations and Concepts (Springer, 2024), pp. 111-130, written from scratch with numpy, scipy and
matplotlib. Every algorithm is a small function meant to be read next to the lecture notes
(../ch04-single-layer-regression.html):

* Section 4.1.1, basis functions: poly_basis, gauss_basis, sigmoid_basis and design_matrix build the
  design matrix (4.15) with the constant 'bias' column phi_0 = 1 first.
* Sections 4.1.2-4.1.3, maximum likelihood: fit_ml solves the normal equations (4.14) through the
  singular value decomposition (the pseudo-inverse (4.16), which is also the minimum-norm solution
  when Phi has dependent columns); fit_normal_equations solves (4.14) directly, to show the loss of
  accuracy when Phi^T Phi is nearly singular; sigma2_ml is (4.20).
* Section 4.1.4, geometry: projection_matrix is Phi (Phi^T Phi)^{-1} Phi^T of (4.59).
* Section 4.1.5, sequential learning: lms runs the LMS update (4.22) (stochastic gradient descent on
  (4.11)), with a constant or decreasing learning rate; batch_gd is ordinary gradient descent.
* Section 4.1.6, regularised least squares: fit_ridge is (4.27), via the SVD (shrinkage factors).
* Section 4.1.7, multiple outputs: fit_ml applied to a target matrix T gives (4.31)-(4.32).
* Section 4.2, decision theory: lq_risk_grid / lq_minimiser compute the expected Minkowski loss
  (4.40) of a one-dimensional conditional density and its minimiser; log_risk_grid is the q -> 0
  limit E[ln|y - t|] derived in the notes (the minimiser of the Minkowski risk tends to the minimiser
  of this function, which is NOT the mode in general).
* Section 4.3, the bias-variance trade-off: bias_variance_experiment is the experiment of Figures 4.7
  and 4.8 ((4.50)-(4.52)); ridge_bias_variance_exact gives the same quantities in closed form for a
  fixed design.
* JSRandom: the seeded generator DK.rng of the browser demos (mulberry32 uniforms and Box-Muller
  normals, the second normal of each pair cached), so that sine_data and line_data produce the same
  numbers as basis-regression-demo.html and lms-demo.html.

Figures reproduced: 4.2, 4.3, 4.5, 4.6, 4.7, 4.8. Run `python3 ch04_single_layer_regression.py
--list`, `python3 ch04_single_layer_regression.py` (all figures, under a minute) or
`python3 ch04_single_layer_regression.py --fig 4.7 4.8`. Files go to ../img/ch04/.

Figures not reproduced by code, and why:
* Figure 4.1 (the linear model drawn as a network with one layer of weights) and Figure 4.4 (the
  same with K outputs) are network schematics; the notes redraw them as inline SVG.

Settings chosen where the book gives none:
* Figure 4.2: polynomials x^j, j = 1..10, on [-1, 1]; 11 Gaussians (4.4) with centres equally spaced
  on [-1, 1] and s = 0.1; 11 sigmoids (4.5) with the same centres and s = 0.1.
* Figure 4.3: N = 3 and two basis vectors, so that the subspace S is a plane in R^3; the vectors are
  phi_1 = (1, 0.1, 0.05) and phi_2 = (0.15, 1, 0.2) (the values of two basis functions at three
  inputs), and t = (0.7, 0.6, 0.95). The figure is
  computed: y is Phi w_ML from (4.14), and the residual t - y is drawn to show it is orthogonal to S.
* Figure 4.5: the population version of the sinusoidal problem: h(x) = sin(2 pi x) and Gaussian noise
  of standard deviation 0.3, with p(t | x0) drawn at x0 = 0.35.
* Figures 4.7 and 4.8: x uniform on (0, 1), t = sin(2 pi x) + noise of standard deviation 0.3
  (the sinusoidal data of Section 1.2), L = 100 data sets of N = 25 points (data set l uses
  JSRandom(4000 + l)), 24 Gaussian basis functions (4.4) with centres equally spaced on [0, 1] and
  s = 0.1, plus the bias; the regularised error (4.26) penalises all 25 weights, bias included, as
  written in the book. The integrals over x in (4.51)-(4.52) are averages over 200 equally spaced
  points of [0, 1]; the test error is the mean squared error on 1000 fresh points (JSRandom(5000)),
  averaged over the 100 fits.
"""
import math
import os

import numpy as np

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, RED, savefig

CH = 4
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
M32 = 0xFFFFFFFF
NOISE_SD = 0.3


# =========================================================================== the demos' generator
def _imul(a, b):
    """32-bit integer multiplication with wrap-around, as Math.imul (result as unsigned)."""
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    u() is the mulberry32 generator; norm() is the Box-Muller transform that returns
    r cos(theta) and caches r sin(theta) for the next call, as in JavaScript.
    """

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def _pos(self):
        x = 0.0
        while x == 0.0:
            x = self.u()
        return x

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        r = math.sqrt(-2.0 * math.log(self._pos()))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def sine_data(seed, N, sd=NOISE_SD):
    """N points x ~ U(0, 1), t = sin(2 pi x) + sd * noise, drawn point by point (x then its noise)."""
    rng = JSRandom(seed)
    x, t = np.empty(N), np.empty(N)
    for n in range(N):
        x[n] = rng.u()
        t[n] = math.sin(2 * math.pi * x[n]) + sd * rng.norm()
    return x, t


LINE_W = (-0.3, 0.8)   # true intercept and slope of the LMS demo's data


def line_data(seed, N, sd=0.2, xmax=1.0):
    """N points x ~ U(0, xmax), t = -0.3 + 0.8 x + sd * noise (the data of lms-demo.html)."""
    rng = JSRandom(seed)
    x, t = np.empty(N), np.empty(N)
    for n in range(N):
        x[n] = xmax * rng.u()
        t[n] = LINE_W[0] + LINE_W[1] * x[n] + sd * rng.norm()
    return x, t


# =========================================================================== Section 4.1.1: basis functions
def sigmoid(a):
    return common.sigmoid(a)


def centres(M, lo=0.0, hi=1.0):
    """M - 1 equally spaced centres on [lo, hi] for the non-constant basis functions."""
    if M - 1 == 1:
        return np.array([(lo + hi) / 2])
    return lo + (hi - lo) * np.arange(M - 1) / (M - 2)


def poly_basis(x, M):
    """phi_j(x) = x^j, j = 0..M-1 (Chapter 1's polynomial)."""
    x = np.asarray(x, float)
    return np.stack([x ** j for j in range(M)], axis=1)


def gauss_basis(x, M, s, lo=0.0, hi=1.0):
    """Bias column plus M - 1 Gaussian basis functions (4.4) with centres on [lo, hi] and width s."""
    x = np.asarray(x, float)
    mu = centres(M, lo, hi)
    return np.column_stack([np.ones_like(x), np.exp(-(x[:, None] - mu[None, :]) ** 2 / (2 * s * s))])


def sigmoid_basis(x, M, s, lo=0.0, hi=1.0):
    """Bias column plus M - 1 sigmoidal basis functions (4.5)."""
    x = np.asarray(x, float)
    mu = centres(M, lo, hi)
    return np.column_stack([np.ones_like(x), sigmoid((x[:, None] - mu[None, :]) / s)])


def design_matrix(x, kind, M, s=0.1):
    """The N x M design matrix (4.15): Phi[n, j] = phi_j(x_n), column 0 the constant 1."""
    if kind == "poly":
        return poly_basis(x, M)
    if kind == "gauss":
        return gauss_basis(x, M, s)
    if kind == "sigmoid":
        return sigmoid_basis(x, M, s)
    raise ValueError(kind)


# =========================================================================== Sections 4.1.2-4.1.7: fitting
def log_likelihood(w, sigma2, Phi, t):
    """ln p(t | X, w, sigma^2) of (4.10)."""
    N = len(t)
    ED = 0.5 * np.sum((t - Phi @ w) ** 2)
    return -0.5 * N * math.log(sigma2) - 0.5 * N * math.log(2 * math.pi) - ED / sigma2


def grad_log_likelihood(w, sigma2, Phi, t):
    """The gradient (4.12), as a column vector: (1/sigma^2) sum_n (t_n - w^T phi_n) phi_n."""
    return Phi.T @ (t - Phi @ w) / sigma2


def pinv(Phi, rcond=1e-12):
    """The Moore-Penrose pseudo-inverse from the SVD Phi = U diag(d) V^T (small d treated as 0)."""
    U, d, Vt = np.linalg.svd(Phi, full_matrices=False)
    keep = d > rcond * d.max()
    dinv = np.where(keep, 1.0 / np.where(keep, d, 1.0), 0.0)
    return (Vt.T * dinv) @ U.T


def fit_ml(Phi, t):
    """w_ML = Phi^dagger t (4.14), (4.16); t may be a vector or an N x K matrix T, giving (4.31)."""
    return pinv(Phi) @ t


def fit_normal_equations(Phi, t):
    """w = (Phi^T Phi)^{-1} Phi^T t by solving the normal equations directly (loses accuracy when
    Phi^T Phi is nearly singular, because its condition number is the square of Phi's)."""
    return np.linalg.solve(Phi.T @ Phi, Phi.T @ t)


def fit_ridge(Phi, t, lam):
    """The regularised least-squares solution (4.27), w = (lambda I + Phi^T Phi)^{-1} Phi^T t,
    computed from the SVD: w = sum_i d_i / (d_i^2 + lambda) (u_i^T t) v_i."""
    if lam == 0:
        return fit_ml(Phi, t)
    U, d, Vt = np.linalg.svd(Phi, full_matrices=False)
    return Vt.T @ ((d / (d * d + lam)) * (U.T @ t)) if np.ndim(t) == 1 else Vt.T @ ((d / (d * d + lam))[:, None] * (U.T @ t))


def sigma2_ml(Phi, t, w):
    """The maximum likelihood noise variance (4.20)."""
    return float(np.mean((t - Phi @ w) ** 2))


def projection_matrix(Phi):
    """P = Phi (Phi^T Phi)^{-1} Phi^T of (4.59), the orthogonal projector onto the columns of Phi."""
    return Phi @ pinv(Phi)


def weighted_ls(Phi, t, r):
    """Minimiser of the weighted error (4.60): w = (Phi^T R Phi)^{-1} Phi^T R t, R = diag(r)."""
    sr = np.sqrt(r)
    return fit_ml(Phi * sr[:, None], t * sr)


# =========================================================================== Section 4.1.5: sequential learning
def lms(Phi, t, eta, epochs, w0=None, schedule="constant", order="cyclic", seed=1, record=True):
    """The LMS algorithm (4.22): w <- w + eta_tau (t_n - w^T phi_n) phi_n, one data point at a time.

    schedule 'constant' uses eta_tau = eta; 'decay' uses eta_tau = eta / (1 + tau / N), which satisfies
    the Robbins-Monro conditions sum eta_tau = infinity, sum eta_tau^2 < infinity. order 'cyclic' visits
    n = 0, 1, ..., N-1 in every epoch; 'random' shuffles each epoch with JSRandom(seed) (Fisher-Yates, as
    in the demo). Returns the final weights and (if record) the array of all iterates.
    """
    N, M = Phi.shape
    w = np.zeros(M) if w0 is None else np.array(w0, float)
    path = [w.copy()] if record else None
    rng = JSRandom(seed)
    tau = 0
    for _ in range(epochs):
        idx = list(range(N))
        if order == "random":
            for i in range(N - 1, 0, -1):
                j = int(math.floor(rng.u() * (i + 1)))
                idx[i], idx[j] = idx[j], idx[i]
        for n in idx:
            et = eta if schedule == "constant" else eta / (1.0 + tau / N)
            r = t[n]
            for j in range(M):
                r -= w[j] * Phi[n, j]
            for j in range(M):
                w[j] += et * r * Phi[n, j]
            tau += 1
            if record:
                path.append(w.copy())
    return w, (np.array(path) if record else None)


def batch_gd(Phi, t, eta, steps, w0=None):
    """Gradient descent on the mean squared error: w <- w + (eta / N) Phi^T (t - Phi w)."""
    N, M = Phi.shape
    w = np.zeros(M) if w0 is None else np.array(w0, float)
    path = [w.copy()]
    for _ in range(steps):
        w = w + (eta / N) * (Phi.T @ (t - Phi @ w))
        path.append(w.copy())
    return w, np.array(path)


# =========================================================================== Section 4.2: decision theory
def density(kind, t):
    """The conditional densities p(t | x) used for the Minkowski-loss demo."""
    from scipy import stats
    if kind == "gauss":
        return stats.norm.pdf(t, 1.0, 0.8)
    if kind == "gamma":
        return stats.gamma.pdf(t, 2.0)
    if kind == "bimodal":
        return 0.3 * stats.norm.pdf(t, -1.0, 0.2) + 0.7 * stats.norm.pdf(t, 1.5, 0.7)
    raise ValueError(kind)


DENSITY_RANGE = {"gauss": (-3.0, 5.0), "gamma": (0.0, 16.0), "bimodal": (-3.0, 5.0)}


def lq_risk_grid(kind, q, n=1201):
    """E|y - t|^q on the grid t_k (y = t_k), by the trapezoidal rule with the cell around y = t_k
    integrated exactly (p taken constant there). Same algorithm as minkowski-loss-demo.html."""
    lo, hi = DENSITY_RANGE[kind]
    t = np.linspace(lo, hi, n)
    h = t[1] - t[0]
    p = density(kind, t)
    wts = np.full(n, h)
    wts[0] = wts[-1] = h / 2
    D = np.abs(t[:, None] - t[None, :])
    G = (D ** q * (p * wts)[None, :]).sum(1)
    # replace the two half-cells next to y by the exact integral of |u|^q p(y) over (-h, h)
    G += -(h / 2) * (h ** q) * (np.r_[p[1:], 0.0] + np.r_[0.0, p[:-1]]) + 2 * h ** (1 + q) / (1 + q) * p
    return t, G


def log_risk_grid(kind, n=1201):
    """E[ln|y - t|] on the same grid, with the singular cell integrated exactly: int_{-h}^{h} ln|u| du = 2h(ln h - 1)."""
    lo, hi = DENSITY_RANGE[kind]
    t = np.linspace(lo, hi, n)
    h = t[1] - t[0]
    p = density(kind, t)
    wts = np.full(n, h)
    wts[0] = wts[-1] = h / 2
    D = np.abs(t[:, None] - t[None, :])
    with np.errstate(divide="ignore"):
        LD = np.where(D > 0, np.log(np.where(D > 0, D, 1.0)), 0.0)
    G = (LD * (p * wts)[None, :]).sum(1)
    G += -(h / 2) * math.log(h) * (np.r_[p[1:], 0.0] + np.r_[0.0, p[:-1]]) + 2 * h * (math.log(h) - 1) * p
    return t, G


def grid_argmin(t, G):
    """Minimiser of G on the grid, refined by a parabola through the three points around the minimum."""
    k = int(np.argmin(G))
    if 0 < k < len(t) - 1:
        a, b, c = G[k - 1], G[k], G[k + 1]
        den = a - 2 * b + c
        if den > 0:
            return float(t[k] + 0.5 * (a - c) / den * (t[1] - t[0]))
    return float(t[k])


def lq_minimiser(kind, q, n=1201):
    if q == 0:
        return grid_argmin(*log_risk_grid(kind, n))
    return grid_argmin(*lq_risk_grid(kind, q, n))


# =========================================================================== Section 4.3: bias and variance
def bias_variance(F, h):
    """(bias)^2 and variance of (4.51)-(4.52) from fits F (L x n grid values) and the truth h."""
    fbar = F.mean(0)                                   # (4.50)
    return float(np.mean((fbar - h) ** 2)), float(np.mean(((F - fbar) ** 2).mean(0)))


BV = dict(L=100, N=25, M=25, s=0.1, sd=NOISE_SD, seed=4000, test_seed=5000, Ntest=1000)


def bias_variance_experiment(lnlams, L=BV["L"], N=BV["N"], M=BV["M"], s=BV["s"], sd=BV["sd"]):
    """The experiment behind Figures 4.7 and 4.8 for every value of ln lambda in lnlams."""
    xg = np.linspace(0, 1, 200)
    hg = np.sin(2 * np.pi * xg)
    Pg = gauss_basis(xg, M, s)
    sets = [sine_data(BV["seed"] + l, N, sd) for l in range(L)]
    xt, tt = sine_data(BV["test_seed"], BV["Ntest"], sd)
    Pt = gauss_basis(xt, M, s)
    Ps = [gauss_basis(x, M, s) for x, _ in sets]
    out = dict(xg=xg, h=hg, lnlam=np.asarray(lnlams, float), fits={}, bias2=[], var=[], test=[])
    for ll in lnlams:
        lam = math.exp(ll)
        W = np.array([fit_ridge(P, t, lam) for P, (_, t) in zip(Ps, sets)])
        F = W @ Pg.T
        b2, v = bias_variance(F, hg)
        out["fits"][float(ll)] = F
        out["bias2"].append(b2)
        out["var"].append(v)
        out["test"].append(float(np.mean((W @ Pt.T - tt[None, :]) ** 2)))
    for k in ("bias2", "var", "test"):
        out[k] = np.array(out[k])
    return out


def ridge_bias_variance_exact(Phi, h_train, lam, sd, Pg):
    """Closed form for a FIXED design: the ridge fit is f = S t with S = Pg (lambda I + Phi^T Phi)^{-1} Phi^T,
    so E_D[f] = S h_train and var = sd^2 * (row norms of S)^2 at each grid point."""
    A = np.linalg.solve(lam * np.eye(Phi.shape[1]) + Phi.T @ Phi, Phi.T)
    S = Pg @ A
    return S @ h_train, sd ** 2 * np.sum(S * S, 1)


# =========================================================================== figures
def fig_4_2(out_dir=None):
    """Polynomial, Gaussian and sigmoidal basis functions."""
    x = np.linspace(-1, 1, 400)
    mu = np.linspace(-1, 1, 11)
    fig, ax = common.panel_grid(1, 3, size=(2.7, 2.3))
    ax = ax[0]
    cols = [RED, BLUE, GREEN, MAGENTA, ORANGE, "#17becf", "#7b4fbf", GREY, "#8c564b", "#bcbd22", "#1f1f1f"]
    for j in range(1, 11):
        ax[0].plot(x, x ** j, color=cols[(j - 1) % len(cols)], lw=1.2)
    for j, m in enumerate(mu):
        ax[1].plot(x, np.exp(-(x - m) ** 2 / (2 * 0.1 ** 2)), color=cols[j % len(cols)], lw=1.2)
        ax[2].plot(x, sigmoid((x - m) / 0.1), color=cols[j % len(cols)], lw=1.2)
    ax[0].set_ylim(-1, 1)
    for a in ax:
        a.set_xlim(-1, 1)
        a.set_xticks([-1, 0, 1])
    ax[1].set_ylim(0, 1)
    ax[2].set_ylim(0, 1)
    ax[0].set_title(r"polynomials $x^j$")
    ax[1].set_title(r"Gaussians (4.4), $s=0.1$")
    ax[2].set_title(r"sigmoids (4.5), $s=0.1$")
    fig.tight_layout()
    savefig(fig, CH, "fig4-2", out_dir)


GEOM = dict(phi1=np.array([1.0, 0.1, 0.05]), phi2=np.array([0.15, 1.0, 0.2]), t=np.array([0.7, 0.6, 0.95]))


def fig_4_3(out_dir=None):
    """The least-squares fit as the orthogonal projection of t onto the subspace S spanned by the columns of Phi."""
    import matplotlib.pyplot as plt
    p1, p2, t = GEOM["phi1"], GEOM["phi2"], GEOM["t"]
    Phi = np.column_stack([p1, p2])
    w = fit_ml(Phi, t)
    y = Phi @ w
    fig = plt.figure(figsize=(4.6, 4.0))
    ax = fig.add_subplot(111, projection="3d")
    a, b = np.meshgrid(np.linspace(-0.1, 1.15, 12), np.linspace(-0.1, 1.15, 12))
    P = a[..., None] * p1 + b[..., None] * p2
    ax.plot_surface(P[..., 0], P[..., 1], P[..., 2], color="#9ecae1", alpha=0.25, linewidth=0, shade=False)
    o = np.zeros(3)

    def arrow(v, c, lab, off=(0, 0, 0), lw=2.0, start=o):
        ax.quiver(*start, *(v - start), color=c, lw=lw, arrow_length_ratio=0.08)
        ax.text(*(v + np.array(off)), lab, color=c, fontsize=12)

    arrow(p1, BLUE, r"$\varphi_1$", (0.03, 0.03, 0.05))
    arrow(p2, BLUE, r"$\varphi_2$", (0.03, 0.03, 0.05))
    arrow(t, RED, r"$\mathbf{t}$", (0.02, 0.02, 0.06))
    arrow(y, GREEN, r"$\mathbf{y}$", (0.03, 0.03, -0.18))
    ax.plot(*np.column_stack([y, t]), color=GREY, ls="--", lw=1.4)
    # right-angle mark between the residual and the plane
    r = (t - y) / np.linalg.norm(t - y)
    u = (p2 - p2 @ (y / np.linalg.norm(y)) * y / np.linalg.norm(y))
    u = u / np.linalg.norm(u)
    e = 0.12
    sq = np.array([y + e * u, y + e * u + e * r, y + e * r])
    ax.plot(*sq.T, color=GREY, lw=1.0)
    ax.text(*(P[-1, -1] + np.array([0, 0, 0.03])), r"$\mathcal{S}$", color="#3b6fa0", fontsize=14)
    ax.set_xlabel(r"$t_1$", labelpad=-8)
    ax.set_ylabel(r"$t_2$", labelpad=-8)
    ax.set_zlabel(r"$t_3$", labelpad=-8)
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    for axis in (ax.xaxis, ax.yaxis, ax.zaxis):
        axis.set_pane_color((1, 1, 1, 0))
    ax.set_xlim(0, 1.1)
    ax.set_ylim(0, 1.1)
    ax.set_zlim(0, 1.0)
    ax.view_init(elev=16, azim=-58)
    ax.set_box_aspect((1, 1, 0.9))
    fig.subplots_adjust(0, 0, 1, 1)
    savefig(fig, CH, "fig4-3", out_dir)
    return dict(w=w, y=y, residual_dot=(t - y) @ Phi)


def fig_4_5(out_dir=None):
    """The regression function E[t|x] and the conditional density p(t|x0) at one input."""
    import matplotlib.pyplot as plt
    x = np.linspace(0, 1, 300)
    h = np.sin(2 * np.pi * x)
    x0, sd = 0.35, NOISE_SD
    fig, ax = plt.subplots(figsize=(4.4, 3.0))
    ax.fill_between(x, h - 2 * sd, h + 2 * sd, color=RED, alpha=0.08, lw=0)
    ax.plot(x, h, color=RED, lw=2, label=r"$f^\star(x)=\mathbb{E}[t\,|\,x]$")
    tt = np.linspace(-3.2 * sd, 3.2 * sd, 300) + math.sin(2 * math.pi * x0)
    dens = np.exp(-(tt - math.sin(2 * math.pi * x0)) ** 2 / (2 * sd * sd)) / math.sqrt(2 * math.pi * sd * sd)
    ax.axvline(x0, color=GREY, lw=0.8, ls=":")
    ax.plot(x0 + 0.12 * dens / dens.max(), tt, color=BLUE, lw=1.6, label=r"$p(t\,|\,x_0)$")
    ax.plot([x0], [math.sin(2 * math.pi * x0)], "o", color=BLUE, ms=4)
    ax.text(x0 - 0.01, -1.55, r"$x_0$", ha="right", fontsize=10)
    rng = JSRandom(11)
    xs = np.array([rng.u() for _ in range(40)])
    ts = np.sin(2 * np.pi * xs) + sd * np.array([rng.norm() for _ in range(40)])
    ax.plot(xs, ts, "o", mfc="none", mec=GREY, ms=3.5, mew=0.8, label="samples from $p(x,t)$")
    ax.set_xlim(0, 1)
    ax.set_ylim(-1.7, 1.7)
    ax.set_xlabel("$x$")
    ax.set_ylabel("$t$")
    ax.legend(loc="upper right", fontsize=8, frameon=False)
    fig.tight_layout()
    savefig(fig, CH, "fig4-5", out_dir)


def fig_4_6(out_dir=None):
    """The Minkowski loss |f - t|^q for q = 0.3, 1, 2, 10."""
    u = np.linspace(-2, 2, 801)
    fig, ax = common.panel_grid(1, 4, size=(2.25, 2.0))
    for a, q in zip(ax[0], (0.3, 1, 2, 10)):
        a.plot(u, np.abs(u) ** q, color=RED, lw=1.8)
        a.set_xlim(-2, 2)
        a.set_ylim(0, 2)
        a.set_xlabel(r"$f-t$")
        a.set_ylabel(r"$|f-t|^{%g}$" % q)
        a.text(0, 1.75, r"$q=%g$" % q, ha="center", fontsize=10)
    fig.tight_layout()
    savefig(fig, CH, "fig4-6", out_dir)


BV_SHOW = (3.0, 1.0, -3.0)


def fig_4_7(out_dir=None):
    """20 of the 100 ridge fits (left) and their average against sin(2 pi x) (right) for ln lambda = 3, 1, -3."""
    res = bias_variance_experiment(BV_SHOW)
    fig, ax = common.panel_grid(3, 2, size=(2.8, 1.95))
    for i, ll in enumerate(BV_SHOW):
        F = res["fits"][ll]
        for l in range(20):
            ax[i, 0].plot(res["xg"], F[l], color=RED, lw=0.7, alpha=0.8)
        ax[i, 1].plot(res["xg"], res["h"], color=GREEN, lw=1.8)
        ax[i, 1].plot(res["xg"], F.mean(0), color=RED, lw=1.8)
        ax[i, 0].text(0.62, 1.05, r"$\ln\lambda=%g$" % ll, fontsize=10)
        for a in ax[i]:
            a.set_xlim(0, 1)
            a.set_ylim(-1.5, 1.5)
            a.set_xticks([0, 1])
            a.set_yticks([-1, 0, 1])
            a.set_ylabel("$t$", labelpad=-2)
        if i == 2:
            ax[i, 0].set_xlabel("$x$", labelpad=-6)
            ax[i, 1].set_xlabel("$x$", labelpad=-6)
    fig.tight_layout()
    savefig(fig, CH, "fig4-7", out_dir)
    return res


def fig_4_8(out_dir=None):
    """(bias)^2, variance, their sum and the test error against ln lambda."""
    import matplotlib.pyplot as plt
    ll = np.linspace(-3, 3, 61)
    res = bias_variance_experiment(ll)
    s = res["bias2"] + res["var"]
    k = int(np.argmin(s))
    kt = int(np.argmin(res["test"]))
    fig, ax = plt.subplots(figsize=(4.4, 3.0))
    ax.plot(ll, res["bias2"], color=BLUE, label=r"$(\mathrm{bias})^2$")
    ax.plot(ll, res["var"], color=RED, label="variance")
    ax.plot(ll, s, color=MAGENTA, label=r"$(\mathrm{bias})^2+\mathrm{variance}$")
    ax.plot(ll, res["test"], color="black", label="test error")
    ax.axvline(ll[k], color=GREY, ls=":", lw=0.9)
    ax.set_xlim(-3, 3)
    ax.set_ylim(0, 0.25)
    ax.set_yticks([0, 0.1, 0.2])
    ax.set_xlabel(r"$\ln\lambda$")
    ax.legend(fontsize=8, frameon=False, loc="upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig4-8", out_dir)
    print("  minimum of bias^2 + variance at ln lambda = %.2f (value %.4f); minimum test error at ln lambda = %.2f (value %.4f)"
          % (ll[k], s[k], ll[kt], res["test"][kt]))
    return dict(ln_min=float(ll[k]), ln_min_test=float(ll[kt]), res=res)


def bias_variance_minimum(step=0.01):
    """ln lambda minimising (bias)^2 + variance and the test error, on a fine grid (for the notes)."""
    ll = np.arange(-1.5, 1.5 + 1e-9, step)
    res = bias_variance_experiment(ll)
    s = res["bias2"] + res["var"]
    return float(ll[np.argmin(s)]), float(ll[np.argmin(res["test"])]), float(s.min()), float(res["test"].min())


FIGURES = {"4.2": fig_4_2, "4.3": fig_4_3, "4.5": fig_4_5, "4.6": fig_4_6, "4.7": fig_4_7, "4.8": fig_4_8}

if __name__ == "__main__":
    common.run(FIGURES, CH)
