"""Bishop and Bishop (2024), Chapter 1: The Deep Learning Revolution.

Reproduces the numerical figures of Chapter 1 of C. M. Bishop and H. Bishop, Deep Learning:
Foundations and Concepts (Springer, 2024), written from scratch with numpy and matplotlib, and
holds the small algorithms that the lecture notes (ch01-deep-learning-revolution.html) and the
three browser demos use:

* Section 1.2, the tutorial example: polynomial design matrices (1.1), least squares for the
  sum-of-squares error (1.2) and for the regularised error (1.4), both solved by a Householder QR
  factorisation of the (augmented) design matrix (lsq_qr), never by forming Phi^T Phi, because the
  M = 9 design matrix on [0, 1] has condition number about 1.5e7 and Phi^T Phi about 2e14;
  the RMS error (1.3); the hat matrix and the effective number of parameters; the exact
  leave-one-out shortcut; S-fold cross-validation with a seeded partition.
* Section 1.3, history: the perceptron with the step function (1.7) and 0/1 targets, its
  maximum margin (for the convergence bound), a two-layer tanh/sigmoid network trained by
  backpropagation on XOR, and the gradient signal through a deep chain of layers with and without
  residual connections.
* JSRandom: the seeded generator DK.rng of courses/demo-kit.js (mulberry32 + Box-Muller). All data
  and initial weights of the demos come from it, so curve-fitting-demo.html,
  model-selection-demo.html and neural-history-demo.html reproduce the numbers computed here.

The book's own training set. The book does not print its ten training targets, but the M = 9
column of its Table 1.1 determines them: with N = 10 points x_n = (n - 1)/9 the M = 9 polynomial
interpolates the data, so t_n = y(x_n, w*) (BOOK_T below). These targets reproduce the M = 0, 1, 3
columns of Table 1.1 to the printed two decimals, which confirms both the inputs and the targets.
The book does not give its noise level or its test set; we use noise standard deviation 0.2 and a
test set of 100 points x = (n - 1)/99 with targets sin(2 pi x) + 0.2 * JSRandom(SEED_TEST) normals.

A finding about Table 1.2. With these targets, the columns ln lambda = -18 and ln lambda = 0 of the
book's Table 1.2 are reproduced exactly (to the printed decimals) only if lambda is multiplied by
N = 10 in (1.4), i.e. if the minimised error is (1/N) sum {y - t}^2 / 2 + lambda/2 ||w||^2; with
(1.4) as printed they are reproduced at ln lambda = -18 + ln 10 and 0 + ln 10. Our Figures 1.9
and 1.10 use (1.4) exactly as printed (red / solid) and show the book's convention as well
(dashed), see fig1_9 and fig1_10.

Figures: 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10 (and Tables 1.1 and 1.2, printed by --tables).
Skipped (not computed from data, a model or a simulation):
* 1.1 photographs of skin lesions; 1.2 protein structure reprinted from Jumper et al. (2021);
  1.3 photographs of synthetic faces; 1.12 drawing of two neurons; 1.14 photographs of the
  Mark 1 perceptron hardware; 1.16 compute-growth chart reprinted from OpenAI (its data are not
  in the book).
* 1.11 (S-fold cross-validation), 1.13 (a single neuron) and 1.15 (a two-layer network) are
  schematic diagrams; they are redrawn as inline SVG in the lecture notes.

Run `python3 ch01_deep_learning_revolution.py --list`, `python3 ch01_deep_learning_revolution.py`
(all figures, a few seconds) or `--fig 1.6 1.10`. Files go to ../img/ch01/.
"""
import math
import os
import sys

import numpy as np

import common

plt = common.plt
CH = 1
HERE = os.path.dirname(os.path.abspath(__file__))

# ============================================================================= seeded generator
M32 = 0xFFFFFFFF


def _imul(a, b):
    """Math.imul: 32-bit integer multiplication, result as an unsigned 32-bit int."""
    return ((a & M32) * (b & M32)) & M32


def _to_signed(a):
    return a - (1 << 32) if a & 0x80000000 else a


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    Uniforms come from mulberry32; normals from the basic Box-Muller transform, the second
    normal of each pair being cached, exactly as in the JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def _pos(self):
        x = 0.0
        while x == 0.0:
            x = self.u()
        return x

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        r = math.sqrt(-2.0 * math.log(self._pos()))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)

    def shuffle(self, a):
        """Fisher-Yates shuffle in place, as ML.shuffle in courses/ml/ml-kit.js."""
        for i in range(len(a) - 1, 0, -1):
            j = int(math.floor(self.u() * (i + 1)))
            a[i], a[j] = a[j], a[i]
        return a


# ============================================================================= Section 1.2 data
# Table 1.1, M = 9 column (the book's printed coefficients).
BOOK_W9 = np.array([0.26, -66.13, 1665.69, -15566.61, 76321.23, -217389.15, 370626.48,
                    -372051.47, 202540.70, -46080.94])
BOOK_X = np.arange(10) / 9.0
NOISE_SD = 0.2
SEED_TEST = 2
SEED_N15 = 15
SEED_N100 = 100


def design(x, M):
    """Polynomial design matrix Phi[n, j] = x_n^j, j = 0..M, for (1.1)."""
    x = np.asarray(x, dtype=float)
    P = np.ones((len(x), M + 1))
    for j in range(1, M + 1):
        P[:, j] = P[:, j - 1] * x
    return P


def poly(w, x):
    """y(x, w) = sum_j w_j x^j (1.1), evaluated with the same powers as design()."""
    return design(x, len(w) - 1) @ np.asarray(w, dtype=float)


BOOK_T = poly(BOOK_W9, BOOK_X)   # the book's ten training targets, recovered from Table 1.1


def sine_data(N, seed, sd=NOISE_SD):
    """N points x_n = (n-1)/(N-1) on [0, 1] with t_n = sin(2 pi x_n) + sd * normal (JSRandom(seed))."""
    x = np.arange(N) / (N - 1.0)
    r = JSRandom(seed)
    t = np.array([math.sin(2 * math.pi * xi) + sd * r.norm() for xi in x])
    return x, t


def test_set():
    return sine_data(100, SEED_TEST)


# ============================================================================= least squares
def lsq_qr(A, b):
    """Least-squares solution of A w ~ b by Householder QR (A has full column rank).

    The browser demos use the same algorithm (function lsq in the pages), so the two agree to
    rounding error even when A is badly conditioned."""
    A = np.array(A, dtype=float)
    b = np.array(b, dtype=float)
    n, p = A.shape
    for k in range(p):
        x = A[k:, k]
        nx = math.sqrt(float(x @ x))
        if nx == 0.0:
            continue
        alpha = -nx if x[0] >= 0 else nx
        v = x.copy()
        v[0] -= alpha
        vv = float(v @ v)
        if vv == 0.0:
            continue
        A[k:, k:] -= np.outer(v, (2.0 / vv) * (v @ A[k:, k:]))
        b[k:] -= (2.0 / vv) * (v @ b[k:]) * v
    w = np.zeros(p)
    for k in range(p - 1, -1, -1):
        w[k] = (b[k] - A[k, k + 1:p] @ w[k + 1:]) / A[k, k]
    return w


def fit(x, t, M, lam=0.0, pen0=True):
    """Minimiser of the regularised error (1.4) for the order-M polynomial (lam = 0: (1.2)).

    With lam > 0 the problem is ordinary least squares for the augmented system
    [Phi; sqrt(lam) D] w ~ [t; 0], D = I (or diag(0,1,...,1) when pen0 is False)."""
    P = design(x, M)
    if lam <= 0:
        return lsq_qr(P, t)
    d = np.ones(M + 1)
    if not pen0:
        d[0] = 0.0
    A = np.vstack([P, math.sqrt(lam) * np.diag(d)[d > 0]])
    b = np.concatenate([t, np.zeros(int((d > 0).sum()))])
    return lsq_qr(A, b)


def rms(w, x, t):
    """E_RMS (1.3)."""
    r = poly(w, x) - np.asarray(t)
    return math.sqrt(float(r @ r) / len(r))


def hat_matrix(x, M, lam=0.0, pen0=True):
    """H with y_hat = H t for the (regularised) least-squares fit, via the SVD."""
    P = design(x, M)
    D = np.eye(M + 1)
    if not pen0:
        D[0, 0] = 0
    return P @ np.linalg.solve(P.T @ P + lam * D, P.T) if lam > 0 else P @ np.linalg.pinv(P)


def eff_params(x, M, lam):
    """Effective number of parameters tr H(lambda) = sum_i s_i^2 / (s_i^2 + lambda)."""
    s = np.linalg.svd(design(x, M), compute_uv=False)
    return float(np.sum(s ** 2 / (s ** 2 + lam)))


def loo_shortcut(x, t, M, lam=0.0):
    """Leave-one-out residuals (t_n - y_hat_n) / (1 - H_nn) of a linear smoother."""
    H = hat_matrix(x, M, lam)
    return (t - H @ t) / (1.0 - np.diag(H))


def loo_bruteforce(x, t, M, lam=0.0):
    out = np.zeros(len(x))
    for n in range(len(x)):
        keep = np.arange(len(x)) != n
        w = fit(x[keep], t[keep], M, lam)
        out[n] = t[n] - poly(w, x[n:n + 1])[0]
    return out


def folds(N, S, seed):
    """Seeded partition of 0..N-1 into S groups of (nearly) equal size: shuffle, then deal out
    consecutive blocks (the block sizes differ by at most one)."""
    idx = JSRandom(seed).shuffle(list(range(N)))
    sizes = [N // S + (1 if k < N % S else 0) for k in range(S)]
    out, pos = [], 0
    for s in sizes:
        out.append(sorted(idx[pos:pos + s]))
        pos += s
    return out


def cv_rms(x, t, M, lam, S, seed):
    """S-fold cross-validation estimate sqrt(mean squared validation residual over all points)."""
    sq = 0.0
    for g in folds(len(x), S, seed):
        keep = np.ones(len(x), bool)
        keep[g] = False
        w = fit(x[keep], t[keep], M, lam)
        r = poly(w, x[g]) - t[g]
        sq += float(r @ r)
    return math.sqrt(sq / len(x))


def cv_curve(x, t, M, lnlams, S, seed):
    """S-fold cross-validation RMS error for each ln lambda (M fixed)."""
    return np.array([cv_rms(x, t, M, math.exp(ll), S, seed) for ll in lnlams])


def optimism_mc(N, R, seed, Ms=range(10), sd=NOISE_SD):
    """Monte Carlo of the training error and the in-sample prediction error, x_n = (n-1)/(N-1).

    Replicate r draws N training noises, then N fresh noises at the same inputs, from one
    JSRandom(seed) stream (in that order). For every order M the least-squares fit to the
    training targets gives the training mean squared error and the mean squared error against the
    fresh targets. Returns two arrays (len(Ms)) averaged over the R replicates."""
    x = np.arange(N) / (N - 1.0)
    f = np.sin(2 * np.pi * x)
    r = JSRandom(seed)
    tr = np.zeros(len(Ms))
    te = np.zeros(len(Ms))
    for _ in range(R):
        e1 = np.array([r.norm() for _ in range(N)])
        e2 = np.array([r.norm() for _ in range(N)])
        t1, t2 = f + sd * e1, f + sd * e2
        for i, M in enumerate(Ms):
            y = poly(fit(x, t1, M), x)
            tr[i] += np.mean((y - t1) ** 2)
            te[i] += np.mean((y - t2) ** 2)
    return tr / R, te / R


def optimism_theory(N, Ms=range(10), sd=NOISE_SD):
    """Exact expectations: bias^2 + sd^2 (N - p)/N (training) and bias^2 + sd^2 (N + p)/N
    (in-sample prediction), p = M + 1, bias^2 = ||(I - H) f||^2 / N."""
    x = np.arange(N) / (N - 1.0)
    f = np.sin(2 * np.pi * x)
    tr, te = [], []
    for M in Ms:
        H = hat_matrix(x, M)
        b2 = float(np.sum((f - H @ f) ** 2)) / N
        p = M + 1
        tr.append(b2 + sd ** 2 * (N - p) / N)
        te.append(b2 + sd ** 2 * (N + p) / N)
    return np.array(tr), np.array(te)


# ============================================================================= Section 1.3
def step(a):
    """The perceptron's step function (1.7): 1 if a > 0, else 0."""
    return (np.asarray(a) > 0).astype(float)


def perceptron(X, t, max_epochs=1000, w0=None):
    """Rosenblatt's perceptron with 0/1 targets and the step function (1.7).

    X has a leading column of ones (the bias input x_0 = 1). Patterns are visited in order,
    and on each one w <- w + (t_n - f(w^T x_n)) x_n, which changes w only on a mistake.
    Returns (w, number of updates, epochs used, converged)."""
    w = np.zeros(X.shape[1]) if w0 is None else np.array(w0, float)
    k = 0
    for ep in range(max_epochs):
        changed = False
        for n in range(len(X)):
            e = t[n] - (1.0 if w @ X[n] > 0 else 0.0)
            if e != 0:
                w = w + e * X[n]
                k += 1
                changed = True
        if not changed:
            return w, k, ep + 1, True
    return w, k, max_epochs, False


def max_margin(X, t):
    """Largest margin gamma* = max over unit w of min_n s_n w^T x_n (s_n = 2 t_n - 1).

    Solved through its dual: gamma* is the distance from the origin to the convex hull of the
    points s_n x_n, i.e. min ||sum_n alpha_n s_n x_n|| over the probability simplex (for any unit w
    and any alpha in the simplex, min_n s_n w^T x_n <= w^T sum_n alpha_n s_n x_n <= ||sum ...||).
    Returns (gamma, unit w) with gamma = min_n s_n w^T x_n for the returned w, or None if the
    data are not linearly separable (then the hull contains the origin)."""
    from scipy.optimize import minimize
    A = (2 * np.asarray(t) - 1.0)[:, None] * X
    n = len(A)
    res = minimize(lambda a: float(np.sum((A.T @ a) ** 2)), np.ones(n) / n, jac=lambda a: 2 * A @ (A.T @ a),
                   method="SLSQP", bounds=[(0, None)] * n,
                   constraints=[{"type": "eq", "fun": lambda a: a.sum() - 1.0, "jac": lambda a: np.ones(n)}],
                   options={"ftol": 1e-16, "maxiter": 2000})
    v = A.T @ res.x
    nv = float(np.linalg.norm(v))
    if nv < 1e-9:
        return None
    u = v / nv
    gamma = float((A @ u).min())
    if gamma <= 0 or gamma < nv * (1 - 1e-6):
        return None
    return gamma, u


def perceptron_data(N, margin, seed, angle=0.6, offset=0.15):
    """N points uniform on [-1, 1]^2 labelled by the line cos(angle) x1 + sin(angle) x2 = offset;
    points closer than `margin` to the line are redrawn. Returns (X with bias column, t in {0, 1}).
    The browser demo draws the same points with DK.rng(seed)."""
    r = JSRandom(seed)
    c, s = math.cos(angle), math.sin(angle)
    X, t = [], []
    while len(X) < N:
        x1, x2 = 2 * r.u() - 1, 2 * r.u() - 1
        d = c * x1 + s * x2 - offset
        if abs(d) < margin:
            continue
        X.append([1.0, x1, x2])
        t.append(1.0 if d > 0 else 0.0)
    return np.array(X), np.array(t)


XOR_X = np.array([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]])
XOR_T = np.array([0.0, 1.0, 1.0, 0.0])


def mlp_init(H, seed, scale=1.0):
    """Weights of a 2-H-1 network: W1 (H x 3, bias first), w2 (H + 1, bias first), N(0, scale^2)
    from JSRandom(seed) in the order W1 row by row, then w2."""
    r = JSRandom(seed)
    W1 = np.array([[scale * r.norm() for _ in range(3)] for _ in range(H)])
    w2 = np.array([scale * r.norm() for _ in range(H + 1)])
    return W1, w2


def mlp_forward(W1, w2, X):
    a1 = W1[:, 0] + X @ W1[:, 1:].T          # pre-activations (1.5) of the hidden units
    z = np.tanh(a1)                            # hidden activations (1.6), f = tanh
    a2 = w2[0] + z @ w2[1:]
    y = common.sigmoid(a2)                     # output unit, logistic sigmoid
    return a1, z, a2, y


def mlp_loss(W1, w2, X, t):
    """Mean cross-entropy -1/N sum {t ln y + (1 - t) ln(1 - y)}, computed stably from a2."""
    a2 = mlp_forward(W1, w2, X)[2]
    return float(np.mean(np.logaddexp(0, -a2) * t + np.logaddexp(0, a2) * (1 - t)))


def mlp_grad(W1, w2, X, t):
    """Backpropagation: output error y - t, hidden error (1 - z^2) w2 (y - t)."""
    a1, z, a2, y = mlp_forward(W1, w2, X)
    N = len(X)
    d2 = (y - t) / N
    g2 = np.concatenate([[d2.sum()], z.T @ d2])
    d1 = (1 - z ** 2) * np.outer(d2, w2[1:])
    g1 = np.column_stack([d1.sum(0), d1.T @ X])
    return g1, g2


def mlp_train(W1, w2, X, t, lr, steps):
    """Full-batch gradient descent; returns final weights and the loss after every step."""
    W1, w2 = W1.copy(), w2.copy()
    hist = [mlp_loss(W1, w2, X, t)]
    for _ in range(steps):
        g1, g2 = mlp_grad(W1, w2, X, t)
        W1 -= lr * g1
        w2 -= lr * g2
        hist.append(mlp_loss(W1, w2, X, t))
    return W1, w2, np.array(hist)


def deep_chain_signal(L, width, seed, act="sigmoid", residual=False, gain=1.0):
    """Backpropagated signal ||dE/dz^(l)|| at every layer l = 0..L of a deep chain.

    z^(0) is a random input; each layer computes a = W z^(l-1) (W with N(0, gain^2 / width)
    entries from JSRandom(seed)), and z^(l) = f(a) (plain) or z^(l) = z^(l-1) + f(a) - f(0)
    (residual). The output error dE/dz^(L) is the all-ones vector divided by sqrt(width), so
    its norm is 1. Returns the norms from layer L down to layer 0 (index l)."""
    r = JSRandom(seed)
    z = np.array([r.norm() for _ in range(width)])
    Ws, As = [], []
    f = (lambda a: common.sigmoid(a)) if act == "sigmoid" else np.tanh
    fp = (lambda a: common.sigmoid(a) * (1 - common.sigmoid(a))) if act == "sigmoid" else (lambda a: 1 - np.tanh(a) ** 2)
    f0 = float(f(np.zeros(1))[0])
    for _ in range(L):
        W = np.array([[gain * r.norm() / math.sqrt(width) for _ in range(width)] for _ in range(width)])
        a = W @ z
        Ws.append(W)
        As.append(a)
        z = z + f(a) - f0 if residual else f(a)
    g = np.ones(width) / math.sqrt(width)
    norms = [0.0] * (L + 1)
    norms[L] = float(np.linalg.norm(g))
    for l in range(L - 1, -1, -1):
        back = Ws[l].T @ (fp(As[l]) * g)
        g = g + back if residual else back
        norms[l] = float(np.linalg.norm(g))
    return np.array(norms)


# ============================================================================= plotting helpers
XG = np.linspace(0, 1, 400)


def _sine_axes(ax, lab=None):
    ax.plot(XG, np.sin(2 * np.pi * XG), color=common.GREEN, lw=1.6)
    ax.set_xlim(-0.05, 1.05)
    ax.set_ylim(-1.5, 1.5)
    ax.set_xticks([0, 1])
    ax.set_yticks([-1, 1])
    ax.set_xlabel("$x$")
    ax.set_ylabel("$t$", rotation=0)
    if lab:
        ax.text(0.62, 0.86, lab, transform=ax.transAxes, fontsize=10)


def _points(ax, x, t, ms=6):
    ax.plot(x, t, "o", mfc="none", mec=common.BLUE, mew=1.4, ms=ms)


# ============================================================================= figures
def fig1_4(out_dir=None):
    """Figure 1.4: the ten training points and the generating curve sin(2 pi x)."""
    fig, ax = plt.subplots(figsize=(3.6, 2.7))
    _sine_axes(ax)
    _points(ax, BOOK_X, BOOK_T)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-4", out_dir)


def fig1_5(out_dir=None):
    """Figure 1.5: the sum-of-squares error as the vertical displacements of the points from y(x, w)."""
    w = fit(BOOK_X, BOOK_T, 3)
    fig, ax = plt.subplots(figsize=(4.2, 3.0))
    ax.plot(XG, poly(w, XG), color=common.RED, lw=1.6)
    y = poly(w, BOOK_X)
    for xn, tn, yn in zip(BOOK_X, BOOK_T, y):
        ax.plot([xn, xn], [tn, yn], color=common.GREEN, lw=2.0, solid_capstyle="butt")
    _points(ax, BOOK_X, BOOK_T)
    n = 5
    ax.annotate("$t_n$", (BOOK_X[n], BOOK_T[n]), xytext=(BOOK_X[n] - 0.1, BOOK_T[n] - 0.3), fontsize=11)
    ax.annotate("$y(x_n,\\mathbf{w})$", (BOOK_X[n], y[n]), xytext=(BOOK_X[n] + 0.06, y[n] + 0.25), fontsize=11,
                arrowprops=dict(arrowstyle="-", color="0.4", lw=0.6))
    ax.axvline(BOOK_X[n], color="0.6", lw=0.6, ls=":", zorder=0)
    ax.set_xlim(-0.05, 1.05)
    ax.set_ylim(-1.5, 1.6)
    ax.set_xticks([BOOK_X[n]])
    ax.set_xticklabels(["$x_n$"])
    ax.set_yticks([])
    ax.set_xlabel("$x$")
    ax.set_ylabel("$t$", rotation=0)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-5", out_dir)


def fig1_6(out_dir=None):
    """Figure 1.6: least-squares polynomials of order M = 0, 1, 3, 9 fitted to the ten points."""
    fig, axes = common.panel_grid(2, 2, size=(3.2, 2.5))
    for ax, M in zip(axes.ravel(), (0, 1, 3, 9)):
        _sine_axes(ax, "$M=%d$" % M)
        ax.plot(XG, poly(fit(BOOK_X, BOOK_T, M), XG), color=common.RED, lw=1.6)
        _points(ax, BOOK_X, BOOK_T)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-6", out_dir)


def rms_vs_M(x=BOOK_X, t=BOOK_T, Ms=range(10)):
    xt, tt = test_set()
    tr, te = [], []
    for M in Ms:
        w = fit(x, t, M)
        tr.append(rms(w, x, t))
        te.append(rms(w, xt, tt))
    return np.array(tr), np.array(te)


def fig1_7(out_dir=None):
    """Figure 1.7: training and test RMS error (1.3) against the order M."""
    tr, te = rms_vs_M()
    fig, ax = plt.subplots(figsize=(4.4, 2.8))
    M = np.arange(10)
    ax.plot(M, tr, "-o", color=common.RED, ms=5, label="Training")
    ax.plot(M, te, "-o", color=common.BLUE, ms=5, label="Test")
    ax.set_xticks([0, 3, 6, 9])
    ax.set_ylim(0, 1.05)
    ax.set_yticks([0, 0.5, 1])
    ax.set_xlabel("$M$")
    ax.set_ylabel("$E_{\\mathrm{RMS}}$")
    ax.legend(frameon=False, loc="upper left", bbox_to_anchor=(1.0, 1.0))
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-7", out_dir)


def fig1_8(out_dir=None):
    """Figure 1.8: the M = 9 polynomial fitted to N = 15 and N = 100 points."""
    fig, axes = common.panel_grid(1, 2, size=(3.2, 2.5))
    for ax, (N, seed) in zip(axes[0], ((15, SEED_N15), (100, SEED_N100))):
        x, t = sine_data(N, seed)
        _sine_axes(ax, "$N=%d$" % N)
        ax.plot(XG, poly(fit(x, t, 9), XG), color=common.RED, lw=1.6)
        _points(ax, x, t, ms=6 if N < 50 else 4.5)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-8", out_dir)


def fig1_9(out_dir=None):
    """Figure 1.9: M = 9 fitted with the regularised error (1.4), ln lambda = -18 and 0."""
    fig, axes = common.panel_grid(1, 2, size=(3.2, 2.5))
    for ax, ll in zip(axes[0], (-18, 0)):
        _sine_axes(ax, "$\\ln\\lambda=%d$" % ll)
        ax.plot(XG, poly(fit(BOOK_X, BOOK_T, 9, math.exp(ll)), XG), color=common.RED, lw=1.6)
        ax.plot(XG, poly(fit(BOOK_X, BOOK_T, 9, len(BOOK_X) * math.exp(ll)), XG), color=common.RED, lw=1.1, ls="--", alpha=0.75)
        _points(ax, BOOK_X, BOOK_T)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-9", out_dir)


LNLAM = np.linspace(-35, 2, 186)


def rms_vs_lambda(scale=1.0, lnlam=LNLAM):
    xt, tt = test_set()
    tr, te = [], []
    for ll in lnlam:
        w = fit(BOOK_X, BOOK_T, 9, scale * math.exp(ll))
        tr.append(rms(w, BOOK_X, BOOK_T))
        te.append(rms(w, xt, tt))
    return np.array(tr), np.array(te)


def fig1_10(out_dir=None):
    """Figure 1.10: training and test RMS error against ln lambda for M = 9."""
    tr, te = rms_vs_lambda()
    tr10, te10 = rms_vs_lambda(scale=len(BOOK_X))
    fig, ax = plt.subplots(figsize=(4.6, 2.8))
    ax.plot(LNLAM, tr, color=common.RED, label="Training")
    ax.plot(LNLAM, te, color=common.BLUE, label="Test")
    ax.plot(LNLAM, tr10, color=common.RED, lw=1.0, ls="--", alpha=0.7, label="Training, $\\lambda\\to N\\lambda$")
    ax.plot(LNLAM, te10, color=common.BLUE, lw=1.0, ls="--", alpha=0.7, label="Test, $\\lambda\\to N\\lambda$")
    ax.set_xlim(-35, 2)
    ax.set_xticks([-30, -20, -10, 0])
    ax.set_ylim(0, 0.85)
    ax.set_yticks([0, 0.5])
    ax.set_xlabel("$\\ln\\lambda$")
    ax.set_ylabel("$E_{\\mathrm{RMS}}$")
    ax.legend(frameon=False, loc="upper left", bbox_to_anchor=(1.0, 1.0), fontsize=8.5)
    fig.tight_layout()
    return common.savefig(fig, CH, "fig1-10", out_dir)


# ============================================================================= tables and export
def table_1_1():
    return {M: fit(BOOK_X, BOOK_T, M) for M in (0, 1, 3, 9)}


def table_1_2(scale=1.0):
    return {ll: fit(BOOK_X, BOOK_T, 9, scale * math.exp(ll)) for ll in (-18, 0)}


def print_tables():
    print("Table 1.1 (least squares on the recovered training set)")
    for M, w in table_1_1().items():
        print("  M = %d:" % M, " ".join("%.2f" % v for v in w))
    for scale, lab in ((1.0, "(1.4) as printed"), (10.0, "lambda -> N lambda (the book's Table 1.2)")):
        print("Table 1.2,", lab)
        for ll, w in table_1_2(scale).items():
            print("  ln lambda = %d:" % ll, " ".join("%.2f" % v for v in w))


FIGURES = {"1.4": fig1_4, "1.5": fig1_5, "1.6": fig1_6, "1.7": fig1_7, "1.8": fig1_8,
           "1.9": fig1_9, "1.10": fig1_10}

if __name__ == "__main__":
    if "--tables" in sys.argv:
        print_tables()
        sys.exit(0)
    common.run(FIGURES, CH)
