"""Wasserman (2006), Chapter 8: Nonparametric Inference Using Orthogonal Functions.

Companion code for Chapter 8 of L. Wasserman, All of Nonparametric Statistics (Springer, 2006),
pp. 183-195, written from scratch with numpy, scipy and matplotlib, to be read next to the lecture
notes (../ch08-orthogonal-functions.html). The methods the chapter is about (the cosine basis, the
coefficients (8.6), the noise estimate (8.12), the risk estimates (8.15) and (8.18), the nested,
monotone and constant modulators, the pooled-adjacent-violators algorithm, the confidence ball
(8.24) and the band (8.25), Gram-Schmidt for irregular designs, and the orthogonal series density
estimator (8.28)-(8.30)) are implemented here; library versions (numpy's QR, scikit-learn's isotonic
regression) are only used to confirm the implementations. Local linear regression and the tube
formula are imported from the Chapter 5 module.

Methods
* Section 8.2: cos_basis (8.2), coefficients (8.6), sigma2_tail ((8.12), with the factor n that makes it
  consistent; sigma2_tail_printed is the formula as printed), nss_risk (8.18), sure (8.15) and its
  unbiased version, pav (pooled adjacent violators), mon_modulator (minimises (8.19)), const_modulator
  (the James-Stein modulator), react (the whole procedure), weights_grid ((8.11) on a grid), band
  ((8.25) with c from the tube formula (5.102)), ball_radius (Theorem 8.23 with the variance of the
  pivot derived in the notes; ball_tau2_printed is the formula as printed).
* Section 8.3: gram_schmidt (orthonormal basis for the empirical measure, by modified Gram-Schmidt with
  re-orthogonalisation, dropping dependent columns), react_irregular.
* Section 8.4: density_coefficients (8.28), density_risk ((8.30) for nested modulators), react_density,
  surgery; kde and kde_lscv (a Gaussian kernel density estimator with least squares cross-validation,
  for comparison).
* Section 8.6: tensor_design, tensor_fit (tensor product cosine bases, least squares, Cp and
  leave-one-out scores).

Data. The Doppler data of Example 8.20 are simulated (n = 1000, sigma = 0.1) with the same seed as
Figure 5.6 of the Chapter 5 module, so the two figures show the same sample. The CMB data of Example
8.21 are the 899 WMAP power-spectrum points (wmap.dat) from the book's website, treated as a regular
design x_i = i/n. Glass (glass.dat) and motorcycle (motor.dat) data come from the same website; the air
quality data of Exercise 14 are R's datasets::airquality (complete cases), exported once with Rscript
by the Chapter 5 module.

Figures. Figures 8.1 and 8.2 are the only figures of the chapter; both are reproduced. Nothing is
skipped. Supplementary figures for the gaps and the exercises:
  8.S1 Lemma 8.4 and the boundary: decay of cosine coefficients and of the bias B_J for four functions,
       and the equivalent kernel (8.11) inside the interval and at the boundary;
  8.S2 the three modulator classes on the Doppler data (b_j and the fits) and the band (8.25);
  8.S3 the confidence ball (8.24): the pivot standardised with the printed and the derived variance;
  8.S4 Exercise 7 (glass): the empirical orthonormal basis, react against local linear, bands;
  8.S5 Exercise 8 (motorcycle): react fit, band, and a display of the confidence ball;
  8.S6 Exercise 9: Doppler at sigma = 0.1, 1, 3, local linear against react, with bands;
  8.S7 Exercise 10: Cauchy errors, and a robust version using block medians;
  8.S8 Exercise 11: react density against kernel density estimates for mixtures of two Normals;
  8.S9 Exercise 12: the greedy modulator, its estimated and its actual risk;
  8.S10 Exercise 13: the tensor product fit and its SURE surface over (J1, J2).
Exercise 14 (air quality) is the table returned by airquality_tensor().
Run `python3 ch08_orthogonal_functions.py --list`, `python3 ch08_orthogonal_functions.py` (all figures,
about a minute on one CPU thread) or `python3 ch08_orthogonal_functions.py --fig 8.1`.
Files go to ../img/ch08/; the data used by the browser demonstrations go to ../data/ch08-*.json.
"""
import json
import math
import os

import numpy as np
from scipy import stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig
import ch05_nonparametric_regression as c5

CH = 8
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
SEED_DOPPLER = c5.SEED_DOPPLER       # the sample of Figure 5.6
SEED_EX9 = 809
SEED_EX10 = 810
SEED_EX11 = 811                      # JSRandom seed shared with the density demo
SEED_EX12 = 812
SEED_EX13 = 813
SEED_BALL = 823


# =========================================================================== Section 8.2: the basis
def cos_basis(x, J):
    """The cosine basis (8.2) at the points x: an (len(x), J) matrix with columns phi_1, ..., phi_J,
    phi_1 = 1 and phi_j(x) = sqrt(2) cos((j - 1) pi x)."""
    x = np.asarray(x, float)
    P = math.sqrt(2.0) * np.cos(np.pi * np.outer(x, np.arange(J)))
    P[:, 0] = 1.0
    return P


def regular_design(n):
    return np.arange(1, n + 1) / n


def coefficients(y, x=None):
    """Z_j = n^{-1} sum_i Y_i phi_j(x_i), j = 1, ..., n (8.6); regular design x_i = i/n by default."""
    n = len(y)
    x = regular_design(n) if x is None else x
    return cos_basis(x, n).T @ np.asarray(y, float) / n


def sigma2_tail(Z, Jn=None):
    """The noise estimate (8.12) with the factor that makes it consistent:
    sigma^2 = (n / J_n) sum_{j > n - J_n} Z_j^2, J_n = n/4 by default. Since Z_j is about
    N(theta_j, sigma^2 / n) and theta_j is about 0 for large j, each Z_j^2 estimates sigma^2 / n."""
    n = len(Z)
    Jn = n // 4 if Jn is None else Jn
    return float(n / Jn * np.sum(np.asarray(Z)[n - Jn:] ** 2))


def sigma2_tail_printed(Z, Jn=None):
    """(8.12) exactly as printed: (n - J_n)^{-1} sum_{i = n - J_n + 1}^n Z_i^2 (not consistent)."""
    n = len(Z)
    Jn = n // 4 if Jn is None else Jn
    return float(np.sum(np.asarray(Z)[n - Jn:] ** 2) / (n - Jn))


def nss_risk(Z, s2, positive=True):
    """R(J) = J sigma^2/n + sum_{j > J} (Z_j^2 - sigma^2/n)_+ for J = 1, ..., n (8.18).
    With positive=False the positive part is dropped (the unbiased SURE)."""
    n = len(Z)
    s = s2 / n
    t = np.asarray(Z) ** 2 - s
    if positive:
        t = np.maximum(t, 0.0)
    tail = np.concatenate([np.cumsum(t[::-1])[::-1][1:], [0.0]])     # sum over j > J
    return np.arange(1, n + 1) * s + tail


def sure(b, Z, s2, positive=True):
    """The (modified) SURE (8.15) of the modulator b; positive=False gives the unbiased version."""
    n = len(Z)
    s = s2 / n
    t = np.asarray(Z) ** 2 - s
    if positive:
        t = np.maximum(t, 0.0)
    b = np.asarray(b, float)
    return float(np.sum(t * (1 - b) ** 2) + s * np.sum(b ** 2))


def true_risk(b, theta, s2):
    """The risk (8.14) of b Z when Z_j ~ N(theta_j, sigma^2/n) independently."""
    n = len(theta)
    b = np.asarray(b, float)
    return float(np.sum(np.asarray(theta) ** 2 * (1 - b) ** 2) + s2 / n * np.sum(b ** 2))


def pav(y, w):
    """Weighted least squares fit of a non-increasing sequence to y (weights w > 0) by the
    pooled-adjacent-violators algorithm: returns the fitted vector."""
    vals, wts, cnt = [], [], []
    for yi, wi in zip(np.asarray(y, float), np.asarray(w, float)):
        vals.append(yi); wts.append(wi); cnt.append(1)
        while len(vals) > 1 and vals[-2] < vals[-1]:          # a violation of b_1 >= b_2 >= ...
            W = wts[-2] + wts[-1]
            v = (wts[-2] * vals[-2] + wts[-1] * vals[-1]) / W
            c = cnt[-2] + cnt[-1]
            vals[-2:] = [v]; wts[-2:] = [W]; cnt[-2:] = [c]
    return np.repeat(vals, cnt)


def mon_modulator(Z, s2):
    """The monotone modulator minimising (8.19): fit g_j = (Z_j^2 - s)/Z_j^2 with weights Z_j^2 by a
    non-increasing sequence (PAV), then clip to [0, 1] (clipping an isotonic fit solves the
    box-constrained problem)."""
    n = len(Z)
    s = s2 / n
    Z2 = np.maximum(np.asarray(Z) ** 2, 1e-300)
    g = (Z2 - s) / Z2
    return np.clip(pav(g, Z2), 0.0, 1.0)


def const_modulator(Z, s2, positive=True):
    """The constant modulator b minimising (8.15) over (b, ..., b): b = S / (S + n s) with
    S = sum (Z_j^2 - s)_+ (positive=True); with positive=False, b = (1 - n s / sum Z_j^2)_+, the
    positive-part James-Stein estimator."""
    n = len(Z)
    s = s2 / n
    Z2 = np.asarray(Z) ** 2
    if positive:
        S = np.sum(np.maximum(Z2 - s, 0))
        return np.full(n, S / (S + n * s))
    return np.full(n, max(0.0, 1 - n * s / np.sum(Z2)))


def react(y, x=None, cls="nss", s2=None, Jn=None):
    """The react procedure on a regular design: returns dict with Z, sigma^2, b, J (NSS), risk curve."""
    Z = coefficients(y, x)
    s2 = sigma2_tail(Z, Jn) if s2 is None else s2
    out = {"Z": Z, "s2": s2, "risk": nss_risk(Z, s2)}
    out["J"] = int(np.argmin(out["risk"]) + 1)
    n = len(Z)
    if cls == "nss":
        out["b"] = (np.arange(n) < out["J"]).astype(float)
    elif cls == "mon":
        out["b"] = mon_modulator(Z, s2)
    elif cls == "cons":
        out["b"] = const_modulator(Z, s2)
    else:
        raise ValueError(cls)
    out["Rhat"] = sure(out["b"], Z, s2)
    return out


def fit_grid(xg, Z, b):
    """r_n(x) = sum_j b_j Z_j phi_j(x) (8.9) on the points xg (only the non-zero b_j are used)."""
    k = int(np.max(np.nonzero(np.asarray(b) > 0)[0]) + 1) if np.any(np.asarray(b) > 0) else 1
    return cos_basis(xg, k) @ (np.asarray(b)[:k] * np.asarray(Z)[:k])


def weights_grid(xg, x, b):
    """The weights l_i(x) = n^{-1} sum_j b_j phi_j(x) phi_j(x_i) of (8.11), one row per point of xg."""
    n = len(x)
    k = int(np.max(np.nonzero(np.asarray(b) > 0)[0]) + 1)
    return (cos_basis(xg, k) * np.asarray(b)[:k]) @ cos_basis(x, k).T / n


def band(xg, x, Z, b, s2, alpha=0.05):
    """The band (8.25): r_n(x) +- c sigma ||l(x)||, c from the tube formula (5.102) with kappa_0 of
    the weights (8.11). Returns fit, lo, hi, the exact ||l(x)||, the approximation (8.26), kappa_0, c."""
    Lg = weights_grid(xg, x, b)
    fit = fit_grid(xg, Z, b)
    nl = np.linalg.norm(Lg, axis=1)
    n = len(x)
    k = int(np.max(np.nonzero(np.asarray(b) > 0)[0]) + 1)
    approx = np.sqrt((cos_basis(xg, k) ** 2) @ (np.asarray(b)[:k] ** 2) / n)
    k0 = c5.tube_kappa0(Lg, xg)
    c = c5.tube_c(k0, alpha)
    s = math.sqrt(s2)
    return {"fit": fit, "lo": fit - c * s * nl, "hi": fit + c * s * nl, "nl": nl, "nl_approx": approx,
            "kappa0": k0, "c": c}


# =========================================================================== confidence ball (8.24)
def ball_tau2(Z, b, s2, Jn=None):
    """Variance of L - R (L = ||bZ - theta||^2, R the unbiased SURE with sigma^2 from (8.12)), as
    derived in the notes: 2 s^2 sum_j ((2b_j - 1) - A c_j)^2 + 4 s sum_j theta_j^2 ((1 - b_j) + A c_j)^2,
    with s = sigma^2/n, A = sum_k (2 b_k - 1), c_j = 1/J_n on the last J_n coordinates, and theta_j^2
    estimated by Z_j^2 - s. This is Var(L - R) itself (the book's tau^2/n)."""
    n = len(Z)
    Jn = n // 4 if Jn is None else Jn
    s = s2 / n
    b = np.asarray(b, float)
    c = np.zeros(n); c[n - Jn:] = 1.0 / Jn
    A = np.sum(2 * b - 1)
    th2 = np.asarray(Z) ** 2 - s
    return float(2 * s * s * np.sum(((2 * b - 1) - A * c) ** 2) + 4 * s * np.sum(th2 * ((1 - b) + A * c) ** 2))


def ball_tau2_printed(Z, b, s2, Jn=None):
    """tau^2 / n with tau^2 as printed in Theorem 8.23."""
    n = len(Z)
    Jn = n // 4 if Jn is None else Jn
    s = s2 / n
    b = np.asarray(b, float)
    c = np.zeros(n); c[n - Jn:] = 1.0 / Jn
    th2 = np.asarray(Z) ** 2 - s
    tau2 = 2 * s2 * s2 / n * np.sum(((2 * b - 1) * (1 - c)) ** 2) + 4 * s2 * np.sum(th2 * ((1 - b) + (2 * b - 1) * c) ** 2)
    return float(tau2 / n)


def ball_radius(Z, b, s2, alpha=0.05, Jn=None, printed=False):
    """s_n with s_n^2 = R(b) + z_alpha sqrt(Var(L - R)) (8.24), using the unbiased SURE."""
    v = ball_tau2_printed(Z, b, s2, Jn) if printed else ball_tau2(Z, b, s2, Jn)
    R = sure(b, Z, s2, positive=False)
    return math.sqrt(max(R + stats.norm.ppf(1 - alpha) * math.sqrt(max(v, 0.0)), 0.0))


def ball_coverage(reps=1000, n=1000, sigma=0.1, cls="nss", seed=SEED_BALL):
    """Simulated coverage of the ball (8.24) for the Doppler coefficients theta = E Z, with the printed
    and the derived variance; also the standardised pivots."""
    x = regular_design(n)
    P = cos_basis(x, n)
    r = c5.doppler(x)
    theta = P.T @ r / n
    rng = np.random.default_rng(seed)
    out = {"mine": 0, "printed": 0, "piv_mine": [], "piv_printed": [], "piv_fixed": []}
    bfix = (np.arange(n) < 160).astype(float)
    for _ in range(reps):
        y = r + sigma * rng.standard_normal(n)
        Z = P.T @ y / n
        s2 = sigma2_tail(Z)
        R = react(y, cls=cls, s2=s2)
        b = R["b"]
        L = np.sum((b * Z - theta) ** 2)
        Rh = sure(b, Z, s2, positive=False)
        vm, vp = ball_tau2(Z, b, s2), ball_tau2_printed(Z, b, s2)
        z = stats.norm.ppf(0.95)
        out["mine"] += L <= Rh + z * math.sqrt(vm)
        out["printed"] += L <= Rh + z * math.sqrt(vp)
        out["piv_mine"].append((L - Rh) / math.sqrt(vm))
        out["piv_printed"].append((L - Rh) / math.sqrt(vp))
        Lf = np.sum((bfix * Z - theta) ** 2)
        out["piv_fixed"].append((Lf - sure(bfix, Z, s2, positive=False)) / math.sqrt(ball_tau2(Z, bfix, s2)))
    out["mine"] /= reps
    out["printed"] /= reps
    for k in ("piv_mine", "piv_printed", "piv_fixed"):
        out[k] = np.array(out[k])
    return out


# =========================================================================== Section 8.3: irregular designs
def gram_schmidt(G, tol=1e-3):
    """Modified Gram-Schmidt, applied twice per column, on the columns of G (n x p) with the inner
    product <f, g> = n^{-1} sum_i f(x_i) g(x_i). The process stops at the first column whose remaining
    norm is below tol times its original norm (that column, and all later ones, are numerically
    dependent on the earlier ones at these design points). Returns Phi (n x q, Phi^T Phi / n = I), the kept column indices and the
    upper-triangular R with G[:, kept] = Phi R, so that phi(x) = g_kept(x) R^{-1} off the design."""
    n, p = G.shape
    Q, kept, Rcols = [], [], []
    for k in range(p):
        v = G[:, k].astype(float).copy()
        norm0 = math.sqrt(v @ v / n)
        coef = np.zeros(len(Q))
        for _ in range(2):
            for i, q in enumerate(Q):
                a = q @ v / n
                coef[i] += a
                v -= a * q
        nv = math.sqrt(v @ v / n)
        if norm0 == 0 or nv < tol * norm0:
            break
        Q.append(v / nv)
        kept.append(k)
        Rcols.append(np.concatenate([coef, [nv]]))
    q = len(Q)
    R = np.zeros((q, q))
    for j, col in enumerate(Rcols):
        R[:len(col), j] = col
    return np.array(Q).T, kept, R


def rescale(x):
    x = np.asarray(x, float)
    return (x - x.min()) / (x.max() - x.min())


def react_irregular(x, y, p=None, Jn=None, Jmax=None):
    """react on an irregular design (Section 8.3). The cosine functions g_1, ..., g_p at the rescaled
    x's are orthonormalised for the empirical measure (gram_schmidt); dependent columns (from ties)
    are dropped. The coordinates are Z_j = n^{-1} sum Y_i phi_j(x_i), j <= q, plus the residual of Y
    on all q functions, which carries n - q pure-noise coordinates. sigma^2 is (8.12) over the last
    J_n = n/4 of the n coordinates (the residual's n - q coordinates contribute RSS_q / n), and the
    risk of the nested modulator J <= q is J s + sum_{J < j <= q} (Z_j^2 - s)_+ + (RSS_q/n - (n - q) s)_+.
    Jmax restricts the search to J <= Jmax (for the glass data, J <= sqrt(n): see the notes)."""
    x = np.asarray(x, float); y = np.asarray(y, float)
    n = len(y)
    u = rescale(x)
    p = min(len(np.unique(u)), n // 2) if p is None else p
    Phi, kept, Rm = gram_schmidt(cos_basis(u, p))
    q = Phi.shape[1]
    Z = Phi.T @ y / n
    rss = float(np.sum((y - Phi @ Z) ** 2))
    Jn = n // 4 if Jn is None else Jn
    ntail_res = n - q                                   # residual coordinates, all in the tail
    tail = rss / n + (np.sum(Z[q - (Jn - ntail_res):] ** 2) if Jn > ntail_res else 0.0)
    if Jn < ntail_res:                                  # tail inside the residual: use its average
        tail = rss / n * Jn / ntail_res
    s2 = n / Jn * tail
    s = s2 / n
    t = np.maximum(Z ** 2 - s, 0)
    rest = max(rss / n - (n - q) * s, 0.0)
    risk = np.array([J * s + np.sum(t[J:]) + rest for J in range(1, q + 1)])
    Jmax = q if Jmax is None else min(Jmax, q)
    J = int(np.argmin(risk[:Jmax]) + 1)
    return {"u": u, "Phi": Phi, "kept": kept, "R": Rm, "Z": Z, "s2": s2, "risk": risk, "J": J, "q": q,
            "rss": rss, "xmin": float(x.min()), "xmax": float(x.max())}


def irregular_eval(fit, xg, J):
    """phi_1, ..., phi_J of an irregular-design fit at new points xg (original scale): (len(xg), J)."""
    ug = (np.asarray(xg, float) - fit["xmin"]) / (fit["xmax"] - fit["xmin"])
    kept = fit["kept"][:J]
    Gg = cos_basis(ug, max(kept) + 1)[:, kept]
    return np.linalg.solve(fit["R"][:J, :J].T, Gg.T).T


def irregular_band(fit, xg, J, alpha=0.05):
    """Fit and band (8.25) for the NSS modulator with J terms on an irregular design. The weights are
    l_i(x) = n^{-1} sum_{j <= J} phi_j(x) phi_j(x_i), so ||l(x)||^2 = n^{-1} sum_j phi_j(x)^2 exactly."""
    n = fit["Phi"].shape[0]
    Pg = irregular_eval(fit, xg, J)
    Lg = Pg @ fit["Phi"][:, :J].T / n
    f = Pg @ fit["Z"][:J]
    nl = np.linalg.norm(Lg, axis=1)
    k0 = c5.tube_kappa0(Lg, xg)
    c = c5.tube_c(k0, alpha)
    s = math.sqrt(fit["s2"])
    return {"fit": f, "lo": f - c * s * nl, "hi": f + c * s * nl, "nl": nl, "kappa0": k0, "c": c}


# =========================================================================== Section 8.4: density estimation
def density_coefficients(X, n_terms=None):
    """Z_j = n^{-1} sum_i phi_j(X_i) (8.28) and sigma_j^2 = n^{-2} sum_i (phi_j(X_i) - Z_j)^2, X in (0, 1)."""
    X = np.asarray(X, float)
    n = len(X)
    k = n if n_terms is None else n_terms
    P = cos_basis(X, k)
    Z = P.mean(0)
    s2j = np.sum((P - Z) ** 2, axis=0) / n ** 2
    return Z, s2j


def density_risk(Z, s2j):
    """(8.30) for the nested modulators: R(J) = sum_{j<=J} sigma_j^2 + sum_{j>J} (Z_j^2 - sigma_j^2)_+."""
    t = np.maximum(Z ** 2 - s2j, 0)
    tail = np.concatenate([np.cumsum(t[::-1])[::-1][1:], [0.0]])
    return np.cumsum(s2j) + tail


def surgery(f, grid):
    """Remove the negative part and renormalise to integrate to 1 (trapezoid rule on grid)."""
    g = np.maximum(f, 0)
    return g / np.trapz(g, grid)


def react_density(X, a, b, grid):
    """react density estimate (NSS) for data on (a, b): rescale to (0, 1), estimate, map back.
    Returns the raw and the corrected estimate on grid, J and the risk curve."""
    u = (np.asarray(X, float) - a) / (b - a)
    Z, s2j = density_coefficients(u)
    risk = density_risk(Z, s2j)
    J = int(np.argmin(risk) + 1)
    ug = (np.asarray(grid) - a) / (b - a)
    raw = cos_basis(ug, J) @ Z[:J] / (b - a)
    return {"raw": raw, "fhat": surgery(raw, grid), "J": J, "risk": risk, "Z": Z, "s2j": s2j}


def kde(X, grid, h):
    X = np.asarray(X, float)
    return stats.norm.pdf((np.asarray(grid)[:, None] - X[None, :]) / h).mean(1) / h


def kde_lscv_score(X, h):
    """Least squares cross-validation score int fhat^2 - 2 n^{-1} sum_i fhat_{-i}(X_i) of the Gaussian
    kernel estimator, in closed form (the convolution of two N(0, h^2) kernels is N(0, 2h^2))."""
    X = np.asarray(X, float)
    n = len(X)
    D = X[:, None] - X[None, :]
    int_f2 = np.sum(stats.norm.pdf(D, scale=math.sqrt(2) * h)) / n ** 2
    K = stats.norm.pdf(D, scale=h)
    loo = (np.sum(K) - n * stats.norm.pdf(0, scale=h)) / (n * (n - 1))
    return float(int_f2 - 2 * loo)


def kde_hgrid():
    return np.exp(np.linspace(np.log(0.05), np.log(2.0), 60))


def kde_lscv(X):
    hs = kde_hgrid()
    sc = np.array([kde_lscv_score(X, h) for h in hs])
    return float(hs[np.argmin(sc)]), hs, sc


def density_range(X):
    """The interval (a, b) the react density estimate uses for data on the real line."""
    lo, hi = float(np.min(X)), float(np.max(X))
    pad = 0.1 * (hi - lo)
    return lo - pad, hi + pad


def mixture_sample(n, mu, seed=SEED_EX11):
    """n draws from (1/2) N(0, 1) + (1/2) N(mu, 1) with the demos' generator (DK.rng): for each i a
    uniform picks the component, then one Normal draw."""
    g = c5.JSRandom(seed)
    out = np.empty(n)
    for i in range(n):
        comp = g.u() < 0.5
        out[i] = (0.0 if comp else mu) + g.norm()
    return out


def mixture_pdf(x, mu):
    return 0.5 * stats.norm.pdf(x) + 0.5 * stats.norm.pdf(x, mu)


def density_compare(n=1000, mu=2.0, seed=SEED_EX11):
    X = mixture_sample(n, mu, seed)
    a, b = density_range(X)
    grid = np.linspace(a, b, 801)
    R = react_density(X, a, b, grid)
    h, _, _ = kde_lscv(X)
    hn = 1.06 * np.std(X, ddof=1) * n ** (-0.2)
    truth = mixture_pdf(grid, mu)
    ise = lambda f: float(np.trapz((f - truth) ** 2, grid))
    return {"X": X, "a": a, "b": b, "grid": grid, "react": R, "h": h, "hnr": hn,
            "kde": kde(X, grid, h), "kde_nr": kde(X, grid, hn), "truth": truth,
            "ise_react": ise(R["fhat"]), "ise_kde": ise(kde(X, grid, h)), "ise_nr": ise(kde(X, grid, hn))}


# =========================================================================== Section 8.6: tensor products
def tensor_design(U, Js):
    """Tensor-product cosine design for the columns of U (each in [0, 1]) with Js[k] functions in
    coordinate k: all products phi_{j1}(u1) ... phi_{jd}(ud), j_k <= Js[k]."""
    import itertools
    bases = [cos_basis(U[:, k], Js[k]) for k in range(U.shape[1])]
    cols = []
    for idx in itertools.product(*[range(J) for J in Js]):
        c = np.ones(len(U))
        for k, j in enumerate(idx):
            c = c * bases[k][:, j]
        cols.append(c)
    return np.array(cols).T


def tensor_fit(U, y, Js, s2=None):
    """Least squares on the tensor design: fitted values, p, RSS, the leave-one-out score (shortcut for
    a projection) and, given sigma^2, the SURE (Mallows' Cp) RSS/n + 2 sigma^2 p / n - sigma^2."""
    X = tensor_design(U, Js)
    Q, _ = np.linalg.qr(X)
    H = Q @ Q.T
    f = H @ y
    rss = float(np.sum((y - f) ** 2))
    n = len(y)
    p = X.shape[1]
    loo = float(np.mean(((y - f) / (1 - np.diag(H))) ** 2))
    out = {"f": f, "p": p, "rss": rss, "loo": loo, "X": X}
    if s2 is not None:
        out["sure"] = rss / n + 2 * s2 * p / n - s2
    return out


def ex13_data(n=1000, seed=SEED_EX13, independent=True):
    """Exercise 13: Y = x1 + cos(x2) + N(0, 1). As printed, x1i = x2i = i/n (the same covariate twice);
    with independent=True the covariates are iid Uniform(0, 1), which is presumably what is intended."""
    rng = np.random.default_rng(seed)
    if independent:
        U = rng.uniform(size=(n, 2))
    else:
        t = regular_design(n)
        U = np.column_stack([t, t])
    y = U[:, 0] + np.cos(U[:, 1]) + rng.standard_normal(n)
    return U, y


def ex13_surface(Jmax=8):
    U, y = ex13_data()
    S = np.zeros((Jmax, Jmax))
    for a in range(Jmax):
        for b in range(Jmax):
            S[a, b] = tensor_fit(U, y, (a + 1, b + 1), s2=1.0)["sure"]
    return U, y, S


def airquality_tensor():
    """Exercise 14: ozone on (solar radiation, wind, temperature), n = 111 complete cases. Candidates:
    the full tensor product with J terms in each coordinate (J^3 functions), and the main-effects
    (additive) and two-way models of the same ANOVA expansion. Scores: leave-one-out CV."""
    d = c5.airquality()
    U = np.column_stack([rescale(d[c].to_numpy(float)) for c in ("Solar.R", "Wind", "Temp")])
    y = d["Ozone"].to_numpy(float)
    rows = []
    for J in (1, 2, 3, 4):
        X = tensor_design(U, (J, J, J))
        import itertools
        idx = list(itertools.product(range(J), range(J), range(J)))
        order = np.array([sum(1 for t in i if t > 0) for i in idx])
        for name, keep in (("additive", order <= 1), ("two-way", order <= 2), ("full tensor", order <= 3)):
            Xs = X[:, keep]
            if name != "full tensor" and J == 1:
                continue
            Q, _ = np.linalg.qr(Xs)
            f = Q @ (Q.T @ y)
            h = np.sum(Q ** 2, 1)
            loo = float(np.mean(((y - f) / (1 - h)) ** 2)) if np.all(h < 1 - 1e-10) else np.inf
            rss = float(np.sum((y - f) ** 2))
            rows.append({"J": J, "model": name, "p": int(Xs.shape[1]), "rss": rss, "loo": loo,
                         "r2": 1 - rss / float(np.sum((y - y.mean()) ** 2))})
    return rows


# =========================================================================== data and helpers for figures
def doppler_data(n=1000, sigma=0.1, seed=SEED_DOPPLER):
    return c5.doppler_data(n, sigma, seed)


def cmb_regular():
    x, y, _ = c5.cmb()
    return x, y


def _style(ax, xl=None, yl=None):
    if xl:
        ax.set_xlabel(xl)
    if yl:
        ax.set_ylabel(yl)


def local_linear_cv(x, y, hs):
    sel = c5.select_h(x, y, hs, "ll")
    return sel["hbest"], sel


# =========================================================================== figures
def fig_8_1(out=None):
    """Doppler: true function, 1000 data points, estimated NSS risk against J, react fit."""
    x, y = doppler_data()
    R = react(y)
    xg = np.linspace(0, 1, 2001)
    fig, ax = common.panel_grid(2, 2, size=(3.3, 2.5))
    ax[0, 0].plot(xg, c5.doppler(xg), color="k", lw=1.2)
    ax[0, 1].plot(x, y, ".", ms=1.6, color=GREY)
    J = np.arange(1, len(y) + 1)
    ax[1, 0].plot(J, R["risk"], color=BLUE)
    ax[1, 0].axvline(R["J"], color=RED, ls="--", lw=1)
    ax[1, 0].text(R["J"] + 20, ax[1, 0].get_ylim()[1] * 0.9, "J = %d" % R["J"], color=RED, fontsize=8, va="top")
    ax[1, 1].plot(xg, fit_grid(xg, R["Z"], R["b"]), color=RED, lw=1)
    for a in (ax[0, 0], ax[0, 1], ax[1, 1]):
        a.set_ylim(-0.75, 0.75)
        _style(a, "x")
    _style(ax[1, 0], "number of terms J", "estimated risk")
    ax[0, 0].set_title("Doppler function")
    ax[0, 1].set_title("n = 1000, sigma = 0.1")
    ax[1, 0].set_title(r"$\hat R(J)$, (8.18)")
    ax[1, 1].set_title("react fit (NSS)")
    fig.tight_layout()
    savefig(fig, CH, "fig8-1", out)


def cmb_react():
    x, y = cmb_regular()
    R = react(y)
    n = len(y)
    risk = R["risk"]
    loc = [k + 1 for k in range(1, 99) if risk[k] < risk[k - 1] and risk[k] < risk[k + 1]]
    second = [j for j in loc if j > 20][0]
    return x, y, R, second


def fig_8_2(out=None):
    """CMB data: NSS fit with J = 6, the estimated risk, the fit with the second local minimum (J = 40)."""
    x, y, R, J2 = cmb_react()
    n = len(y)
    t = regular_design(n)
    tg = np.linspace(1 / n, 1, 1800)
    xg = 1 + tg * n          # multipole 2..900 corresponds to i = 1..n
    fig, ax = common.panel_grid(2, 2, size=(3.3, 2.5))
    for a, J in ((ax[0, 0], R["J"]), (ax[1, 0], J2)):
        b = (np.arange(n) < J).astype(float)
        a.plot(x, y, ".", ms=1.4, color=GREY, alpha=.6)
        a.plot(xg, fit_grid(tg, R["Z"], b), color=RED, lw=1.2)
        a.set_ylim(-1500, 7500)
        a.set_title("NSS fit, J = %d" % J)
        _style(a, "multipole", "power")
    a = ax[0, 1]
    a.plot(np.arange(1, 51), R["risk"][:50] / 1e6, color=BLUE)
    a.axvline(R["J"], color=RED, ls="--", lw=1)
    a.axvline(J2, color=GREY, ls=":", lw=1)
    a.set_title(r"estimated risk $\hat R(J)$")
    _style(a, "J", r"$\times 10^6$")
    ax[1, 1].axis("off")
    ax[1, 1].text(0.02, 0.6, "sigma-hat (8.12): %.0f\nglobal minimum J = %d\nlocal minimum J = %d" % (math.sqrt(R["s2"]), R["J"], J2),
                  transform=ax[1, 1].transAxes, fontsize=9, va="center")
    fig.tight_layout()
    savefig(fig, CH, "fig8-2", out)


def test_functions():
    """Four functions for Lemma 8.4 and the boundary: a periodic smooth one (r'(0) = r'(1) = 0), one
    with r'(0) != r'(1), the Doppler function and a step."""
    return [("cos-smooth: exp(cos 2 pi x)", lambda x: np.exp(np.cos(2 * np.pi * x)), PURPLE),
            ("r(x) = x (r' != 0 at 0, 1)", lambda x: x, BLUE),
            ("Doppler", c5.doppler, GREEN),
            ("step at 0.4", lambda x: (x > 0.4).astype(float), RED)]


def exact_coefficients(f, J, m=20000):
    """theta_j = int_0^1 phi_j f, j <= J, by the midpoint rule on m points (exact enough here)."""
    u = (np.arange(m) + 0.5) / m
    return cos_basis(u, J).T @ f(u) / m


def fig_8_S1(out=None):
    """Lemma 8.4 and the boundary: bias B_J against J for four functions; equivalent kernels."""
    fig, ax = common.panel_grid(1, 3, size=(3.3, 2.7))
    Jmax = 400
    for lab, f, col in test_functions():
        th = exact_coefficients(f, 4000)
        u = (np.arange(20000) + 0.5) / 20000
        tot = float(np.mean(f(u) ** 2))
        B = tot - np.cumsum(th[:Jmax] ** 2)
        Js = np.arange(1, Jmax + 1)
        ax[0, 0].loglog(Js[1:], np.maximum(B[1:], 1e-16), color=col, label=lab)
        ax[0, 1].loglog(Js[1:], np.abs(th[1:Jmax]) + 1e-18, ".", ms=1.5, color=col)
    ax[0, 0].loglog([10, 300], [1e-3, 1e-3 * (30) ** -3], "k--", lw=.8)
    ax[0, 0].text(60, 4e-6, r"$J^{-3}$", fontsize=8)
    ax[0, 0].set_ylim(1e-14, 1)
    ax[0, 0].legend(fontsize=6.5, loc="lower left")
    _style(ax[0, 0], "J", r"bias $B_J=\sum_{j>J}\theta_j^2$")
    _style(ax[0, 1], "j", r"$|\theta_j|$")
    ax[0, 1].set_ylim(1e-12, 2)
    n = 1000
    x = regular_design(n)
    for x0, col in ((0.5, BLUE), (0.0, RED), (0.1, GREEN)):
        b = (np.arange(n) < 25).astype(float)
        w = weights_grid(np.array([x0]), x, b)[0]
        ax[0, 2].plot(x, n * w, color=col, lw=1, label="x = %.1f" % x0)
    ax[0, 2].legend(fontsize=7)
    _style(ax[0, 2], "x_i", r"$n\,\ell_i(x)$, J = 25")
    ax[0, 0].set_title("Lemma 8.4: the bias")
    ax[0, 1].set_title("coefficient decay")
    ax[0, 2].set_title("equivalent kernel (8.11)")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S1", out)


def doppler_modulators():
    x, y = doppler_data()
    Z = coefficients(y)
    s2 = sigma2_tail(Z)
    out = {}
    for cls in ("nss", "mon", "cons"):
        R = react(y, cls=cls, s2=s2)
        out[cls] = R
    theta = cos_basis(x, len(y)).T @ c5.doppler(x) / len(y)
    for cls in out:
        out[cls]["true_risk"] = true_risk(out[cls]["b"], theta, s2)
    return x, y, out


def fig_8_S2(out=None):
    """The three modulator classes on the Doppler data, and the band (8.25) for the NSS fit."""
    x, y, M = doppler_modulators()
    fig, ax = common.panel_grid(1, 3, size=(3.3, 2.6))
    j = np.arange(1, len(y) + 1)
    for cls, col, lab in (("nss", RED, "NSS"), ("mon", BLUE, "monotone"), ("cons", GREEN, "constant (James-Stein)")):
        ax[0, 0].plot(j[:400], M[cls]["b"][:400], color=col, lw=1.2, label="%s, R-hat = %.4f" % (lab, M[cls]["Rhat"]))
    ax[0, 0].legend(fontsize=6.5, loc="center right")
    _style(ax[0, 0], "j", r"$\hat b_j$")
    ax[0, 0].set_title("the three modulators")
    xg = np.linspace(0, 1, 1001)
    ax[0, 1].plot(xg, c5.doppler(xg), color="k", lw=.8)
    ax[0, 1].plot(xg, fit_grid(xg, M["cons"]["Z"], M["cons"]["b"]), color=GREEN, lw=.8)
    ax[0, 1].plot(xg, fit_grid(xg, M["mon"]["Z"], M["mon"]["b"]), color=BLUE, lw=.8)
    ax[0, 1].set_xlim(0, 0.3)
    ax[0, 1].set_title("fits near 0 (black: truth)")
    _style(ax[0, 1], "x")
    xg2 = np.linspace(0, 1, 1000)
    B = band(xg2, x, M["nss"]["Z"], M["nss"]["b"], M["nss"]["s2"])
    ax[0, 2].fill_between(xg2, B["lo"], B["hi"], color=BLUE, alpha=.3, lw=0)
    ax[0, 2].plot(xg2, B["fit"], color=RED, lw=.8)
    ax[0, 2].plot(xg2, c5.doppler(xg2), color="k", lw=.6)
    ax[0, 2].set_ylim(-.75, .75)
    ax[0, 2].set_title(r"band (8.25): $\kappa_0$=%.0f, c=%.2f" % (B["kappa0"], B["c"]))
    _style(ax[0, 2], "x")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S2", out)


def fig_8_S3(out=None):
    """The confidence ball: distribution of the pivot standardised two ways, and coverage."""
    C = ball_coverage(reps=1000)
    fig, ax = common.panel_grid(1, 2, size=(3.6, 2.7))
    g = np.linspace(-4, 4, 200)
    bins = np.linspace(-4, 5, 46)
    ax[0, 0].hist(C["piv_fixed"], bins=bins, density=True, color=GREY, alpha=.7, label="fixed J = 160")
    ax[0, 0].plot(g, stats.norm.pdf(g), "k", lw=1)
    ax[0, 0].set_title("pivot, derived variance, fixed b")
    ax[0, 0].legend(fontsize=7)
    ax[0, 1].hist(C["piv_printed"], bins=bins, density=True, histtype="step", color=RED, lw=1.3,
                  label="printed tau: cover %.3f" % C["printed"])
    ax[0, 1].hist(C["piv_mine"], bins=bins, density=True, histtype="step", color=BLUE, lw=1.3,
                  label="derived tau: cover %.3f" % C["mine"])
    ax[0, 1].plot(g, stats.norm.pdf(g), "k", lw=1)
    ax[0, 1].set_title("pivot at the selected J (NSS)")
    ax[0, 1].legend(fontsize=7)
    for a in ax[0]:
        _style(a, r"$(L-\hat R)/\mathrm{sd}$")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S3", out)
    return C


def glass_analysis():
    x, y = c5.glass()
    o = np.argsort(x, kind="stable")
    x, y = x[o], y[o]
    F = react_irregular(x, y, Jmax=int(math.sqrt(len(y))))
    F["J_all"] = int(np.argmin(F["risk"]) + 1)
    xg = np.linspace(x.min(), x.max(), 400)
    B = irregular_band(F, xg, F["J"])
    h, sel = local_linear_cv(x, y, c5.log_grid(0.08, 1.5, 60))
    L = c5.ll_matrix(x, xg, h)
    Ld = c5.ll_matrix(x, x, h)
    s2 = c5.sigma2_linear(Ld, y)
    k0 = c5.tube_kappa0(L, xg)
    cll = c5.tube_c(k0)
    lo, hi, _ = c5.linear_band(L, L @ y, math.sqrt(s2), cll)
    return {"x": x, "y": y, "F": F, "xg": xg, "B": B, "h": h, "ll": L @ y, "lo": lo, "hi": hi, "s2_ll": s2,
            "c_ll": cll, "k0_ll": k0, "rice": c5.sigma2_rice(x, y)}


def fig_8_S4(out=None):
    """Exercise 7 (glass): the empirical orthonormal basis, react vs local linear, bands."""
    G = glass_analysis()
    F = G["F"]
    fig, ax = common.panel_grid(1, 3, size=(3.3, 2.7))
    for j, col in zip(range(4), (GREY, BLUE, RED, GREEN)):
        ax[0, 0].plot(G["xg"], irregular_eval(F, G["xg"], 4)[:, j], color=col, lw=1, label=r"$\phi_%d$" % (j + 1))
    ax[0, 0].plot(G["x"], np.full(len(G["x"]), -3.5), "|", color="k", ms=5)
    ax[0, 0].legend(fontsize=7, ncol=2)
    ax[0, 0].set_title("basis orthonormal for $P_n$")
    _style(ax[0, 0], "aluminium")
    a = ax[0, 1]
    a.plot(G["x"], G["y"], ".", ms=3, color=GREY)
    a.fill_between(G["xg"], G["B"]["lo"], G["B"]["hi"], color=BLUE, alpha=.25, lw=0)
    a.plot(G["xg"], G["B"]["fit"], color=BLUE, lw=1.4, label="react, J = %d" % F["J"])
    a.plot(G["xg"], G["ll"], color=RED, lw=1.2, ls="--", label="local linear, h = %.2f" % G["h"])
    a.set_ylim(-8, 16)
    a.set_xlim(G["x"].min() - .05, G["x"].max() + .05)
    a.legend(fontsize=7)
    a.set_title("fits and react band")
    _style(a, "aluminium", "refractive index")
    a = ax[0, 2]
    a.plot(np.arange(1, F["q"] + 1), F["risk"], color=BLUE)
    a.axvline(F["J"], color=RED, ls="--", lw=1)
    a.axvline(F["J_all"], color=GREY, ls=":", lw=1)
    a.axvspan(0, math.sqrt(len(G["y"])), color=RED, alpha=.08, lw=0)
    a.set_title(r"estimated risk, $\hat\sigma$ = %.2f" % math.sqrt(F["s2"]))
    _style(a, "J")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S4", out)


def motor_analysis():
    x, y = c5.motor()
    o = np.argsort(x, kind="stable")
    x, y = x[o], y[o]
    F = react_irregular(x, y)
    xg = np.linspace(x.min(), x.max(), 400)
    B = irregular_band(F, xg, F["J"])
    n = len(y)
    J = F["J"]
    b = np.zeros(F["q"]); b[:J] = 1
    # ball over the q function coordinates and the residual (whose coordinates have b = 0)
    Zfull = np.concatenate([F["Z"], np.zeros(n - F["q"])])
    Zfull[F["q"]:] = math.sqrt(F["rss"] / n / max(n - F["q"], 1))   # only sum of squares matters below
    bfull = np.concatenate([b, np.zeros(n - F["q"])])
    rad = ball_radius(Zfull, bfull, F["s2"])
    return {"x": x, "y": y, "F": F, "xg": xg, "B": B, "radius": rad}


def fig_8_S5(out=None):
    """Exercise 8 (motorcycle): react fit and band; the confidence ball shown through its members."""
    M = motor_analysis()
    F = M["F"]
    J = F["J"]
    fig, ax = common.panel_grid(1, 2, size=(3.6, 2.7))
    a = ax[0, 0]
    a.plot(M["x"], M["y"], ".", ms=3, color=GREY)
    a.fill_between(M["xg"], M["B"]["lo"], M["B"]["hi"], color=BLUE, alpha=.25, lw=0)
    a.plot(M["xg"], M["B"]["fit"], color=BLUE, lw=1.4)
    a.set_title("react, J = %d, c = %.2f" % (J, M["B"]["c"]))
    _style(a, "time (ms)", "acceleration (g)")
    a = ax[0, 1]
    rng = np.random.default_rng(SEED_BALL)
    K = min(F["q"], 2 * J)
    Pg = irregular_eval(F, M["xg"], K)
    th = np.zeros(K); th[:J] = F["Z"][:J]
    for _ in range(30):
        u = rng.standard_normal(K); u /= np.linalg.norm(u)
        a.plot(M["xg"], Pg @ (th + M["radius"] * u), color=BLUE, lw=.5, alpha=.5)
    a.plot(M["x"], M["y"], ".", ms=2.5, color=GREY)
    a.plot(M["xg"], Pg @ th, color=RED, lw=1.4)
    a.set_title("30 functions on the ball's surface\n(directions in the first %d coordinates)" % K, fontsize=9)
    _style(a, "time (ms)")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S5", out)


def ex9_analysis(sigma):
    n = 1000
    x = regular_design(n)
    rng = np.random.default_rng(SEED_EX9)
    e = rng.standard_normal(n)
    y = c5.doppler(x) + sigma * e
    R = react(y)
    xg = np.linspace(0.001, 1, 600)
    B = band(xg, x, R["Z"], R["b"], R["s2"])
    hs = c5.log_grid(0.002, 0.3, 50)
    h, _ = local_linear_cv(x, y, hs)
    L = c5.ll_matrix(x, xg, h)
    s2 = c5.sigma2_linear(c5.ll_matrix(x, x, h), y)
    k0 = c5.tube_kappa0(L, xg)
    c = c5.tube_c(k0)
    lo, hi, _ = c5.linear_band(L, L @ y, math.sqrt(s2), c)
    truth = c5.doppler(xg)
    return {"x": x, "y": y, "xg": xg, "R": R, "B": B, "h": h, "ll": L @ y, "lo": lo, "hi": hi, "c_ll": c,
            "ise_react": float(np.mean((B["fit"] - truth) ** 2)), "ise_ll": float(np.mean((L @ y - truth) ** 2)),
            "s_react": math.sqrt(R["s2"]), "s_ll": math.sqrt(s2)}


def fig_8_S6(out=None):
    """Exercise 9: Doppler at three noise levels, local linear vs react, with bands."""
    fig, ax = common.panel_grid(2, 3, size=(3.1, 2.4))
    for k, sigma in enumerate((0.1, 1.0, 3.0)):
        A = ex9_analysis(sigma)
        for row, (f, lo, hi, lab, col) in enumerate(((A["B"]["fit"], A["B"]["lo"], A["B"]["hi"], "react, J = %d" % A["R"]["J"], BLUE),
                                                     (A["ll"], A["lo"], A["hi"], "local linear, h = %.3f" % A["h"], RED))):
            a = ax[row, k]
            a.fill_between(A["xg"], lo, hi, color=col, alpha=.25, lw=0)
            a.plot(A["xg"], f, color=col, lw=.9)
            a.plot(A["xg"], c5.doppler(A["xg"]), color="k", lw=.6)
            a.set_ylim(-1.2 - sigma, 1.2 + sigma)
            a.set_title("sigma = %g: %s" % (sigma, lab), fontsize=8.5)
    fig.tight_layout()
    savefig(fig, CH, "fig8-S6", out)


def block_medians(y, m):
    n = len(y) // m * m
    return np.median(np.asarray(y[:n]).reshape(-1, m), axis=1)


def ex10_analysis(m=5):
    n = 1000
    x = regular_design(n)
    rng = np.random.default_rng(SEED_EX10)
    y = c5.doppler(x) + 0.1 * rng.standard_cauchy(n)
    R = react(y)
    xg = np.linspace(0.001, 1, 800)
    ym = block_medians(y, m)
    nb = len(ym)
    Rm = react(ym)        # block centres are equally spaced; treated as the regular design k/nb
    xb = (np.arange(nb) + 1) / nb
    # map block k (covering x = (km - m + 1)/n .. km/n) to its centre
    fit_m = fit_grid(np.clip(xg + (m - 1) / (2 * n), 0, 1), Rm["Z"], Rm["b"])
    truth = c5.doppler(xg)
    hs = c5.log_grid(0.002, 0.3, 50)
    h, _ = local_linear_cv(x, y, hs)
    ll = c5.ll_matrix(x, xg, h) @ y
    return {"x": x, "y": y, "xg": xg, "R": R, "Rm": Rm, "fit": fit_grid(xg, R["Z"], R["b"]), "fit_m": fit_m,
            "ll": ll, "h": h, "ise": float(np.mean((fit_grid(xg, R["Z"], R["b"]) - truth) ** 2)),
            "ise_m": float(np.mean((fit_m - truth) ** 2)), "ise_ll": float(np.mean((ll - truth) ** 2)), "m": m}


def fig_8_S7(out=None):
    """Exercise 10: Cauchy errors; react on the raw data and on block medians."""
    A = ex10_analysis()
    fig, ax = common.panel_grid(1, 3, size=(3.2, 2.6))
    ax[0, 0].plot(A["x"], A["y"], ".", ms=1.5, color=GREY)
    ax[0, 0].set_ylim(-3, 3)
    ax[0, 0].set_title("Cauchy errors (clipped view)")
    ax[0, 1].plot(A["xg"], A["fit"], color=RED, lw=.8)
    ax[0, 1].plot(A["xg"], c5.doppler(A["xg"]), "k", lw=.6)
    ax[0, 1].set_title("react on raw data, J = %d" % A["R"]["J"])
    ax[0, 2].plot(A["xg"], A["fit_m"], color=BLUE, lw=.9)
    ax[0, 2].plot(A["xg"], c5.doppler(A["xg"]), "k", lw=.6)
    ax[0, 2].set_title("react on medians of %d, J = %d" % (A["m"], A["Rm"]["J"]))
    for a in ax[0, 1:]:
        a.set_ylim(-1, 1)
    fig.tight_layout()
    savefig(fig, CH, "fig8-S7", out)


def fig_8_S8(out=None):
    """Exercise 11: react density vs kernel density estimates, mixtures with mu = 0, ..., 5."""
    fig, ax = common.panel_grid(2, 3, size=(3.1, 2.3))
    for a, mu in zip(ax.ravel(), range(6)):
        D = density_compare(mu=float(mu))
        a.plot(D["grid"], D["truth"], "k", lw=1)
        a.plot(D["grid"], D["react"]["fhat"], color=BLUE, lw=1.1, label="react (J=%d)" % D["react"]["J"])
        a.plot(D["grid"], D["kde"], color=RED, lw=1, ls="--", label="kernel, LSCV")
        a.set_title(r"$\mu$ = %d" % mu)
        a.legend(fontsize=6.5, loc="upper right")
        a.set_xlim(-4, mu + 4)
    fig.tight_layout()
    savefig(fig, CH, "fig8-S8", out)


def greedy_modulator(Z, s2):
    """Exercise 12: minimise the estimated risk (8.15) coordinate by coordinate:
    b_j = (Z_j^2 - s)_+ / ((Z_j^2 - s)_+ + s)."""
    s = s2 / len(Z)
    t = np.maximum(np.asarray(Z) ** 2 - s, 0)
    return t / (t + s)


def ex12_simulation(reps=200, n=1000, sigma=0.1, seed=SEED_EX12):
    x = regular_design(n)
    P = cos_basis(x, n)
    theta = P.T @ c5.doppler(x) / n
    rng = np.random.default_rng(seed)
    rows = {"greedy": [], "nss": [], "oracle": []}
    est = {"greedy": [], "nss": [], "greedy_u": [], "nss_u": []}
    s = sigma ** 2 / n
    bstar = theta ** 2 / (theta ** 2 + s)
    for _ in range(reps):
        Z = P.T @ (c5.doppler(x) + sigma * rng.standard_normal(n)) / n
        s2 = sigma2_tail(Z)
        bg = greedy_modulator(Z, s2)
        R = nss_risk(Z, s2); J = int(np.argmin(R) + 1)
        bn = (np.arange(n) < J).astype(float)
        rows["greedy"].append(np.sum((bg * Z - theta) ** 2)); est["greedy"].append(sure(bg, Z, s2))
        rows["nss"].append(np.sum((bn * Z - theta) ** 2)); est["nss"].append(sure(bn, Z, s2))
        est["greedy_u"].append(sure(bg, Z, s2, positive=False)); est["nss_u"].append(sure(bn, Z, s2, positive=False))
        rows["oracle"].append(np.sum((bstar * Z - theta) ** 2))
    return {k: np.array(v) for k, v in rows.items()}, {k: np.array(v) for k, v in est.items()}, true_risk(bstar, theta, sigma ** 2)


def fig_8_S9(out=None):
    """Exercise 12: the greedy modulator; estimated against actual loss."""
    L, E, Rstar = ex12_simulation()
    fig, ax = common.panel_grid(1, 2, size=(3.6, 3.0))
    for a, key, ttl in ((ax[0, 0], "", "modified SURE (8.15)"), (ax[0, 1], "_u", "unbiased SURE")):
        a.plot(E["greedy" + key] * 1e3, L["greedy"] * 1e3, ".", ms=3, color=RED, label="greedy")
        a.plot(E["nss" + key] * 1e3, L["nss"] * 1e3, ".", ms=3, color=BLUE, label="NSS")
        lim = [0, 7.5]
        a.plot(lim, lim, "k:", lw=.8)
        if key:
            lim = [-6, 7.5]
        a.axhline(Rstar * 1e3, color=GREEN, ls="--", lw=1, label="oracle R(b*)")
        a.set_xlim(lim); a.set_ylim(lim)
        a.set_title(ttl)
        a.legend(fontsize=7, loc="lower right")
        _style(a, r"estimated risk ($\times 10^{-3}$)", r"loss $\|\hat b Z-\theta\|^2$ ($\times 10^{-3}$)")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S9", out)


def fig_8_S10(out=None):
    """Exercise 13: SURE over (J1, J2) and the fitted surface."""
    U, y, S = ex13_surface()
    i, j = np.unravel_index(np.argmin(S), S.shape)
    fig, ax = common.panel_grid(1, 2, size=(3.4, 2.9))
    im = ax[0, 0].imshow(S - S.min(), origin="lower", extent=[0.5, 8.5, 0.5, 8.5], cmap="viridis")
    ax[0, 0].plot(j + 1, i + 1, "r*", ms=10)
    fig.colorbar(im, ax=ax[0, 0], shrink=.8)
    _style(ax[0, 0], r"$J_2$", r"$J_1$")
    ax[0, 0].set_title("SURE minus its minimum")
    g = np.linspace(0, 1, 41)
    G1, G2 = np.meshgrid(g, g, indexing="ij")
    Ug = np.column_stack([G1.ravel(), G2.ravel()])
    X = tensor_design(U, (i + 1, j + 1))
    beta = np.linalg.lstsq(X, y, rcond=None)[0]
    F = (tensor_design(Ug, (i + 1, j + 1)) @ beta).reshape(G1.shape)
    k = 20     # the slices x2 = 0.5 and x1 = 0.5
    ax[0, 1].plot(g, F[:, k], color=BLUE, lw=1.2, label=r"fit at $x_2=0.5$, against $x_1$")
    ax[0, 1].plot(g, g + np.cos(0.5), "--", color=BLUE, lw=.8)
    ax[0, 1].plot(g, F[k, :], color=RED, lw=1.2, label=r"fit at $x_1=0.5$, against $x_2$")
    ax[0, 1].plot(g, 0.5 + np.cos(g), "--", color=RED, lw=.8)
    ax[0, 1].legend(fontsize=6.5)
    ax[0, 1].set_title("two slices (dashed: truth)")
    _style(ax[0, 1], r"$x_1$ or $x_2$")
    fig.tight_layout()
    savefig(fig, CH, "fig8-S10", out)


# =========================================================================== JSON for the demos
def export_json():
    """../data/ch08-react.json (regression demo) and ../data/ch08-noise.json (the Normal draws)."""
    os.makedirs(DATA_OUT, exist_ok=True)
    rnd = lambda v, k=6: [round(float(t), k) for t in v]
    x, y = doppler_data()
    e = (y - c5.doppler(x)) / 0.1
    xc, yc = cmb_regular()
    xg, yg = c5.glass(); o = np.argsort(xg, kind="stable")
    xm, ym = c5.motor(); om = np.argsort(xm, kind="stable")
    D = {"doppler_e": rnd(e, 6),
         "cmb": {"x": rnd(xc, 0), "y": rnd(yc, 4)},
         "glass": {"x": rnd(xg[o], 4), "y": rnd(yg[o], 4)},
         "motor": {"x": rnd(xm[om], 4), "y": rnd(ym[om], 4)}}
    with open(os.path.join(DATA_OUT, "ch08-react.json"), "w") as f:
        json.dump(D, f, separators=(",", ":"))
    print("wrote ../data/ch08-react.json")


def _with_export(fun):
    def wrapped(out_dir=None):
        fun(out_dir)
        export_json()
    wrapped.__doc__ = fun.__doc__
    return wrapped


FIGURES = {
    "8.1": fig_8_1, "8.2": fig_8_2,
    "8.S1": fig_8_S1, "8.S2": fig_8_S2, "8.S3": fig_8_S3, "8.S4": fig_8_S4, "8.S5": fig_8_S5,
    "8.S6": fig_8_S6, "8.S7": fig_8_S7, "8.S8": fig_8_S8, "8.S9": fig_8_S9, "8.S10": _with_export(fig_8_S10),
}

if __name__ == "__main__":
    common.run(FIGURES, CH)
