"""James et al. (2021, 2023), Chapter 4: Classification.

Reproduces the figures and tables of Chapter 4 of G. James, D. Witten, T. Hastie, R. Tibshirani
(and J. Taylor), An Introduction to Statistical Learning, with Applications in R (2nd edition, 2021)
and with Applications in Python (2023), Springer, written from scratch with numpy, scipy and
matplotlib. Every method is a small function meant to be read next to the lecture notes
(../ch04-classification.html):

* Section 4.3, logistic regression: logistic_fit (Newton-Raphson, i.e. iteratively reweighted
  least squares, for the likelihood (4.5), with standard errors from the inverse information),
  softmax_fit and baseline_fit (multinomial logistic regression in the softmax coding (4.13) and
  the baseline coding (4.10)-(4.11)).
* Section 4.4, generative models: lda_fit / lda_proba ((4.20)-(4.24)), qda_fit / qda_proba (4.28),
  nb_fit / nb_proba (naive Bayes (4.29)-(4.30) with Gaussian or categorical margins), knn_predict.
* Assessment: confusion (Tables 4.4-4.9), roc (Figure 4.8) and auc (the area as a Mann-Whitney
  probability).
* Section 4.6, generalised linear models: sum_contrasts (the coding behind Table 4.10 and Figures
  4.13 and 4.15), ols_fit and poisson_fit (Newton-Raphson for the Poisson likelihood (4.38)).
* Section 4.5.2: scenario_data and the six-method simulation of Figures 4.11 and 4.12.

The data are the book's Default, Bikeshare, Smarket and Caravan files (read by common.isl_data).
`python3 ch04_classification.py --list` lists the figures; `--fig tables` prints Tables 4.1-4.11 with
the numbers quoted in the text and writes the demonstration data to ../data/.

Figures reproduced: all fifteen, 4.1-4.15. Figure 4.5 (two bivariate Gaussian densities) is drawn
from the formula (4.23); Figures 4.4, 4.6, 4.9, 4.10, 4.11 and 4.12 are simulations whose settings the
book does not give in full, so the settings below are mine (chosen to match the book's description
and the ranges of its axes). Nothing is skipped.

Settings chosen where the book gives none:
* Figure 4.1: all 333 defaulters and a random 15% of the non-defaulters (numpy seed 1) in the scatter plot.
* Figure 4.3, left: the default rate in each balance bin of width 100, for students and non-students
  separately (bins with fewer than 20 people are dropped), smoothed by a logistic fit on balance
  within each group; dashed lines are the overall rates of the two groups.
* Figure 4.4: mu = (-1.25, 1.25), sigma = 1, 20 training points per class (numpy seed 4), test error
  from the exact normal tail probabilities.
* Figure 4.6: three classes with means (0, 0.5), (-1.5, -1.6) and (1.6, -1.2), common covariance
  [[1, 0.5], [0.5, 1]] scaled by 0.62, 20 points per class (numpy seed 4); error rates from
  200,000 test points per class (numpy seed 104).
* Figure 4.9: 50 points per class (numpy seed 9); means (-1, -0.4) and (1, 0.4), unit variances;
  left correlation 0.7 in both classes, right 0.7 (orange) and -0.7 (blue).
* Figure 4.10: class 1: X1 ~ N(0, 1), X2 ~ N(1, 0.6^2) mixed with N(2.5, 0.5^2) (weights 0.8 / 0.2),
  X3 in {1, 2, 3} with probabilities (0.25, 0.45, 0.30); class 2: X1 ~ N(1.8, 0.8^2),
  X2 ~ N(-0.8, 0.9^2), X3 with probabilities (0.6, 0.25, 0.15); 100 points each (seed 10);
  Gaussian kernel density estimates with Silverman's bandwidth.
* Figures 4.11 and 4.12: 100 training sets per scenario (seeds 1000 s + r), test sets of 5,000 points,
  KNN-CV chooses K in 1..15 by leave-one-out cross-validation; scenario_data documents each scenario.
* Figure 4.14: jitter of +-0.3 hours (seed 14); the green curve is the smoothing spline of bikers
  on hour (equivalently, of the hourly means weighted by their counts) with the smoothing
  parameter chosen by generalised cross-validation (scipy's make_smoothing_spline).
"""
import json
import os

import numpy as np
from scipy import stats
from scipy.interpolate import make_smoothing_spline

import common
from common import BLUE, GREEN, GREY, ORANGE, PURPLE, RED, savefig

plt = common.plt
CH = 4
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
CACHE_DIR = os.path.join(HERE, "cache")
MONTHS = ["Jan", "Feb", "March", "April", "May", "June", "July", "Aug", "Sept", "Oct", "Nov", "Dec"]
WEATHER = ["clear", "cloudy/misty", "light rain/snow", "heavy rain/snow"]
OR, BL = "#e8862a", "#3b78c2"      # the book's orange and blue


# =========================================================================== data
def load_default():
    """The Default data: y (1 = default), balance, income (in dollars), student (1 = Yes)."""
    D = common.isl_data("Default")
    return dict(y=(D.default == "Yes").astype(int).values, balance=D.balance.values.astype(float),
                income=D.income.values.astype(float), student=(D.student == "Yes").astype(int).values)


def load_smarket():
    return common.isl_data("Smarket")


def load_bikeshare():
    return common.isl_data("Bikeshare", index_col=0)


def sum_contrasts(values, levels):
    """Columns for the 'sum to zero' coding of a factor (R's contr.sum): level j < L gets the unit
    vector e_j, the last level gets (-1, ..., -1). The implied coefficient of the last level is minus
    the sum of the others, so the level effects sum to zero."""
    values = np.asarray(values)
    M = np.zeros((len(values), len(levels) - 1))
    for j, lev in enumerate(levels[:-1]):
        M[values == lev, j] = 1.0
    M[values == levels[-1], :] = -1.0
    return M


def treatment_contrasts(values, levels):
    """Dummy (one-hot without the first level) coding, R's default contr.treatment."""
    values = np.asarray(values)
    return np.column_stack([(values == lev).astype(float) for lev in levels[1:]])


def bikeshare_design(B, coding="sum"):
    """Design matrix of bikers ~ mnth + hr + workingday + temp + weathersit, as in Section 4.6.
    Returns X and the column names (intercept first)."""
    code = sum_contrasts if coding == "sum" else treatment_contrasts
    hours = list(range(24))
    Xm = code(B.mnth.values, MONTHS)
    Xh = code(B.hr.values, hours)
    Xw = treatment_contrasts(B.weathersit.values, WEATHER)
    X = np.column_stack([np.ones(len(B)), Xm, Xh, B.workingday.values, B.temp.values, Xw])
    lm = MONTHS[:-1] if coding == "sum" else MONTHS[1:]
    lh = hours[:-1] if coding == "sum" else hours[1:]
    names = (["intercept"] + ["mnth[%s]" % m for m in lm] + ["hr[%d]" % h for h in lh]
             + ["workingday", "temp"] + ["weathersit[%s]" % w for w in WEATHER[1:]])
    return X, names


def full_levels(beta, names, prefix):
    """All level effects of a sum-coded factor: the reported ones plus minus their sum."""
    b = np.array([beta[i] for i, n in enumerate(names) if n.startswith(prefix + "[")])
    return np.r_[b, -b.sum()]


# =========================================================================== logistic regression
def logistic_fit(X, y, tol=1e-12, maxit=100):
    """Maximise the Bernoulli log-likelihood (4.5) by Newton-Raphson (iteratively reweighted least
    squares). X must contain the intercept column. Returns coefficients, standard errors (square
    roots of the diagonal of the inverse Fisher information X^T W X), z = beta / se, two-sided
    p-values, the log-likelihood, the deviance -2 loglik and the number of iterations."""
    X = np.asarray(X, float)
    y = np.asarray(y, float)
    beta = np.zeros(X.shape[1])
    for it in range(1, maxit + 1):
        eta = X @ beta
        p = common.sigmoid(eta)
        w = p * (1 - p)
        g = X.T @ (y - p)                       # score (gradient of the log-likelihood)
        H = X.T @ (X * w[:, None])              # information (minus the Hessian)
        step = np.linalg.solve(H, g)
        beta = beta + step
        if np.max(np.abs(step)) < tol * (1 + np.max(np.abs(beta))):
            break
    p = common.sigmoid(X @ beta)
    H = X.T @ (X * (p * (1 - p))[:, None])
    se = np.sqrt(np.diag(np.linalg.inv(H)))
    ll = np.sum(y * np.log(p) + (1 - y) * np.log1p(-p))
    z = beta / se
    return dict(beta=beta, se=se, z=z, p=2 * stats.norm.sf(np.abs(z)), loglik=ll, deviance=-2 * ll,
                iterations=it, fitted=p)


def softmax(A):
    A = A - A.max(1, keepdims=True)
    E = np.exp(A)
    return E / E.sum(1, keepdims=True)


def softmax_fit(X, g, K, maxit=200, tol=1e-11):
    """Multinomial logistic regression in the softmax coding (4.13), fitted by Newton's method with
    the K-th coefficient vector fixed at zero (the softmax model is unchanged by adding the same
    vector to every class, so one class must be pinned down). Returns a (p, K) coefficient matrix."""
    n, p = X.shape
    B = np.zeros((p, K - 1))
    Y = np.eye(K)[g]
    for _ in range(maxit):
        P = softmax(np.c_[X @ B, np.zeros(n)])
        grad = (X.T @ (Y - P)[:, :K - 1]).T.ravel()
        H = np.zeros(((K - 1) * p, (K - 1) * p))
        for a in range(K - 1):
            for b in range(K - 1):
                w = P[:, a] * ((a == b) - P[:, b])
                H[a * p:(a + 1) * p, b * p:(b + 1) * p] = X.T @ (X * w[:, None])
        step = np.linalg.solve(H, grad).reshape(K - 1, p).T
        B = B + step
        if np.max(np.abs(step)) < tol:
            break
    return np.c_[B, np.zeros(p)]


# =========================================================================== generative classifiers
def lda_fit(X, g, K=None, priors=None):
    """LDA: class means, pooled covariance with divisor n - K ((4.20) for p >= 1) and priors n_k / n (4.21)."""
    X = np.atleast_2d(np.asarray(X, float))
    if X.shape[0] != len(g):
        X = X.T
    K = K or int(np.max(g)) + 1
    n, p = X.shape
    mu = np.array([X[g == k].mean(0) for k in range(K)])
    R = X - mu[g]
    S = R.T @ R / (n - K)
    pi = np.array([np.mean(g == k) for k in range(K)]) if priors is None else np.asarray(priors, float)
    return dict(mu=mu, Sigma=S, pi=pi, K=K)


def lda_scores(fit, X):
    """Discriminant functions delta_k(x) = x^T S^-1 mu_k - mu_k^T S^-1 mu_k / 2 + log pi_k  (4.24)."""
    X = np.atleast_2d(np.asarray(X, float))
    Si_mu = np.linalg.solve(fit["Sigma"], fit["mu"].T)          # p x K
    return X @ Si_mu - 0.5 * np.sum(fit["mu"].T * Si_mu, 0) + np.log(fit["pi"])


def lda_proba(fit, X):
    return softmax(lda_scores(fit, X))


def lda_direction(fit):
    """The coefficients of R's lda() (MASS) for two classes: the discriminant direction
    S^-1 (mu_2 - mu_1), scaled so that its within-class variance is 1."""
    a = np.linalg.solve(fit["Sigma"], fit["mu"][1] - fit["mu"][0])
    return a / np.sqrt(a @ fit["Sigma"] @ a)


def qda_fit(X, g, K=None):
    """QDA: one mean and one covariance (divisor n_k - 1) per class, priors n_k / n."""
    X = np.asarray(X, float)
    K = K or int(np.max(g)) + 1
    mu = np.array([X[g == k].mean(0) for k in range(K)])
    S = np.array([np.cov(X[g == k].T, ddof=1).reshape(X.shape[1], X.shape[1]) for k in range(K)])
    pi = np.array([np.mean(g == k) for k in range(K)])
    return dict(mu=mu, Sigmas=S, pi=pi, K=K)


def qda_scores(fit, X):
    """delta_k(x) = -(x - mu_k)^T S_k^-1 (x - mu_k) / 2 - log|S_k| / 2 + log pi_k  (4.28)."""
    X = np.atleast_2d(np.asarray(X, float))
    out = np.empty((X.shape[0], fit["K"]))
    for k in range(fit["K"]):
        D = X - fit["mu"][k]
        Si = np.linalg.inv(fit["Sigmas"][k])
        out[:, k] = (-0.5 * np.einsum("ij,jk,ik->i", D, Si, D) - 0.5 * np.linalg.slogdet(fit["Sigmas"][k])[1]
                     + np.log(fit["pi"][k]))
    return out


def qda_proba(fit, X):
    return softmax(qda_scores(fit, X))


def nb_fit(X, g, K=None, categorical=(), ddof=1):
    """Naive Bayes (4.29)-(4.30): each quantitative column gets a normal density per class (mean and
    standard deviation with divisor n_k - ddof); each categorical column gets the class's relative
    frequencies of its levels. Returns the fitted margins and the priors."""
    X = np.asarray(X, float)
    K = K or int(np.max(g)) + 1
    marg = []
    for j in range(X.shape[1]):
        if j in categorical:
            levels = np.unique(X[:, j])
            marg.append(("cat", levels, np.array([[np.mean(X[g == k, j] == v) for v in levels] for k in range(K)])))
        else:
            marg.append(("norm", np.array([X[g == k, j].mean() for k in range(K)]),
                         np.array([X[g == k, j].std(ddof=ddof) for k in range(K)])))
    return dict(margins=marg, pi=np.array([np.mean(g == k) for k in range(K)]), K=K)


def nb_scores(fit, X):
    X = np.atleast_2d(np.asarray(X, float))
    out = np.tile(np.log(fit["pi"]), (X.shape[0], 1))
    for j, m in enumerate(fit["margins"]):
        if m[0] == "cat":
            idx = np.searchsorted(m[1], X[:, j])
            out += np.log(m[2][:, idx].T)
        else:
            out += stats.norm.logpdf(X[:, j][:, None], m[1][None, :], m[2][None, :])
    return out


def nb_proba(fit, X):
    return softmax(nb_scores(fit, X))


def knn_predict(Xtr, gtr, Xte, k, K=None):
    """K-nearest-neighbour vote (Euclidean distance). Neighbours are taken in order of distance, with
    exact distance ties resolved by the training index (a stable sort, as scikit-learn does); a tied
    vote goes to the lowest class label."""
    K = K or int(np.max(gtr)) + 1
    d2 = (np.sum(Xte ** 2, 1)[:, None] - 2 * Xte @ Xtr.T + np.sum(Xtr ** 2, 1)[None, :])
    nn = np.argsort(d2, axis=1, kind="stable")[:, :k]
    votes = np.zeros((Xte.shape[0], K))
    for c in range(K):
        votes[:, c] = np.sum(gtr[nn] == c, 1)
    return votes.argmax(1)


def knn_loo_choice(X, g, ks=range(1, 16)):
    """Choose K by leave-one-out cross-validation (each point is classified by its K nearest
    other training points); the smallest K among the best is returned."""
    d2 = np.sum((X[:, None, :] - X[None, :, :]) ** 2, 2)
    np.fill_diagonal(d2, np.inf)
    order = np.argsort(d2, axis=1, kind="stable")
    best, bestk = np.inf, 1
    for k in ks:
        if k > len(g) - 1:
            break
        votes = np.stack([np.sum(g[order[:, :k]] == c, 1) for c in (0, 1)], 1)
        err = np.mean(votes.argmax(1) != g)
        if err < best - 1e-12:
            best, bestk = err, k
    return bestk


# =========================================================================== assessment
def confusion(pred, truth):
    """2 x 2 table [[TN, FN], [FP, TP]]: rows are predicted (0, 1), columns are true (0, 1), as in Table 4.4."""
    pred = np.asarray(pred).astype(int)
    truth = np.asarray(truth).astype(int)
    return np.array([[np.sum((pred == a) & (truth == b)) for b in (0, 1)] for a in (0, 1)])


def rates(C):
    TN, FN, FP, TP = C[0, 0], C[0, 1], C[1, 0], C[1, 1]
    N, P = TN + FP, FN + TP
    return dict(error=(FN + FP) / (N + P), sensitivity=TP / P, specificity=TN / N, fpr=FP / N,
                ppv=TP / (TP + FP) if TP + FP else np.nan, npv=TN / (TN + FN) if TN + FN else np.nan)


def roc(score, y):
    """The ROC curve: (false positive rate, true positive rate) for every threshold, from the
    highest score down; tied scores enter together."""
    score = np.asarray(score, float)
    y = np.asarray(y).astype(int)
    order = np.argsort(-score, kind="stable")
    s, t = score[order], y[order]
    last = np.r_[np.where(np.diff(s) != 0)[0], len(s) - 1]
    tp = np.cumsum(t)[last]
    fp = np.cumsum(1 - t)[last]
    return np.r_[0, fp / fp[-1]], np.r_[0, tp / tp[-1]]


def auc(score, y):
    """Area under the ROC curve = Pr(score of a random positive > score of a random negative)
    + Pr(tie) / 2 (the Mann-Whitney form), computed from ranks."""
    y = np.asarray(y).astype(int)
    r = stats.rankdata(score)
    n1, n0 = y.sum(), len(y) - y.sum()
    return (r[y == 1].sum() - n1 * (n1 + 1) / 2) / (n1 * n0)


# =========================================================================== GLMs
def ols_fit(X, y):
    X = np.asarray(X, float)
    beta, *_ = np.linalg.lstsq(X, y, rcond=None)
    r = y - X @ beta
    n, p = X.shape
    s2 = r @ r / (n - p)
    se = np.sqrt(s2 * np.diag(np.linalg.inv(X.T @ X)))
    return dict(beta=beta, se=se, t=beta / se, fitted=X @ beta, sigma2=s2)


def poisson_fit(X, y, tol=1e-12, maxit=100):
    """Poisson regression with log link: maximise sum_i [y_i eta_i - exp(eta_i)] ((4.38) without
    the constant log y_i!) by Newton-Raphson. Starts from the log of the mean count."""
    X = np.asarray(X, float)
    y = np.asarray(y, float)
    beta = np.zeros(X.shape[1])
    beta[0] = np.log(y.mean())
    for it in range(1, maxit + 1):
        mu = np.exp(X @ beta)
        step = np.linalg.solve(X.T @ (X * mu[:, None]), X.T @ (y - mu))
        beta = beta + step
        if np.max(np.abs(step)) < tol * (1 + np.max(np.abs(beta))):
            break
    mu = np.exp(X @ beta)
    se = np.sqrt(np.diag(np.linalg.inv(X.T @ (X * mu[:, None]))))
    with np.errstate(divide="ignore", invalid="ignore"):
        dev = 2 * np.sum(np.where(y > 0, y * np.log(y / mu), 0) - (y - mu))
    pearson = np.sum((y - mu) ** 2 / mu) / (len(y) - X.shape[1])
    return dict(beta=beta, se=se, z=beta / se, fitted=mu, deviance=dev, dispersion=pearson, iterations=it)


# =========================================================================== shared fits
_CACHE = {}


def default_fits():
    """The fits used in Sections 4.3-4.4 on the Default data (cached)."""
    if "default" in _CACHE:
        return _CACHE["default"]
    D = load_default()
    y = D["y"]
    one = np.ones(len(y))
    L1 = logistic_fit(np.c_[one, D["balance"]], y)
    L2 = logistic_fit(np.c_[one, D["student"]], y)
    L3 = logistic_fit(np.c_[one, D["balance"], D["income"] / 1000, D["student"]], y)
    Xls = np.c_[D["balance"], D["student"]]
    lda = lda_fit(Xls, y)
    p_lda = lda_proba(lda, Xls)[:, 1]
    Lbs = logistic_fit(np.c_[one, Xls], y)
    Xnb = np.c_[D["balance"], D["income"], D["student"]]
    nb = nb_fit(Xnb, y, categorical=(2,))
    p_nb = nb_proba(nb, Xnb)[:, 1]
    out = dict(D=D, y=y, L1=L1, L2=L2, L3=L3, lda=lda, p_lda=p_lda, Lbs=Lbs, nb=nb, p_nb=p_nb)
    _CACHE["default"] = out
    return out


def bikeshare_fits():
    if "bike" in _CACHE:
        return _CACHE["bike"]
    B = load_bikeshare()
    X, names = bikeshare_design(B, "sum")
    y = B.bikers.values.astype(float)
    out = dict(B=B, X=X, names=names, y=y, lm=ols_fit(X, y), pois=poisson_fit(X, y))
    _CACHE["bike"] = out
    return out


# =========================================================================== figures 4.1-4.3
def fig_4_1(out_dir=None):
    """The Default data: balance against income, and boxplots of balance and income by default."""
    D = load_default()
    y = D["y"]
    rng = np.random.default_rng(1)
    keep = (y == 1) | (rng.random(len(y)) < 0.15)
    fig = plt.figure(figsize=(10.5, 3.9))
    gs = fig.add_gridspec(1, 4, width_ratios=[2, 0.06, 1, 1], wspace=0.35)
    ax = fig.add_subplot(gs[0])
    for v, col, mk in ((0, BL, "o"), (1, OR, "+")):
        s = keep & (y == v)
        ax.scatter(D["balance"][s], D["income"][s], s=9 if v == 0 else 18, marker=mk, facecolors="none" if v == 0 else col,
                   edgecolors=col if v == 0 else None, c=None if v == 0 else col, linewidths=0.7, alpha=0.85)
    ax.set_xlabel("Balance")
    ax.set_ylabel("Income")
    for k, var in enumerate(("balance", "income")):
        a = fig.add_subplot(gs[2 + k])
        bp = a.boxplot([D[var][y == 0], D[var][y == 1]], widths=0.6, patch_artist=True, flierprops=dict(markersize=2.5))
        for patch, col in zip(bp["boxes"], (BL, OR)):
            patch.set_facecolor(col)
            patch.set_alpha(0.75)
        for med in bp["medians"]:
            med.set_color("black")
        a.set_xticks([1, 2], ["No", "Yes"])
        a.set_xlabel("Default")
        a.set_ylabel(var.capitalize())
    savefig(fig, CH, "fig4-1", out_dir)


def fig_4_2(out_dir=None):
    """Probability of default against balance: linear regression (left) and logistic regression (right)."""
    F = default_fits()
    D, y = F["D"], F["y"]
    x = D["balance"]
    Xl = np.c_[np.ones(len(x)), x]
    b_lin = np.linalg.lstsq(Xl, y, rcond=None)[0]
    grid = np.linspace(0, 2700, 300)
    fig, axes = common.panel_grid(1, 2, size=(4.4, 3.6))
    for ax, kind in zip(axes[0], ("linear", "logistic")):
        ax.plot(x[y == 0], np.zeros(np.sum(y == 0)), "|", color=OR, ms=7, alpha=0.4, rasterized=True)
        ax.plot(x[y == 1], np.ones(np.sum(y == 1)), "|", color=OR, ms=7, alpha=0.4, rasterized=True)
        f = b_lin[0] + b_lin[1] * grid if kind == "linear" else common.sigmoid(F["L1"]["beta"][0] + F["L1"]["beta"][1] * grid)
        ax.plot(grid, f, color=BL, lw=2)
        ax.axhline(0, color=GREY, lw=0.6, ls=":")
        ax.axhline(1, color=GREY, lw=0.6, ls=":")
        ax.set_xlabel("Balance")
        ax.set_ylabel("Probability of Default")
        ax.set_ylim(-0.12, 1.08)
    savefig(fig, CH, "fig4-2", out_dir)


def fig_4_3(out_dir=None):
    """Confounding: default rate against balance for students and non-students, and balance by student status."""
    F = default_fits()
    D, y = F["D"], F["y"]
    fig = plt.figure(figsize=(9.2, 3.9))
    gs = fig.add_gridspec(1, 2, width_ratios=[1.6, 1], wspace=0.3)
    ax = fig.add_subplot(gs[0])
    grid = np.linspace(500, 2600, 300)
    for s, col in ((1, OR), (0, BL)):
        m = D["student"] == s
        fit = logistic_fit(np.c_[np.ones(m.sum()), D["balance"][m]], y[m])
        ax.plot(grid, common.sigmoid(fit["beta"][0] + fit["beta"][1] * grid), color=col, lw=2,
                label="students" if s else "non-students")
        bins = np.arange(0, 2800, 100)
        idx = np.digitize(D["balance"][m], bins)
        cnt = np.bincount(idx, minlength=len(bins) + 1)
        rate = np.bincount(idx, weights=y[m], minlength=len(bins) + 1) / np.maximum(cnt, 1)
        ok = cnt >= 20
        ax.plot((bins[np.minimum(np.arange(len(cnt)), len(bins) - 1)] - 50)[ok], rate[ok], "o", color=col, ms=3.2, alpha=0.6)
        ax.axhline(y[m].mean(), color=col, ls="--", lw=1.4)
    ax.set_xlim(500, 2600)
    ax.set_xlabel("Credit Card Balance")
    ax.set_ylabel("Default Rate")
    ax.legend(loc="upper left", frameon=False)
    a = fig.add_subplot(gs[1])
    bp = a.boxplot([D["balance"][D["student"] == 0], D["balance"][D["student"] == 1]], widths=0.6, patch_artist=True,
                   flierprops=dict(markersize=2.5))
    for patch, col in zip(bp["boxes"], (BL, OR)):
        patch.set_facecolor(col)
        patch.set_alpha(0.75)
    for med in bp["medians"]:
        med.set_color("black")
    a.set_xticks([1, 2], ["No", "Yes"])
    a.set_xlabel("Student Status")
    a.set_ylabel("Credit Card Balance")
    savefig(fig, CH, "fig4-3", out_dir)


# =========================================================================== figures 4.4-4.6
def lda_1d_example(seed=4, n=20, mu=(-1.25, 1.25), sigma=1.0):
    """Section 4.4.1: two N(mu_k, 1) classes, equal priors. Returns the samples, the LDA boundary
    (4.19 with estimates), and the exact Bayes and LDA test error rates."""
    rng = np.random.default_rng(seed)
    x1 = rng.normal(mu[0], sigma, n)
    x2 = rng.normal(mu[1], sigma, n)
    m1, m2 = x1.mean(), x2.mean()
    s2 = (np.sum((x1 - m1) ** 2) + np.sum((x2 - m2) ** 2)) / (2 * n - 2)
    # equal class sizes: the boundary is the midpoint of the sample means (pi_1 = pi_2)
    b = 0.5 * (m1 + m2) + s2 * np.log(1.0) / (m2 - m1)
    bayes = 0.5 * stats.norm.cdf((mu[0] - 0) / sigma) + 0.5 * stats.norm.sf((mu[1] - 0) / sigma)
    # error of the rule "class 2 if x > b": class 1 errs above b, class 2 below b
    lda_err = 0.5 * stats.norm.sf((b - mu[0]) / sigma) + 0.5 * stats.norm.cdf((b - mu[1]) / sigma)
    return dict(x1=x1, x2=x2, boundary=b, bayes=bayes, lda=lda_err, s2=s2)


def fig_4_4(out_dir=None):
    """Two one-dimensional normal classes: densities with the Bayes boundary, and a sample of 20 per class with the LDA boundary."""
    E = lda_1d_example()
    x = np.linspace(-4, 4, 400)
    fig, axes = common.panel_grid(1, 2, size=(4.3, 3.3))
    ax = axes[0, 0]
    ax.plot(x, stats.norm.pdf(x, -1.25), color=GREEN, lw=2)
    ax.plot(x, stats.norm.pdf(x, 1.25), color=PURPLE, lw=2)
    ax.axvline(0, color="black", ls="--", lw=1.6)
    ax.set_yticks([])
    ax = axes[0, 1]
    bins = np.arange(-4, 4.01, 0.5)
    ax.hist(E["x1"], bins=bins, color=GREEN, alpha=0.55, edgecolor="white")
    ax.hist(E["x2"], bins=bins, color=PURPLE, alpha=0.55, edgecolor="white")
    ax.axvline(0, color="black", ls="--", lw=1.6)
    ax.axvline(E["boundary"], color="black", lw=1.8)
    savefig(fig, CH, "fig4-4", out_dir)


def fig_4_5(out_dir=None):
    """Two bivariate Gaussian densities: uncorrelated with equal variances, and correlation 0.7."""
    from mpl_toolkits.mplot3d import Axes3D  # noqa: F401
    g = np.linspace(-2.6, 2.6, 61)
    X1, X2 = np.meshgrid(g, g)
    pts = np.c_[X1.ravel(), X2.ravel()]
    fig = plt.figure(figsize=(9.4, 4.2))
    for k, rho in enumerate((0.0, 0.7)):
        ax = fig.add_subplot(1, 2, k + 1, projection="3d")
        S = np.array([[1, rho], [rho, 1.0]])
        Z = np.exp(common.gauss_logpdf(pts, np.zeros(2), S)).reshape(X1.shape)
        ax.plot_surface(X1, X2, Z, cmap="viridis", rstride=2, cstride=2, linewidth=0.15, edgecolor="k", alpha=0.95, rasterized=True)
        ax.set_xlabel("$x_1$")
        ax.set_ylabel("$x_2$")
        ax.set_zticks([])
        ax.view_init(elev=30, azim=-15)
        ax.set_box_aspect(None, zoom=0.86)
    fig.subplots_adjust(wspace=0.02)
    savefig(fig, CH, "fig4-5", out_dir)


SIX_MU = np.array([[0.0, 0.5], [-1.5, -1.6], [1.6, -1.2]])
SIX_S = 0.62 * np.array([[1.0, 0.5], [0.5, 1.0]])


def lda_3class_example(seed=4, n=20, ntest=200000):
    """Section 4.4.2 example: three Gaussian classes with a common covariance; returns the training
    sample, the LDA fit and the Bayes and LDA test error rates estimated on a large test sample."""
    rng = np.random.default_rng(seed)
    L = np.linalg.cholesky(SIX_S)
    X = np.vstack([SIX_MU[k] + rng.standard_normal((n, 2)) @ L.T for k in range(3)])
    g = np.repeat(np.arange(3), n)
    fit = lda_fit(X, g)
    rt = np.random.default_rng(seed + 100)
    Xt = np.vstack([SIX_MU[k] + rt.standard_normal((ntest, 2)) @ L.T for k in range(3)])
    gt = np.repeat(np.arange(3), ntest)
    true = dict(mu=SIX_MU, Sigma=SIX_S, pi=np.ones(3) / 3, K=3)
    bayes = np.mean(lda_scores(true, Xt).argmax(1) != gt)
    lda = np.mean(lda_scores(fit, Xt).argmax(1) != gt)
    return dict(X=X, g=g, fit=fit, true=true, bayes=bayes, lda=lda)


def _boundaries(ax, score_fn, xlim, ylim, color, ls, lw=1.6, n=400):
    gx = np.linspace(*xlim, n)
    gy = np.linspace(*ylim, n)
    GX, GY = np.meshgrid(gx, gy)
    S = score_fn(np.c_[GX.ravel(), GY.ravel()])
    lab = S.argmax(1).reshape(GX.shape)
    K = S.shape[1]
    for a in range(K):
        for b in range(a + 1, K):
            D = (S[:, a] - S[:, b]).reshape(GX.shape)
            # draw the a-b boundary only where a and b are the two best classes
            mask = (lab == a) | (lab == b)
            D = np.where(mask, D, np.nan)
            ax.contour(GX, GY, D, levels=[0], colors=color, linestyles=ls, linewidths=lw)
    return lab


def fig_4_6(out_dir=None):
    """Three Gaussian classes with a common covariance: 95% ellipses and Bayes boundaries; a sample of 20 per class with LDA boundaries."""
    E = lda_3class_example()
    cols = (GREEN, ORANGE, BLUE)
    xlim, ylim = (-4.2, 4.2), (-4.2, 3.2)
    fig, axes = common.panel_grid(1, 2, size=(4.3, 3.9))
    r95 = np.sqrt(stats.chi2.ppf(0.95, 2))
    for j, ax in enumerate(axes[0]):
        if j == 0:
            for k in range(3):
                ex, ey = common.ellipse_xy(SIX_MU[k], SIX_S, r95)
                ax.plot(ex, ey, color=cols[k], lw=2)
        else:
            for k in range(3):
                m = E["g"] == k
                ax.scatter(E["X"][m, 0], E["X"][m, 1], color=cols[k], s=16, edgecolor="none")
            _boundaries(ax, lambda P: lda_scores(E["fit"], P), xlim, ylim, "black", "solid")
        _boundaries(ax, lambda P: lda_scores(E["true"], P), xlim, ylim, "black", "dashed", lw=1.2)
        ax.set_xlim(xlim)
        ax.set_ylim(ylim)
        ax.set_xlabel("$X_1$")
        ax.set_ylabel("$X_2$")
    savefig(fig, CH, "fig4-6", out_dir)


# =========================================================================== figures 4.7-4.8
def threshold_curves(p, y, thresholds):
    out = {"overall": [], "defaulters": [], "nondefaulters": []}
    for t in thresholds:
        r = rates(confusion(p > t, y))
        out["overall"].append(r["error"])
        out["defaulters"].append(1 - r["sensitivity"])
        out["nondefaulters"].append(r["fpr"])
    return {k: np.array(v) for k, v in out.items()}


def fig_4_7(out_dir=None):
    """Error rates of LDA on the Default data as the threshold on the posterior probability of default varies."""
    F = default_fits()
    t = np.linspace(0, 0.5, 201)
    C = threshold_curves(F["p_lda"], F["y"], t)
    fig, ax = plt.subplots(figsize=(5.0, 3.8))
    ax.plot(t, C["overall"], color="black", lw=2, label="overall error")
    ax.plot(t, C["defaulters"], color=BLUE, ls="--", lw=2, label="defaulters missed")
    ax.plot(t, C["nondefaulters"], color=ORANGE, ls=":", lw=2.4, label="non-defaulters flagged")
    ax.set_xlabel("Threshold")
    ax.set_ylabel("Error Rate")
    ax.legend(frameon=False)
    savefig(fig, CH, "fig4-7", out_dir)


def fig_4_8(out_dir=None):
    """ROC curve of LDA (balance and student) on the Default training data."""
    F = default_fits()
    fpr, tpr = roc(F["p_lda"], F["y"])
    fig, ax = plt.subplots(figsize=(4.2, 4.2))
    ax.plot(fpr, tpr, color=BLUE, lw=2)
    ax.plot([0, 1], [0, 1], color=GREY, ls=":", lw=1.4)
    ax.set_xlabel("False positive rate")
    ax.set_ylabel("True positive rate")
    ax.set_title("ROC Curve")
    ax.set_aspect("equal")
    ax.text(0.55, 0.12, "AUC = %.3f" % auc(F["p_lda"], F["y"]), transform=ax.transAxes)
    savefig(fig, CH, "fig4-8", out_dir)


# =========================================================================== figures 4.9-4.10
def lda_qda_example(rho2, seed=9, n=50):
    """Figure 4.9: two classes, means (-1, -0.4) and (1, 0.4), unit variances; class 1 (orange) has
    correlation 0.7, class 2 (blue) correlation rho2."""
    rng = np.random.default_rng(seed)
    mus = np.array([[-1.0, -0.4], [1.0, 0.4]])
    Ss = np.array([[[1, 0.7], [0.7, 1.0]], [[1, rho2], [rho2, 1.0]]])
    X = np.vstack([rng.multivariate_normal(mus[k], Ss[k], n) for k in range(2)])
    g = np.repeat([0, 1], n)
    true = dict(mu=mus, Sigmas=Ss, pi=np.array([0.5, 0.5]), K=2)
    return dict(X=X, g=g, lda=lda_fit(X, g), qda=qda_fit(X, g), true=true)


def fig_4_9(out_dir=None):
    """Bayes (purple dashed), LDA (black dotted) and QDA (green solid) boundaries, equal and unequal covariances."""
    xlim, ylim = (-4, 3), (-4, 3)
    fig, axes = common.panel_grid(1, 2, size=(4.1, 4.0))
    for ax, rho2 in zip(axes[0], (0.7, -0.7)):
        E = lda_qda_example(rho2)
        gx, gy = np.meshgrid(np.linspace(*xlim, 90), np.linspace(*ylim, 90))
        lab = qda_scores(E["qda"], np.c_[gx.ravel(), gy.ravel()]).argmax(1)
        ax.scatter(gx.ravel(), gy.ravel(), s=1.6, c=np.where(lab == 0, OR, BL), alpha=0.45, linewidths=0, rasterized=True)
        for k, col in ((0, OR), (1, BL)):
            m = E["g"] == k
            ax.scatter(E["X"][m, 0], E["X"][m, 1], color=col, s=14, edgecolor="none")
        _boundaries(ax, lambda P: qda_scores(E["true"], P), xlim, ylim, PURPLE, "dashed", lw=2)
        _boundaries(ax, lambda P: lda_scores(E["lda"], P), xlim, ylim, "black", "dotted", lw=2)
        _boundaries(ax, lambda P: qda_scores(E["qda"], P), xlim, ylim, GREEN, "solid", lw=2)
        ax.set_xlim(xlim)
        ax.set_ylim(ylim)
        ax.set_xlabel("$X_1$")
        ax.set_ylabel("$X_2$")
    savefig(fig, CH, "fig4-9", out_dir)


def nb_toy(seed=10, n=100):
    """Figure 4.10 toy example: two quantitative predictors and one with three levels, per class."""
    rng = np.random.default_rng(seed)
    c1 = dict(x1=rng.normal(0, 1, n),
              x2=np.where(rng.random(n) < 0.8, rng.normal(1, 0.6, n), rng.normal(2.5, 0.5, n)),
              x3=rng.choice([1, 2, 3], n, p=[0.25, 0.45, 0.30]))
    c2 = dict(x1=rng.normal(1.8, 0.8, n), x2=rng.normal(-0.8, 0.9, n), x3=rng.choice([1, 2, 3], n, p=[0.6, 0.25, 0.15]))
    xs = (0.4, 1.5, 1)
    f = []
    for c in (c1, c2):
        k1 = stats.gaussian_kde(c["x1"], bw_method="silverman")
        k2 = stats.gaussian_kde(c["x2"], bw_method="silverman")
        f.append([k1(xs[0])[0], k2(xs[1])[0], np.mean(c["x3"] == xs[2])])
    f = np.array(f)
    num = 0.5 * f.prod(1)
    return dict(c=(c1, c2), f=f, post=num / num.sum(), xstar=xs)


def fig_4_10(out_dir=None):
    """Naive Bayes toy example: the estimated one-dimensional densities f_kj of three predictors in two classes."""
    T = nb_toy()
    fig, axes = common.panel_grid(2, 3, size=(2.9, 2.2))
    for k, c in enumerate(T["c"]):
        col = (OR, BL)[k]
        for j, key in enumerate(("x1", "x2")):
            ax = axes[k, j]
            ax.hist(c[key], bins=np.arange(-4, 4.5, 0.5), density=True, color=col, alpha=0.45, edgecolor="white")
            kde = stats.gaussian_kde(c[key], bw_method="silverman")
            g = np.linspace(-4, 4.5, 300)
            ax.plot(g, kde(g), color="black", lw=1.4)
            ax.set_title(r"$\hat f_{%d%d}$" % (k + 1, j + 1))
            ax.set_yticks([])
        ax = axes[k, 2]
        ax.bar([1, 2, 3], [np.mean(c["x3"] == v) for v in (1, 2, 3)], color=col, alpha=0.75)
        ax.set_xticks([1, 2, 3])
        ax.set_title(r"$\hat f_{%d3}$" % (k + 1))
        ax.set_yticks([])
    axes[0, 0].set_ylabel("class $k=1$")
    axes[1, 0].set_ylabel("class $k=2$")
    fig.tight_layout()
    savefig(fig, CH, "fig4-10", out_dir)


# =========================================================================== figures 4.11-4.12 (Section 4.5.2)
SCENARIOS = {
    1: "20 per class; independent N(0, 1) inputs, class means (0, 0) and (0.75, 0.75)",
    2: "as 1 with within-class correlation -0.5",
    3: "50 per class; bivariate t with 2 degrees of freedom, scale matrix with correlation -0.5, means as 1",
    4: "50 per class; normal, correlation 0.5 in class 1 and -0.5 in class 2, means (0, 0) and (0.5, 0.5)",
    5: "100 points in all; X ~ N(0, I_2), Pr(Y = 1 | x) = sigmoid(0.7 [3 x1 + 1.2 x2^2 - 1.2 + 1.5 sin(3 x2) + 1.5 sin(2 x1 x2)])",
    6: "6 per class; normal with diagonal covariances diag(1, 4) and diag(4, 1), means (0, 0) and (0.5, -0.5)",
}


def scenario_data(s, n, rng):
    """Draw n points per class (Scenario 5: 2n points in all) for scenario s of Section 4.5.2."""
    if s in (1, 2, 3):
        rho = 0.0 if s == 1 else -0.5
        S = np.array([[1, rho], [rho, 1.0]])
        L = np.linalg.cholesky(S)
        mus = np.array([[0, 0], [0.75, 0.75]])
        Z = rng.standard_normal((2 * n, 2)) @ L.T
        if s == 3:
            Z = Z / np.sqrt(rng.chisquare(2, 2 * n) / 2)[:, None]
        g = np.repeat([0, 1], n)
        return Z + mus[g], g
    if s == 4:
        g = np.repeat([0, 1], n)
        X = np.empty((2 * n, 2))
        for k, rho in ((0, 0.5), (1, -0.5)):
            X[g == k] = rng.multivariate_normal([0.5 * k, 0.5 * k], [[1, rho], [rho, 1]], n)
        return X, g
    if s == 5:
        X = rng.standard_normal((2 * n, 2))
        f = 0.7 * (3 * X[:, 0] + 1.2 * X[:, 1] ** 2 - 1.2 + 1.5 * np.sin(3 * X[:, 1]) + 1.5 * np.sin(2 * X[:, 0] * X[:, 1]))
        g = (rng.random(2 * n) < common.sigmoid(f)).astype(int)
        return X, g
    if s == 6:
        g = np.repeat([0, 1], n)
        sd = np.array([[1.0, 2.0], [2.0, 1.0]])
        mus = np.array([[0, 0], [0.5, -0.5]])
        return mus[g] + rng.standard_normal((2 * n, 2)) * sd[g], g
    raise ValueError(s)


SCEN_N = {1: 20, 2: 20, 3: 50, 4: 50, 5: 50, 6: 6}
METHODS = ["KNN-1", "KNN-CV", "LDA", "Logistic", "NBayes", "QDA"]


def scenario_errors(s, reps=100, ntest=5000):
    out = np.zeros((reps, len(METHODS)))
    rt = np.random.default_rng(999 + s)
    Xt, gt = scenario_data(s, ntest // 2, rt)
    for r in range(reps):
        rng = np.random.default_rng(1000 * s + r)
        while True:
            X, g = scenario_data(s, SCEN_N[s], rng)
            if g.min() != g.max() and min(np.sum(g == 0), np.sum(g == 1)) >= 3:
                break
        preds = []
        preds.append(knn_predict(X, g, Xt, 1, 2))
        preds.append(knn_predict(X, g, Xt, knn_loo_choice(X, g), 2))
        preds.append(lda_scores(lda_fit(X, g), Xt).argmax(1))
        H = np.c_[np.ones(len(g)), X]
        try:
            b = logistic_fit(H, g, maxit=60)["beta"]
        except np.linalg.LinAlgError:
            b = np.zeros(3)
        if not np.all(np.isfinite(b)):
            b = np.zeros(3)
        preds.append((np.c_[np.ones(len(gt)), Xt] @ b > 0).astype(int))
        preds.append(nb_scores(nb_fit(X, g), Xt).argmax(1))
        preds.append(qda_scores(qda_fit(X, g), Xt).argmax(1))
        out[r] = [np.mean(p != gt) for p in preds]
    return out


def scenario_results(use_cache=True):
    path = os.path.join(CACHE_DIR, "ch04_scenarios.npz")
    if use_cache and os.path.exists(path):
        z = np.load(path)
        return {s: z["s%d" % s] for s in range(1, 7)}
    with np.errstate(over="ignore", divide="ignore", invalid="ignore"):
        res = {s: scenario_errors(s) for s in range(1, 7)}
    os.makedirs(CACHE_DIR, exist_ok=True)
    np.savez(path, **{"s%d" % s: v for s, v in res.items()})
    return res


def _scenario_fig(scen, name, out_dir):
    R = scenario_results()
    cols = ["#7fbf7b", "#2ca25f", BL, "#5aa0d8", "#c49ed8", OR]
    fig, axes = common.panel_grid(1, 3, size=(3.4, 3.4))
    for ax, s in zip(axes[0], scen):
        bp = ax.boxplot(R[s], widths=0.6, patch_artist=True, flierprops=dict(markersize=3))
        for patch, c in zip(bp["boxes"], cols):
            patch.set_facecolor(c)
        for med in bp["medians"]:
            med.set_color("black")
        ax.set_xticks(range(1, 7), METHODS, rotation=55, ha="right", fontsize=8)
        ax.set_title("SCENARIO %d" % s)
    fig.tight_layout()
    savefig(fig, CH, name, out_dir)


def fig_4_11(out_dir=None):
    """Test error rates of six methods over 100 training sets: the three linear scenarios."""
    _scenario_fig((1, 2, 3), "fig4-11", out_dir)


def fig_4_12(out_dir=None):
    """Test error rates of six methods over 100 training sets: the three non-linear scenarios."""
    _scenario_fig((4, 5, 6), "fig4-12", out_dir)


# =========================================================================== figures 4.13-4.15
def _coef_profiles(beta, names, out_dir, name, ylim_m=None, ylim_h=None):
    cm = full_levels(beta, names, "mnth")
    ch = full_levels(beta, names, "hr")
    fig, axes = common.panel_grid(1, 2, size=(4.4, 3.5))
    ax = axes[0, 0]
    ax.plot(range(1, 13), cm, "o-", color=BLUE, ms=5)
    ax.set_xticks(range(1, 13), ["J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"])
    ax.set_xlabel("Month")
    ax.set_ylabel("Coefficient")
    ax = axes[0, 1]
    ax.plot(range(24), ch, "o-", color=BLUE, ms=4)
    ax.set_xticks(range(0, 24, 5))
    ax.set_xlabel("Hour")
    ax.set_ylabel("Coefficient")
    savefig(fig, CH, name, out_dir)


def fig_4_13(out_dir=None):
    """Least squares on the Bikeshare data: the month and hour coefficients (sum-to-zero coding)."""
    F = bikeshare_fits()
    _coef_profiles(F["lm"]["beta"], F["names"], out_dir, "fig4-13")


def bikers_by_hour_spline(B, log=False):
    """Smoothing spline of bikers (or log bikers) on hour. With 24 distinct hours, the spline fitted
    to all 8,645 points minimises sum_i (y_i - f(h_i))^2 + lam J(f), which equals
    sum_h n_h (ybar_h - f(h))^2 + lam J(f) up to a constant, so it is the weighted spline of the
    hourly means."""
    y = np.log(B.bikers.values) if log else B.bikers.values.astype(float)
    h = B.hr.values
    hs = np.arange(24)
    n = np.array([np.sum(h == v) for v in hs])
    m = np.array([y[h == v].mean() for v in hs])
    return make_smoothing_spline(hs.astype(float), m, w=n / n.mean())


def fig_4_14(out_dir=None):
    """Bikers against hour (jittered) with a smoothing spline, on the original and the log scale."""
    B = load_bikeshare()
    rng = np.random.default_rng(14)
    hj = B.hr.values + rng.uniform(-0.3, 0.3, len(B))
    fig, axes = common.panel_grid(1, 2, size=(4.4, 3.5))
    g = np.linspace(0, 23, 300)
    for ax, lg in zip(axes[0], (False, True)):
        y = np.log(B.bikers.values) if lg else B.bikers.values
        ax.scatter(hj, y, s=2, color=GREY, alpha=0.25, linewidths=0, rasterized=True)
        ax.plot(g, bikers_by_hour_spline(B, lg)(g), color=GREEN, lw=2.4)
        ax.set_xlabel("Hour")
        ax.set_ylabel("Log(Number of Bikers)" if lg else "Number of Bikers")
    savefig(fig, CH, "fig4-14", out_dir)


def fig_4_15(out_dir=None):
    """Poisson regression on the Bikeshare data: the month and hour coefficients (sum-to-zero coding)."""
    F = bikeshare_fits()
    _coef_profiles(F["pois"]["beta"], F["names"], out_dir, "fig4-15")


# =========================================================================== tables and demo data
def book_numbers():
    """The numbers quoted in Sections 4.3-4.6, recomputed."""
    F = default_fits()
    y = F["y"]
    b1, b2, b3 = F["L1"]["beta"], F["L2"]["beta"], F["L3"]["beta"]
    C5 = confusion(F["p_lda"] > 0.5, y)
    C2 = confusion(F["p_lda"] > 0.2, y)
    E1 = lda_1d_example()
    E3 = lda_3class_example()
    G = bikeshare_fits()
    B = G["B"]
    sel = lambda hrs, ms, w: B.bikers[(B.hr.isin(hrs)) & (B.mnth.isin(ms)) & (B.weathersit == w)]
    s1 = sel([1, 2, 3], ["Dec", "Jan", "Feb"], "light rain/snow")
    s2 = sel([7, 8, 9], ["April", "May", "June"], "clear")
    D = F["D"]
    Xl = np.c_[np.ones(len(y)), D["balance"]]
    lin = np.linalg.lstsq(Xl, y, rcond=None)[0]
    return {
        "p(1000)": common.sigmoid(b1[0] + b1[1] * 1000), "p(2000)": common.sigmoid(b1[0] + b1[1] * 2000),
        "Pr(default|student)": common.sigmoid(b2[0] + b2[1]), "Pr(default|non-student)": common.sigmoid(b2[0]),
        "student 1500/40k": common.sigmoid(b3 @ [1, 1500, 40, 1]), "non-student 1500/40k": common.sigmoid(b3 @ [1, 1500, 40, 0]),
        "mean fitted linear": np.mean(Xl @ lin), "mean fitted logistic": np.mean(F["L1"]["fitted"]),
        "LDA train error": rates(C5)["error"], "LDA sensitivity": rates(C5)["sensitivity"],
        "LDA specificity": rates(C5)["specificity"], "LDA error at 0.2": rates(C2)["error"],
        "AUC LDA": auc(F["p_lda"], y), "AUC logistic (balance, student)": auc(F["Lbs"]["fitted"], y),
        "Bayes error 1-D": E1["bayes"], "LDA error 1-D": E1["lda"],
        "Bayes error 3-class": E3["bayes"], "LDA error 3-class": E3["lda"],
        "NB posterior toy (book's densities)": 0.368 * 0.484 * 0.226 / (0.368 * 0.484 * 0.226 + 0.030 * 0.130 * 0.616),
        "NB posterior toy (this simulation)": nb_toy()["post"][0],
        "negative linear fits": np.mean(G["lm"]["fitted"] < 0),
        "bikers 1-3 AM winter rain mean": s1.mean(), "sd": s1.std(ddof=1),
        "bikers 7-9 AM spring clear mean": s2.mean(), "sd ": s2.std(ddof=1),
        "exp(cloudy)": np.exp(G["pois"]["beta"][G["names"].index("weathersit[cloudy/misty]")]),
        "exp(rain - cloudy)": np.exp(G["pois"]["beta"][G["names"].index("weathersit[light rain/snow]")]
                                     - G["pois"]["beta"][G["names"].index("weathersit[cloudy/misty]")]),
        "Poisson Pearson dispersion": G["pois"]["dispersion"],
    }


def print_tables():
    F = default_fits()
    y = F["y"]

    def table(fit, names, title):
        print(title)
        for n, b, s, z, p in zip(names, fit["beta"], fit["se"], fit["z"], fit["p"]):
            print("  %-14s %10.4f %9.4f %8.2f %9.4f" % (n, b, s, z, p))
    table(F["L1"], ["Intercept", "balance"], "Table 4.1")
    table(F["L2"], ["Intercept", "student[Yes]"], "Table 4.2")
    table(F["L3"], ["Intercept", "balance", "income", "student[Yes]"], "Table 4.3")
    for t, name in ((0.5, "Table 4.4 (LDA, threshold 0.5)"), (0.2, "Table 4.5 (LDA, threshold 0.2)")):
        print(name, confusion(F["p_lda"] > t, y).tolist())
    for t, name in ((0.5, "Table 4.8 (naive Bayes, 0.5)"), (0.2, "Table 4.9 (naive Bayes, 0.2)")):
        print(name, confusion(F["p_nb"] > t, y).tolist())
    G = bikeshare_fits()
    for key, title, lab in (("lm", "Table 4.10 (least squares)", "t"), ("pois", "Table 4.11 (Poisson)", "z")):
        f = G[key]
        print(title)
        for n in ["intercept", "workingday", "temp"] + ["weathersit[%s]" % w for w in WEATHER[1:]]:
            i = G["names"].index(n)
            print("  %-28s %9.3f %8.3f %8.2f" % (n, f["beta"][i], f["se"][i], f[lab][i]))
    for k, v in book_numbers().items():
        print("  %-40s %.4f" % (k, v))
    R = scenario_results()
    print("Section 4.5.2 mean test errors (KNN-1, KNN-CV, LDA, Logistic, NBayes, QDA):")
    for s in range(1, 7):
        print("  scenario %d: %s" % (s, " ".join("%.3f" % v for v in R[s].mean(0))))


def export_demo_data():
    """JSON for the three demonstrations (all under 1 MB)."""
    os.makedirs(DATA_OUT, exist_ok=True)
    F = default_fits()
    D = F["D"]
    J = dict(note="Default data (James et al. 2021): balance (dollars), income (thousands of dollars), student (1 = Yes), default (1 = Yes).",
             balance=[round(v, 2) for v in D["balance"]], income=[round(v / 1000, 3) for v in D["income"]],
             student=D["student"].tolist(), y=F["y"].tolist())
    with open(os.path.join(DATA_OUT, "ch04-default.json"), "w") as f:
        json.dump(J, f, separators=(",", ":"))
    G = bikeshare_fits()
    B = G["B"]
    J = dict(note="Bikeshare data (James et al. 2021): month 0-11, hour 0-23, workingday, temp (normalised), weather 0-3, bikers.",
             months=MONTHS, weather=WEATHER, mnth=[MONTHS.index(m) for m in B.mnth], hr=B.hr.astype(int).tolist(),
             workingday=B.workingday.astype(int).tolist(), temp=[round(v, 2) for v in B.temp],
             weathersit=[WEATHER.index(w) for w in B.weathersit], bikers=B.bikers.astype(int).tolist())
    with open(os.path.join(DATA_OUT, "ch04-bikeshare.json"), "w") as f:
        json.dump(J, f, separators=(",", ":"))
    for name in ("ch04-default.json", "ch04-bikeshare.json"):
        print("wrote ../data/%s (%.0f kB)" % (name, os.path.getsize(os.path.join(DATA_OUT, name)) / 1024))


def tables(out_dir=None):
    """Print Tables 4.1-4.11 and the numbers quoted in the text, and export the demonstration data (no figure)."""
    print_tables()
    export_demo_data()


FIGURES = {"4.1": fig_4_1, "4.2": fig_4_2, "4.3": fig_4_3, "4.4": fig_4_4, "4.5": fig_4_5, "4.6": fig_4_6,
           "4.7": fig_4_7, "4.8": fig_4_8, "4.9": fig_4_9, "4.10": fig_4_10, "4.11": fig_4_11, "4.12": fig_4_12,
           "4.13": fig_4_13, "4.14": fig_4_14, "4.15": fig_4_15, "tables": tables}

if __name__ == "__main__":
    common.run(FIGURES, CH)
