"""James et al. (2021, 2023), Chapter 3: Linear Regression.

Reproduces the figures and tables of Chapter 3 of G. James, D. Witten, T. Hastie and R. Tibshirani,
An Introduction to Statistical Learning with Applications in R, 2nd edition (Springer, 2021), and of
its Python edition (James, Witten, Hastie, Tibshirani and Taylor, 2023), which share the chapter's
text and figures. Everything is written from scratch with numpy, scipy and matplotlib (statsmodels
only for the lowess smoother of the residual plots). The functions are meant to be read next to the
lecture notes (../ch03-linear-regression.html):

* ols: least squares through a QR decomposition, with standard errors (3.8), t-statistics (3.14),
  RSE (3.15)/(3.25), R^2 (3.17), the overall F-statistic (3.23), leverages (3.37), studentized
  residuals and Cook's distances; simple_ls gives the closed forms (3.4) and (3.8).
* f_nested: the partial F-statistic (3.24) for nested models; intervals: confidence and
  prediction intervals for f(x0) and Y at x0; vif: variance inflation factors.
* model_matrix: a small model-matrix builder (numeric columns, treatment-coded factors with the
  first level as baseline, products for interactions, raw powers).
* knn_reg: K-nearest-neighbour regression (Section 3.5).
* tables(): prints Tables 3.1-3.11 and the numbers quoted in the text, and writes the data files
  of the three demonstrations (../data/ch03-*.json).

Data: Advertising, Credit, Auto (the 392 cars with known horsepower) and Boston from the book's
CSV files (common.isl_data). Auto's observation labels in Figure 3.9 are the row numbers of the
original 397-row file, as in R's ISLR2 package, so they match the book (323, 330, 334, ...).

Figures reproduced: all twenty, 3.1-3.20 (Figure 3.2 as one figure with both panels).
Figures computed from data: 3.1, 3.2, 3.5-3.9, 3.14, 3.15. Simulated (the book does not give its
settings; ours are chosen to match the printed numbers and pictures):
* Figure 3.3: 100 X ~ U(-2, 2), Y = 2 + 3X + N(0, 2^2) noise, ten extra data sets (numpy seed 3).
* Figure 3.4: 30 points, X1, X2 ~ U(0, 1), Y = 1 + 2 X1 + 3 X2 + N(0, 0.5^2) (seed 4).
* Figure 3.10: x ~ U(0, 1), Y = 1 + 2x + e with AR(1) errors e_t = rho e_{t-1} + N(0, 1 - rho^2),
  n = 100, rho = 0, 0.5, 0.9 (seed 10).
* Figure 3.11: n = 1000, x ~ U(0, 1), Y = (10 + 20x) exp(N(0, 0.15^2)) (seed 11).
* Figures 3.12-3.13: 40 points x ~ N(0.3, 1.1^2), Y = 1 + 2x + N(0, 0.75^2) sorted by x
  (seed 151), observation 20 moved up by 4.6 (an outlier with ordinary x); Figure 3.13 adds
  observation 41 at (4, 15); its centre panel has 100 correlated pairs (correlation 0.95, seed 13)
  and one red point at (0.7, -1).
* Figure 3.16: 64 inputs on an 8 x 8 grid of [0, 1]^2, Y = 1 + 2 X1 + 3 X2 + N(0, 0.5^2) (seed 16).
* Figures 3.17-3.18: 50 points x ~ U(-1, 1), f(x) = 2.2 + 1.7x, noise sd 0.3 (seed 19); test
  MSE on 10,000 test points; K = 1, ..., 9.
* Figure 3.19: f(x) = 2.15 + 1.55 tanh(1.6x) (slightly non-linear) and
  f(x) = 2.2 + 1.4 sin(2.2x) (strongly non-linear), noise sd 0.22, 50 training points (seed 19).
* Figure 3.20: the strongly non-linear f of Figure 3.19 in the first input plus p - 1 noise inputs,
  all U(-1, 1), 50 training points, 10,000 test points (seed 20).

usage: python3 ch03_linear_regression.py [--fig 3.1 3.9 tables] [--list]
"""
import json
import os

import numpy as np
from scipy import stats

import common
from common import BLUE, GREEN, GREY, ORANGE, RED, savefig

plt = common.plt
CH = 3
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
LIGHTBLUE = "#7fb2e5"


# =========================================================================== data
def advertising():
    """The Advertising data: 200 markets, budgets (thousands of dollars) and sales (thousands of units)."""
    return common.isl_data("Advertising", index_col=0)


def credit():
    return common.isl_data("Credit")


def auto():
    """The 392 cars of the Auto data with known horsepower; the index keeps the original row numbers
    (1-based, as R's ISLR2 package does), which are the labels of Figure 3.9."""
    df = common.isl_data("Auto", na_values="?")
    df.index = np.arange(1, len(df) + 1)
    return df.dropna()


def boston():
    return common.isl_data("Boston", index_col=0)


def carseats():
    return common.isl_data("Carseats")


# =========================================================================== least squares
def model_matrix(df, terms, intercept=True):
    """Model matrix for a list of terms.

    A term is a column name (numeric: used as is; text: treatment dummies, first level in sorted
    order is the baseline), a tuple of names (an interaction: products of the columns of each
    name), or ("pow", name, d) (the raw power name**d). Returns (X, column names)."""
    cols, names = [], []
    if intercept:
        cols.append(np.ones(len(df)))
        names.append("Intercept")

    def enc(name):
        v = df[name]
        if v.dtype == object or str(v.dtype) == "category":
            levels = sorted(v.unique())
            return [(v == lev).to_numpy(float) for lev in levels[1:]], ["%s[%s]" % (name, lev) for lev in levels[1:]]
        return [v.to_numpy(float)], [name]

    for t in terms:
        if isinstance(t, tuple) and t[0] == "pow":
            cols.append(df[t[1]].to_numpy(float) ** t[2])
            names.append("%s^%d" % (t[1], t[2]))
        elif isinstance(t, tuple):
            parts = [enc(n) for n in t]
            prod_c, prod_n = parts[0]
            for c2, n2 in parts[1:]:
                prod_c = [a * b for a in prod_c for b in c2]
                prod_n = [a + ":" + b for a in prod_n for b in n2]
            cols += prod_c
            names += prod_n
        else:
            c, n = enc(t)
            cols += c
            names += n
    return np.column_stack(cols), names


def ols(X, y, names=None):
    """Least squares fit of y on the columns of X (X includes the intercept column if wanted).

    Returns a dict with the coefficients, standard errors (3.8), t-statistics (3.14) and two-sided
    p-values, RSS, TSS, RSE (3.25), R^2 (3.17), adjusted R^2, the F-statistic (3.23) with its
    p-value, the inverse of X^T X, fitted values, residuals, leverages h_i (the diagonal of the
    hat matrix, (3.37) in simple regression), externally studentized residuals and Cook's distances."""
    X = np.asarray(X, float)
    y = np.asarray(y, float)
    n, k = X.shape
    Q, R = np.linalg.qr(X)
    beta = np.linalg.solve(R, Q.T @ y)
    fitted = X @ beta
    resid = y - fitted
    rss = float(resid @ resid)
    has_int = np.allclose(X[:, 0], 1.0)
    tss = float(((y - y.mean()) ** 2).sum()) if has_int else float(y @ y)
    dfr = n - k
    s2 = rss / dfr
    Rinv = np.linalg.inv(R)
    XtXinv = Rinv @ Rinv.T
    se = np.sqrt(s2 * np.diag(XtXinv))
    t = beta / se
    p = 2 * stats.t.sf(np.abs(t), dfr)
    h = (Q ** 2).sum(1)
    p_pred = k - 1 if has_int else k
    F = ((tss - rss) / p_pred) / s2 if p_pred > 0 else np.nan
    s2_i = (rss - resid ** 2 / (1 - h)) / (dfr - 1)          # sigma^2 without observation i
    stud = resid / np.sqrt(s2_i * (1 - h))
    cooks = resid ** 2 / (k * s2) * h / (1 - h) ** 2
    return dict(beta=beta, se=se, t=t, p=p, rss=rss, tss=tss, rse=np.sqrt(s2), s2=s2, df=dfr, n=n, k=k,
                r2=1 - rss / tss, adj_r2=1 - (rss / dfr) / (tss / (n - 1)) if has_int else np.nan,
                F=F, F_p=stats.f.sf(F, p_pred, dfr) if p_pred > 0 else np.nan, XtXinv=XtXinv,
                fitted=fitted, resid=resid, h=h, stud=stud, cooks=cooks, names=names, X=X, y=y)


def fit_df(df, response, terms, intercept=True):
    X, names = model_matrix(df, terms, intercept)
    return ols(X, df[response].to_numpy(float), names)


def simple_ls(x, y):
    """The closed forms of simple linear regression: (3.4) for the estimates and (3.8) for their
    standard errors (with sigma^2 estimated by RSE^2), the RSE (3.15) and R^2 (3.17)."""
    x, y = np.asarray(x, float), np.asarray(y, float)
    n = len(x)
    xb, yb = x.mean(), y.mean()
    sxx = ((x - xb) ** 2).sum()
    b1 = ((x - xb) * (y - yb)).sum() / sxx
    b0 = yb - b1 * xb
    rss = ((y - b0 - b1 * x) ** 2).sum()
    rse = np.sqrt(rss / (n - 2))
    se1 = rse / np.sqrt(sxx)
    se0 = rse * np.sqrt(1 / n + xb ** 2 / sxx)
    return dict(b0=b0, b1=b1, se0=se0, se1=se1, rse=rse, r2=1 - rss / ((y - yb) ** 2).sum(), sxx=sxx, n=n)


def f_nested(small, big):
    """Partial F-statistic (3.24) comparing a model with a larger model that contains it."""
    q = small["df"] - big["df"]
    F = ((small["rss"] - big["rss"]) / q) / (big["rss"] / big["df"])
    return F, stats.f.sf(F, q, big["df"]), q


def intervals(fit, x0, level=0.95):
    """Estimate of f(x0) = x0^T beta, its confidence interval and the prediction interval for a new
    Y at x0 (x0 includes the leading 1). Both use the t quantile with n - p - 1 degrees of freedom:
    se_f = RSE sqrt(x0^T (X^T X)^{-1} x0) and se_pred = RSE sqrt(1 + x0^T (X^T X)^{-1} x0)."""
    x0 = np.atleast_2d(np.asarray(x0, float))
    f = x0 @ fit["beta"]
    q = np.einsum("ij,jk,ik->i", x0, fit["XtXinv"], x0)
    tq = stats.t.ppf(0.5 + level / 2, fit["df"])
    sef = fit["rse"] * np.sqrt(q)
    sep = fit["rse"] * np.sqrt(1 + q)
    return dict(fit=f, ci=np.c_[f - tq * sef, f + tq * sef], pi=np.c_[f - tq * sep, f + tq * sep], se_f=sef, se_pred=sep, tq=tq)


def vif(Xp):
    """Variance inflation factors 1 / (1 - R^2_{X_j | X_-j}) of the columns of Xp (no intercept column)."""
    Xp = np.asarray(Xp, float)
    out = []
    for j in range(Xp.shape[1]):
        others = np.column_stack([np.ones(len(Xp)), np.delete(Xp, j, axis=1)])
        out.append(1 / (1 - ols(others, Xp[:, j])["r2"]))
    return np.array(out)


def coef_ci(fit, level=0.95):
    tq = stats.t.ppf(0.5 + level / 2, fit["df"])
    return np.c_[fit["beta"] - tq * fit["se"], fit["beta"] + tq * fit["se"]]


def knn_reg(xtr, ytr, xte, K):
    """K-nearest-neighbour regression: the average response of the K closest training inputs
    (Euclidean distance; ties broken by the order of the training data)."""
    xtr = np.atleast_2d(np.asarray(xtr, float).T).T if np.ndim(xtr) == 1 else np.asarray(xtr, float)
    xte = np.atleast_2d(np.asarray(xte, float).T).T if np.ndim(xte) == 1 else np.asarray(xte, float)
    out = np.empty(len(xte))
    for s in range(0, len(xte), 2000):
        d = ((xte[s:s + 2000, None, :] - xtr[None, :, :]) ** 2).sum(-1)
        idx = np.argsort(d, axis=1, kind="stable")[:, :K]
        out[s:s + 2000] = np.asarray(ytr)[idx].mean(1)
    return out


def smooth(x, y, frac=0.6):
    """Lowess smooth of y against x (for the red trend lines of the residual plots)."""
    from statsmodels.nonparametric.smoothers_lowess import lowess
    s = lowess(y, x, frac=frac, it=3)
    return s[:, 0], s[:, 1]


# =========================================================================== simulations
def outlier_data(with_41=False):
    """The data of Figures 3.12 and 3.13: 40 points with an outlier (observation 20, ordinary x)
    and, for Figure 3.13, a 41st point with high leverage that is also an outlier."""
    rng = np.random.default_rng(151)
    x = np.sort(rng.normal(0.3, 1.1, 40))
    y = 1 + 2 * x + rng.normal(0, 0.75, 40)
    y[19] = 1 + 2 * x[19] + 4.6
    if with_41:
        x = np.append(x, 4.0)
        y = np.append(y, 15.0)
    return x, y


def knn_sim_1d(f, n=50, sd=0.3, seed=19, ntest=10000):
    rng = np.random.default_rng(seed)
    x = rng.uniform(-1, 1, n)
    y = f(x) + rng.normal(0, sd, n)
    xt = rng.uniform(-1, 1, ntest)
    yt = f(xt) + rng.normal(0, sd, ntest)
    return x, y, xt, yt


def f_linear(x):
    return 2.2 + 1.7 * x


def f_slight(x):
    return 2.15 + 1.55 * np.tanh(1.6 * x)


def f_strong(x):
    return 2.2 + 1.4 * np.sin(2.2 * x)


def knn_vs_ls_mse(x, y, xt, yt, Ks=range(1, 10)):
    X = np.c_[np.ones(len(x)), x] if np.ndim(x) == 1 else np.c_[np.ones(len(x)), x]
    Xt = np.c_[np.ones(len(xt)), xt] if np.ndim(xt) == 1 else np.c_[np.ones(len(xt)), xt]
    b = ols(X, y)["beta"]
    ls = float(np.mean((yt - Xt @ b) ** 2))
    kn = {K: float(np.mean((yt - knn_reg(x, y, xt, K)) ** 2)) for K in Ks}
    return ls, kn


def fig_3_20_data(seed=20, n=50, ntest=10000, ps=(1, 2, 3, 4, 10, 20)):
    rng = np.random.default_rng(seed)
    Xall = rng.uniform(-1, 1, (n, max(ps)))
    y = f_strong(Xall[:, 0]) + rng.normal(0, 0.22, n)
    Xt = rng.uniform(-1, 1, (ntest, max(ps)))
    yt = f_strong(Xt[:, 0]) + rng.normal(0, 0.22, ntest)
    return {p: knn_vs_ls_mse(Xall[:, :p], y, Xt[:, :p], yt) for p in ps}


# =========================================================================== tables and text numbers
def book_numbers():
    """Every number of the chapter's tables and text, recomputed (a dict; printed by tables())."""
    A = advertising()
    out = {}
    tv = fit_df(A, "sales", ["TV"])
    out["T3.1"] = tv
    out["T3.3_radio"] = fit_df(A, "sales", ["radio"])
    out["T3.3_newspaper"] = fit_df(A, "sales", ["newspaper"])
    full = fit_df(A, "sales", ["TV", "radio", "newspaper"])
    out["T3.4"] = full
    out["T3.5"] = A[["TV", "radio", "newspaper", "sales"]].corr().to_numpy()
    tvr = fit_df(A, "sales", ["TV", "radio"])
    out["TV_radio"] = tvr
    out["pred_100_20"] = intervals(tvr, [1, 100, 20])
    out["T3.9"] = fit_df(A, "sales", ["TV", "radio", ("TV", "radio")])
    out["ci_full"] = coef_ci(full)
    out["vif_adv"] = vif(A[["TV", "radio", "newspaper"]].to_numpy())
    C = credit()
    out["T3.7"] = fit_df(C, "Balance", ["Own"])
    pm = C.copy()
    pm["own_pm"] = np.where(C["Own"] == "Yes", 1.0, -1.0)
    out["own_pm"] = fit_df(pm, "Balance", ["own_pm"])
    out["T3.8"] = fit_df(C, "Balance", ["Region"])
    out["region_F"] = f_nested(fit_df(C, "Balance", []), out["T3.8"])
    out["fig3.7_additive"] = fit_df(C, "Balance", ["Income", "Student"])
    out["fig3.7_interaction"] = fit_df(C, "Balance", ["Income", "Student", ("Income", "Student")])
    out["T3.11_m1"] = fit_df(C, "Balance", ["Age", "Limit"])
    out["T3.11_m2"] = fit_df(C, "Balance", ["Rating", "Limit"])
    out["vif_credit"] = vif(C[["Age", "Rating", "Limit"]].to_numpy())
    out["vif_credit_2"] = vif(C[["Age", "Limit"]].to_numpy())
    out["r2_age_rating_limit"] = fit_df(C, "Balance", ["Age", "Rating", "Limit"])["r2"]
    Au = auto()
    out["auto_lin"] = fit_df(Au, "mpg", ["horsepower"])
    out["T3.10"] = fit_df(Au, "mpg", ["horsepower", ("pow", "horsepower", 2)])
    x, y = outlier_data()
    out["outlier_with"] = ols(np.c_[np.ones(40), x], y)
    out["outlier_without"] = ols(np.c_[np.ones(39), np.delete(x, 19)], np.delete(y, 19))
    x, y = outlier_data(True)
    out["lev_with"] = ols(np.c_[np.ones(41), x], y)
    out["lev_without41"] = ols(np.c_[np.ones(40), x[:40]], y[:40])
    return out


def tables(out_dir=None):
    """Print Tables 3.1-3.11 and the numbers in the text, and export the demonstrations' data."""
    B = book_numbers()

    def show(title, fit):
        print("\n" + title)
        print("%-22s %10s %10s %9s %10s" % ("", "Coef", "SE", "t", "p"))
        for nm, b, s, t, p in zip(fit["names"], fit["beta"], fit["se"], fit["t"], fit["p"]):
            print("%-22s %10.4f %10.4f %9.2f %10.3g" % (nm, b, s, t, p))
        print("RSE %.4f  R2 %.4f  adjR2 %.4f  F %.2f (p %.3g)" % (fit["rse"], fit["r2"], fit["adj_r2"], fit["F"], fit["F_p"]))

    show("Table 3.1 / 3.2: sales ~ TV", B["T3.1"])
    print("95% CIs:", coef_ci(B["T3.1"]).round(4).tolist())
    show("Table 3.3 (top): sales ~ radio", B["T3.3_radio"])
    show("Table 3.3 (bottom): sales ~ newspaper", B["T3.3_newspaper"])
    show("Table 3.4 / 3.6: sales ~ TV + radio + newspaper", B["T3.4"])
    print("Table 3.5 correlations:\n", np.round(B["T3.5"], 4))
    print("TV + radio: R2 %.5f RSE %.4f" % (B["TV_radio"]["r2"], B["TV_radio"]["rse"]))
    P = B["pred_100_20"]
    print("TV=100, radio=20: fit %.4f CI %s PI %s" % (P["fit"][0], P["ci"][0].round(4), P["pi"][0].round(4)))
    print("95% CIs (full model):", B["ci_full"].round(4).tolist())
    print("VIF (TV, radio, newspaper):", B["vif_adv"].round(4))
    show("Table 3.9: sales ~ TV * radio", B["T3.9"])
    show("Table 3.7: Balance ~ Own", B["T3.7"])
    show("Balance ~ own (+1/-1 coding)", B["own_pm"])
    show("Table 3.8: Balance ~ Region", B["T3.8"])
    print("F test for region: F %.4f p %.4f" % B["region_F"][:2])
    show("Figure 3.7 left: Balance ~ Income + Student", B["fig3.7_additive"])
    show("Figure 3.7 right: Balance ~ Income * Student", B["fig3.7_interaction"])
    show("Table 3.10: mpg ~ horsepower + horsepower^2", B["T3.10"])
    print("R2 linear %.4f quadratic %.4f" % (B["auto_lin"]["r2"], B["T3.10"]["r2"]))
    show("Table 3.11 model 1: Balance ~ Age + Limit", B["T3.11_m1"])
    show("Table 3.11 model 2: Balance ~ Rating + Limit", B["T3.11_m2"])
    print("VIF (age, rating, limit):", B["vif_credit"].round(3), " R2 %.4f" % B["r2_age_rating_limit"])
    print("VIF (age, limit):", B["vif_credit_2"].round(4), " R2 %.4f" % B["T3.11_m1"]["r2"])
    for k in ("outlier_with", "outlier_without", "lev_with", "lev_without41"):
        f = B[k]
        print("%-16s b0 %.3f b1 %.3f RSE %.3f R2 %.3f" % (k, f["beta"][0], f["beta"][1], f["rse"], f["r2"]))
    print("Fig 3.12: studentized residual of obs 20 = %.2f" % B["outlier_with"]["stud"][19])
    L = B["lev_with"]
    print("Fig 3.13: obs 41 leverage %.3f stud %.2f; obs 20 leverage %.3f stud %.2f" % (L["h"][40], L["stud"][40], L["h"][19], L["stud"][19]))
    export_demo_data()


def export_demo_data():
    """Data and reference numbers for the three demonstrations (../data/ch03-*.json)."""
    os.makedirs(DATA_OUT, exist_ok=True)
    A = advertising()
    adv = {"TV": A["TV"].tolist(), "radio": A["radio"].tolist(), "newspaper": A["newspaper"].tolist(),
           "sales": A["sales"].tolist(), "ref": {}}
    for v in ("TV", "radio", "newspaper"):
        f = fit_df(A, "sales", [v])
        x0 = {"TV": 100.0, "radio": 20.0, "newspaper": 30.0}[v]
        I = intervals(f, [1, x0])
        adv["ref"][v] = dict(beta=f["beta"].tolist(), se=f["se"].tolist(), t=f["t"].tolist(), rse=f["rse"], r2=f["r2"],
                             F=f["F"], x0=x0, fit=float(I["fit"][0]), ci=I["ci"][0].tolist(), pi=I["pi"][0].tolist(),
                             r=float(np.corrcoef(A[v], A["sales"])[0, 1]))
    with open(os.path.join(DATA_OUT, "ch03-advertising.json"), "w") as fh:
        json.dump(adv, fh, separators=(",", ":"))
    x, y = outlier_data(True)
    L = ols(np.c_[np.ones(41), x], y)
    S = ols(np.c_[np.ones(40), x[:40]], y[:40])
    lev = {"x": np.round(x, 5).tolist(), "y": np.round(y, 5).tolist(),
           "ref": {"with41": dict(beta=L["beta"].tolist(), h41=float(L["h"][40]), stud41=float(L["stud"][40]), cook41=float(L["cooks"][40]),
                                  h20=float(L["h"][19]), stud20=float(L["stud"][19]), rse=L["rse"], r2=L["r2"]),
                   "without41": dict(beta=S["beta"].tolist(), h20=float(S["h"][19]), stud20=float(S["stud"][19]), rse=S["rse"], r2=S["r2"])}}
    # the rounded data are what the demo uses: recompute the reference on them
    xr, yr = np.array(lev["x"]), np.array(lev["y"])
    Lr = ols(np.c_[np.ones(41), xr], yr)
    lev["ref"]["rounded_with41"] = dict(beta=Lr["beta"].tolist(), h41=float(Lr["h"][40]), stud41=float(Lr["stud"][40]), cook41=float(Lr["cooks"][40]))
    with open(os.path.join(DATA_OUT, "ch03-leverage.json"), "w") as fh:
        json.dump(lev, fh, separators=(",", ":"))
    C = credit()
    cr = {k: C[k].tolist() for k in ("Age", "Limit", "Rating", "Balance")}
    m1 = fit_df(C, "Balance", ["Age", "Limit"])
    m2 = fit_df(C, "Balance", ["Rating", "Limit"])
    m3 = fit_df(C, "Balance", ["Age", "Rating", "Limit"])
    cr["ref"] = {"age_limit": dict(beta=m1["beta"].tolist(), se=m1["se"].tolist(), rss=m1["rss"], vif=vif(C[["Age", "Limit"]].to_numpy()).tolist()),
                 "rating_limit": dict(beta=m2["beta"].tolist(), se=m2["se"].tolist(), rss=m2["rss"], vif=vif(C[["Rating", "Limit"]].to_numpy()).tolist()),
                 "age_rating_limit": dict(beta=m3["beta"].tolist(), se=m3["se"].tolist(), vif=vif(C[["Age", "Rating", "Limit"]].to_numpy()).tolist(), r2=m3["r2"]),
                 "cor_limit_rating": float(np.corrcoef(C["Limit"], C["Rating"])[0, 1]),
                 "cor_limit_age": float(np.corrcoef(C["Limit"], C["Age"])[0, 1])}
    with open(os.path.join(DATA_OUT, "ch03-credit.json"), "w") as fh:
        json.dump(cr, fh, separators=(",", ":"))
    print("wrote ../data/ch03-advertising.json, ch03-leverage.json, ch03-credit.json")


# =========================================================================== figures
def _segments(ax, x, y, yh, color=GREY, lw=0.6):
    for a, b, c in zip(x, y, yh):
        ax.plot([a, a], [b, c], color=color, lw=lw, zorder=1)


def fig_3_1(out_dir=None):
    """Advertising: least-squares fit of sales on TV, with the residuals."""
    A = advertising()
    f = simple_ls(A["TV"], A["sales"])
    fig, ax = plt.subplots(figsize=(5.4, 4.0))
    yh = f["b0"] + f["b1"] * A["TV"]
    _segments(ax, A["TV"], A["sales"], yh)
    ax.scatter(A["TV"], A["sales"], s=14, color=RED, zorder=3)
    g = np.array([0, 300])
    ax.plot(g, f["b0"] + f["b1"] * g, color=BLUE, lw=2.2, zorder=2)
    ax.set_xlabel("TV")
    ax.set_ylabel("Sales")
    ax.set_xlim(-8, 305)
    savefig(fig, CH, "fig3-1", out_dir)
    return f


def rss_grid(x, y, b0s, b1s):
    """RSS(beta0, beta1) on a grid for a simple regression."""
    B0, B1 = np.meshgrid(b0s, b1s)
    r = y[None, None, :] - B0[..., None] - B1[..., None] * x[None, None, :]
    return B0, B1, (r ** 2).sum(-1)


def fig_3_2(out_dir=None):
    """Advertising: RSS as a function of (beta0, beta1), contour and surface."""
    A = advertising()
    x, y = A["TV"].to_numpy(), A["sales"].to_numpy()
    f = simple_ls(x, y)
    B0, B1, R = rss_grid(x, y, np.linspace(5, 9, 161), np.linspace(0.03, 0.065, 161))
    fig = plt.figure(figsize=(9.6, 4.2))
    ax = fig.add_subplot(1, 2, 1)
    cs = ax.contour(B0, B1, R / 1000, levels=[2.11, 2.15, 2.2, 2.3, 2.5, 3.0], colors=BLUE, linewidths=1.0)
    ax.clabel(cs, fmt="%g", fontsize=8)
    ax.plot(f["b0"], f["b1"], "o", color=RED, ms=6)
    ax.set_xlabel(r"$\beta_0$")
    ax.set_ylabel(r"$\beta_1$")
    ax2 = fig.add_subplot(1, 2, 2, projection="3d")
    b0s, b1s = np.linspace(5, 9, 40), np.linspace(0.03, 0.065, 40)
    G0, G1, RR = rss_grid(x, y, b0s, b1s)
    ax2.plot_surface(G0, G1, RR / 1000, cmap="Blues_r", alpha=0.85, edgecolor="none")
    ax2.scatter([f["b0"]], [f["b1"]], [f["rse"] ** 2 * (f["n"] - 2) / 1000], color=RED, s=30, depthshade=False)
    ax2.set_xlabel(r"$\beta_0$")
    ax2.set_ylabel(r"$\beta_1$")
    ax2.set_zlabel("RSS (thousands)")
    ax2.view_init(elev=22, azim=-125)
    savefig(fig, CH, "fig3-2", out_dir)


def fig_3_3(out_dir=None):
    """Simulated Y = 2 + 3X + e: least-squares lines versus the population regression line."""
    rng = np.random.default_rng(3)
    n = 100
    g = np.array([-2.0, 2.0])
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 4.0))
    x = rng.uniform(-2, 2, n)
    y = 2 + 3 * x + rng.normal(0, 2, n)
    f = simple_ls(x, y)
    for ax in axes:
        ax.set_xlim(-2.1, 2.1)
        ax.set_ylim(-10, 10)
        ax.set_xlabel("X")
        ax.set_ylabel("Y")
    axes[0].scatter(x, y, s=12, facecolors="none", edgecolors="k", lw=0.7)
    axes[0].plot(g, 2 + 3 * g, color=RED, lw=2.2, label="population line")
    axes[0].plot(g, f["b0"] + f["b1"] * g, color=BLUE, lw=2.2, label="least squares")
    axes[0].legend(loc="upper left", frameon=False)
    fits = []
    for _ in range(10):
        xs = rng.uniform(-2, 2, n)
        ys = 2 + 3 * xs + rng.normal(0, 2, n)
        fs = simple_ls(xs, ys)
        fits.append((fs["b0"], fs["b1"]))
        axes[1].plot(g, fs["b0"] + fs["b1"] * g, color=LIGHTBLUE, lw=1.0)
    axes[1].plot(g, 2 + 3 * g, color=RED, lw=2.2)
    axes[1].plot(g, f["b0"] + f["b1"] * g, color="#0b2f7a", lw=2.2)
    savefig(fig, CH, "fig3-3", out_dir)
    return f, fits


def fig_3_4(out_dir=None):
    """Two predictors: the least-squares plane and the vertical residuals (simulated)."""
    rng = np.random.default_rng(4)
    n = 30
    X = rng.uniform(0, 1, (n, 2))
    y = 1 + 2 * X[:, 0] + 3 * X[:, 1] + rng.normal(0, 0.5, n)
    f = ols(np.c_[np.ones(n), X], y)
    b = f["beta"]
    fig = plt.figure(figsize=(5.4, 4.6))
    ax = fig.add_subplot(111, projection="3d")
    g = np.linspace(0, 1, 12)
    G1, G2 = np.meshgrid(g, g)
    ax.plot_surface(G1, G2, b[0] + b[1] * G1 + b[2] * G2, color=LIGHTBLUE, alpha=0.35, edgecolor="none")
    ax.plot_wireframe(G1, G2, b[0] + b[1] * G1 + b[2] * G2, color=BLUE, linewidth=0.3, alpha=0.6)
    for i in range(n):
        ax.plot([X[i, 0]] * 2, [X[i, 1]] * 2, [y[i], f["fitted"][i]], color=GREY, lw=0.7)
    ax.scatter(X[:, 0], X[:, 1], y, color=RED, s=14, depthshade=False)
    ax.set_xlabel("$X_1$")
    ax.set_ylabel("$X_2$")
    ax.set_zlabel("$Y$")
    ax.set_xticklabels([]); ax.set_yticklabels([]); ax.set_zticklabels([])
    ax.view_init(elev=16, azim=-60)
    savefig(fig, CH, "fig3-4", out_dir)


def fig_3_5(out_dir=None):
    """Advertising: the plane of sales on TV and radio; residuals above (red) and below (dark)."""
    A = advertising()
    f = fit_df(A, "sales", ["TV", "radio"])
    b = f["beta"]
    fig = plt.figure(figsize=(6.0, 5.0))
    ax = fig.add_subplot(111, projection="3d")
    g1, g2 = np.linspace(0, 300, 16), np.linspace(0, 50, 16)
    G1, G2 = np.meshgrid(g1, g2)
    ax.plot_surface(G1, G2, b[0] + b[1] * G1 + b[2] * G2, color=LIGHTBLUE, alpha=0.35, edgecolor="none")
    ax.plot_wireframe(G1, G2, b[0] + b[1] * G1 + b[2] * G2, color=BLUE, linewidth=0.3, alpha=0.6)
    up = f["resid"] >= 0
    for i in range(len(A)):
        ax.plot([A["TV"][i + 1]] * 2, [A["radio"][i + 1]] * 2, [A["sales"][i + 1], f["fitted"][i]], color=GREY, lw=0.5)
    ax.scatter(A["TV"][up], A["radio"][up], A["sales"][up], color=RED, s=10, depthshade=False)
    ax.scatter(A["TV"][~up], A["radio"][~up], A["sales"][~up], color="#6b0f0f", s=10, depthshade=False)
    ax.set_xlabel("TV")
    ax.set_ylabel("Radio")
    ax.set_zlabel("Sales")
    ax.view_init(elev=14, azim=-118)
    savefig(fig, CH, "fig3-5", out_dir)
    return f


def fig_3_6(out_dir=None):
    """Credit: scatterplot matrix of the quantitative variables."""
    C = credit()
    cols = ["Balance", "Age", "Cards", "Education", "Income", "Limit", "Rating"]
    k = len(cols)
    fig, axes = plt.subplots(k, k, figsize=(9.6, 9.6))
    for i, a in enumerate(cols):
        for j, b in enumerate(cols):
            ax = axes[i, j]
            ax.set_xticks([]); ax.set_yticks([])
            if i == j:
                ax.text(0.5, 0.5, a, ha="center", va="center", fontsize=11, transform=ax.transAxes)
            else:
                ax.scatter(C[b], C[a], s=2.5, facecolors="none", edgecolors=BLUE if (i + j) % 2 else "#8b1a1a", lw=0.4)
    fig.subplots_adjust(wspace=0.04, hspace=0.04)
    savefig(fig, CH, "fig3-6", out_dir)


def fig_3_7(out_dir=None):
    """Credit: balance on income for students and non-students, without and with an interaction."""
    C = credit()
    add = fit_df(C, "Balance", ["Income", "Student"])
    inter = fit_df(C, "Balance", ["Income", "Student", ("Income", "Student")])
    g = np.array([0, 150])
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 4.0))
    b = add["beta"]
    axes[0].plot(g, b[0] + b[2] + b[1] * g, color=RED, lw=2.2, label="student")
    axes[0].plot(g, b[0] + b[1] * g, color="k", lw=2.2, label="non-student")
    b = inter["beta"]
    axes[1].plot(g, b[0] + b[2] + (b[1] + b[3]) * g, color=RED, lw=2.2, label="student")
    axes[1].plot(g, b[0] + b[1] * g, color="k", lw=2.2, label="non-student")
    for ax in axes:
        ax.set_xlabel("Income")
        ax.set_ylabel("Balance")
        ax.set_ylim(150, 1500)
        ax.set_xlim(0, 150)
    axes[1].legend(loc="upper left", frameon=False)
    savefig(fig, CH, "fig3-7", out_dir)
    return add, inter


def fig_3_8(out_dir=None):
    """Auto: mpg on horsepower, polynomial fits of degree 1, 2 and 5."""
    Au = auto()
    x, y = Au["horsepower"].to_numpy(float), Au["mpg"].to_numpy(float)
    g = np.linspace(x.min(), x.max(), 300)
    xc, xs = x.mean(), x.std()
    fig, ax = plt.subplots(figsize=(5.6, 4.4))
    ax.scatter(x, y, s=12, facecolors="none", edgecolors=GREY, lw=0.6)
    for d, col, lab in ((1, ORANGE, "Linear"), (2, BLUE, "Degree 2"), (5, GREEN, "Degree 5")):
        X = np.vander((x - xc) / xs, d + 1, increasing=True)   # centred and scaled powers: same fit, better conditioning
        f = ols(X, y)
        ax.plot(g, np.vander((g - xc) / xs, d + 1, increasing=True) @ f["beta"], color=col, lw=2, label=lab)
    ax.set_xlabel("Horsepower")
    ax.set_ylabel("Miles per gallon")
    ax.legend(frameon=False)
    savefig(fig, CH, "fig3-8", out_dir)


def _resid_panel(ax, fit, labels, title, nlab=3):
    ax.scatter(fit["fitted"], fit["resid"], s=12, facecolors="none", edgecolors=GREY, lw=0.6)
    sx, sy = smooth(fit["fitted"], fit["resid"])
    ax.plot(sx, sy, color=RED, lw=1.8)
    ax.axhline(0, color="k", lw=0.6, ls=":")
    for i in np.argsort(-np.abs(fit["resid"]))[:nlab]:
        ax.annotate(str(labels[i]), (fit["fitted"][i], fit["resid"][i]), fontsize=8, xytext=(3, 2), textcoords="offset points")
    ax.set_title(title)
    ax.set_xlabel("Fitted values")
    ax.set_ylabel("Residuals")


def fig_3_9(out_dir=None):
    """Auto: residuals against fitted values for the linear and the quadratic fit."""
    Au = auto()
    lin = fit_df(Au, "mpg", ["horsepower"])
    quad = fit_df(Au, "mpg", ["horsepower", ("pow", "horsepower", 2)])
    labels = Au.index.to_numpy()
    fig, axes = plt.subplots(1, 2, figsize=(9.4, 4.2))
    _resid_panel(axes[0], lin, labels, "Residual plot for linear fit")
    _resid_panel(axes[1], quad, labels, "Residual plot for quadratic fit")
    savefig(fig, CH, "fig3-9", out_dir)
    return [labels[i] for i in np.argsort(-np.abs(lin["resid"]))[:3]], [labels[i] for i in np.argsort(-np.abs(quad["resid"]))[:3]]


def ar1_residuals(rho, n=100, seed=10):
    rng = np.random.default_rng(seed + int(round(10 * rho)))
    x = rng.uniform(0, 1, n)
    e = np.empty(n)
    e[0] = rng.normal()
    for t in range(1, n):
        e[t] = rho * e[t - 1] + rng.normal(0, np.sqrt(1 - rho ** 2))
    y = 1 + 2 * x + e
    return ols(np.c_[np.ones(n), x], y)["resid"]


def fig_3_10(out_dir=None):
    """Residuals of regressions on simulated time series with AR(1) errors, rho = 0, 0.5, 0.9."""
    fig, axes = plt.subplots(3, 1, figsize=(6.4, 7.2), sharex=True)
    lag1 = {}
    for ax, rho in zip(axes, (0.0, 0.5, 0.9)):
        r = ar1_residuals(rho)
        lag1[rho] = float(np.corrcoef(r[:-1], r[1:])[0, 1])
        ax.plot(np.arange(1, 101), r, color="k", lw=0.8)
        ax.scatter(np.arange(1, 101), r, s=10, facecolors="none", edgecolors=GREY, lw=0.7)
        ax.axhline(0, color=GREY, lw=0.6, ls=":")
        ax.set_title(r"$\rho=%.1f$" % rho)
        ax.set_ylabel("Residual")
    axes[-1].set_xlabel("Observation")
    savefig(fig, CH, "fig3-10", out_dir)
    return lag1


def _quantile_band(x, r, q, nb=12):
    """Smooth curve through the q-quantile of r in nb bins of x (for the outer lines of Figure 3.11)."""
    edges = np.quantile(x, np.linspace(0, 1, nb + 1))
    mid, val = [], []
    for a, b in zip(edges[:-1], edges[1:]):
        m = (x >= a) & (x <= b)
        mid.append(np.median(x[m]))
        val.append(np.quantile(r[m], q))
    mid, val = np.array(mid), np.array(val)
    c = np.polyfit(mid, val, 2)
    g = np.linspace(x.min(), x.max(), 100)
    return g, np.polyval(c, g)


def fig_3_11(out_dir=None):
    """Heteroscedasticity: residual plots for Y and for log Y (simulated)."""
    rng = np.random.default_rng(11)
    n = 1000
    x = rng.uniform(0, 1, n)
    y = (10 + 20 * x) * np.exp(rng.normal(0, 0.15, n))
    fig, axes = plt.subplots(1, 2, figsize=(9.6, 4.2))
    for ax, resp, title in ((axes[0], y, "Response Y"), (axes[1], np.log(y), "Response log(Y)")):
        f = ols(np.c_[np.ones(n), x], resp)
        _resid_panel(ax, f, np.arange(1, n + 1), title)
        for q in (0.05, 0.95):
            g, v = _quantile_band(f["fitted"], f["resid"], q)
            ax.plot(g, v, color=BLUE, lw=1.6)
    savefig(fig, CH, "fig3-11", out_dir)


def fig_3_12(out_dir=None):
    """An outlier with an ordinary x value: fits with and without it, residuals, studentized residuals."""
    x, y = outlier_data()
    f = ols(np.c_[np.ones(40), x], y)
    f0 = ols(np.c_[np.ones(39), np.delete(x, 19)], np.delete(y, 19))
    g = np.array([x.min() - 0.3, x.max() + 0.3])
    fig, axes = plt.subplots(1, 3, figsize=(11.4, 3.8))
    cols = np.array(["k"] * 40, dtype=object)
    cols[19] = RED
    axes[0].scatter(x, y, s=22, facecolors="none", edgecolors=cols, lw=0.9)
    axes[0].plot(g, f["beta"][0] + f["beta"][1] * g, color=RED, lw=2)
    axes[0].plot(g, f0["beta"][0] + f0["beta"][1] * g, color=BLUE, lw=2, ls="--")
    axes[0].annotate("20", (x[19], y[19]), color=RED, xytext=(-14, 2), textcoords="offset points", fontsize=9)
    axes[0].set_xlabel("X")
    axes[0].set_ylabel("Y")
    for ax, v, lab in ((axes[1], f["resid"], "Residuals"), (axes[2], f["stud"], "Studentized Residuals")):
        ax.scatter(f["fitted"], v, s=22, facecolors="none", edgecolors=cols, lw=0.9)
        ax.annotate("20", (f["fitted"][19], v[19]), color=RED, xytext=(-14, 2), textcoords="offset points", fontsize=9)
        ax.axhline(0, color=GREY, lw=0.6, ls=":")
        ax.set_xlabel("Fitted Values")
        ax.set_ylabel(lab)
    savefig(fig, CH, "fig3-12", out_dir)
    return f, f0


def fig_3_13(out_dir=None):
    """High leverage: observation 41, a high-leverage point in two dimensions, studentized residuals
    against leverage."""
    x, y = outlier_data(True)
    f = ols(np.c_[np.ones(41), x], y)
    f0 = ols(np.c_[np.ones(40), x[:40]], y[:40])
    fig, axes = plt.subplots(1, 3, figsize=(11.4, 3.8))
    cols = np.array(["k"] * 41, dtype=object)
    cols[[19, 40]] = RED
    g = np.array([x.min() - 0.3, 4.3])
    axes[0].scatter(x, y, s=22, facecolors="none", edgecolors=cols, lw=0.9)
    axes[0].plot(g, f["beta"][0] + f["beta"][1] * g, color=RED, lw=2)
    axes[0].plot(g, f0["beta"][0] + f0["beta"][1] * g, color=BLUE, lw=2, ls="--")
    for i, lab in ((19, "20"), (40, "41")):
        axes[0].annotate(lab, (x[i], y[i]), color=RED, xytext=(-14, 2), textcoords="offset points", fontsize=9)
    axes[0].set_xlabel("X")
    axes[0].set_ylabel("Y")
    rng = np.random.default_rng(13)
    z = rng.normal(size=(100, 2))
    x1 = z[:, 0]
    x2 = 0.95 * z[:, 0] + np.sqrt(1 - 0.95 ** 2) * z[:, 1]
    axes[1].scatter(x1, x2, s=18, facecolors="none", edgecolors="k", lw=0.7)
    axes[1].scatter([0.7], [-1.0], s=30, facecolors="none", edgecolors=RED, lw=1.2)
    ex, ey = common.ellipse_xy(np.zeros(2), np.array([[1, 0.95], [0.95, 1]]), nsd=np.sqrt(stats.chi2.ppf(0.95, 2)))
    axes[1].plot(ex, ey, color=BLUE, ls="--", lw=1.2)
    axes[1].set_xlabel("$X_1$")
    axes[1].set_ylabel("$X_2$")
    axes[2].scatter(f["h"], f["stud"], s=22, facecolors="none", edgecolors=cols, lw=0.9)
    for i, lab in ((19, "20"), (40, "41")):
        axes[2].annotate(lab, (f["h"][i], f["stud"][i]), color=RED, xytext=(-14, 2), textcoords="offset points", fontsize=9)
    axes[2].axhline(0, color=GREY, lw=0.6, ls=":")
    axes[2].set_xlabel("Leverage")
    axes[2].set_ylabel("Studentized Residuals")
    savefig(fig, CH, "fig3-13", out_dir)
    # leverage of the red point in the two-predictor panel, against the average (p + 1)/n
    X2 = np.c_[np.ones(101), np.append(x1, 0.7), np.append(x2, -1.0)]
    h2 = ols(X2, rng.normal(size=101))["h"]
    return f, f0, float(h2[-1])


def fig_3_14(out_dir=None):
    """Credit: age against limit (not collinear) and rating against limit (collinear)."""
    C = credit()
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 4.0))
    axes[0].scatter(C["Limit"], C["Age"], s=10, facecolors="none", edgecolors=RED, lw=0.6)
    axes[0].set_ylabel("Age")
    axes[1].scatter(C["Limit"], C["Rating"], s=10, facecolors="none", edgecolors=RED, lw=0.6)
    axes[1].set_ylabel("Rating")
    for ax in axes:
        ax.set_xlabel("Limit")
    savefig(fig, CH, "fig3-14", out_dir)
    return float(np.corrcoef(C["Limit"], C["Age"])[0, 1]), float(np.corrcoef(C["Limit"], C["Rating"])[0, 1])


def profile_rss(df, response, a, b, ba, bb):
    """RSS(beta_a, beta_b) minimised over the intercept, on a grid (for Figure 3.15)."""
    y = df[response].to_numpy(float)
    xa, xb = df[a].to_numpy(float), df[b].to_numpy(float)
    yc, xac, xbc = y - y.mean(), xa - xa.mean(), xb - xb.mean()
    A, Bm = np.meshgrid(ba, bb)
    r = yc[None, None, :] - A[..., None] * xac[None, None, :] - Bm[..., None] * xbc[None, None, :]
    return A, Bm, (r ** 2).sum(-1)


def fig_3_15(out_dir=None):
    """Credit: RSS contours for balance on (limit, age) and on (limit, rating), +-4 standard errors."""
    C = credit()
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 4.2))
    out = {}
    for ax, other in zip(axes, ("Age", "Rating")):
        f = fit_df(C, "Balance", ["Limit", other])
        bl, bo = f["beta"][1], f["beta"][2]
        sl, so = f["se"][1], f["se"][2]
        ba = np.linspace(bl - 4 * sl, bl + 4 * sl, 241)
        bb = np.linspace(bo - 4 * so, bo + 4 * so, 241)
        A, Bm, R = profile_rss(C, "Balance", "Limit", other, ba, bb)
        cs = ax.contour(A, Bm, R / 1e6, levels=[21.25, 21.5, 21.8], colors=BLUE, linewidths=1.0)
        ax.clabel(cs, fmt="%g", fontsize=8)
        ax.plot(bl, bo, "o", color="k", ms=5)
        ax.axvline(bl, color="k", ls=":", lw=0.8, ymax=0.5)
        ax.axhline(bo, color="k", ls=":", lw=0.8, xmax=0.5)
        ax.set_xlabel(r"$\beta_{\mathrm{Limit}}$")
        ax.set_ylabel(r"$\beta_{\mathrm{%s}}$" % other)
        out[other] = f["rss"] / 1e6
    savefig(fig, CH, "fig3-15", out_dir)
    return out


def fig_3_16(out_dir=None):
    """KNN regression with two predictors (64 observations on a grid): K = 1 and K = 9."""
    rng = np.random.default_rng(16)
    g = (np.arange(8) + 0.5) / 8
    G1, G2 = np.meshgrid(g, g)
    X = np.c_[G1.ravel(), G2.ravel()]
    y = 1 + 2 * X[:, 0] + 3 * X[:, 1] + rng.normal(0, 0.5, 64)
    s = np.linspace(0, 1, 60)
    S1, S2 = np.meshgrid(s, s)
    grid = np.c_[S1.ravel(), S2.ravel()]
    fig = plt.figure(figsize=(10.4, 4.6))
    for k, K in enumerate((1, 9)):
        ax = fig.add_subplot(1, 2, k + 1, projection="3d")
        Z = knn_reg(X, y, grid, K).reshape(S1.shape)
        ax.plot_surface(S1, S2, Z, cmap="RdYlBu", rstride=1, cstride=1, linewidth=0, antialiased=False, alpha=0.9)
        ax.scatter(X[:, 0], X[:, 1], y, color=ORANGE, s=10, depthshade=False)
        ax.set_xlabel("$x_1$")
        ax.set_ylabel("$x_2$")
        ax.set_zlabel("y")
        ax.set_xticklabels([]); ax.set_yticklabels([]); ax.set_zticklabels([])
        ax.view_init(elev=24, azim=-60)
        ax.set_title("K = %d" % K)
    savefig(fig, CH, "fig3-16", out_dir)


def fig_3_17(out_dir=None):
    """KNN regression in one dimension, true f linear: K = 1 and K = 9."""
    x, y, _, _ = knn_sim_1d(f_linear)
    g = np.linspace(-1, 1, 801)
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 4.0))
    for ax, K in zip(axes, (1, 9)):
        ax.scatter(x, y, s=14, color=RED, zorder=3)
        ax.plot(g, f_linear(g), color="k", lw=2)
        ax.step(g, knn_reg(x, y, g, K), where="mid", color=BLUE, lw=1.4)
        ax.set_xlabel("x")
        ax.set_ylabel("y")
        ax.set_title("K = %d" % K)
    savefig(fig, CH, "fig3-17", out_dir)


def _mse_panel(ax, ls, kn):
    Ks = sorted(kn)
    ax.plot([1 / K for K in Ks], [kn[K] for K in Ks], "-o", color=GREEN, ms=3, lw=1.4)
    ax.axhline(ls, color="k", ls="--", lw=1)
    ax.set_xscale("log")
    ax.set_xticks([0.2, 0.5, 1.0])
    ax.set_xticklabels(["0.2", "0.5", "1.0"])
    ax.set_xlabel("1/K")
    ax.set_ylabel("Mean Squared Error")


def fig_3_18(out_dir=None):
    """True f linear: least squares fit, and test MSE of KNN against 1/K."""
    x, y, xt, yt = knn_sim_1d(f_linear)
    ls, kn = knn_vs_ls_mse(x, y, xt, yt)
    b = ols(np.c_[np.ones(len(x)), x], y)["beta"]
    g = np.array([-1, 1])
    fig, axes = plt.subplots(1, 2, figsize=(9.0, 4.0))
    axes[0].scatter(x, y, s=14, color=RED, zorder=3)
    axes[0].plot(g, f_linear(g), color="k", lw=2)
    axes[0].plot(g, b[0] + b[1] * g, color=BLUE, lw=2, ls="--")
    axes[0].set_xlabel("x")
    axes[0].set_ylabel("y")
    _mse_panel(axes[1], ls, kn)
    axes[1].set_ylim(0, max(kn.values()) * 1.15)
    savefig(fig, CH, "fig3-18", out_dir)
    return ls, kn


def fig_3_19(out_dir=None):
    """Slightly and strongly non-linear f: KNN fits (K = 1, 9) and test MSE against 1/K."""
    g = np.linspace(-1, 1, 801)
    fig, axes = plt.subplots(2, 2, figsize=(9.0, 7.6))
    res = {}
    for row, (f, name) in enumerate(((f_slight, "slight"), (f_strong, "strong"))):
        x, y, xt, yt = knn_sim_1d(f, sd=0.22, seed=19)
        ax = axes[row, 0]
        ax.scatter(x, y, s=12, color=GREY, zorder=3)
        ax.plot(g, f(g), color="k", lw=2.2)
        ax.step(g, knn_reg(x, y, g, 1), where="mid", color=BLUE, lw=1.0)
        ax.step(g, knn_reg(x, y, g, 9), where="mid", color=RED, lw=1.4)
        ax.set_xlabel("x")
        ax.set_ylabel("y")
        ls, kn = knn_vs_ls_mse(x, y, xt, yt)
        _mse_panel(axes[row, 1], ls, kn)
        axes[row, 1].set_ylim(0, max(max(kn.values()), ls) * 1.15)
        res[name] = (ls, kn)
    savefig(fig, CH, "fig3-19", out_dir)
    return res


def fig_3_20(out_dir=None):
    """Test MSE of least squares and KNN as noise predictors are added (p = 1, 2, 3, 4, 10, 20)."""
    res = fig_3_20_data()
    fig, axes = plt.subplots(1, 6, figsize=(12.0, 3.0), sharey=True)
    for ax, (p, (ls, kn)) in zip(axes, res.items()):
        _mse_panel(ax, ls, kn)
        ax.set_ylim(0, 1.0)
        ax.set_title("p=%d" % p)
        if p != 1:
            ax.set_ylabel("")
    savefig(fig, CH, "fig3-20", out_dir)
    return res


# =========================================================================== applied exercises (Python edition)
def exercise_results():
    """Computations for the applied Exercises 8-15 (the Python edition's random generators)."""
    R = {}
    Au = auto()
    f8 = fit_df(Au, "mpg", ["horsepower"])
    R["ex8"] = dict(fit=f8, pred98=intervals(f8, [1, 98]))
    Ax = Au.drop(columns="name")
    preds9 = [c for c in Ax.columns if c != "mpg"]
    f9 = fit_df(Ax, "mpg", preds9)
    R["ex9"] = dict(fit=f9, cor=Ax.corr(), max_lev=int(Au.index[np.argmax(f9["h"])]), lev=float(f9["h"].max()),
                    max_stud=float(np.abs(f9["stud"]).max()), n_stud3=int((np.abs(f9["stud"]) > 3).sum()))
    R["ex9_inter"] = fit_df(Ax, "mpg", preds9 + [("displacement", "weight")])
    T = Ax.assign(log_hp=np.log(Ax["horsepower"]), sqrt_hp=np.sqrt(Ax["horsepower"]))
    R["ex9_trans"] = {k: fit_df(T, "mpg", [v])["r2"] for k, v in (("hp", "horsepower"), ("log_hp", "log_hp"), ("sqrt_hp", "sqrt_hp"))}
    Cs = carseats()
    fa = fit_df(Cs, "Sales", ["Price", "Urban", "US"])
    fe = fit_df(Cs, "Sales", ["Price", "US"])
    R["ex10"] = dict(fa=fa, fe=fe, ci=coef_ci(fe), F=f_nested(fe, fa), max_h=float(fe["h"].max()),
                     max_stud=float(np.abs(fe["stud"]).max()), hbar=3 / 400)
    rng = np.random.default_rng(1)
    x = rng.normal(size=100)
    y = 2 * x + rng.normal(size=100)
    yx = ols(x[:, None], y)
    xy = ols(y[:, None], x)
    n = 100
    t_formula = np.sqrt(n - 1) * (x @ y) / np.sqrt((x @ x) * (y @ y) - (x @ y) ** 2)
    yx1 = ols(np.c_[np.ones(n), x], y)
    xy1 = ols(np.c_[np.ones(n), y], x)
    R["ex11"] = dict(yx=yx, xy=xy, t_formula=t_formula, t_int=(yx1["t"][1], xy1["t"][1]))
    rng = np.random.default_rng(1)
    x = rng.normal(size=100)
    yd = 3 * x + rng.normal(size=100)
    perm = rng.permutation(x)                     # same sum of squares as x
    R["ex12"] = dict(diff=((x @ yd) / (x @ x), (x @ yd) / (yd @ yd)), same=((x @ perm) / (x @ x), (x @ perm) / (perm @ perm)))
    out13 = {}
    for label, sd in (("original", 0.5), ("less", 0.1), ("more", 1.0)):
        rng = np.random.default_rng(1)
        x = rng.normal(size=100)
        eps = rng.normal(0, sd, 100)
        y = -1 + 0.5 * x + eps
        f = ols(np.c_[np.ones(100), x], y)
        fq = ols(np.c_[np.ones(100), x, x ** 2], y)
        out13[label] = dict(fit=f, ci=coef_ci(f), quad_p=float(fq["p"][2]), quad_F=f_nested(f, fq))
    R["ex13"] = out13
    rng = np.random.default_rng(10)
    x1 = rng.uniform(0, 1, size=100)
    x2 = 0.5 * x1 + rng.normal(size=100) / 10
    y = 2 + 2 * x1 + 0.3 * x2 + rng.normal(size=100)
    e14 = dict(cor=float(np.corrcoef(x1, x2)[0, 1]),
               both=ols(np.c_[np.ones(100), x1, x2], y), only1=ols(np.c_[np.ones(100), x1], y), only2=ols(np.c_[np.ones(100), x2], y))
    x1, x2, y = np.append(x1, 0.1), np.append(x2, 0.8), np.append(y, 6)
    e14.update(both_g=ols(np.c_[np.ones(101), x1, x2], y), only1_g=ols(np.c_[np.ones(101), x1], y), only2_g=ols(np.c_[np.ones(101), x2], y))
    R["ex14"] = e14
    Bo = boston()
    preds = [c for c in Bo.columns if c != "crim"]
    uni = {p: fit_df(Bo, "crim", [p]) for p in preds}
    multi = fit_df(Bo, "crim", preds)
    cubic = {}
    for p in preds:
        if p == "chas":
            continue
        small = fit_df(Bo, "crim", [p])
        big = fit_df(Bo, "crim", [p, ("pow", p, 2), ("pow", p, 3)])
        cubic[p] = f_nested(small, big)
    R["ex15"] = dict(uni=uni, multi=multi, cubic=cubic, preds=preds)
    return R


def exercises(out_dir=None):
    """Print the results of the applied exercises and draw the plot of Exercise 15(c)."""
    R = exercise_results()
    f = R["ex8"]["fit"]
    P = R["ex8"]["pred98"]
    print("Ex 8: beta %s  se %s  RSE %.4f  R2 %.4f; at 98: %.4f CI %s PI %s" % (f["beta"].round(4), f["se"].round(4), f["rse"], f["r2"], P["fit"][0], P["ci"][0].round(3), P["pi"][0].round(3)))
    f = R["ex9"]["fit"]
    print("Ex 9: F %.1f (p %.3g); coefficients:" % (f["F"], f["F_p"]))
    for nm, b, s, p in zip(f["names"], f["beta"], f["se"], f["p"]):
        print("   %-14s %10.4f %9.4f %10.3g" % (nm, b, s, p))
    print("   RSE %.3f R2 %.4f; max leverage %.3f (row %d), max |stud| %.2f, %d with |stud|>3" % (f["rse"], f["r2"], R["ex9"]["lev"], R["ex9"]["max_lev"], R["ex9"]["max_stud"], R["ex9"]["n_stud3"]))
    fi = R["ex9_inter"]
    print("   displacement:weight %.6f (p %.3g), R2 %.4f" % (fi["beta"][-1], fi["p"][-1], fi["r2"]))
    print("   R2 of mpg on hp, log hp, sqrt hp:", {k: round(v, 4) for k, v in R["ex9_trans"].items()})
    e = R["ex10"]
    for k in ("fa", "fe"):
        print("Ex 10 %s: %s  beta %s  se %s  p %s  R2 %.4f  RSE %.4f" % (k, e[k]["names"], e[k]["beta"].round(4), e[k]["se"].round(4), e[k]["p"].round(4), e[k]["r2"], e[k]["rse"]))
    print("   CI (e):", e["ci"].round(4).tolist(), " max leverage %.4f vs (p+1)/n %.4f, max |stud| %.2f" % (e["max_h"], e["hbar"], e["max_stud"]))
    e = R["ex11"]
    print("Ex 11: y~x+0: b %.4f se %.4f t %.3f p %.3g; x~y+0: b %.4f se %.4f t %.3f; formula t %.3f; with intercept t %.4f %.4f"
          % (e["yx"]["beta"][0], e["yx"]["se"][0], e["yx"]["t"][0], e["yx"]["p"][0], e["xy"]["beta"][0], e["xy"]["se"][0], e["xy"]["t"][0], e["t_formula"], *e["t_int"]))
    print("Ex 12:", R["ex12"])
    for k, v in R["ex13"].items():
        print("Ex 13 %-8s beta %s RSE %.4f R2 %.4f CI %s quadratic p %.3f" % (k, v["fit"]["beta"].round(4), v["fit"]["rse"], v["fit"]["r2"], v["ci"].round(4).tolist(), v["quad_p"]))
    e = R["ex14"]
    print("Ex 14: cor %.4f" % e["cor"])
    for k in ("both", "only1", "only2", "both_g", "only1_g", "only2_g"):
        print("   %-8s beta %s se %s p %s; last obs h %.3f stud %.2f" % (k, e[k]["beta"].round(4), e[k]["se"].round(4), e[k]["p"].round(4), e[k]["h"][-1], e[k]["stud"][-1]))
    e = R["ex15"]
    print("Ex 15: univariate slopes and p; multiple coefficient and p; cubic F p")
    for j, p in enumerate(e["preds"]):
        u = e["uni"][p]
        c = e["cubic"].get(p, (np.nan, np.nan))
        print("   %-8s %10.4f %10.3g   %10.4f %10.3g   %10.3g" % (p, u["beta"][1], u["p"][1], e["multi"]["beta"][j + 1], e["multi"]["p"][j + 1], c[1]))
    print("   multiple R2 %.4f RSE %.4f F %.2f" % (e["multi"]["r2"], e["multi"]["rse"], e["multi"]["F"]))
    fig, ax = plt.subplots(figsize=(5.6, 4.4))
    us = [e["uni"][p]["beta"][1] for p in e["preds"]]
    ms = e["multi"]["beta"][1:]
    ax.scatter(us, ms, color=BLUE, s=24)
    for p, a, b in zip(e["preds"], us, ms):
        ax.annotate(p, (a, b), fontsize=8, xytext=(4, 3), textcoords="offset points")
    ax.axhline(0, color=GREY, lw=0.6, ls=":")
    ax.axvline(0, color=GREY, lw=0.6, ls=":")
    ax.set_xlabel("simple regression coefficient")
    ax.set_ylabel("multiple regression coefficient")
    savefig(fig, CH, "ex3-15", out_dir)
    return R


FIGURES = {"3.1": fig_3_1, "3.2": fig_3_2, "3.3": fig_3_3, "3.4": fig_3_4, "3.5": fig_3_5, "3.6": fig_3_6,
           "3.7": fig_3_7, "3.8": fig_3_8, "3.9": fig_3_9, "3.10": fig_3_10, "3.11": fig_3_11, "3.12": fig_3_12,
           "3.13": fig_3_13, "3.14": fig_3_14, "3.15": fig_3_15, "3.16": fig_3_16, "3.17": fig_3_17,
           "3.18": fig_3_18, "3.19": fig_3_19, "3.20": fig_3_20, "tables": tables, "exercises": exercises}

if __name__ == "__main__":
    common.run(FIGURES, CH)
