"""Wasserman (2004), Chapter 13: Linear and Logistic Regression.

Companion code for Chapter 13 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 209-229, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch13-regression.html):

* Data: stars (log surface temperature and log light intensity of the stars in the cluster CYG OB1,
  Figure 13.1), florida (the 2000 presidential election in Florida, county by county, Example 13.6),
  crime (47 states in 1960, Example 13.14), carmileage and coris (the book's own data files,
  Exercises 13.6, 13.7 and 13.11, Example 13.17).
* Sections 13.1-13.4, simple linear regression: simple_fit (least squares (13.5)-(13.6), the unbiased
  and the maximum likelihood variance estimates (13.7) and (13.10), the covariance matrix (13.11) and
  the standard errors (13.12)-(13.13)), predict (the fitted mean, its standard error and the
  prediction standard error xi_n of (13.16)) and pred_coverage_sim (the coverage experiment of
  Exercise 13.10).
* Section 13.5, multiple regression: ols (the normal equations (13.20), the covariance (13.21), the
  hat matrix and leave-one-out residuals) and coef_table (Wald statistics and p-values).
* Section 13.6, model selection: subset_scores (Mallows' Cp (13.26), AIC (13.27), BIC (13.31) and
  leave-one-out cross-validation (13.28)-(13.29) for every subset of covariates, both with sigma from
  the full model, as in the book, and with sigma estimated in each model, as R does), stepwise
  (forward and backward search with any score), zheng_loh (the Zheng-Loh method), optimism_exact (the
  expected training error and the prediction risk of Theorem 13.15 in closed form) and kfold_cv.
* Section 13.7, logistic regression: irls (Newton-Raphson, which is the reweighted least squares
  algorithm of p. 224), mm_logistic (the bounded-curvature algorithm of Boehning and Lindsay, a
  slower but monotone alternative), grad_logistic (gradient ascent), logistic_table (coefficients,
  standard errors from the Fisher information, Wald and likelihood ratio tests), backward_aic_logistic
  (Exercise 13.11) and boot_logistic (a parametric bootstrap of the coefficients).
* Section 13.9 and Exercise 13.9: aic_bic_select (the probability that AIC or BIC picks the smaller of
  two Normal models) and kl_risk (the Kullback-Leibler risk of the selected density).
* JSRandom, js_uniforms and js_normals: the seeded generator of the browser demonstrations (DK.rng in
  courses/demo-kit.js), so that the simulations on the pages can be reproduced here number for number.

The chapter has three figures, all reproduced, so nothing is skipped: 13.1 (the star data with the
least squares line; the book's line is reproduced to three digits when the five stars with log
temperature below 4.0, which fall outside the book's plot, are left out), 13.2 (the election data on
the raw and log scales, with residuals) and 13.3 (the logistic function).
Six supplementary figures illustrate the gaps filled in the notes:
  13.S1 confidence band against prediction band, Palm Beach county, and Exercise 13.10;
  13.S2 the optimism of the training error, Cp and cross-validation (Theorem 13.15);
  13.S3 Cp, AIC, BIC and cross-validation on the crime and car mileage data (Section 13.6);
  13.S4 AIC against BIC for two Normal models (Exercise 13.9);
  13.S5 logistic regression on the CORIS data: Newton against slower algorithms, and a fitted curve;
  13.S6 the car mileage data on the raw and log scales (Exercise 13.6).
Run `python3 ch13_regression.py --list`, `python3 ch13_regression.py` (the JSON files and all nine figures, a few
seconds on one CPU thread) or `python3 ch13_regression.py --fig 13.2`.
Files go to ../img/ch13/. The data used by the demonstrations go to ../data/ch13-prediction.json,
../data/ch13-selection.json and ../data/ch13-coris.json (export_json).

Sources of the data that are not on the book's website. The star data are the Hertzsprung-Russell
diagram of the star cluster CYG OB1 (Humphreys 1978), as tabulated by Rousseeuw and Leroy (1987,
Table 3 of Chapter 2); 47 stars. The election data are the certified county returns of the 2000
presidential election in Florida (Florida Division of Elections), 67 counties; their totals are
2,912,790 votes for Bush and 17,484 for Buchanan. The crime data are the 47 states of Ehrlich (1973)
and Vandaele (1978), as distributed with Venables and Ripley (2002); the book's response is the crime
rate divided by 10, and its "Wealth" is the variable GDP of that table.
"""
import itertools
import json
import math
import os

import numpy as np
from scipy import integrate, stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig

CH = 13
HERE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(HERE, "data")
M32 = 0xFFFFFFFF


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js): u() is
    mulberry32 and norm() the Box-Muller transform that returns r cos(t) and keeps r sin(t) for the
    next call, exactly as in JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed), vectorised (mulberry32 adds a constant
    to its state at each call, so the j-th output is a hash of seed + j * 0x6D2B79F5 mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def js_normals(seed, n):
    """The first n values of DK.rng(seed).norm() (Box-Muller pairs: cos first, then the cached sin)."""
    npair = (n + 1) // 2
    u = js_uniforms(seed, 2 * npair)
    u1, u2 = u[0::2], u[1::2]
    if np.any(u1 == 0.0):
        raise RuntimeError("a zero uniform: use JSRandom for this stream")
    r = np.sqrt(-2.0 * np.log(u1))
    th = 2.0 * np.pi * u2
    out = np.empty(2 * npair)
    out[0::2] = r * np.cos(th)
    out[1::2] = r * np.sin(th)
    return out[:n]


# =========================================================================== data
def stars(all_stars=False):
    """The CYG OB1 star data: arrays (x, y) = (log light intensity, log surface temperature).

    Figure 13.1 plots log temperature between 4.0 and 4.6, so it leaves out the five stars with
    log temperature below 4.0 (the four red giants and star 7); with those 42 stars the least squares
    line is 3.581 + 0.166 x, the book's line. all_stars=True returns all 47."""
    d = np.genfromtxt(os.path.join(DATA, "stars_cyg_ob1.csv"), delimiter=",", skip_header=1)
    te, li = d[:, 1], d[:, 2]
    keep = np.ones(len(te), bool) if all_stars else te >= 4.0
    return li[keep], te[keep]


def florida():
    """County, Bush votes, Buchanan votes for the 67 Florida counties (2000)."""
    import pandas as pd
    return pd.read_csv(os.path.join(DATA, "florida2000.csv"))


CRIME_NAMES = ["Age", "Southern", "Education", "Expenditure", "Labor", "Males", "Population",
               "U1", "U2", "Wealth"]
_CRIME_COLS = ["M", "So", "Ed", "Po1", "LF", "M.F", "Pop", "U1", "U2", "GDP"]


def crime():
    """Example 13.14: (X, y, names) with the 10 covariates of the book's table (no intercept column)
    and y = crime rate / 10, the scale of the book's coefficients."""
    import pandas as pd
    d = pd.read_csv(os.path.join(DATA, "uscrime.csv"))
    return d[_CRIME_COLS].to_numpy(float), d["y"].to_numpy(float) / 10.0, list(CRIME_NAMES)


def carmileage():
    """The passenger car mileage data (82 cars): a DataFrame with name, VOL, HP, MPG, SP, WT. The
    file has a text header; each data line ends with the five numbers."""
    import pandas as pd
    lines = open(common.aos_file("carmileage.dat")).read().split("\n")
    i0 = [k for k, l in enumerate(lines) if l.startswith("MAKE")][0]
    rows = []
    for l in lines[i0 + 1:]:
        t = l.split()
        if len(t) < 6:
            continue
        rows.append([" ".join(t[:-5])] + [float(v) for v in t[-5:]])
    return pd.DataFrame(rows, columns=["name", "VOL", "HP", "MPG", "SP", "WT"])


CORIS_NAMES = ["sbp", "tobacco", "ldl", "adiposity", "famhist", "typea", "obesity", "alcohol", "age"]


def coris():
    """The CORIS data (Example 13.17): (X, y, names), X the 9 covariates (462 x 9), y = chd."""
    import pandas as pd
    d = pd.read_csv(common.aos_file("coris.dat"), skiprows=4, header=None)
    d = d.iloc[:, 1:].to_numpy(float)
    return d[:, :9], d[:, 9], list(CORIS_NAMES)


# =========================================================================== 13.1-13.4 simple regression
def simple_fit(x, y):
    """Least squares (13.5)-(13.6), sigma^2 by (13.7) and (13.10), covariance (13.11), se (13.12)-(13.13)."""
    x, y = np.asarray(x, float), np.asarray(y, float)
    n = len(x)
    xb, yb = x.mean(), y.mean()
    sxx = np.sum((x - xb) ** 2)
    b1 = np.sum((x - xb) * (y - yb)) / sxx
    b0 = yb - b1 * xb
    res = y - b0 - b1 * x
    rss = float(np.sum(res ** 2))
    s2 = rss / (n - 2)
    sX2 = sxx / n
    cov = s2 / (n * sX2) * np.array([[np.mean(x ** 2), -xb], [-xb, 1.0]])
    return dict(n=n, b0=b0, b1=b1, res=res, rss=rss, s2=s2, s2_mle=rss / n, sigma=math.sqrt(s2), xbar=xb,
                sxx=sxx, sX=math.sqrt(sX2), cov=cov, se0=math.sqrt(cov[0, 0]), se1=math.sqrt(cov[1, 1]))


def predict(fit, xs):
    """At x*: the prediction (13.15), the standard error of the fitted mean sqrt(V(Y*_hat)) and the
    prediction standard error xi_n of (13.16), all with sigma replaced by its unbiased estimate."""
    xs = np.asarray(xs, float)
    yhat = fit["b0"] + fit["b1"] * xs
    C = fit["cov"]
    se_mean = np.sqrt(C[0, 0] + xs ** 2 * C[1, 1] + 2 * xs * C[0, 1])
    xi = np.sqrt(se_mean ** 2 + fit["s2"])
    return yhat, se_mean, xi


def xi_formula(x, xs, s2):
    """(13.16) exactly as printed: xi^2 = s2 (sum (x_i - x*)^2 / (n sum (x_i - xbar)^2) + 1)."""
    x = np.asarray(x, float)
    n = len(x)
    return math.sqrt(s2 * (np.sum((x - xs) ** 2) / (n * np.sum((x - x.mean()) ** 2)) + 1))


def naive_coverage(fit, xs, z=2.0):
    """Exercise 13.10(a): P(Y* in Y*_hat +/- z s), s = sd of the fitted mean, from the Normal
    approximation P(|N(0, 1 + sigma^2/s^2)| < z), with sigma and s at their estimates."""
    _, s, _ = predict(fit, xs)
    return 2 * stats.norm.cdf(z / np.sqrt(1 + fit["s2"] / s ** 2)) - 1


def pred_coverage_sim(x, b0, b1, sigma, xs, z, R, seed):
    """The coverage experiment of the prediction demo (Exercise 13.10), number for number.

    The design x is held fixed and the truth is Y = b0 + b1 x + sigma e. Replicate r uses the next
    n + 1 normals of DK.rng(seed): n for the data, then one for the new response Y* at x*. Returns the
    proportions of replicates in which (i) Y* lies in Y*_hat +/- z s (s the estimated sd of the fitted
    mean), (ii) Y* lies in Y*_hat +/- z xi_n, (iii) the mean b0 + b1 x* lies in Y*_hat +/- z s."""
    x = np.asarray(x, float)
    n = len(x)
    e = js_normals(seed, R * (n + 1)).reshape(R, n + 1)
    mu = b0 + b1 * x
    Y = mu[None, :] + sigma * e[:, :n]
    Ystar = b0 + b1 * xs + sigma * e[:, n]
    xb = x.mean()
    sxx = np.sum((x - xb) ** 2)
    B1 = ((x - xb)[None, :] * (Y - Y.mean(1, keepdims=True))).sum(1) / sxx
    B0 = Y.mean(1) - B1 * xb
    rss = ((Y - B0[:, None] - B1[:, None] * x[None, :]) ** 2).sum(1)
    s2 = rss / (n - 2)
    se_mean = np.sqrt(s2 * (1.0 / n + (xs - xb) ** 2 / sxx))
    xi = np.sqrt(se_mean ** 2 + s2)
    yh = B0 + B1 * xs
    return dict(naive=float(np.mean(np.abs(Ystar - yh) < z * se_mean)),
                pred=float(np.mean(np.abs(Ystar - yh) < z * xi)),
                mean=float(np.mean(np.abs(b0 + b1 * xs - yh) < z * se_mean)))


# =========================================================================== 13.5 multiple regression
def add_intercept(X):
    X = np.asarray(X, float)
    if X.ndim == 1:
        X = X[:, None]
    return np.column_stack([np.ones(len(X)), X])


def ols(X, y):
    """Least squares (13.20) with covariance (13.21), sigma^2 = RSS/(n - k), the diagonal of the hat
    matrix U = X (X'X)^{-1} X' of (13.30) and the leave-one-out residuals of (13.29). X includes any
    intercept column."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    n, k = X.shape
    XtX = X.T @ X
    beta = np.linalg.solve(XtX, X.T @ y)
    G = np.linalg.inv(XtX)
    fitted = X @ beta
    res = y - fitted
    rss = float(res @ res)
    s2 = rss / (n - k) if n > k else float("nan")
    h = np.einsum("ij,jk,ik->i", X, G, X)
    return dict(beta=beta, cov=s2 * G, se=np.sqrt(s2 * np.diag(G)), s2=s2, rss=rss, res=res, fitted=fitted,
                h=h, loo=res / (1 - h), n=n, k=k)


def coef_table(X, y):
    """(beta_j, se_j, W_j = beta_j / se_j, two-sided p-value) for every column; the p-values use the
    t distribution with n - k degrees of freedom, as regression programs do (the Normal p-values of
    the Wald test (13.23) are also returned)."""
    f = ols(X, y)
    W = f["beta"] / f["se"]
    return dict(beta=f["beta"], se=f["se"], W=W, p_t=2 * stats.t.sf(np.abs(W), f["n"] - f["k"]),
                p_norm=2 * stats.norm.sf(np.abs(W)), fit=f)


def loo_brute(X, y):
    """Leave-one-out cross-validation (13.28) by refitting n times."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    out = 0.0
    for i in range(len(y)):
        m = np.ones(len(y), bool)
        m[i] = False
        b = np.linalg.lstsq(X[m], y[m], rcond=None)[0]
        out += (y[i] - X[i] @ b) ** 2
    return out


def kfold_cv(X, y, K, seed=0):
    """K-fold cross-validation (sum of squared prediction errors over all held-out points)."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    idx = np.random.default_rng(seed).permutation(len(y))
    out = 0.0
    for fold in np.array_split(idx, K):
        m = np.ones(len(y), bool)
        m[fold] = False
        b = np.linalg.lstsq(X[m], y[m], rcond=None)[0]
        out += np.sum((y[fold] - X[fold] @ b) ** 2)
    return out


# =========================================================================== 13.6 model selection
def scores_for(X, y, cols, s2_full, intercept=True):
    """All the scores of Section 13.6 for the model with the covariates `cols` (indices into X).

    |S| counts the columns of X_S, the intercept included. 'cp' is (13.26); 'aic' and 'bic' are
    (13.27) and (13.31) with sigma^2 fixed at its full-model estimate, as the book does (larger is
    better); 'cv' is (13.29); 'aicR' and 'bicR' are R's versions n log(RSS/n) + 2|S| and
    n log(RSS/n) + |S| log n, in which sigma is estimated in each model (smaller is better)."""
    y = np.asarray(y, float)
    n = len(y)
    Xs = X[:, list(cols)] if len(cols) else np.zeros((n, 0))
    if intercept:
        Xs = add_intercept(Xs) if len(cols) else np.ones((n, 1))
    f = ols(Xs, y)
    p = Xs.shape[1]
    ll = -0.5 * n * math.log(2 * math.pi * s2_full) - f["rss"] / (2 * s2_full)
    return dict(cols=tuple(cols), p=p, rss=f["rss"], cp=f["rss"] + 2 * p * s2_full, aic=ll - p,
                bic=ll - 0.5 * p * math.log(n), cv=float(np.sum(f["loo"] ** 2)),
                aicR=n * math.log(f["rss"] / n) + 2 * p, bicR=n * math.log(f["rss"] / n) + p * math.log(n))


def full_s2(X, y):
    return ols(add_intercept(X), y)["s2"]


def subset_scores(X, y):
    """scores_for() for all 2^k subsets of the k covariates (the intercept is always in)."""
    X = np.asarray(X, float)
    k = X.shape[1]
    s2 = full_s2(X, y)
    out = []
    for size in range(k + 1):
        for cols in itertools.combinations(range(k), size):
            out.append(scores_for(X, y, cols, s2))
    return out


def best(rows, key, larger=False):
    return max(rows, key=lambda r: r[key]) if larger else min(rows, key=lambda r: r[key])


def stepwise(X, y, key="aicR", direction="backward", larger=False):
    """Greedy forward or backward search with the score `key` (Section 13.6). Returns the list of
    steps; each step is (current model, its score, [(variable changed, score after the change), ...])
    sorted from best to worst, the last step being the one where no change improves the score."""
    X = np.asarray(X, float)
    k = X.shape[1]
    s2 = full_s2(X, y)
    sign = -1.0 if larger else 1.0
    cur = list(range(k)) if direction == "backward" else []
    steps = []
    while True:
        here = scores_for(X, y, cur, s2)[key]
        moves = []
        cand = cur if direction == "backward" else [j for j in range(k) if j not in cur]
        for j in cand:
            new = [c for c in cur if c != j] if direction == "backward" else sorted(cur + [j])
            moves.append((j, scores_for(X, y, new, s2)[key]))
        moves.sort(key=lambda t: sign * t[1])
        steps.append((list(cur), here, moves))
        if not moves or sign * moves[0][1] >= sign * here:
            break
        j = moves[0][0]
        cur = [c for c in cur if c != j] if direction == "backward" else sorted(cur + [j])
    return steps


def zheng_loh(X, y):
    """The Zheng-Loh method (p. 222): order the covariates by |W_j| in the full model and choose the
    number j of leading covariates that minimises RSS(j) + j sigma^2 log n (sigma^2 from the full
    model). Returns (chosen columns, order, criterion values for j = 0..k, |W| in that order)."""
    X = np.asarray(X, float)
    n, k = X.shape
    t = coef_table(add_intercept(X), y)
    W = np.abs(t["W"][1:])
    order = list(np.argsort(-W, kind="stable"))
    s2 = t["fit"]["s2"]
    crit = []
    for j in range(k + 1):
        cols = order[:j]
        rss = ols(add_intercept(X[:, cols]) if j else np.ones((n, 1)), y)["rss"]
        crit.append(rss + j * s2 * math.log(n))
    jhat = int(np.argmin(crit))
    return sorted(order[:jhat]), order, np.array(crit), W[order]


def optimism_exact(X, mu, sigma, nested):
    """Theorem 13.15 in closed form for least squares fits with fixed design X (no intercept column
    added), mean vector mu and noise sd sigma. For each model in `nested` (a list of column lists,
    an intercept always included) returns (|S|, E R_tr(S), R(S)), with
      E R_tr(S) = ||(I - H_S) mu||^2 + (n - |S|) sigma^2,  R(S) = ||(I - H_S) mu||^2 + (n + |S|) sigma^2."""
    n = len(mu)
    out = []
    for cols in nested:
        Xs = add_intercept(X[:, cols]) if len(cols) else np.ones((n, 1))
        p = Xs.shape[1]
        b = np.linalg.lstsq(Xs, mu, rcond=None)[0]
        bias2 = float(np.sum((mu - Xs @ b) ** 2))
        out.append((p, bias2 + (n - p) * sigma ** 2, bias2 + (n + p) * sigma ** 2))
    return np.array(out)


def optimism_sim(X, mu, sigma, nested, R, seed=0):
    """Monte Carlo averages of the training error, Cp (with sigma^2 from the largest model) and the
    leave-one-out score for the nested models of optimism_exact."""
    rng = np.random.default_rng(seed)
    n = len(mu)
    out = np.zeros((len(nested), 3))
    designs = [add_intercept(X[:, c]) if len(c) else np.ones((n, 1)) for c in nested]
    Ps = []
    for Xs in designs:
        G = np.linalg.inv(Xs.T @ Xs)
        H = Xs @ G @ Xs.T
        Ps.append((H, np.diag(H), Xs.shape[1]))
    Hf, _, pf = Ps[-1]
    for _ in range(R):
        y = mu + sigma * rng.standard_normal(n)
        s2 = float(np.sum((y - Hf @ y) ** 2)) / (n - pf)
        for m, (H, h, p) in enumerate(Ps):
            r = y - H @ y
            rss = float(r @ r)
            out[m] += (rss, rss + 2 * p * s2, float(np.sum((r / (1 - h)) ** 2)))
    return out / R


# =========================================================================== 13.7 logistic regression
def loglik_logistic(X, y, b):
    eta = X @ b
    return float(np.sum(y * eta - np.logaddexp(0.0, eta)))


def irls(X, y, beta0=None, tol=1e-10, maxit=50):
    """Newton-Raphson for the logistic likelihood (13.35), written as the reweighted least squares
    algorithm of p. 224 (with Z in place of Y in step 3): Z = eta + (Y - p)/(p(1 - p)),
    W = diag(p(1 - p)), beta <- (X'WX)^{-1} X'WZ. Stops when max |change in beta| < tol.
    Returns beta, the inverse Fisher information (the estimated covariance), and the history of
    log-likelihoods and coefficient vectors."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    b = np.zeros(X.shape[1]) if beta0 is None else np.array(beta0, float)
    hist_l, hist_b = [loglik_logistic(X, y, b)], [b.copy()]
    for it in range(maxit):
        eta = X @ b
        p = common.sigmoid(eta)
        w = p * (1 - p)
        z = eta + (y - p) / w
        A = X.T @ (w[:, None] * X)
        bn = np.linalg.solve(A, X.T @ (w * z))
        done = np.max(np.abs(bn - b)) < tol
        b = bn
        hist_l.append(loglik_logistic(X, y, b))
        hist_b.append(b.copy())
        if done:
            break
    p = common.sigmoid(X @ b)
    info = X.T @ ((p * (1 - p))[:, None] * X)
    return dict(beta=b, cov=np.linalg.inv(info), info=info, loglik=hist_l[-1], hist_l=np.array(hist_l),
                hist_b=np.array(hist_b), iters=len(hist_l) - 1, p=p)


def mm_logistic(X, y, iters, beta0=None):
    """Bounded-curvature (minorise-maximise) iteration: since p(1 - p) <= 1/4, the Hessian is bounded
    below by -X'X/4, which gives the monotone update beta <- beta + 4 (X'X)^{-1} X'(y - p)."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    b = np.zeros(X.shape[1]) if beta0 is None else np.array(beta0, float)
    G = 4.0 * np.linalg.inv(X.T @ X)
    ls = [loglik_logistic(X, y, b)]
    for _ in range(iters):
        b = b + G @ (X.T @ (y - common.sigmoid(X @ b)))
        ls.append(loglik_logistic(X, y, b))
    return b, np.array(ls)


def grad_logistic(X, y, iters, beta0=None):
    """Gradient ascent with the fixed step 4 / lambda_max(X'X), which also increases the likelihood
    at every step."""
    X, y = np.asarray(X, float), np.asarray(y, float)
    b = np.zeros(X.shape[1]) if beta0 is None else np.array(beta0, float)
    step = 4.0 / np.linalg.eigvalsh(X.T @ X)[-1]
    ls = [loglik_logistic(X, y, b)]
    for _ in range(iters):
        b = b + step * (X.T @ (y - common.sigmoid(X @ b)))
        ls.append(loglik_logistic(X, y, b))
    return b, np.array(ls)


def logistic_table(X, y):
    """Example 13.17: beta_j, se_j (square roots of the diagonal of the inverse Fisher information),
    W_j, the Wald p-value, and the likelihood ratio statistic and p-value for dropping column j.
    X includes the intercept column."""
    X = np.asarray(X, float)
    f = irls(X, y)
    se = np.sqrt(np.diag(f["cov"]))
    W = f["beta"] / se
    lrt = np.zeros(X.shape[1])
    for j in range(X.shape[1]):
        g = irls(np.delete(X, j, axis=1), y)
        lrt[j] = 2 * (f["loglik"] - g["loglik"])
    return dict(beta=f["beta"], se=se, W=W, p=2 * stats.norm.sf(np.abs(W)), lrt=lrt,
                p_lrt=stats.chi2.sf(lrt, 1), loglik=f["loglik"], fit=f)


def aic_logistic(X, y, cols):
    """R's AIC for a logistic model: -2 loglik + 2 (number of coefficients), intercept included."""
    Xs = add_intercept(X[:, cols]) if len(cols) else np.ones((len(y), 1))
    return -2 * irls(Xs, y)["loglik"] + 2 * Xs.shape[1]


def backward_aic_logistic(X, y):
    """Exercise 13.11: backward stepwise logistic regression with AIC. Returns the steps as in
    stepwise()."""
    k = X.shape[1]
    cur = list(range(k))
    steps = []
    while True:
        here = aic_logistic(X, y, cur)
        moves = sorted(((j, aic_logistic(X, y, [c for c in cur if c != j])) for j in cur), key=lambda t: t[1])
        steps.append((list(cur), here, moves))
        if not moves or moves[0][1] >= here:
            break
        cur = [c for c in cur if c != moves[0][0]]
    return steps


def boot_logistic(X, y, R, seed, tol=1e-10):
    """Parametric bootstrap of the logistic coefficients, number for number with the CORIS demo:
    fit beta_hat; replicate r draws y*_i = 1{u < p_i} with the next n uniforms of DK.rng(seed) and
    refits by irls() started at beta_hat. Returns the R x k array of refitted coefficients."""
    f = irls(X, y)
    p = f["p"]
    n = len(y)
    U = js_uniforms(seed, R * n).reshape(R, n)
    out = np.zeros((R, X.shape[1]))
    for r in range(R):
        ys = (U[r] < p).astype(float)
        out[r] = irls(X, ys, beta0=f["beta"], tol=tol)["beta"]
    return out


# =========================================================================== Exercise 13.9: AIC and BIC
def aic_bic_select(n, theta, crit="aic"):
    """P(J_n = 0): the probability that the model N(0, 1) is chosen over N(theta, 1), theta free.
    AIC1 - AIC0 = n Xbar^2 / 2 - 1 and BIC1 - BIC0 = n Xbar^2 / 2 - log(n) / 2, so M0 is chosen when
    n Xbar^2 < c with c = 2 (AIC) or log n (BIC); sqrt(n) Xbar ~ N(sqrt(n) theta, 1)."""
    c = 2.0 if crit == "aic" else math.log(n)
    m = math.sqrt(n) * theta
    return float(stats.norm.cdf(math.sqrt(c) - m) - stats.norm.cdf(-math.sqrt(c) - m))


def kl_risk(n, theta, crit="aic"):
    """E D(phi_theta, f_n_hat): D(phi_a, phi_b) = (a - b)^2 / 2; the selected density is phi_0 if
    n Xbar^2 < c, else phi_Xbar."""
    c = 2.0 if crit == "aic" else math.log(n)
    sd = 1.0 / math.sqrt(n)
    a = math.sqrt(c / n)
    # chosen M0 on |xbar| < a: loss theta^2/2; otherwise (xbar - theta)^2/2
    p0 = stats.norm.cdf((a - theta) / sd) - stats.norm.cdf((-a - theta) / sd)
    # E[(xbar - theta)^2 1{|xbar| >= a}] = sd^2 - E[(xbar-theta)^2 1{|xbar|<a}]
    lo, hi = (-a - theta) / sd, (a - theta) / sd
    inner = sd ** 2 * ((stats.norm.cdf(hi) - stats.norm.cdf(lo)) - (hi * stats.norm.pdf(hi) - lo * stats.norm.pdf(lo)))
    return 0.5 * theta ** 2 * p0 + 0.5 * (sd ** 2 - inner)


# =========================================================================== figures
def fig_13_1(out_dir=None):
    """Figure 13.1: log surface temperature against log light intensity, with the least squares line."""
    x, y = stars()
    f = simple_fit(x, y)
    fig, ax = common.panel_grid(1, 1, size=(4.2, 3.6))
    ax = ax[0, 0]
    ax.plot(x, y, "o", color="k", ms=3.2)
    xx = np.linspace(3.85, 5.85, 10)
    ax.plot(xx, f["b0"] + f["b1"] * xx, color=RED, lw=1.4)
    ax.set_xlim(3.85, 5.85)
    ax.set_ylim(3.98, 4.64)
    ax.set_xlabel("log light intensity (X)")
    ax.set_ylabel("log surface temperature (Y)")
    ax.text(0.03, 0.95, r"$\widehat r(x)=%.3f+%.3f\,x$" % (f["b0"], f["b1"]), transform=ax.transAxes, va="top", fontsize=9)
    fig.tight_layout()
    return savefig(fig, CH, "fig13-1", out_dir)


def election_fits():
    d = florida()
    keep = (d.county != "Palm Beach").to_numpy()
    raw = simple_fit(d.bush[keep], d.buchanan[keep])
    lg = simple_fit(np.log(d.bush[keep]), np.log(d.buchanan[keep]))
    return d, keep, raw, lg


def fig_13_2(out_dir=None):
    """Figure 13.2: Buchanan against Bush votes by county, raw and log scale, with residuals."""
    d, keep, raw, lg = election_fits()
    fig, axes = common.panel_grid(2, 2, size=(3.4, 2.7))
    for row, (f, tr, lab) in enumerate(((raw, lambda v: v, ""), (lg, np.log, "log "))):
        x, y = tr(d.bush.to_numpy(float)), tr(d.buchanan.to_numpy(float))
        ax = axes[row, 0]
        ax.plot(x[keep], y[keep], "o", color="k", ms=2.4)
        ax.plot(x[~keep], y[~keep], "o", color=RED, ms=3.5)
        ax.annotate("Palm Beach", (x[~keep][0], y[~keep][0]), xytext=(-8, -2), textcoords="offset points",
                    ha="right", va="center", fontsize=7.5, color=RED)
        xx = np.linspace(x.min(), x.max(), 10)
        ax.plot(xx, f["b0"] + f["b1"] * xx, color=BLUE, lw=1.3)
        ax.set_xlabel(lab + "Bush")
        ax.set_ylabel(lab + "Buchanan")
        ax = axes[row, 1]
        ax.plot(x[keep], f["res"], "o", color="k", ms=2.4)
        ax.axhline(0, color=GREY, ls="--", lw=1)
        ax.set_xlabel(lab + "Bush")
        ax.set_ylabel("residuals")
        if row == 0:
            axes[row, 0].ticklabel_format(style="plain")
            axes[row, 1].ticklabel_format(style="plain")
    for a, l in zip(axes.ravel(), "abcd"):
        common.panel_label(a, l, loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-2", out_dir)


def fig_13_3(out_dir=None):
    """Figure 13.3: the logistic function e^x / (1 + e^x)."""
    fig, ax = common.panel_grid(1, 1, size=(4.0, 2.6))
    ax = ax[0, 0]
    x = np.linspace(-6, 6, 400)
    ax.plot(x, common.sigmoid(x), color="k", lw=1.6)
    ax.axhline(0, color=GREY, lw=0.6)
    ax.axhline(1, color=GREY, lw=0.6, ls="--")
    ax.axhline(0.5, color=GREY, lw=0.6, ls=":")
    ax.set_xlim(-6, 6)
    ax.set_ylim(-0.05, 1.05)
    ax.set_yticks([0, 0.5, 1])
    ax.set_xlabel("$x$")
    ax.set_ylabel(r"$p=e^x/(1+e^x)$")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-3", out_dir)


def fig_13_s1(out_dir=None):
    """Supplementary: confidence and prediction bands for the election data on the log scale, with
    Palm Beach; and the coverage of the two intervals for a new response (Exercise 13.10)."""
    d, keep, raw, lg = election_fits()
    x, y = np.log(d.bush.to_numpy(float)), np.log(d.buchanan.to_numpy(float))
    fig, axes = common.panel_grid(1, 2, size=(4.1, 3.2))
    ax = axes[0, 0]
    xx = np.linspace(6.5, 13.0, 300)
    yh, sm, xi = predict(lg, xx)
    z = stats.norm.ppf(0.975)
    ax.fill_between(xx, yh - z * xi, yh + z * xi, color=BLUE, alpha=0.12, lw=0, label="95% prediction band")
    ax.fill_between(xx, yh - z * sm, yh + z * sm, color=BLUE, alpha=0.35, lw=0, label="95% band for the mean")
    ax.plot(xx, yh, color=BLUE, lw=1.2)
    ax.plot(x[keep], y[keep], "o", color="k", ms=2.2)
    ax.plot(x[~keep], y[~keep], "o", color=RED, ms=4)
    xs = x[~keep][0]
    ax.annotate("Palm Beach", (xs, y[~keep][0]), xytext=(-8, 0), textcoords="offset points", ha="right",
                va="center", fontsize=8, color=RED)
    ax.set_xlabel("log Bush")
    ax.set_ylabel("log Buchanan")
    ax.legend(fontsize=7.5, frameon=False, loc="lower right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    xg = np.linspace(6.5, 13.0, 200)
    for zz, col in ((2.0, RED), (1.0, ORANGE)):
        ax.plot(xg, naive_coverage(lg, xg, zz), color=col, label=r"$\widehat Y_*\pm%g\,s$" % zz)
        ax.axhline(2 * stats.norm.cdf(zz) - 1, color=col, ls=":", lw=1)
    xsim = np.array([7.0, 8.5, 10.0, 11.5, 12.5])
    for xv in xsim:
        r2 = pred_coverage_sim(x[keep], lg["b0"], lg["b1"], lg["sigma"], xv, 2.0, 4000, 13)
        ax.plot(xv, r2["naive"], "o", color=RED, ms=4)
        ax.plot(xv, r2["pred"], "s", color=BLUE, ms=4)
    ax.plot([], [], "s", color=BLUE, ms=4, label=r"$\widehat Y_*\pm2\,\widehat\xi_n$ (simulated)")
    ax.set_ylim(0, 1.02)
    ax.set_xlabel(r"$x_*$ (log Bush)")
    ax.set_ylabel(r"coverage of a new $Y_*$")
    ax.legend(fontsize=7.5, frameon=False, loc="center right")
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S1", out_dir)


def crime_truth():
    """The 'truth' of the optimism experiment: the fitted 6-variable model of Example 13.16."""
    X, y, names = crime()
    sel = [names.index(v) for v in ("Age", "Education", "Expenditure", "Males", "U1", "U2")]
    f = ols(add_intercept(X[:, sel]), y)
    return X, f["fitted"], math.sqrt(f["s2"]), names


def crime_nested():
    """Nested models: covariates added in decreasing order of |W| in the full model."""
    X, y, names = crime()
    _, order, _, _ = zheng_loh(X, y)
    return [sorted(order[:j]) for j in range(X.shape[1] + 1)], order


def fig_13_s2(out_dir=None):
    """Supplementary: Theorem 13.15 on the crime design; expected training error, prediction risk,
    and the averages of Cp and leave-one-out cross-validation."""
    X, mu, sigma, names = crime_truth()
    nested, order = crime_nested()
    ex = optimism_exact(X, mu, sigma, nested)
    sim = optimism_sim(X, mu, sigma, nested, R=2000, seed=13)
    fig, axes = common.panel_grid(1, 2, size=(4.1, 3.1))
    ax = axes[0, 0]
    p = ex[:, 0]
    ax.plot(p, ex[:, 2], color="k", lw=1.6, label=r"prediction risk $R(S)$")
    ax.plot(p, ex[:, 1], color=RED, lw=1.4, label=r"$E\,\widehat R_{tr}(S)$")
    ax.plot(p, sim[:, 0], "o", color=RED, ms=3)
    ax.plot(p, sim[:, 1], "s", color=BLUE, ms=3.5, label=r"mean of $C_p$ (13.26)")
    ax.plot(p, sim[:, 2], "^", color=GREEN, ms=3.5, label="mean of LOO CV (13.29)")
    ax.set_xlabel(r"$|S|$ (intercept + covariates by $|W_j|$)")
    ax.set_ylabel("sum of squared errors")
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    ax.plot(p, (ex[:, 2] - ex[:, 1]) / sigma ** 2, color="k", lw=1.4, label=r"exact: $2|S|$")
    ax.plot(p, (ex[:, 2] - sim[:, 0]) / sigma ** 2, "o", color=RED, ms=3.5, label="simulated optimism")
    ax.set_xlabel(r"$|S|$")
    ax.set_ylabel(r"$\{R(S)-E\,\widehat R_{tr}(S)\}/\sigma^2$")
    ax.legend(fontsize=7.5, frameon=False, loc="upper left", bbox_to_anchor=(0.08, 1))
    common.panel_label(ax, "b", loc="lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S2", out_dir)


def fig_13_s3(out_dir=None):
    """Supplementary: all-subsets scores for the crime data (Cp, BIC on the Cp scale and LOO CV), the
    backward AIC search of Example 13.16, and Cp against BIC for the car data (Exercise 13.7)."""
    X, y, names = crime()
    rows = subset_scores(X, y)
    s2 = full_s2(X, y)
    n = len(y)
    fig, axes = common.panel_grid(1, 3, size=(3.5, 3.1))
    ax = axes[0, 0]
    P = np.array([r["p"] for r in rows])
    jit = (np.random.default_rng(1).random(len(P)) - 0.5) * 0.35
    ax.plot(P + jit, [r["cp"] / 1e4 for r in rows], ".", color=GREY, ms=2, alpha=0.5)
    sizes = sorted(set(P))
    for key, col, lab, conv in (("cp", BLUE, r"$C_p$", lambda r: r["cp"]),
                                ("bic", ORANGE, r"BIC as $RSS+|S|\widehat\sigma^2\log n$", lambda r: r["rss"] + r["p"] * s2 * math.log(n)),
                                ("cv", GREEN, "LOO CV", lambda r: r["cv"])):
        v = [min(conv(r) for r in rows if r["p"] == s) / 1e4 for s in sizes]
        ax.plot(sizes, v, "-o", color=col, ms=3, lw=1.3, label=lab)
        j = int(np.argmin(v))
        ax.plot(sizes[j], v[j], "*", color=col, ms=11, mec="k", mew=0.5)
    ax.set_xlabel(r"$|S|$")
    ax.set_ylabel(r"score / $10^4$")
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    steps = stepwise(X, y, "aicR", "backward")
    cur_vals = [s[1] for s in steps]
    for i, (cur, here, moves) in enumerate(steps):
        ax.plot([i] * len(moves), [m[1] for m in moves], "_", color=GREY, ms=9)
    ax.plot(range(len(steps)), cur_vals, "-o", color=BLUE, ms=4)
    for i in range(len(steps) - 1):
        ax.annotate("$-$" + names[steps[i][2][0][0]], (i + 0.5, (cur_vals[i] + cur_vals[i + 1]) / 2), fontsize=7,
                    ha="left", va="bottom", color=BLUE, rotation=0)
    ax.set_xlabel("backward step")
    ax.set_ylabel(r"AIC $= n\log(RSS/n)+2|S|$")
    ax.set_ylim(304, 330)
    common.panel_label(ax, "b", loc="upper left")
    ax = axes[0, 2]
    car = carmileage()
    Xc = car[["VOL", "HP", "SP", "WT"]].to_numpy(float)
    rc = subset_scores(Xc, car.MPG.to_numpy(float))
    s2c = full_s2(Xc, car.MPG.to_numpy(float))
    nc = len(car)
    Pc = np.array([r["p"] for r in rc])
    ax.plot(Pc - 0.08, [r["cp"] for r in rc], "o", color=BLUE, ms=3.5, label=r"$C_p$")
    ax.plot(Pc + 0.08, [r["rss"] + r["p"] * s2c * math.log(nc) for r in rc], "s", color=ORANGE, ms=3.5, label="BIC (Cp scale)")
    ax.set_xlabel(r"$|S|$ (car data, all 16 subsets)")
    ax.set_ylabel("score")
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "c", loc="lower left")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S3", out_dir)


def fig_13_s4(out_dir=None):
    """Supplementary: Exercise 13.9, AIC against BIC for N(0,1) versus N(theta,1)."""
    ns = np.unique(np.round(np.logspace(0.5, 4, 60)).astype(int))
    fig, axes = common.panel_grid(1, 2, size=(4.1, 3.1))
    ax = axes[0, 0]
    for th, col in ((0.0, "k"), (0.1, BLUE), (0.3, RED)):
        ax.plot(ns, [aic_bic_select(n, th, "aic") for n in ns], color=col, label=r"AIC, $\theta=%g$" % th)
        ax.plot(ns, [aic_bic_select(n, th, "bic") for n in ns], color=col, ls="--", label=r"BIC, $\theta=%g$" % th)
    ax.axhline(stats.chi2.cdf(2, 1), color=GREY, lw=0.8, ls=":")
    ax.set_xscale("log")
    ax.set_xlabel("$n$")
    ax.set_ylabel(r"$P(J_n=0)$")
    ax.legend(fontsize=6.5, frameon=False, loc="center left", ncol=1)
    common.panel_label(ax, "a", loc="lower right")
    ax = axes[0, 1]
    for th, col in ((0.0, "k"), (0.1, BLUE), (0.3, RED)):
        ax.plot(ns, [n * kl_risk(n, th, "aic") for n in ns], color=col)
        ax.plot(ns, [n * kl_risk(n, th, "bic") for n in ns], color=col, ls="--")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_xlabel("$n$")
    ax.set_ylabel(r"$n\times E\,D(\phi_\theta,\widehat f_n)$")
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S4", out_dir)


def fig_13_s5(out_dir=None):
    """Supplementary: logistic regression on CORIS; convergence of Newton (IRLS) against the
    bounded-curvature and gradient algorithms, and the fitted probability of chd against age."""
    X, y, names = coris()
    Xr = add_intercept(X)
    Z = (X - X.mean(0)) / X.std(0)
    Xs = add_intercept(Z)
    fig, axes = common.panel_grid(1, 2, size=(4.1, 3.1))
    ax = axes[0, 0]
    for XX, ls_, lab in ((Xs, "-", "standardised"), (Xr, "--", "raw units")):
        f = irls(XX, y, tol=1e-13)
        lmax = f["loglik"]
        gap = np.maximum(lmax - f["hist_l"], 1e-16)
        ax.plot(np.arange(len(gap)) + 1, gap, ls_, color=RED, marker="o", ms=3, label="Newton (IRLS), " + lab)
        _, lm = mm_logistic(XX, y, 1000)
        ax.plot(np.arange(len(lm)) + 1, np.maximum(lmax - lm, 1e-16), ls_, color=BLUE, lw=1.2, label="bounded curvature, " + lab)
        _, lg = grad_logistic(XX, y, 1000)
        ax.plot(np.arange(len(lg)) + 1, np.maximum(lmax - lg, 1e-16), ls_, color=GREEN, lw=1.2, label="gradient ascent, " + lab)
    ax.set_yscale("log")
    ax.set_xscale("log")
    ax.set_ylim(1e-13, 1e3)
    ax.set_xlim(1, 1001)
    ax.set_xlabel("iteration + 1")
    ax.set_ylabel(r"$\ell(\widehat\beta)-\ell(\beta^{s})$")
    ax.legend(fontsize=6, frameon=False, loc="lower left")
    common.panel_label(ax, "a", loc="upper right")
    ax = axes[0, 1]
    age = X[:, names.index("age")]
    A = add_intercept(age)
    f = irls(A, y)
    bins = np.arange(15, 70, 5)
    mid, prop, nb = [], [], []
    for lo in bins[:-1]:
        m = (age >= lo) & (age < lo + 5)
        if m.sum():
            mid.append(lo + 2.5)
            prop.append(y[m].mean())
            nb.append(m.sum())
    aa = np.linspace(14, 65, 200)
    eta = f["beta"][0] + f["beta"][1] * aa
    se = np.sqrt(f["cov"][0, 0] + aa ** 2 * f["cov"][1, 1] + 2 * aa * f["cov"][0, 1])
    z = stats.norm.ppf(0.975)
    ax.fill_between(aa, common.sigmoid(eta - z * se), common.sigmoid(eta + z * se), color=RED, alpha=0.18, lw=0)
    ax.plot(aa, common.sigmoid(eta), color=RED, label="logistic fit: logit p = %.2f + %.4f age" % tuple(f["beta"]))
    jit = (np.random.default_rng(2).random(len(y)) - 0.5) * 0.06
    ax.plot(age, y + jit, "|", color=GREY, ms=4, alpha=0.6)
    ax.plot(mid, prop, "o", color="k", ms=4, label="proportion with chd (5-year bins)")
    ax.set_xlabel("age")
    ax.set_ylabel("P(chd = 1)")
    ax.set_ylim(-0.06, 1.06)
    ax.legend(fontsize=6.5, frameon=False, loc="upper left")
    common.panel_label(ax, "b", loc="lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S5", out_dir)


def fig_13_s6(out_dir=None):
    """Supplementary: Exercise 13.6, MPG and log(MPG) against horsepower, with residuals."""
    car = carmileage()
    x = car.HP.to_numpy(float)
    fig, axes = common.panel_grid(2, 2, size=(3.4, 2.6))
    for row, (yy, lab) in enumerate(((car.MPG.to_numpy(float), "MPG"), (np.log(car.MPG.to_numpy(float)), "log MPG"))):
        f = simple_fit(x, yy)
        ax = axes[row, 0]
        ax.plot(x, yy, "o", color="k", ms=2.4)
        xx = np.linspace(40, 330, 10)
        ax.plot(xx, f["b0"] + f["b1"] * xx, color=BLUE)
        ax.set_xlabel("horsepower")
        ax.set_ylabel(lab)
        ax = axes[row, 1]
        ax.plot(f["b0"] + f["b1"] * x, f["res"], "o", color="k", ms=2.4)
        ax.axhline(0, color=GREY, ls="--", lw=1)
        ax.set_xlabel("fitted " + lab)
        ax.set_ylabel("residuals")
    for a, l in zip(axes.ravel(), "abcd"):
        common.panel_label(a, l, loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig13-S6", out_dir)


# =========================================================================== JSON for the demos
def export_json():
    """Write the data of the three demonstrations to ../data/ch13-*.json."""
    out = os.path.normpath(os.path.join(HERE, "..", "data"))
    os.makedirs(out, exist_ok=True)
    d = florida()
    xs, ys = stars()
    car = carmileage()
    pred = {
        "election": {"county": d.county.tolist(), "bush": d.bush.astype(int).tolist(),
                     "buchanan": d.buchanan.astype(int).tolist()},
        "stars": {"x": [round(v, 2) for v in xs], "y": [round(v, 2) for v in ys]},
        "cars": {"hp": car.HP.astype(int).tolist(), "mpg": car.MPG.round(1).tolist()},
    }
    X, y, names = crime()
    sel = {
        "crime": {"names": names, "X": X.astype(int).tolist(), "y": y.round(1).tolist()},
        "cars": {"names": ["VOL", "HP", "SP", "WT"], "X": car[["VOL", "HP", "SP", "WT"]].to_numpy().tolist(),
                 "y": car.MPG.round(1).tolist()},
    }
    Xc, yc, nc = coris()
    cor = {"names": nc, "X": [[float(v) for v in r] for r in Xc], "y": yc.astype(int).tolist()}
    for name, obj in (("ch13-prediction.json", pred), ("ch13-selection.json", sel), ("ch13-coris.json", cor)):
        with open(os.path.join(out, name), "w") as f:
            json.dump(obj, f, separators=(",", ":"))
        print("wrote", os.path.join(out, name), os.path.getsize(os.path.join(out, name)), "bytes")


FIGURES = {"13.1": fig_13_1, "13.2": fig_13_2, "13.3": fig_13_3, "13.S1": fig_13_s1, "13.S2": fig_13_s2,
           "13.S3": fig_13_s3, "13.S4": fig_13_s4, "13.S5": fig_13_s5, "13.S6": fig_13_s6}

if __name__ == "__main__":
    export_json()
    common.run(FIGURES, CH)
