"""Chapter 9 lab: support vector machines (Python version).

Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie, Tibshirani
and Taylor (2023), An Introduction to Statistical Learning with Applications in Python, Section 9.6,
and the applied exercises of Section 9.7.

The code is my own, written with numpy, pandas, statsmodels, matplotlib and scikit-learn (not the
book's ISLP package). It asks the same questions of the same data as the Python lab of the book:
  9.6.1 the support vector classifier on 50 simulated points (C = 10 and 0.1), its coefficients,
        five-fold cross-validation over C, test predictions, and a barely separable data set;
  9.6.2 the support vector machine with a radial kernel, and cross-validation over C and gamma;
  9.6.3 ROC curves on the training and test halves;
  9.6.4 three classes, one-versus-one;
  9.6.5 the Khan gene-expression data (four tumour types, 2308 genes).
The random seeds are those of the book's Python lab, so the printed numbers are the book's.

Both editions fit with the same solver: R's e1071::svm() and scikit-learn's SVC both call LIBSVM.
The R lab simulates different data (R's random numbers), so the section "The R lab's data in
Python" reads the R lab's data (written by ch09_lab.R to cache/) and refits the R lab's models here:
the support vectors, coefficients and tables agree. The Khan data come from the R package ISLR2,
exported once by ch09_lab.R to cache/.

Run: python3 ch09_lab.py        (about 20 seconds; it runs ch09_lab.R first if cache/ lacks its data)
Figures go to ../img/ch09/lab-py-*.svg and the key numbers to cache/ch09_lab_py.csv.
"""
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import subprocess

import numpy as np
import pandas as pd
import statsmodels.api as sm
import sklearn.model_selection as skm
from scipy.stats import norm
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import SVC

import common
import matplotlib.pyplot as plt

np.set_printoptions(precision=4, suppress=True)
HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.join(HERE, "cache")
KEY = {}                       # key numbers, saved for the course's own records


def keep(name, value):
    KEY[name] = np.atleast_1d(np.asarray(value, dtype=float)).ravel()


# ======================================================================== helpers
def confusion_table(predicted, truth):
    """Counts of (predicted, true) labels, predicted classes in the rows."""
    return pd.crosstab(pd.Series(predicted, name="Predicted"), pd.Series(truth, name="Truth"))


def linear_coefficients(model):
    """beta_0 and beta of a fitted linear SVC from its dual solution: beta = sum_i (alpha_i y_i) x_i,
    the representation (9.18)-(9.19); dual_coef_ holds alpha_i y_i for the support vectors."""
    beta = (model.dual_coef_ @ model.support_vectors_).ravel()
    return model.intercept_[0], beta


def plot_svm(X, y, model, ax, title=None, colours=("#7b4fbf", "#1f5fbf", "#2ca02c")):
    """Shade the predicted class over a grid, draw the margins of a two-class fit (f = -1, 0, 1)
    and mark the support vectors with a plus sign."""
    lo, hi = X.min(0) - 0.5, X.max(0) + 0.5
    g1, g2 = np.meshgrid(np.linspace(lo[0], hi[0], 300), np.linspace(lo[1], hi[1], 300))
    G = np.column_stack([g1.ravel(), g2.ravel()])
    classes = list(model.classes_)
    pred = np.searchsorted(classes, model.predict(G)).reshape(g1.shape)
    from matplotlib.colors import ListedColormap
    cmap = ListedColormap([c for c in colours[:len(classes)]])
    ax.contourf(g1, g2, pred, levels=np.arange(len(classes) + 1) - 0.5, cmap=cmap, alpha=0.12)
    if len(classes) == 2:
        f = model.decision_function(G).reshape(g1.shape)
        ax.contour(g1, g2, f, levels=[-1, 0, 1], colors="k", linestyles=["--", "-", "--"], linewidths=[1, 1.5, 1])
    sv = np.zeros(len(y), bool)
    sv[model.support_] = True
    for k, c in enumerate(classes):
        m = (np.asarray(y) == c)
        ax.scatter(X[m & ~sv, 0], X[m & ~sv, 1], s=22, color=colours[k], edgecolors="none")
        ax.scatter(X[m & sv, 0], X[m & sv, 1], s=60, color=colours[k], marker="+", linewidths=1.6)
    ax.set(xlabel="$X_1$", ylabel="$X_2$", title=title)


def e1071_scale(Xtrain):
    """The scaling e1071::svm() applies by default (scale = TRUE): centre and divide by the standard
    deviation (divisor n - 1) of the training data; returns a function applying it to any rows."""
    m, s = Xtrain.mean(0), Xtrain.std(0, ddof=1)
    return lambda Z: (np.asarray(Z, float) - m) / s


def qrand(n, k, start=1):
    """n numbers in [0, 1) from stream k: frac(i * sqrt(p_k)), i = start, ..., start + n - 1, p_k the
    k-th prime (a Weyl sequence). The exercises use it so that R and Python get identical data."""
    a = np.sqrt([2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37][k - 1]) % 1
    return (np.arange(start, start + n) * a) % 1


def cv_error(make_model, X, y, K=10):
    """K-fold cross-validated misclassification rate, row i in fold (i mod K) (as in the R lab)."""
    fold = np.arange(len(y)) % K
    wrong = 0
    for k in range(K):
        m = make_model().fit(X[fold != k], y[fold != k])
        wrong += np.sum(m.predict(X[fold == k]) != y[fold == k])
    return wrong / len(y)


def read_r_data():
    """The R lab's data (simulated with R's random numbers) and the Khan data, written by ch09_lab.R."""
    need = ["ch09_rlab_linear.csv", "ch09_rlab_radial.csv", "ch09_khan_xtrain.csv"]
    if not all(os.path.exists(os.path.join(CACHE, f)) for f in need):
        subprocess.run(["/usr/local/bin/Rscript", "ch09_lab.R"], cwd=HERE, check=True, capture_output=True)
    rd = {f[10:-4]: pd.read_csv(os.path.join(CACHE, f)) for f in
          ["ch09_rlab_linear.csv", "ch09_rlab_lintest.csv", "ch09_rlab_radial.csv", "ch09_rlab_three.csv"]}
    khan = {k: pd.read_csv(os.path.join(CACHE, "ch09_khan_%s.csv" % k)).to_numpy() for k in ("xtrain", "xtest", "ytrain", "ytest")}
    khan["ytrain"], khan["ytest"] = khan["ytrain"].ravel(), khan["ytest"].ravel()
    return rd, khan


# ======================================================================== 9.6.1 support vector classifier
def lab_svc():
    rng = np.random.default_rng(1)
    X = rng.standard_normal((50, 2))
    y = np.array([-1] * 25 + [1] * 25)
    X[y == 1] += 1                                  # the classes overlap: not linearly separable

    svm_linear = SVC(C=10, kernel="linear").fit(X, y)
    svm_linear_small = SVC(C=0.1, kernel="linear").fit(X, y)
    b0, beta = linear_coefficients(svm_linear)
    print("C = 10:  beta =", beta, " beta_0 = %.4f" % b0, " coef_ =", svm_linear.coef_.ravel())
    print("support vectors per class: C = 10", svm_linear.n_support_, "  C = 0.1", svm_linear_small.n_support_)
    print("margin 1/||beta||: C = 10 %.4f,  C = 0.1 %.4f" % (1 / np.linalg.norm(beta),
                                                          1 / np.linalg.norm(linear_coefficients(svm_linear_small)[1])))
    keep("py_svc10_coef", beta); keep("py_svc10_nsv", svm_linear.n_support_); keep("py_svc01_nsv", svm_linear_small.n_support_)

    fig, ax = plt.subplots(1, 2, figsize=(9, 4.3))
    plot_svm(X, y, svm_linear, ax[0], "C = 10")
    plot_svm(X, y, svm_linear_small, ax[1], "C = 0.1")
    common.savefig(fig, 9, "lab-py-svc", formats=("svg",))

    kfold = skm.KFold(5, random_state=0, shuffle=True)
    grid = skm.GridSearchCV(svm_linear, {"C": [0.001, 0.01, 0.1, 1, 5, 10, 100]},
                            refit=True, cv=kfold, scoring="accuracy").fit(X, y)
    print("best C:", grid.best_params_, " CV accuracy:", grid.cv_results_["mean_test_score"])
    keep("py_svc_cv_acc", grid.cv_results_["mean_test_score"])

    X_test = rng.standard_normal((20, 2))
    y_test = np.array([-1] * 10 + [1] * 10)
    X_test[y_test == 1] += 1
    best_ = grid.best_estimator_
    print(confusion_table(best_.predict(X_test), y_test))
    svm_ = SVC(C=0.001, kernel="linear").fit(X, y)
    print(confusion_table(svm_.predict(X_test), y_test))
    keep("py_svc_test_right", [np.sum(best_.predict(X_test) == y_test), np.sum(svm_.predict(X_test) == y_test)])

    X[y == 1] += 1.9                                # now the classes are (just) linearly separable
    svm_ = SVC(C=1e5, kernel="linear").fit(X, y)
    print(confusion_table(svm_.predict(X), y))
    svm_small = SVC(C=0.1, kernel="linear").fit(X, y)
    print("separable data: support vectors with C = 1e5:", svm_.n_support_.sum(), " with C = 0.1:", svm_small.n_support_.sum())
    print("training errors: %d and %d" % (np.sum(svm_.predict(X) != y), np.sum(svm_small.predict(X) != y)))
    print("margin 1/||beta||: C = 1e5 %.4f,  C = 0.1 %.4f" % (1 / np.linalg.norm(linear_coefficients(svm_)[1]),
                                                           1 / np.linalg.norm(linear_coefficients(svm_small)[1])))
    keep("py_sep_nsv", [svm_.n_support_.sum(), svm_small.n_support_.sum()])


# ======================================================================== 9.6.2 and 9.6.3 radial kernel, ROC curves
def lab_svm_roc():
    rng = np.random.default_rng(1)
    rng.standard_normal((50, 2)); rng.standard_normal((20, 2))   # the draws of Section 9.6.1 come first
    X = rng.standard_normal((200, 2))
    X[:100] += 2
    X[100:150] -= 2
    y = np.array([1] * 150 + [2] * 50)              # class 2 sits between the two halves of class 1
    X_train, X_test, y_train, y_test = skm.train_test_split(X, y, test_size=0.5, random_state=0)

    svm_rbf = SVC(kernel="rbf", gamma=1, C=1).fit(X_train, y_train)
    svm_rbf_big = SVC(kernel="rbf", gamma=1, C=1e5).fit(X_train, y_train)
    print("radial, gamma = 1: support vectors C = 1:", svm_rbf.n_support_, " C = 1e5:", svm_rbf_big.n_support_)
    print("training errors: C = 1: %d, C = 1e5: %d" % (np.sum(svm_rbf.predict(X_train) != y_train),
                                                      np.sum(svm_rbf_big.predict(X_train) != y_train)))
    keep("py_rbf_nsv", svm_rbf.n_support_)
    fig, ax = plt.subplots(1, 2, figsize=(9, 4.3))
    plot_svm(X_train, y_train, svm_rbf, ax[0], r"radial, $\gamma$ = 1, C = 1")
    plot_svm(X_train, y_train, svm_rbf_big, ax[1], r"radial, $\gamma$ = 1, C = $10^5$")
    common.savefig(fig, 9, "lab-py-radial", formats=("svg",))

    kfold = skm.KFold(5, random_state=0, shuffle=True)
    grid = skm.GridSearchCV(svm_rbf, {"C": [0.1, 1, 10, 100, 1000], "gamma": [0.5, 1, 2, 3, 4]},
                            refit=True, cv=kfold, scoring="accuracy").fit(X_train, y_train)
    print("best:", grid.best_params_)
    print(pd.DataFrame(grid.cv_results_["mean_test_score"].reshape(5, 5),
                       index=pd.Index([0.1, 1, 10, 100, 1000], name="C"), columns=pd.Index([0.5, 1, 2, 3, 4], name="gamma")))
    best_svm = grid.best_estimator_
    print(confusion_table(best_svm.predict(X_test), y_test))
    keep("py_rbf_cv_acc", grid.cv_results_["mean_test_score"])
    keep("py_rbf_best", [grid.best_params_["C"], grid.best_params_["gamma"]])
    keep("py_rbf_test_err", np.mean(best_svm.predict(X_test) != y_test))

    # ROC curves from the decision values f(x): positive values predict class 2
    svm_flex = SVC(kernel="rbf", gamma=50, C=1).fit(X_train, y_train)
    fig, ax = plt.subplots(1, 2, figsize=(9, 4.3))
    aucs = []
    for k, (Xs, ys, name) in enumerate([(X_train, y_train, "Training data"), (X_test, y_test, "Test data")]):
        for model, colour, lab in [(best_svm, "k", r"$\gamma$ = %g" % best_svm.gamma), (svm_flex, "r", r"$\gamma$ = 50")]:
            score = model.decision_function(Xs)
            fpr, tpr, _ = roc_curve(ys == 2, score)
            auc = roc_auc_score(ys == 2, score)
            aucs.append(auc)
            ax[k].plot(fpr, tpr, colour, label="%s (AUC %.3f)" % (lab, auc))
        ax[k].plot([0, 1], [0, 1], ":", color="0.6")
        ax[k].set(xlabel="False positive rate", ylabel="True positive rate", title=name)
        ax[k].legend(loc="lower right")
    common.savefig(fig, 9, "lab-py-roc", formats=("svg",))
    print("AUC train (best, gamma 50), test (best, gamma 50):", np.round(aucs, 4))
    keep("py_roc_auc", aucs)
    return X, y


def lab_multiclass(X, y):
    rng = np.random.default_rng(123)
    X = np.vstack([X, rng.standard_normal((50, 2))])
    y = np.hstack([y, [0] * 50])
    X[y == 0, 1] += 2
    svm_rbf_3 = SVC(kernel="rbf", C=10, gamma=1, decision_function_shape="ovo").fit(X, y)
    print("three classes: support vectors per class", svm_rbf_3.n_support_,
          " training error %.3f" % np.mean(svm_rbf_3.predict(X) != y))
    # one-versus-one by hand: the three pairwise votes give the same labels as predict()
    D = svm_rbf_3.decision_function(X)              # columns: (0 vs 1), (0 vs 2), (1 vs 2); positive = first class
    votes = np.zeros((len(y), 3))
    for col, (a, b) in enumerate([(0, 1), (0, 2), (1, 2)]):
        votes[:, a] += D[:, col] > 0
        votes[:, b] += D[:, col] <= 0
    print("hand-counted votes reproduce predict():", np.array_equal(svm_rbf_3.classes_[votes.argmax(1)], svm_rbf_3.predict(X)))
    keep("py_three_nsv", svm_rbf_3.n_support_)
    fig, ax = plt.subplots(figsize=(4.8, 4.3))
    plot_svm(X, y, svm_rbf_3, ax, "three classes, one-versus-one", colours=("#2ca02c", "#7b4fbf", "#1f5fbf"))
    common.savefig(fig, 9, "lab-py-three", formats=("svg",))


# ======================================================================== 9.6.5 gene expression
def lab_khan(khan):
    print("Khan:", khan["xtrain"].shape, khan["xtest"].shape)
    print("training classes:", np.bincount(khan["ytrain"])[1:], " test classes:", np.bincount(khan["ytest"])[1:])
    khan_linear = SVC(kernel="linear", C=10).fit(khan["xtrain"], khan["ytrain"])
    print(confusion_table(khan_linear.predict(khan["xtrain"]), khan["ytrain"]))
    print(confusion_table(khan_linear.predict(khan["xtest"]), khan["ytest"]))
    print("support vectors per class:", khan_linear.n_support_)
    keep("khan_raw_nsv", khan_linear.n_support_)
    keep("khan_raw_test_err", np.sum(khan_linear.predict(khan["xtest"]) != khan["ytest"]))
    # with every gene standardised first, as e1071::svm() does by default in the R lab
    sc = e1071_scale(khan["xtrain"])
    khan_scaled = SVC(kernel="linear", C=10).fit(sc(khan["xtrain"]), khan["ytrain"])
    print("standardised genes: support vectors", khan_scaled.n_support_, " test errors",
          np.sum(khan_scaled.predict(sc(khan["xtest"])) != khan["ytest"]))
    keep("khan_nsv", khan_scaled.n_support_)
    keep("khan_test_table", confusion_table(khan_scaled.predict(sc(khan["xtest"])), khan["ytest"]).to_numpy())
    # one-versus-all (Section 9.4.2): SVC itself always fits the K(K - 1)/2 pairwise classifiers, even with
    # decision_function_shape="ovr"; OneVsRestClassifier fits K "class k against the rest" classifiers
    ova = OneVsRestClassifier(SVC(kernel="linear", C=10)).fit(khan["xtrain"], khan["ytrain"])
    print("one-versus-all: test errors", np.sum(ova.predict(khan["xtest"]) != khan["ytest"]))
    keep("khan_ova_pred", ova.predict(khan["xtest"]))


# ======================================================================== the R lab's data in Python
def r_lab_in_python(rd):
    d = rd["linear"]
    X, y = d[["x1", "x2"]].to_numpy(), d["y"].to_numpy()
    fit10 = SVC(C=10, kernel="linear").fit(X, y)                       # svm(..., cost = 10, scale = FALSE)
    print("R lab data, cost 10: support vectors (R's numbering)", np.sort(fit10.support_) + 1)
    b0, beta = linear_coefficients(fit10)
    print("  beta_0 = %.6f, beta = %s, margin 1/||beta|| = %.4f" % (b0, beta, 1 / np.linalg.norm(beta)))
    keep("rlab_sv10", np.sort(fit10.support_) + 1); keep("rlab_coef10", np.r_[b0, beta])
    keep("rlab_sv01", np.sort(SVC(C=0.1, kernel="linear").fit(X, y).support_) + 1)
    t = rd["lintest"]
    Xt, yt = t[["x1", "x2"]].to_numpy(), t["y"].to_numpy()
    keep("rlab_test_right", [np.sum(SVC(C=c, kernel="linear").fit(X, y).predict(Xt) == yt) for c in (0.1, 0.01)])
    Xs = X.copy()
    Xs[y == 1] += 0.5                                                   # the barely separable version
    sc = e1071_scale(Xs)                                                # the R lab leaves scale = TRUE here
    big, one = SVC(C=1e5, kernel="linear").fit(sc(Xs), y), SVC(C=1, kernel="linear").fit(sc(Xs), y)
    keep("rlab_sep_nsv", [big.n_support_.sum(), one.n_support_.sum()])
    keep("rlab_sep_err", [np.sum(big.predict(sc(Xs)) != y), np.sum(one.predict(sc(Xs)) != y)])

    r = rd["radial"]
    X, y, pos = r[["x1", "x2"]].to_numpy(), r["y"].to_numpy(), r["train"].to_numpy()
    tr = np.argsort(np.where(pos > 0, pos, 10 ** 6))[:np.sum(pos > 0)]   # training rows in R's sample() order
    sc = e1071_scale(X[tr])
    m = SVC(kernel="rbf", gamma=1, C=1).fit(sc(X[tr]), y[tr])
    print("R lab radial data, gamma 1, cost 1: support vectors", m.n_support_)
    keep("rlab_rbf_nsv", m.n_support_)
    best = SVC(kernel="rbf", gamma=0.5, C=1).fit(sc(X[tr]), y[tr])   # the parameters R's tune() chose
    te = pos == 0
    print(confusion_table(best.predict(sc(X[te])), y[te]))
    keep("rlab_rbf_test_table", confusion_table(best.predict(sc(X[te])), y[te]).to_numpy())
    aucs = []
    for g in (2, 50):
        mg = SVC(kernel="rbf", gamma=g, C=1).fit(sc(X[tr]), y[tr])
        aucs += [roc_auc_score(y[tr] == 2, mg.decision_function(sc(X[tr]))), roc_auc_score(y[te] == 2, mg.decision_function(sc(X[te])))]
    keep("rlab_roc_auc", aucs)
    print("R lab ROC: AUC train/test gamma 2: %.4f %.4f, gamma 50: %.4f %.4f" % tuple(aucs))
    three = rd["three"]
    X3, y3 = three[["x1", "x2"]].to_numpy(), three["y"].to_numpy()
    sc3 = e1071_scale(X3)
    m3 = SVC(kernel="rbf", C=10, gamma=1).fit(sc3(X3), y3)
    keep("rlab_three_nsv", m3.n_support_); keep("rlab_three_err", np.sum(m3.predict(sc3(X3)) != y3))


# ======================================================================== exercises 4-8
def exercises():
    # Exercise 4: a curved boundary; 100 training and 100 test points
    n = 200
    x1, x2, z = 4 * qrand(n, 1) - 2, 4 * qrand(n, 2) - 2, norm.ppf(qrand(n, 3))
    y = np.where(x2 - (x1 ** 2 - 1.5) + 0.3 * z > 0, 1, -1)
    X, tr = np.column_stack([x1, x2]), np.arange(n) < 100
    ex4 = []
    for name, model in [("linear", SVC(kernel="linear", C=1)),
                        ("polynomial, d = 2", SVC(kernel="poly", degree=2, gamma=1, coef0=1, C=1)),
                        ("radial, gamma = 1", SVC(kernel="rbf", gamma=1, C=1))]:
        model.fit(X[tr], y[tr])
        e = [np.mean(model.predict(X[tr]) != y[tr]), np.mean(model.predict(X[~tr]) != y[~tr])]
        print("Exercise 4, %-18s training error %.2f, test error %.2f" % (name, *e))
        ex4 += e
    keep("ex4_err", ex4)

    # Exercise 5: logistic regression with non-linear features against an SVM
    n = 500
    x1, x2, z = qrand(n, 4) - 0.5, qrand(n, 5) - 0.5, norm.ppf(qrand(n, 6))
    y = (x1 ** 2 - x2 ** 2 + 0.02 * z > 0).astype(int)
    X = np.column_stack([x1, x2])
    lin = sm.GLM(y, sm.add_constant(X), family=sm.families.Binomial()).fit()
    Q = np.column_stack([x1, x2, x1 ** 2, x2 ** 2, x1 * x2])
    quad = sm.GLM(y, sm.add_constant(Q), family=sm.families.Binomial()).fit()
    svc = SVC(kernel="linear", C=1).fit(X, y)
    rbf = SVC(kernel="rbf", gamma=10, C=10).fit(X, y)
    e5 = [np.mean((lin.fittedvalues > 0.5) != y), np.mean((quad.fittedvalues > 0.5) != y),
          np.mean(svc.predict(X) != y), np.mean(rbf.predict(X) != y)]
    print("Exercise 5: training errors, logistic linear %.3f, logistic quadratic %.3f, SVC %.3f, radial SVM %.3f" % tuple(e5))
    print("  quadratic logistic coefficients:", quad.params.round(3))
    keep("ex5_err", e5); keep("ex5_quad_coef", quad.params)
    fig, ax = plt.subplots(1, 4, figsize=(12, 3.2), sharey=True)
    for a, lab, title in zip(ax, [lin.fittedvalues > 0.5, quad.fittedvalues > 0.5, svc.predict(X), rbf.predict(X)],
                             ["logistic, linear", "logistic, quadratic", "support vector classifier", "radial SVM"]):
        a.scatter(x1, x2, c=np.where(np.asarray(lab, int) == 1, "#1f5fbf", "#d62728"), s=6)
        a.set(title=title, xlabel="$X_1$")
    ax[0].set_ylabel("$X_2$")
    common.savefig(fig, 9, "lab-py-ex5", formats=("svg",))

    # Exercise 6: barely separable training data; two class-1 points sit just below the true line
    x1, x2 = 4 * qrand(200, 7) - 2, 4 * qrand(200, 8) - 2
    gap = np.abs(x2 - x1) > 0.3
    X = np.column_stack([x1, x2])[gap][:100]
    X = np.vstack([X, [[1.2, 1.05], [1.5, 1.38]]])
    y = np.r_[np.where(X[:100, 1] > X[:100, 0], 1, -1), 1, 1]
    xt1, xt2 = 4 * qrand(2000, 7, start=1001) - 2, 4 * qrand(2000, 8, start=1001) - 2
    Xt, yt = np.column_stack([xt1, xt2]), np.where(xt2 > xt1, 1, -1)
    costs = [0.01, 0.1, 1, 10, 100, 1000, 10000]
    rows = []
    for c in costs:
        m = SVC(kernel="linear", C=c).fit(X, y)
        rows.append([c, np.sum(m.predict(X) != y), cv_error(lambda: SVC(kernel="linear", C=c), X, y),
                     np.mean(m.predict(Xt) != yt)])
    t6 = pd.DataFrame(rows, columns=["cost", "training errors", "CV error", "test error"])
    print("Exercise 6\n", t6.to_string(index=False))
    keep("ex6_table", t6.to_numpy()[:, 1:])

    # Exercise 7: high or low mileage in the Auto data, ten folds by row number
    Auto = common.isl_data("Auto", na_values="?").dropna().reset_index(drop=True)
    y = np.where(Auto["mpg"] > Auto["mpg"].median(), 1, 0)
    X = Auto[["cylinders", "displacement", "horsepower", "weight", "acceleration", "year", "origin"]].to_numpy(float)
    X = e1071_scale(X)(X)
    cv_lin = [cv_error(lambda: SVC(kernel="linear", C=c), X, y) for c in (0.01, 0.1, 1, 10, 100)]
    cv_rbf = [[cv_error(lambda: SVC(kernel="rbf", gamma=g, C=c), X, y) for c in (0.1, 1, 10, 100)] for g in (0.01, 0.1, 1)]
    cv_pol = [[cv_error(lambda: SVC(kernel="poly", degree=d, gamma=1, coef0=1, C=c), X, y) for c in (0.01, 0.1, 1, 10)] for d in (2, 3)]
    print("Exercise 7: CV error, linear (cost 0.01 ... 100):", np.round(cv_lin, 4))
    print("  radial (rows gamma 0.01, 0.1, 1; columns cost 0.1 ... 100):\n", np.round(cv_rbf, 4))
    print("  polynomial (rows degree 2, 3; columns cost 0.01 ... 10):\n", np.round(cv_pol, 4))
    keep("ex7_lin", cv_lin); keep("ex7_rbf", cv_rbf); keep("ex7_pol", cv_pol)

    # Exercise 8: the OJ data, 800 training rows chosen by a fixed permutation
    OJ = common.isl_data("OJ")
    y = OJ["Purchase"].to_numpy()
    X = OJ.drop(columns="Purchase").assign(Store7=(OJ["Store7"] == "Yes").astype(float)).to_numpy(float)
    perm = (np.arange(len(y)) * 377) % len(y)
    tr = np.zeros(len(y), bool)
    tr[perm[:800]] = True
    sc = e1071_scale(X[tr])
    Xtr, Xte, ytr, yte = sc(X[tr]), sc(X[~tr]), y[tr], y[~tr]
    p = X.shape[1]
    grid = [0.01, 0.0316, 0.1, 0.316, 1, 3.16, 10]
    kernels = {"linear": dict(kernel="linear"), "radial": dict(kernel="rbf", gamma=1 / p),
               "polynomial": dict(kernel="poly", degree=2, gamma=1 / p, coef0=0)}
    out8 = []
    for name, kw in kernels.items():
        m = SVC(C=0.01, **kw).fit(Xtr, ytr)
        e0 = [np.mean(m.predict(Xtr) != ytr), np.mean(m.predict(Xte) != yte)]
        cv = [cv_error(lambda: SVC(C=c, **kw), Xtr, ytr) for c in grid]
        cbest = grid[int(np.argmin(cv))]
        mb = SVC(C=cbest, **kw).fit(Xtr, ytr)
        e1 = [np.mean(mb.predict(Xtr) != ytr), np.mean(mb.predict(Xte) != yte)]
        print("Exercise 8, %-10s cost 0.01: %d support vectors, errors train %.4f test %.4f; best cost %g: train %.4f test %.4f"
              % (name, m.n_support_.sum(), *e0, cbest, *e1))
        out8 += [m.n_support_.sum(), *e0, cbest, *e1]
        keep("ex8_cv_" + name, cv)
    keep("ex8", out8)


def main():
    lab_svc()
    X, y = lab_svm_roc()
    lab_multiclass(X, y)
    rd, khan = read_r_data()
    lab_khan(khan)
    r_lab_in_python(rd)
    exercises()
    names, vals = [], []
    for k, v in KEY.items():
        names += [k] if len(v) == 1 else ["%s_%d" % (k, i + 1) for i in range(len(v))]
        vals += list(v)
    os.makedirs(CACHE, exist_ok=True)
    pd.DataFrame({"name": names, "value": vals}).to_csv(os.path.join(CACHE, "ch09_lab_py.csv"), index=False, float_format="%.10g")
    print("wrote %d numbers to cache/ch09_lab_py.csv" % len(vals))


if __name__ == "__main__":
    main()
