"""James et al. (2021, 2023), Chapter 1: Introduction.

Reproduces the data figures of Chapter 1 of G. James, D. Witten, T. Hastie and R. Tibshirani, An
Introduction to Statistical Learning with Applications in R, 2nd edition (Springer, 2021), pp. 1-14,
whose text and figures are shared by the Python edition (James, Witten, Hastie, Tibshirani and Taylor,
2023, pp. 1-13). Everything is written from scratch with numpy, pandas and matplotlib and is meant to
be read next to the lecture notes (../ch01-introduction.html):

* Wage data: load_wage reads the 3000 men; local_linear is the kernel-weighted local linear smoother
  used for the curve of wage against age; ls_line is the least-squares line of wage against year;
  box_stats computes the boxplot statistics the way R's boxplot() does (Tukey's hinges, whiskers to the
  most extreme point within 1.5 box lengths, the rest drawn as outliers).
* Smarket data: load_smarket reads the 1250 trading days of 2001-2005; welch_t compares a lag on the
  Up and Down days; qda_fit and qda_posterior are quadratic discriminant analysis written out from
  Bayes' theorem (the preview of Chapter 4 behind Figure 1.3).
* NCI60 data: load_nci60 reads the 64 x 6830 expression matrix and the 64 labels; pca_scores gives
  the principal component scores of the standardised data by the singular value decomposition;
  single_linkage_groups cuts a single-linkage tree of the two-dimensional scores into four groups.
* data_set_table lists the size (n, p) of every data set of Table 1.1 that is stored as a CSV file.
* export_json writes the data files read by the two browser demonstrations
  (../data/ch01-wage.json and ../data/ch01-smarket.json).

Figures reproduced: 1.1 (Wage), 1.2 (Smarket lags by direction), 1.3 (QDA predicted probabilities),
1.4 (NCI60 principal components). Run `python3 ch01_introduction.py --list`, `python3
ch01_introduction.py` (all figures, a few seconds) or `python3 ch01_introduction.py --fig 1.4`.
Files go to ../img/ch01/.

Figures not reproduced: none; the chapter has four figures, all computed from data, and Table 1.1
is a list of data sets (its sizes are printed by the entry 'T1.1').

Settings chosen where the book gives none:
* Figure 1.1, left: the curve is a local linear fit with a Gaussian kernel of standard deviation
  h = 5 years, evaluated on a grid of ages 18 to 80 (the book does not say which smoother it used;
  any smoother with a span of a few years gives the same picture). Centre: the least-squares line.
  Right: boxplots with R's conventions (box_stats).
* Figure 1.3: QDA with inputs Lag1 and Lag2, trained on 2001-2004, applied to the 252 days of 2005;
  the probability shown is the posterior probability of Down. These are the inputs, years and method
  of the Chapter 4 lab, and they give the 60% (more precisely 59.92%) correct predictions quoted in
  the caption.
* Figure 1.4: principal components of the 64 cell lines after each gene is centred and scaled to unit
  variance (R's prcomp(scale = TRUE)); signs are fixed to the book's orientation (orient_like_book). The
  left panel colours four groups found by single-linkage clustering of the two scores (Z1, Z2); the
  book does not say how its four groups were drawn, and two cell lines at Z1 of about 25 that the
  book colours with the right-hand group join the large group here. The right panel uses the 14
  labels stored with the data.
* NCI60 is not one of the book's CSV files; load_nci60 reads a copy exported once from the R package
  ISLR2 (written to cache/ch01-NCI60.csv by Rscript the first time it is needed).
"""
import json
import os
import subprocess

import numpy as np
import pandas as pd

import common
from common import savefig, BLUE, RED, GREEN, GREY, ORANGE, PURPLE, CYAN, MAGENTA

import matplotlib.pyplot as plt
from matplotlib.lines import Line2D

CH = 1
HERE = os.path.dirname(os.path.abspath(__file__))
JSON_DIR = os.path.normpath(os.path.join(HERE, "..", "data"))
CACHE_DIR = os.path.join(HERE, "cache")
NCI_CSV = os.path.join(CACHE_DIR, "ch01-NCI60.csv")
RSCRIPT = os.environ.get("RSCRIPT", "/usr/local/bin/Rscript")

EDUCATION_LEVELS = ["1. < HS Grad", "2. HS Grad", "3. Some College", "4. College Grad", "5. Advanced Degree"]
POINT = "#9aa8c8"     # the light grey-blue of the scatter plots


# =========================================================================== data
def load_wage():
    """Wage data: 3000 rows; adds 'edu', the education level as an integer 1..5."""
    W = common.isl_data("Wage")
    W["edu"] = W["education"].str[0].astype(int)
    return W


def load_smarket():
    """Smarket data: 1250 trading days (2001-2005), Lag1..Lag5, Volume, Today and Direction."""
    return common.isl_data("Smarket")


def load_nci60():
    """NCI60 data: (X, labels) with X the 64 x 6830 expression matrix (cell lines in rows).

    The data ship with the R package ISLR2 (not as a CSV file on the book's website); the first call
    exports them with Rscript to cache/ch01-NCI60.csv, later calls read that file."""
    if not os.path.exists(NCI_CSV):
        os.makedirs(CACHE_DIR, exist_ok=True)
        code = ('library(ISLR2); d <- data.frame(label = NCI60$labs, NCI60$data, check.names = FALSE); '
                'write.csv(d, "%s", row.names = FALSE)' % NCI_CSV)
        subprocess.run([RSCRIPT, "-e", code], check=True, capture_output=True)
    D = pd.read_csv(NCI_CSV)
    return D.iloc[:, 1:].to_numpy(float), D["label"].to_numpy(str)


# =========================================================================== Wage: smoothers and boxplots
def local_linear(x, y, grid, h):
    """Local linear regression with a Gaussian kernel of standard deviation h.

    At each x0 in grid, minimise sum_i K_h(x_i - x0) (y_i - a - b (x_i - x0))^2 and return a. With
    S_k = sum K (x_i - x0)^k and T_k = sum K (x_i - x0)^k y_i the minimiser is
    a = (S_2 T_0 - S_1 T_1) / (S_0 S_2 - S_1^2)."""
    x, y = np.asarray(x, float), np.asarray(y, float)
    out = np.empty(len(grid))
    for j, x0 in enumerate(np.asarray(grid, float)):
        d = x - x0
        k = np.exp(-0.5 * (d / h) ** 2)
        s0, s1, s2 = k.sum(), (k * d).sum(), (k * d * d).sum()
        t0, t1 = (k * y).sum(), (k * d * y).sum()
        out[j] = (s2 * t0 - s1 * t1) / (s0 * s2 - s1 * s1)
    return out


def ls_line(x, y):
    """Least-squares intercept and slope: b = sum (x - xbar)(y - ybar) / sum (x - xbar)^2, a = ybar - b xbar."""
    x, y = np.asarray(x, float), np.asarray(y, float)
    b = np.sum((x - x.mean()) * (y - y.mean())) / np.sum((x - x.mean()) ** 2)
    return y.mean() - b * x.mean(), b


def fivenum(v):
    """Tukey's five-number summary (R's fivenum): minimum, lower hinge, median, upper hinge, maximum.

    The hinges are the medians of the lower and upper halves of the sorted data, the median being
    included in both halves when n is odd."""
    v = np.sort(np.asarray(v, float))
    n = len(v)
    n4 = np.floor((n + 3) / 2) / 2
    idx = np.array([1, n4, (n + 1) / 2, n + 1 - n4, n]) - 1
    return 0.5 * (v[np.floor(idx).astype(int)] + v[np.ceil(idx).astype(int)])


def box_stats(v, coef=1.5):
    """Boxplot statistics as in R's boxplot.stats: dict with the hinges (box), median, whisker ends
    (the most extreme data points within coef box lengths of the box) and the outliers."""
    v = np.asarray(v, float)
    mn, lo, med, hi, mx = fivenum(v)
    iqr = hi - lo
    inside = v[(v >= lo - coef * iqr) & (v <= hi + coef * iqr)]
    wlo, whi = inside.min(), inside.max()
    out = np.sort(v[(v < wlo) | (v > whi)])
    return dict(med=med, q1=lo, q3=hi, whislo=wlo, whishi=whi, fliers=out, n=len(v))


def draw_boxes(ax, groups, colours, labels, width=0.6, flier_size=2.5, mean=False):
    """Draw boxplots from box_stats (R's conventions) with matplotlib's bxp."""
    stats = []
    for g, lab in zip(groups, labels):
        s = box_stats(g)
        s["label"] = lab
        if mean:
            s["mean"] = float(np.mean(g))
        stats.append(s)
    bp = ax.bxp(stats, widths=width, patch_artist=True, showmeans=mean,
                flierprops=dict(marker="o", markersize=flier_size, markerfacecolor="none",
                                markeredgecolor="black", markeredgewidth=0.6),
                medianprops=dict(color="black", linewidth=2),
                whiskerprops=dict(linestyle=(0, (4, 3)), linewidth=0.8),
                capprops=dict(linewidth=0.8))
    for p, c in zip(bp["boxes"], colours):
        p.set_facecolor(c)
        p.set_edgecolor("black")
        p.set_linewidth(0.8)
    return stats


# =========================================================================== Smarket: t statistics and QDA
def welch_t(a, b):
    """Welch two-sample t statistic (mean(a) - mean(b)) / sqrt(s_a^2/n_a + s_b^2/n_b) and its df."""
    a, b = np.asarray(a, float), np.asarray(b, float)
    va, vb = a.var(ddof=1) / len(a), b.var(ddof=1) / len(b)
    t = (a.mean() - b.mean()) / np.sqrt(va + vb)
    df = (va + vb) ** 2 / (va ** 2 / (len(a) - 1) + vb ** 2 / (len(b) - 1))
    return float(t), float(df)


def qda_fit(X, y):
    """Quadratic discriminant analysis: for each class k the prior (class proportion), mean vector and
    covariance matrix (divisor n_k - 1), as in R's MASS::qda and scikit-learn."""
    X = np.atleast_2d(np.asarray(X, float))
    y = np.asarray(y)
    classes = np.unique(y)
    fit = dict(classes=classes, prior=[], mean=[], cov=[])
    for k in classes:
        Xk = X[y == k]
        fit["prior"].append(len(Xk) / len(X))
        fit["mean"].append(Xk.mean(0))
        fit["cov"].append(np.cov(Xk, rowvar=False).reshape(X.shape[1], X.shape[1]))
    return fit


def qda_posterior(fit, X):
    """Posterior probabilities Pr(Y = k | X = x) (rows of X) by Bayes' theorem with Gaussian classes:
    log pi_k - (1/2) log det S_k - (1/2)(x - m_k)' S_k^{-1} (x - m_k), normalised over k."""
    X = np.atleast_2d(np.asarray(X, float))
    scores = []
    for pk, mk, Sk in zip(fit["prior"], fit["mean"], fit["cov"]):
        D = X - mk
        Sinv = np.linalg.inv(Sk)
        q = np.einsum("ij,jk,ik->i", D, Sinv, D)
        scores.append(np.log(pk) - 0.5 * np.linalg.slogdet(Sk)[1] - 0.5 * q)
    S = np.column_stack(scores)
    S -= S.max(1, keepdims=True)
    P = np.exp(S)
    return P / P.sum(1, keepdims=True)


def smarket_qda(inputs=("Lag1", "Lag2"), train_years=(2001, 2004), test_year=2005):
    """Fit QDA on the training years and return (P(Down) on the test days, test directions, accuracy)."""
    S = load_smarket()
    tr = S["Year"].between(*train_years).to_numpy()
    te = (S["Year"] == test_year).to_numpy()
    fit = qda_fit(S.loc[tr, list(inputs)].to_numpy(), S.loc[tr, "Direction"].to_numpy())
    P = qda_posterior(fit, S.loc[te, list(inputs)].to_numpy())
    pdown = P[:, list(fit["classes"]).index("Down")]
    ytest = S.loc[te, "Direction"].to_numpy()
    pred = np.where(pdown > 0.5, "Down", "Up")
    return pdown, ytest, float(np.mean(pred == ytest))


# =========================================================================== NCI60: principal components
def pca_scores(X, scale=True):
    """Principal component scores (n x min(n, p)) and the proportion of variance explained.

    Columns are centred (and divided by their standard deviation, divisor n - 1, when scale=True, as in
    R's prcomp). Then X = U D V' (thin SVD), the scores are U D and the loadings are the columns of V.
    The sign of each component is arbitrary (v and -v are equally good directions); numpy and R both
    take the signs that LAPACK returns, so they agree here, but see orient_like_book."""
    X = np.asarray(X, float)
    Z = X - X.mean(0)
    if scale:
        Z = Z / Z.std(0, ddof=1)
    U, d, Vt = np.linalg.svd(Z, full_matrices=False)
    scores = U * d
    pve = d ** 2 / np.sum(d ** 2)
    return scores, pve, Vt


def orient_like_book(Z, labels):
    """Fix the arbitrary signs of the first two score columns so that the picture has the book's
    orientation: the leukaemia cell lines on the positive side of Z1, the melanoma lines on the
    negative side of Z2. Returns a copy."""
    Z = np.array(Z, float)
    if Z[labels == "LEUKEMIA", 0].mean() < 0:
        Z[:, 0] = -Z[:, 0]
    if Z[labels == "MELANOMA", 1].mean() > 0:
        Z[:, 1] = -Z[:, 1]
    return Z


def single_linkage_groups(P, k):
    """Cut the single-linkage tree of the rows of P into k groups (labels 0..k-1, numbered by size)."""
    from scipy.cluster.hierarchy import linkage, fcluster
    lab = fcluster(linkage(P, "single"), k, "maxclust") - 1
    order = np.argsort(-np.bincount(lab), kind="stable")
    relabel = np.empty(k, int)
    relabel[order] = np.arange(k)
    return relabel[lab]


# =========================================================================== Table 1.1
TABLE_1_1 = ["Auto", "Bikeshare", "Boston", "BrainCancer", "Caravan", "Carseats", "College", "Credit",
             "Default", "Fund", "Hitters", "Khan", "NCI60", "NYSE", "OJ", "Portfolio", "Publication",
             "Smarket", "USArrests", "Wage", "Weekly"]


def data_set_table():
    """(name, n, p) for the data sets of Table 1.1 stored as CSV files (row-name columns dropped)."""
    rows = []
    for name in TABLE_1_1:
        try:
            D = common.isl_data(name)
        except FileNotFoundError:
            continue
        D = D.drop(columns=[c for c in D.columns if c.startswith("Unnamed")])
        rows.append((name, D.shape[0], D.shape[1]))
    return rows


# =========================================================================== figures
def fig_1_1(out_dir=None):
    """Figure 1.1: Wage against age (with a local linear fit), year (least-squares line) and education."""
    W = load_wage()
    fig, ax = plt.subplots(1, 3, figsize=(10.2, 3.5))
    ax[0].scatter(W["age"], W["wage"], s=4, color=POINT, lw=0)
    grid = np.linspace(18, 80, 125)
    ax[0].plot(grid, local_linear(W["age"], W["wage"], grid, 5.0), color=BLUE, lw=2)
    ax[0].set_xlabel("Age")
    ax[1].scatter(W["year"], W["wage"], s=4, color=POINT, lw=0)
    a, b = ls_line(W["year"], W["wage"])
    ax[1].plot([2003, 2009], [a + b * 2003, a + b * 2009], color=BLUE, lw=2)
    ax[1].set_xlabel("Year")
    ax[1].set_xticks([2003, 2006, 2009])
    groups = [W.loc[W["edu"] == k, "wage"].to_numpy() for k in range(1, 6)]
    cols = ["#3fa9f5", "#2bb5a0", "#ece75f", "#3d8bd9", "#f08a3c"]
    draw_boxes(ax[2], groups, cols, [str(k) for k in range(1, 6)])
    ax[2].set_xlabel("Education Level")
    for a_ in ax:
        a_.set_ylabel("Wage")
        a_.set_yticks([50, 100, 200, 300])
        a_.set_ylim(15, 330)
    fig.tight_layout()
    return savefig(fig, CH, "fig1-1", out_dir)


def fig_1_2(out_dir=None):
    """Figure 1.2: boxplots of the percentage change of the S&P 500 one, two and three days earlier,
    for the days on which the market went Down and Up."""
    S = load_smarket()
    fig, ax = plt.subplots(1, 3, figsize=(10.2, 3.6))
    for j, (lag, title) in enumerate((("Lag1", "Yesterday"), ("Lag2", "Two Days Previous"),
                                      ("Lag3", "Three Days Previous"))):
        groups = [S.loc[S["Direction"] == d, lag].to_numpy() for d in ("Down", "Up")]
        draw_boxes(ax[j], groups, ["#33b5ff", "#d2691e"], ["Down", "Up"], width=0.65, flier_size=3)
        ax[j].set_title(title)
        ax[j].set_xlabel("Today's Direction")
        ax[j].set_ylabel("Percentage change in S&P")
        ax[j].set_ylim(-5, 6.2)
    fig.tight_layout()
    return savefig(fig, CH, "fig1-2", out_dir)


def fig_1_3(out_dir=None):
    """Figure 1.3: QDA (Lag1, Lag2; trained on 2001-2004) predicted probability of Down for the days of
    2005, by the actual direction."""
    pdown, y, acc = smarket_qda()
    fig, ax = plt.subplots(figsize=(4.2, 3.9))
    groups = [pdown[y == d] for d in ("Down", "Up")]
    draw_boxes(ax, groups, ["#33b5ff", "#d2691e"], ["Down", "Up"], width=0.65, flier_size=5)
    ax.set_xlabel("Today's Direction")
    ax.set_ylabel("Predicted Probability")
    fig.tight_layout()
    print("  QDA on Lag1, Lag2: %d test days, accuracy %.4f" % (len(y), acc))
    return savefig(fig, CH, "fig1-3", out_dir)


# 14 distinct (colour, marker) pairs for the 14 labels of NCI60
LABEL_STYLE = {
    "BREAST": (RED, "o"), "CNS": ("#e6a817", "D"), "COLON": (BLUE, "^"), "K562A-repro": ("#00a6a6", "s"),
    "K562B-repro": ("#00a6a6", "D"), "LEUKEMIA": ("#1f9e3a", "^"), "MCF7A-repro": (MAGENTA, "s"),
    "MCF7D-repro": (MAGENTA, "D"), "MELANOMA": ("#1e90ff", "o"), "NSCLC": ("#2b2bd6", "^"),
    "OVARIAN": (ORANGE, "^"), "PROSTATE": ("#8a2be2", "s"), "RENAL": (PURPLE, "D"), "UNKNOWN": (GREY, "o"),
}
GROUP_COLOURS = ["#4b2bd6", "#1e9bf0", "#25d366", RED]


def fig_1_4(out_dir=None):
    """Figure 1.4: the 64 NCI60 cell lines on their first two principal components, coloured by four
    groups (left) and by the 14 labels (right)."""
    X, lab = load_nci60()
    Z, pve, _ = pca_scores(X)
    Z = orient_like_book(Z, lab)
    g = single_linkage_groups(Z[:, :2], 4)
    fig, ax = plt.subplots(1, 2, figsize=(9.4, 4.1), sharey=True)
    for k in range(4):
        m = g == k
        ax[0].scatter(Z[m, 0], Z[m, 1], s=34, color=GROUP_COLOURS[k], lw=0)
    for name, (c, mk) in LABEL_STYLE.items():
        m = lab == name
        ax[1].scatter(Z[m, 0], Z[m, 1], s=34, color=c, marker=mk, lw=0, label=name)
    ax[1].legend(fontsize=6.5, ncol=1, loc="center left", bbox_to_anchor=(1.01, 0.5), frameon=False,
                 handletextpad=0.3, borderaxespad=0)
    for a_ in ax:
        a_.set_xlabel("$Z_1$")
    ax[0].set_ylabel("$Z_2$")
    fig.tight_layout()
    print("  proportion of variance explained by Z1, Z2: %.4f, %.4f; group sizes %s"
          % (pve[0], pve[1], np.bincount(g).tolist()))
    return savefig(fig, CH, "fig1-4", out_dir)


# =========================================================================== data for the demos
def export_json():
    """Write ../data/ch01-wage.json and ../data/ch01-smarket.json for the browser demonstrations."""
    os.makedirs(JSON_DIR, exist_ok=True)
    W = load_wage()
    wage = dict(age=W["age"].astype(int).tolist(), year=W["year"].astype(int).tolist(),
                edu=W["edu"].astype(int).tolist(), wage=[round(float(v), 4) for v in W["wage"]],
                jobclass=[int(s[0]) for s in W["jobclass"]], education=EDUCATION_LEVELS)
    S = load_smarket()
    cols = ["Year", "Lag1", "Lag2", "Lag3", "Lag4", "Lag5", "Volume", "Today"]
    smarket = {c: [float(v) if c != "Year" else int(v) for v in S[c]] for c in cols}
    smarket["up"] = [int(d == "Up") for d in S["Direction"]]
    for name, obj in (("ch01-wage.json", wage), ("ch01-smarket.json", smarket)):
        path = os.path.join(JSON_DIR, name)
        with open(path, "w") as f:
            json.dump(obj, f, separators=(",", ":"))
        print("wrote", os.path.relpath(path, os.getcwd()), "(%d kB)" % (os.path.getsize(path) // 1024))


def fig_table_and_data(out_dir=None):
    """Table 1.1 (sizes printed) and the JSON files of the demonstrations."""
    for name, n, p in data_set_table():
        print("  %-12s n = %5d  p = %4d" % (name, n, p))
    export_json()


FIGURES = {"1.1": fig_1_1, "1.2": fig_1_2, "1.3": fig_1_3, "1.4": fig_1_4, "T1.1": fig_table_and_data}

if __name__ == "__main__":
    common.run(FIGURES, CH)
