"""Applied Statistical Learning in R and Python, Chapter 1 lab (Python version).

Chapter 1 of James et al. (2021, 2023) has no lab of its own. This script is a first tour of Python on
the three data sets the chapter introduces: Wage, Smarket and NCI60. It loads each one, summarises it,
draws a first plot, and reproduces the two numbers quoted in the chapter (the 60% of Figure 1.3 and the
principal components of Figure 1.4). The R version, ch01_lab.R, does the same steps in the same order,
and the lecture notes show the two side by side.

Run from this folder:  python3 ch01_lab.py
Needs numpy, pandas, matplotlib, statsmodels and scikit-learn. The data are read with isl_data() from
common.py (the book's CSV files); NCI60 comes with the R package ISLR2 only, and load_nci60() in
ch01_introduction.py reads a copy exported from it.
Plots are written to ../img/ch01/lab-py-*.svg (and .png)
"""
import os

import numpy as np
import pandas as pd
import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import statsmodels.formula.api as smf  # noqa: E402
from statsmodels.nonparametric.smoothers_lowess import lowess  # noqa: E402
from scipy import stats  # noqa: E402
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis  # noqa: E402

from common import isl_data  # noqa: E402
from ch01_introduction import load_nci60  # noqa: E402

pd.set_option("display.width", 100)
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "img", "ch01")
os.makedirs(OUT, exist_ok=True)


def save(fig, name):
    for ext in ("svg", "png"):
        fig.savefig(os.path.join(OUT, name + "." + ext), dpi=110, bbox_inches="tight",
                    metadata={"Date": None} if ext == "svg" else None)
    plt.close(fig)


## step: vectors
x = np.array([1, 6, 2])
y = np.array([1, 4, 3])
print(len(x))
print(x + y)                    # elementwise
print(x[0])                     # Python counts from 0

## step: matrices
A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])
print(A @ B)                    # matrix product: 19 22 / 43 50
print(A * B)                    # elementwise product: 5 12 / 21 32
print(A.T)                      # transpose
print(A[1, 0])                  # row 2, column 1: the entry a_21 = 3

## step: wage-load
Wage = isl_data("Wage")
print(Wage.shape)               # n = 3000 rows, 11 columns
print(list(Wage.columns))
print(Wage[["year", "age", "education", "wage"]].head(3))

## step: wage-summary
print(Wage["wage"].describe().round(2))
print(Wage["education"].value_counts().sort_index())
print(Wage.groupby("education")["wage"].mean().round(2))

## step: wage-line
fit_year = smf.ols("wage ~ year", data=Wage).fit()
print(fit_year.params)
print(6 * fit_year.params["year"])  # fitted change from 2003 to 2009, in thousands of dollars

## step: wage-plot
fig, ax = plt.subplots(1, 2, figsize=(8.2, 3.8))
ax[0].scatter(Wage["age"], Wage["wage"], s=3, color="steelblue")
smooth = lowess(Wage["wage"], Wage["age"], frac=0.3, it=3, delta=0.01 * np.ptp(Wage["age"]))
ax[0].plot(smooth[:, 0], smooth[:, 1], color="darkblue", lw=2)
ax[0].set_xlabel("Age")
ax[0].set_ylabel("Wage")
groups = [g["wage"].to_numpy() for _, g in Wage.groupby("education")]
ax[1].boxplot(groups, tick_labels=[1, 2, 3, 4, 5], patch_artist=True,
              boxprops=dict(facecolor="lightblue"), flierprops=dict(markersize=3))
ax[1].set_xlabel("Education level")
ax[1].set_ylabel("Wage")
fig.tight_layout()
save(fig, "lab-py-wage")

## step: smarket-load
Smarket = isl_data("Smarket")
print(Smarket.shape)
print(Smarket["Direction"].value_counts().sort_index())
print(Smarket.drop(columns="Direction").corr().round(2))

## step: smarket-lags
print(Smarket.groupby("Direction")[["Lag1", "Lag2", "Lag3"]].mean())
down = Smarket.loc[Smarket["Direction"] == "Down", "Lag1"]
up = Smarket.loc[Smarket["Direction"] == "Up", "Lag1"]
print(stats.ttest_ind(down, up, equal_var=False).statistic)

## step: smarket-plot
fig, ax = plt.subplots(1, 2, figsize=(8.2, 3.8))
bp = ax[0].boxplot([down, up], tick_labels=["Down", "Up"], patch_artist=True, flierprops=dict(markersize=3))
for patch, colour in zip(bp["boxes"], ["deepskyblue", "chocolate"]):
    patch.set_facecolor(colour)
ax[0].set_xlabel("Today's direction")
ax[0].set_ylabel("Yesterday's % change")
ax[1].scatter(np.arange(1, len(Smarket) + 1), Smarket["Volume"], s=3, color="grey")
ax[1].set_xlabel("Day (2001 to 2005)")
ax[1].set_ylabel("Volume (billions of shares)")
fig.tight_layout()
save(fig, "lab-py-smarket")

## step: smarket-qda
train = (Smarket["Year"] < 2005).to_numpy()
X_train, y_train = Smarket.loc[train, ["Lag1", "Lag2"]], Smarket.loc[train, "Direction"]
X_test, y_test = Smarket.loc[~train, ["Lag1", "Lag2"]], Smarket.loc[~train, "Direction"]
qda_fit = QuadraticDiscriminantAnalysis().fit(X_train, y_train)
qda_class = qda_fit.predict(X_test)
p_down = qda_fit.predict_proba(X_test)[:, list(qda_fit.classes_).index("Down")]
print(pd.crosstab(pd.Series(qda_class, name="predicted"), pd.Series(y_test.to_numpy(), name="actual")))
print(np.mean(qda_class == y_test.to_numpy()))

## step: nci-load
nci_data, nci_labs = load_nci60()
print(nci_data.shape)           # 64 cell lines, 6830 genes
print(pd.Series(nci_labs).value_counts().sort_index())

## step: nci-pca
Z = (nci_data - nci_data.mean(axis=0)) / nci_data.std(axis=0, ddof=1)   # as prcomp(scale = TRUE)
U, d, Vt = np.linalg.svd(Z, full_matrices=False)
scores = U * d                  # principal component scores, one row per cell line
pve = d ** 2 / np.sum(d ** 2)
print(pve[:3].round(4))
print(scores[:3, :2].round(3))

## step: nci-plot
fig, ax = plt.subplots(1, 2, figsize=(8.2, 3.8))
codes, names = pd.factorize(pd.Series(nci_labs), sort=True)
ax[0].scatter(scores[:, 0], scores[:, 1], c=codes, cmap="hsv", s=18)
ax[0].set_xlabel("Z1")
ax[0].set_ylabel("Z2")
ax[1].plot(np.arange(1, len(pve) + 1), 100 * np.cumsum(pve), "o-", color="brown", ms=3)
ax[1].set_ylim(0, 100)
ax[1].set_xlabel("Number of components")
ax[1].set_ylabel("Cumulative % of variance")
fig.tight_layout()
save(fig, "lab-py-nci60")
