"""Chapter 4 lab: classification methods, in Python.

Course: Applied Statistical Learning in R and Python. The lab follows the steps of the Python lab of
James, Witten, Hastie, Tibshirani and Taylor (2023), An Introduction to Statistical Learning with
Applications in Python, Section 4.7 (same data, same models, same questions), written for this course
with pandas, statsmodels and scikit-learn (the book's own helper package is not used). The R version,
ch04_lab.R, does the same steps and prints the same labels, so the two outputs can be compared line
by line. Where the two editions split the Caravan data differently, both splits are run here.

Run:  python3 ch04_lab.py        (writes two figures to ../img/ch04/lab4-py-*.svg/png/pdf)
"""
import os

os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
import warnings

import numpy as np
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler

import common


def show(label, value, digits=4):
    if not isinstance(value, str):
        value = " ".join("%.*f" % (digits, v) for v in np.atleast_1d(np.asarray(value, float)))
    print("%s: %s" % (label, value))


def show_table(label, pred, truth):
    """Confusion matrix with predicted classes in rows and true classes in columns (as R's table())."""
    print(label + ":")
    print(pd.crosstab(pd.Series(pred, name="predicted"), pd.Series(np.asarray(truth), name="truth")))


# ================================================================ 4.7.1 the stock market data
print("\n== 4.7.1 Smarket")
Smarket = common.isl_data("Smarket")
show("dim", Smarket.shape, 0)
show("Direction counts (Down, Up)", Smarket.Direction.value_counts().sort_index().values, 0)
cors = Smarket.drop(columns="Direction").corr()
show("cor(Year, Volume)", cors.loc["Year", "Volume"], 3)
lags = ["Lag1", "Lag2", "Lag3", "Lag4", "Lag5", "Today"]
lag_cor = cors.loc[lags, lags].values
show("largest |cor| among Lag1-Lag5 and Today", np.max(np.abs(lag_cor[np.triu_indices(6, 1)])), 3)
fig, ax = common.plt.subplots(figsize=(6.4, 3.4))
ax.plot(Smarket.Volume.values, ".", ms=2.5, color="0.3")
ax.set_xlabel("Day (2001-2005)")
ax.set_ylabel("Volume")
common.savefig(fig, 4, "lab4-py-volume")

# ================================================================ 4.7.2 logistic regression
print("\n== 4.7.2 Logistic regression")
y = (Smarket.Direction == "Up").astype(int)               # 1 = Up, as in R
X = sm.add_constant(Smarket[["Lag1", "Lag2", "Lag3", "Lag4", "Lag5", "Volume"]])
results = sm.GLM(y, X, family=sm.families.Binomial()).fit()
print(pd.DataFrame({"coef": results.params, "std err": results.bse, "z": results.tvalues,
                    "P>|z|": results.pvalues}).round(4))
show("residual deviance", results.deviance, 2)
probs = results.predict()
show("first five fitted probabilities", probs[:5])
labels = np.where(probs > 0.5, "Up", "Down")
show_table("training confusion (all 1250 days)", labels, Smarket.Direction)
show("training accuracy", np.mean(labels == Smarket.Direction))

train = (Smarket.Year < 2005).values
L_train, L_test = Smarket.Direction[train], Smarket.Direction[~train]
show("test days in 2005", np.sum(~train), 0)

results = sm.GLM(y[train], X[train], family=sm.families.Binomial()).fit()
probs = results.predict(X[~train])
labels = np.where(probs > 0.5, "Up", "Down")
show_table("2005 confusion, six predictors", labels, L_test)
show("2005 accuracy, six predictors", np.mean(labels == L_test))

X2 = sm.add_constant(Smarket[["Lag1", "Lag2"]])
results = sm.GLM(y[train], X2[train], family=sm.families.Binomial()).fit()
probs = results.predict(X2[~train])
labels = np.where(probs > 0.5, "Up", "Down")
show_table("2005 confusion, Lag1 + Lag2", labels, L_test)
show("2005 accuracy, Lag1 + Lag2", np.mean(labels == L_test))
show("precision on predicted Up days", np.mean(L_test[labels == "Up"] == "Up"))
show("always-Up accuracy in 2005", np.mean(L_test == "Up"))
newX = sm.add_constant(pd.DataFrame({"Lag1": [1.2, 1.5], "Lag2": [1.1, -0.8]}), has_constant="add")
show("Pr(Up) at (1.2, 1.1) and (1.5, -0.8)", results.predict(newX))

# ================================================================ 4.7.3 LDA
print("\n== 4.7.3 Linear discriminant analysis")
X_train, X_test = Smarket.loc[train, ["Lag1", "Lag2"]], Smarket.loc[~train, ["Lag1", "Lag2"]]
lda = LDA(store_covariance=True).fit(X_train, L_train)     # classes_ = ['Down', 'Up']
show("priors (Down, Up)", lda.priors_)
show("group means Down (Lag1, Lag2)", lda.means_[0])
show("group means Up (Lag1, Lag2)", lda.means_[1])
show("LD1 coefficients (Lag1, Lag2)", lda.scalings_[:, 0])
lda_pred = lda.predict(X_test)
show_table("2005 confusion, LDA", lda_pred, L_test)
show("2005 accuracy, LDA", np.mean(lda_pred == L_test))
lda_prob = lda.predict_proba(X_test)
show("days with Pr(Down) >= 0.5 and < 0.5", [np.sum(lda_prob[:, 0] >= 0.5), np.sum(lda_prob[:, 0] < 0.5)], 0)
show("days with Pr(Down) > 0.9", np.sum(lda_prob[:, 0] > 0.9), 0)
show("largest Pr(Down) in 2005", lda_prob[:, 0].max())

# ================================================================ 4.7.4 QDA
print("\n== 4.7.4 Quadratic discriminant analysis")
qda = QDA(store_covariance=True).fit(X_train, L_train)
show("QDA covariance of class Down", qda.covariance_[0].ravel())
qda_pred = qda.predict(X_test)
show_table("2005 confusion, QDA", qda_pred, L_test)
show("2005 accuracy, QDA", np.mean(qda_pred == L_test))

# ================================================================ 4.7.5 naive Bayes
print("\n== 4.7.5 Naive Bayes")
NB = GaussianNB().fit(X_train, L_train)
show("NB mean and sd of Lag1 in class Down", [NB.theta_[0, 0], np.sqrt(NB.var_[0, 0])])
x_down = X_train.Lag1[L_train == "Down"]
show("by hand: mean, sd (divisor n)", [x_down.mean(), x_down.std(ddof=0)])
nb_pred = NB.predict(X_test)
show_table("2005 confusion, naive Bayes", nb_pred, L_test)
show("2005 accuracy, naive Bayes", np.mean(nb_pred == L_test))
show("first three Pr(Up), naive Bayes", NB.predict_proba(X_test)[:3, 1])
# GaussianNB divides by n; R's naiveBayes() divides by n - 1. Rescaling the variances reproduces R.
n_k = np.array([np.sum(L_train == c) for c in NB.classes_])
NB.var_ = NB.var_ * (n_k / (n_k - 1))[:, None]
nb_pred_r = NB.predict(X_test)
show("2005 accuracy, naive Bayes with divisor n - 1", np.mean(nb_pred_r == L_test))

# ================================================================ 4.7.6 K-nearest neighbours
print("\n== 4.7.6 K-nearest neighbours")
knn1 = KNeighborsClassifier(n_neighbors=1).fit(X_train, L_train)
knn1_pred = knn1.predict(X_test)
show_table("2005 confusion, KNN K = 1", knn1_pred, L_test)
show("2005 accuracy, KNN K = 1", np.mean(knn1_pred == L_test))
knn3 = KNeighborsClassifier(n_neighbors=3).fit(X_train, L_train)
show("2005 accuracy, KNN K = 3", np.mean(knn3.predict(X_test) == L_test))

Caravan = common.isl_data("Caravan")
Purchase = Caravan.Purchase
show("Caravan dim", Caravan.shape, 0)
show("purchase rate", np.mean(Purchase == "Yes"))
features = Caravan.drop(columns="Purchase")
X_std = (features - features.mean()) / features.std()       # divisor n - 1, as R's scale()
show("variance of columns 1 and 2 before / after scaling",
     [features.iloc[:, 0].var(), features.iloc[:, 1].var(), X_std.iloc[:, 0].var(), X_std.iloc[:, 1].var()])


def caravan_knn(X_tr, X_te, y_tr, y_te, label):
    for k in (1, 3, 5):
        pred = KNeighborsClassifier(n_neighbors=k).fit(X_tr, y_tr).predict(X_te)
        yes = pred == "Yes"
        hit = np.sum(yes & (y_te == "Yes"))
        show("%s KNN K = %d: error, predicted Yes, correct Yes, success rate" % (label, k),
             [np.mean(pred != y_te), yes.sum(), hit, hit / max(yes.sum(), 1)])


# (a) the R edition's split: the first 1,000 customers are the test set
test = np.arange(len(Caravan)) < 1000
X_tr, X_te, y_tr, y_te = X_std[~test], X_std[test], Purchase[~test].values, Purchase[test].values
caravan_knn(X_tr, X_te, y_tr, y_te, "Caravan")
show("Caravan null error (always No)", np.mean(y_te != "No"))

yb = (Purchase == "Yes").astype(int)
with warnings.catch_warnings():
    warnings.simplefilter("ignore")                         # some fitted probabilities are 0 or 1
    logit = sm.GLM(yb[~test], sm.add_constant(features[~test]), family=sm.families.Binomial()).fit()
glm_probs = logit.predict(sm.add_constant(features[test]))
for cut in (0.5, 0.25):
    yes = glm_probs.values > cut
    show("Caravan logistic, threshold %.2f: predicted Yes, correct Yes" % cut,
         [yes.sum(), np.sum(yes & (y_te == "Yes"))], 0)

# (b) the Python edition's split: 1,000 test customers drawn at random (random_state=0), with the
#     features standardised by StandardScaler (divisor n)
scaled = pd.DataFrame(StandardScaler().fit_transform(features), columns=features.columns)
Xa, Xb, ya, yb2 = train_test_split(scaled, Purchase, test_size=1000, random_state=0)
caravan_knn(Xa, Xb, ya.values, yb2.values, "Caravan (random split)")
show("Caravan (random split) null error", np.mean(yb2 != "No"))

# ================================================================ 4.7.7 Poisson regression
print("\n== 4.7.7 Linear and Poisson regression on Bikeshare")
Bike = common.isl_data("Bikeshare", index_col=0)
months = ["Jan", "Feb", "March", "April", "May", "June", "July", "Aug", "Sept", "Oct", "Nov", "Dec"]
Bike["mnth"] = pd.Categorical(Bike.mnth, categories=months)
Bike["hr"] = pd.Categorical(Bike.hr, categories=range(24))
Bike["weathersit"] = pd.Categorical(Bike.weathersit,
                                    categories=["clear", "cloudy/misty", "light rain/snow", "heavy rain/snow"])
show("Bikeshare dim", Bike.shape, 0)
M_lm = smf.ols("bikers ~ mnth + hr + workingday + temp + weathersit", data=Bike).fit()
show("lm (treatment coding): intercept, mnthFeb, mnthMarch", M_lm.params.iloc[:3], 3)
f_sum = "bikers ~ C(mnth, Sum) + C(hr, Sum) + workingday + temp + weathersit"
M2_lm = smf.ols(f_sum, data=Bike).fit()
show("lm (sum coding): intercept, Jan, Feb", M2_lm.params.iloc[:3], 3)
show("max |difference| of fitted values between the codings",
     np.max(np.abs(M_lm.fittedvalues - M2_lm.fittedvalues)), 10)


def level_effects(params, factor):
    """All level effects of a sum-coded factor: the reported ones and minus their sum (the last level)."""
    b = params[params.index.str.startswith("C(%s, Sum)" % factor)].values
    return np.r_[b, -b.sum()]


coef_month, coef_hr = level_effects(M2_lm.params, "mnth"), level_effects(M2_lm.params, "hr")
show("lm December and hour-23 effects", [coef_month[11], coef_hr[23]], 3)
show("share of negative fitted values", np.mean(M2_lm.fittedvalues < 0))

M_pois = smf.glm(f_sum, data=Bike, family=sm.families.Poisson()).fit()
keep = ["Intercept", "workingday", "temp"] + [n for n in M_pois.params.index if n.startswith("weathersit")]
print(pd.DataFrame({"coef": M_pois.params[keep], "std err": M_pois.bse[keep], "z": M_pois.tvalues[keep]}).round(4))
coef_month, coef_hr = level_effects(M_pois.params, "mnth"), level_effects(M_pois.params, "hr")
show("Poisson month effects (Jan..Dec)", coef_month, 3)
show("Poisson December and hour-23 effects", [coef_month[11], coef_hr[23]], 4)
show("Poisson deviance", M_pois.deviance, 1)
show("smallest Poisson fitted value", M_pois.fittedvalues.min(), 3)

fig, (ax_m, ax_h) = common.plt.subplots(1, 2, figsize=(9, 3.6))
ax_m.plot(range(12), coef_month, "o-", color="blue")
ax_m.set_xticks(range(12), [m[0] for m in months])
ax_m.set_xlabel("Month")
ax_m.set_ylabel("Coefficient")
ax_h.plot(range(24), coef_hr, "o-", color="blue")
ax_h.set_xlabel("Hour")
ax_h.set_ylabel("Coefficient")
common.savefig(fig, 4, "lab4-py-poisson-coef")


# ================================================================ applied exercises 13, 14 and 16
# (Section 4.8 of the book; the notes give the questions and discuss the answers.)
def nb_divisor_n1(X, y):
    """GaussianNB with the variances rescaled to divisor n_k - 1, as R's naiveBayes()."""
    nb = GaussianNB().fit(X, y)
    n_k = np.array([np.sum(np.asarray(y) == c) for c in nb.classes_])
    nb.var_ = nb.var_ * (n_k / (n_k - 1))[:, None]
    return nb


def logit_fit(X, y):
    return sm.GLM(np.asarray(y).astype(int), sm.add_constant(X), family=sm.families.Binomial()).fit()


print("\n== Exercise 13 (Weekly)")
Weekly = common.isl_data("Weekly")
show("Weekly dim", Weekly.shape, 0)
yw = (Weekly.Direction == "Up").astype(int)
lag6 = ["Lag1", "Lag2", "Lag3", "Lag4", "Lag5", "Volume"]
fit13 = logit_fit(Weekly[lag6], yw)
show("Ex13b p-values (Lag1..Lag5, Volume)", fit13.pvalues.values[1:])
pred13 = np.where(fit13.fittedvalues > 0.5, "Up", "Down")
show_table("Ex13c confusion (all weeks)", pred13, Weekly.Direction)
show("Ex13c accuracy", np.mean(pred13 == Weekly.Direction))
tr13 = (Weekly.Year <= 2008).values
Xtr, Xte = Weekly.loc[tr13, ["Lag2"]], Weekly.loc[~tr13, ["Lag2"]]
Ltr, Lte = Weekly.Direction[tr13].values, Weekly.Direction[~tr13].values
fit13d = logit_fit(Xtr, yw[tr13])
pred13d = np.where(fit13d.predict(sm.add_constant(Xte, has_constant="add")) > 0.5, "Up", "Down")
show_table("Ex13d confusion, logistic on Lag2 (2009-2010)", pred13d, Lte)
show("Ex13d-h accuracy: logistic, LDA, QDA, KNN-1, naive Bayes", [
    np.mean(pred13d == Lte),
    np.mean(LDA().fit(Xtr, Ltr).predict(Xte) == Lte),
    np.mean(QDA().fit(Xtr, Ltr).predict(Xte) == Lte),
    np.mean(KNeighborsClassifier(1).fit(Xtr, Ltr).predict(Xte) == Lte),
    np.mean(nb_divisor_n1(Xtr, Ltr).predict(Xte) == Lte)])
show("Ex13j KNN on Lag2, K = 1..15",
     [np.mean(KNeighborsClassifier(k).fit(Xtr, Ltr).predict(Xte) == Lte) for k in range(1, 16)], 3)
Xq = Weekly[["Lag2"]].assign(Lag2sq=Weekly.Lag2 ** 2)
pq = logit_fit(Xq[tr13], yw[tr13]).predict(sm.add_constant(Xq[~tr13], has_constant="add"))
show("Ex13j LDA on Lag1 + Lag2, logistic on Lag2 + Lag2^2", [
    np.mean(LDA().fit(Weekly.loc[tr13, ["Lag1", "Lag2"]], Ltr).predict(Weekly.loc[~tr13, ["Lag1", "Lag2"]]) == Lte),
    np.mean(np.where(pq > 0.5, "Up", "Down") == Lte)])

print("\n== Exercise 14 (Auto)")
Auto = common.isl_data("Auto", na_values="?").dropna().reset_index(drop=True)    # 392 cars, as in R
Auto["mpg01"] = (Auto.mpg > Auto.mpg.median()).astype(int)
show("Ex14a median mpg, share of 1s", [Auto.mpg.median(), Auto.mpg01.mean()])
v6 = ["cylinders", "displacement", "horsepower", "weight", "acceleration", "year"]
show("Ex14b cor(mpg01, cylinders, displacement, horsepower, weight, acceleration, year)",
     [np.corrcoef(Auto.mpg01, Auto[v])[0, 1] for v in v6], 3)
tr14 = (np.arange(len(Auto)) % 2 == 0)        # rows 1, 3, 5, ... (R's odd rows) train
v14 = ["cylinders", "displacement", "horsepower", "weight"]
Xtr, Xte, ytr, yte = Auto.loc[tr14, v14], Auto.loc[~tr14, v14], Auto.mpg01[tr14].values, Auto.mpg01[~tr14].values
p14 = logit_fit(Xtr, ytr).predict(sm.add_constant(Xte))
show("Ex14d-g test error: LDA, QDA, logistic, naive Bayes", [
    np.mean(LDA().fit(Xtr, ytr).predict(Xte) != yte), np.mean(QDA().fit(Xtr, ytr).predict(Xte) != yte),
    np.mean((p14.values > 0.5) != yte), np.mean(nb_divisor_n1(Xtr, ytr).predict(Xte) != yte)])
Ztr, Zte = (Xtr - Xtr.mean()) / Xtr.std(), (Xte - Xtr.mean()) / Xtr.std()
show("Ex14h KNN test error, K = 1..10 (standardised)",
     [np.mean(KNeighborsClassifier(k).fit(Ztr, ytr).predict(Zte) != yte) for k in range(1, 11)], 3)

print("\n== Exercise 16 (Boston)")
Boston = common.isl_data("Boston", index_col=0)
Boston["crim01"] = (Boston.crim > Boston.crim.median()).astype(int)
tr16 = (np.arange(len(Boston)) % 2 == 0)
for vars16 in (["nox", "rad", "dis", "age", "tax", "indus"], [c for c in Boston.columns if c not in ("crim", "crim01")]):
    Xtr, Xte = Boston.loc[tr16, vars16], Boston.loc[~tr16, vars16]
    ytr, yte = Boston.crim01[tr16].values, Boston.crim01[~tr16].values
    p16 = logit_fit(Xtr, ytr).predict(sm.add_constant(Xte))
    Ztr, Zte = (Xtr - Xtr.mean()) / Xtr.std(), (Xte - Xtr.mean()) / Xtr.std()
    show("Ex16 test error with %d predictors: logistic, LDA, naive Bayes, KNN-1, KNN-5, KNN-10" % len(vars16), [
        np.mean((p16.values > 0.5) != yte), np.mean(LDA().fit(Xtr, ytr).predict(Xte) != yte),
        np.mean(nb_divisor_n1(Xtr, ytr).predict(Xte) != yte)]
        + [np.mean(KNeighborsClassifier(k).fit(Ztr, ytr).predict(Zte) != yte) for k in (1, 5, 10)])
