"""Applied Statistical Learning in R and Python, Chapter 3 lab (Python version): linear regression.

The same analyses as the R lab of James et al. (2021), Section 3.6, and the Python lab of James et
al. (2023), Section 3.6, written independently with pandas and statsmodels (the book's own helper
package is not used): simple regression of medv on lstat in the Boston data (estimates, confidence
intervals, confidence and prediction intervals, diagnostic plots, leverage), multiple regression,
variance inflation factors, an interaction, polynomial and log transformations compared by F-tests,
qualitative predictors in the Carseats data, and a first function. The R version, ch03_lab.R, does
the same steps in the same order and gives the same numbers; the lecture notes show the two side
by side.

Run from this folder:  python3 ch03_lab.py
Needs numpy, pandas, matplotlib, scipy and statsmodels. The data are read with isl_data() from
common.py (the book's CSV files).
Plots are written to ../img/ch03/lab-py-*.svg (and .png)
"""
import os

import numpy as np
import pandas as pd
import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import statsmodels.api as sm  # noqa: E402
import statsmodels.formula.api as smf  # noqa: E402
from statsmodels.stats.anova import anova_lm  # noqa: E402
from statsmodels.stats.outliers_influence import variance_inflation_factor  # noqa: E402
from statsmodels.nonparametric.smoothers_lowess import lowess  # noqa: E402
from scipy import stats  # noqa: E402

from common import isl_data  # noqa: E402

pd.set_option("display.width", 100)
pd.set_option("display.max_columns", 20)
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "img", "ch03")
os.makedirs(OUT, exist_ok=True)


def save(fig, name):
    for ext in ("svg", "png"):
        fig.savefig(os.path.join(OUT, name + "." + ext), dpi=110, bbox_inches="tight",
                    metadata={"Date": None} if ext == "svg" else None)
    plt.close(fig)


def poly(x, degree):
    """Orthogonal polynomials of x, the same basis as R's poly(x, degree): the columns of the
    QR decomposition of the centred powers, scaled to unit length."""
    x = np.asarray(x, float)
    V = np.vander(x - x.mean(), degree + 1, increasing=True)
    Q, R = np.linalg.qr(V)
    Z = Q * np.diag(R)
    Z = Z / np.sqrt((Z ** 2).sum(0))
    return Z[:, 1:]


def coefs(fit, digits=4):
    """The coefficient table (estimate, standard error, t, p), like summary(fit)$coefficients in R."""
    return pd.DataFrame({"coef": fit.params, "std err": fit.bse, "t": fit.tvalues, "P>|t|": fit.pvalues}).round(digits)


## ==== step: boston
Boston = isl_data("Boston", index_col=0)
print(Boston.shape)              # 506 census tracts, 12 predictors and the response medv
print(list(Boston.columns))

## ==== step: simple-fit
X = sm.add_constant(Boston[["lstat"]])        # the model matrix: a column of ones and lstat
y = Boston["medv"]
results = sm.OLS(y, X).fit()
print(results.summary().tables[1])
print("RSE %.3f on %d df, R2 %.4f, F %.1f" % (np.sqrt(results.scale), results.df_resid, results.rsquared, results.fvalue))

## ==== step: simple-extract
print(results.params)
print(results.conf_int(alpha=0.05))           # estimate -/+ t(0.975, 504) * SE
print(pd.Series({"RSE": np.sqrt(results.scale), "R2": results.rsquared}))

## ==== step: simple-predict
new = sm.add_constant(pd.DataFrame({"lstat": [5, 10, 15]}), has_constant="add")
pred = results.get_prediction(new).summary_frame(alpha=0.05)
print(pred[["mean", "mean_ci_lower", "mean_ci_upper"]])   # confidence intervals for f(x0)
print(pred[["mean", "obs_ci_lower", "obs_ci_upper"]])     # prediction intervals for Y at x0

## ==== step: simple-plot
fig, ax = plt.subplots(figsize=(5.8, 4.3))
ax.scatter(Boston["lstat"], Boston["medv"], s=8, color="0.4")
ax.axline((0, results.params["const"]), slope=results.params["lstat"], color="red", lw=3)
ax.set_xlabel("lstat")
ax.set_ylabel("medv")
save(fig, "lab-py-fit")

## ==== step: simple-diagnostics
def diagnostics(fit, name):
    """The four panels that R draws for plot(lm.fit)."""
    infl = fit.get_influence()
    r_std = infl.resid_studentized_internal
    h = infl.hat_matrix_diag
    fv = fit.fittedvalues
    fig, ax = plt.subplots(2, 2, figsize=(8.4, 7.4))
    ax[0, 0].scatter(fv, fit.resid, s=8, facecolors="none", edgecolors="0.3")
    ax[0, 0].plot(*lowess(fit.resid, fv, frac=2 / 3).T, color="red")
    ax[0, 0].set(xlabel="Fitted values", ylabel="Residuals", title="Residuals vs Fitted")
    sm.qqplot(r_std, line="45", ax=ax[0, 1], markersize=3)
    ax[0, 1].set(title="Normal Q-Q", ylabel="Standardized residuals")
    ax[1, 0].scatter(fv, np.sqrt(np.abs(r_std)), s=8, facecolors="none", edgecolors="0.3")
    ax[1, 0].plot(*lowess(np.sqrt(np.abs(r_std)), fv, frac=2 / 3).T, color="red")
    ax[1, 0].set(xlabel="Fitted values", ylabel=r"$\sqrt{|\mathrm{standardized\ residual}|}$", title="Scale-Location")
    ax[1, 1].scatter(h, r_std, s=8, facecolors="none", edgecolors="0.3")
    ax[1, 1].set(xlabel="Leverage", ylabel="Standardized residuals", title="Residuals vs Leverage")
    fig.tight_layout()
    save(fig, name)


diagnostics(results, "lab-py-diagnostics")

## ==== step: simple-leverage
infl = results.get_influence()
h = infl.hat_matrix_diag
print(np.argmax(h), h.max())                  # index 374: Python counts from 0 (R says 375)
r_stud = pd.Series(infl.resid_studentized_external, index=Boston.index)
print(r_stud.abs().sort_values(ascending=False).head(3))

## ==== step: multiple
results1 = smf.ols("medv ~ lstat + age", data=Boston).fit()
print(coefs(results1))

## ==== step: multiple-all
terms = Boston.columns.drop("medv")
results = smf.ols("medv ~ " + " + ".join(terms), data=Boston).fit()
print(coefs(results))
print("RSE %.3f on %d df, R2 %.4f, adjusted R2 %.4f, F %.1f" % (np.sqrt(results.scale), results.df_resid, results.rsquared, results.rsquared_adj, results.fvalue))

## ==== step: vif
Xall = sm.add_constant(Boston[terms])
vif = pd.Series([variance_inflation_factor(Xall.values, i) for i in range(1, Xall.shape[1])], index=terms)
print(vif.round(3))

## ==== step: minus-age
minus_age = terms.drop("age")
print(coefs(smf.ols("medv ~ " + " + ".join(minus_age), data=Boston).fit()))

## ==== step: interaction
print(coefs(smf.ols("medv ~ lstat * age", data=Boston).fit(), 6))

## ==== step: quadratic
lm_fit = smf.ols("medv ~ lstat", data=Boston).fit()
lm_fit2 = smf.ols("medv ~ lstat + I(lstat**2)", data=Boston).fit()
print(coefs(lm_fit2, 5))
print(anova_lm(lm_fit, lm_fit2))

## ==== step: quadratic-age
P2 = poly(Boston["lstat"], 2)
B2 = Boston.assign(p1=P2[:, 0], p2=P2[:, 1])
results3 = smf.ols("medv ~ p1 + p2 + age", data=B2).fit()
print(coefs(results3))
print(anova_lm(results1, results3))

## ==== step: quadratic-diagnostics
diagnostics(lm_fit2, "lab-py-diagnostics-quad")

## ==== step: poly5
P5 = pd.DataFrame(poly(Boston["lstat"], 5), columns=["p%d" % k for k in range(1, 6)], index=Boston.index)
lm_fit5 = sm.OLS(Boston["medv"], sm.add_constant(P5)).fit()
print(coefs(lm_fit5))
raw5 = smf.ols("medv ~ lstat + I(lstat**2) + I(lstat**3) + I(lstat**4) + I(lstat**5)", data=Boston).fit()
print(np.abs(lm_fit5.fittedvalues - raw5.fittedvalues).max())   # same fitted values, different coefficients

## ==== step: log
print(coefs(smf.ols("medv ~ np.log(rm)", data=Boston).fit()))

## ==== step: carseats
Carseats = isl_data("Carseats")
print(Carseats.shape)
allvars = Carseats.columns.drop("Sales")
formula = "Sales ~ " + " + ".join(allvars) + " + Income:Advertising + Price:Age"
fit_cs = smf.ols(formula, data=Carseats).fit()
print(coefs(fit_cs))
print("RSE %.3f on %d df, R2 %.4f, F %.1f" % (np.sqrt(fit_cs.scale), fit_cs.df_resid, fit_cs.rsquared, fit_cs.fvalue))

## ==== step: contrasts
levels = sorted(Carseats["ShelveLoc"].unique())            # Bad, Good, Medium: Bad is the baseline
print(pd.get_dummies(pd.Categorical(levels, categories=levels), drop_first=True, dtype=int).set_axis(levels))

## ==== step: function
def abline(ax, b, m, *args, **kwargs):
    "Add the line with intercept b and slope m to the axes ax."
    xlim = ax.get_xlim()
    ax.plot(xlim, [b + m * xlim[0], b + m * xlim[1]], *args, **kwargs)


fig, ax = plt.subplots(figsize=(5.8, 4.3))
ax.scatter(Boston["lstat"], Boston["medv"], s=8, color="0.4")
abline(ax, lm_fit.params["Intercept"], lm_fit.params["lstat"], "r--", linewidth=3)
plt.close(fig)
print("abline drew a line with intercept %.4f and slope %.4f" % (lm_fit.params["Intercept"], lm_fit.params["lstat"]))
