"""Chapter 10 lab: Deep Learning (Python version).

Course: Applied Statistical Learning in R and Python, which follows James, Witten, Hastie,
Tibshirani and Taylor (2023), An Introduction to Statistical Learning with Applications in Python,
Section 10.9, and the R edition of 2021.

The networks are written in plain PyTorch (torch.nn and a ten-line training loop), without
pytorch_lightning, torchvision or the book's helper package:
  [1] 10.9.1  a single-layer network on the Hitters data, against least squares and the lasso
              (the lasso path is put on glmnet's scale, so it matches the R lab);
  [2] 10.9.2  a two-hidden-layer network and multinomial logistic regression on the MNIST digits
              (loaded with common.mnist);
  [3] 10.9.3  the CIFAR100 network of the book, built and its parameters counted (not trained), and
              a small convolutional network trained on MNIST instead;
  [4] 10.9.6  the NYSE trading-volume forecasts: linear AR models, a recurrent network and a
              nonlinear AR model;
  [5] the applied exercises 6, 7 and 9-12.
The pretrained ResNet50 (10.9.4) and the IMDb models (10.9.5) need downloads (torchvision weights,
the IMDb corpus) and are described in the notes but not run. The NYSE data are not among the book's
CSV files; load_nyse() exports them once from the R package ISLR2 to cache/ch10-NYSE.csv.

The Hitters test set is the one of the R lab (set.seed(13); sample(1:263, 87)), so least squares
and the lasso give the same numbers in both languages. The networks are trained with the same
architecture, optimiser and settings as in the R lab, but random initial weights, minibatches and
dropout masks differ between R and torch, so their results agree only roughly.

Run: python3 ch10_lab.py     (about two minutes; prints the results, writes ../img/ch10/lab-py-*.svg
and the key numbers to cache/ch10_lab_py.csv). The code is my own.
"""
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import subprocess
import time

import numpy as np
import pandas as pd
import sklearn.linear_model as skl
import statsmodels.api as sm
import torch
from torch import nn

import common
from common import plt, savefig

HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.join(HERE, "cache")
torch.set_num_threads(1)
pd.set_option("display.width", 100)
np.set_printoptions(linewidth=100, suppress=True)
KEY = {}


def keep(name, value):
    KEY[name] = np.atleast_1d(np.asarray(value, float))


def step(k, title):
    print("\n== [%s] %s" % (k, title))


# ======================================================================== helpers
def n_params(model):
    return sum(p.numel() for p in model.parameters())


def fit(model, X, y, loss_fn, epochs, batch_size, lr=0.001, monitor=None, seed=0):
    """Minibatch RMSprop (torch's defaults, as the R lab's rmsprop_step). monitor(model) is called
    after every epoch, in evaluation mode (dropout switched off); returns its values as a DataFrame."""
    gen = torch.Generator().manual_seed(seed)
    opt = torch.optim.RMSprop(model.parameters(), lr=lr)
    n, rows = len(y), []
    for epoch in range(1, epochs + 1):
        model.train()
        perm = torch.randperm(n, generator=gen)
        for s in range(0, n, batch_size):
            idx = perm[s:s + batch_size]
            opt.zero_grad()
            loss_fn(model(X[idx]), y[idx]).backward()
            opt.step()
        if monitor is not None:
            model.eval()
            with torch.no_grad():
                rows.append(dict(epoch=epoch, **monitor(model)))
    model.eval()
    return pd.DataFrame(rows)


def mse(out, y):
    return ((out.flatten() - y) ** 2).mean()


def tensor(a):
    return torch.tensor(np.asarray(a, dtype=np.float32))


def lasso_glmnet(X, y, lambdas=None, nlambda=100):
    """The lasso path on glmnet's scale: columns standardised with their 1/n standard deviation,
    objective RSS/(2n) + lambda * sum|beta_j|, glmnet's default lambda sequence. Returns
    (lambdas, intercepts, coefficients on the original scale, one column per lambda)."""
    mx, sx = X.mean(0), X.std(0)
    Z, yc = (X - mx) / sx, y - y.mean()
    if lambdas is None:
        lmax = np.abs(Z.T @ yc).max() / len(y)
        ratio = 1e-4 if len(y) > X.shape[1] else 1e-2
        lambdas = lmax * np.exp(np.linspace(0, np.log(ratio), nlambda))
    _, coefs, _ = skl.lasso_path(Z, yc, alphas=lambdas, tol=1e-12, max_iter=200000)
    B = coefs / sx[:, None]
    return lambdas, y.mean() - mx @ B, B


def load_nyse():
    """The NYSE data (6051 trading days, 1962-1986) of the R package ISLR2, exported once to cache/."""
    path = os.path.join(CACHE, "ch10-NYSE.csv")
    if not os.path.exists(path):
        os.makedirs(CACHE, exist_ok=True)
        subprocess.run([os.environ.get("RSCRIPT", "/usr/local/bin/Rscript"), "-e",
                        'write.csv(ISLR2::NYSE, "%s", row.names = FALSE)' % path], check=True, capture_output=True)
    return pd.read_csv(path)


# Hitters: the test players of the R lab, set.seed(13); sample(1:263, 87) (1-based row numbers)
TESTID = np.array([259, 74, 205, 132, 221, 248, 55, 23, 5, 140, 49, 54, 184, 32, 250, 125, 47, 81, 188, 253, 11, 68,
                   120, 87, 78, 61, 117, 131, 223, 236, 197, 190, 242, 222, 66, 129, 62, 51, 216, 127, 115, 118, 60,
                   244, 76, 251, 42, 182, 227, 119, 199, 15, 157, 113, 14, 185, 121, 160, 31, 146, 258, 179, 233, 239,
                   263, 77, 176, 228, 105, 97, 159, 48, 143, 232, 124, 247, 166, 136, 142, 139, 195, 84, 17, 153, 147,
                   53, 7])


def hitters_matrix(H):
    """R's model.matrix(Salary ~ . - 1): without an intercept the first factor (League) keeps both of its
    dummies, LeagueA and LeagueN; the later factors drop their first level (DivisionW, NewLeagueN)."""
    cols, first = {}, True
    for c in H.columns.drop("Salary"):
        if H[c].dtype == object:
            levels = sorted(H[c].unique())
            for lev in (levels if first else levels[1:]):
                cols[c + lev] = (H[c] == lev).to_numpy(float)
            first = False
        else:
            cols[c] = H[c].to_numpy(float)
    return pd.DataFrame(cols)


if __name__ == "__main__":
    # ================================================================== [1] Hitters
    step(1, "10.9.1 A single-layer network on the Hitters data")
    Hitters = common.isl_data("Hitters").dropna().reset_index(drop=True)
    n = len(Hitters)
    test = np.zeros(n, bool)
    test[TESTID - 1] = True
    keep("testid", TESTID)

    Xlm = pd.get_dummies(Hitters.drop(columns=["Salary"]), drop_first=True, dtype=float)
    Xlm.insert(0, "intercept", 1.0)
    Y = Hitters["Salary"].to_numpy()
    beta = np.linalg.lstsq(Xlm[~test].to_numpy(), Y[~test], rcond=None)[0]
    mae_lm = np.abs(Xlm[test].to_numpy() @ beta - Y[test]).mean()
    print("least squares: test MAE %.4f" % mae_lm)
    keep("hit_lm_mae", mae_lm)

    M = hitters_matrix(Hitters)
    X = ((M - M.mean()) / M.std()).to_numpy()            # R's scale(): 1/(n-1) standard deviations
    print("model matrix:", X.shape, list(M.columns[13:17]))
    Xtr, Ytr, Xte, Yte = X[~test], Y[~test], X[test], Y[test]
    lambdas, _, _ = lasso_glmnet(Xtr, Ytr)
    fold = np.arange(len(Ytr)) % 10                      # training row i in fold i mod 10, as foldid in R
    abs_err = np.zeros((len(Ytr), len(lambdas)))
    for k in range(10):
        _, b0, B = lasso_glmnet(Xtr[fold != k], Ytr[fold != k], lambdas)
        abs_err[fold == k] = np.abs(Ytr[fold == k, None] - (b0 + Xtr[fold == k] @ B))
    best = np.argmin(abs_err.mean(0))
    _, b0, B = lasso_glmnet(Xtr, Ytr, lambdas)
    mae_lasso = np.abs(Yte - (b0[best] + Xte @ B[:, best])).mean()
    print("lasso, fixed folds: lambda.min %.4f  test MAE %.4f  nonzero coefficients %d"
          % (lambdas[best], mae_lasso, np.sum(B[:, best] != 0)))
    keep("hit_lasso_lambda", lambdas[best])
    keep("hit_lasso_mae", mae_lasso)
    keep("hit_lasso_nz", np.sum(B[:, best] != 0))

    torch.manual_seed(1)
    hit_model = nn.Sequential(nn.Linear(X.shape[1], 50), nn.ReLU(), nn.Dropout(0.4), nn.Linear(50, 1))
    print("parameters:", n_params(hit_model))           # 50 * 21 + 51 = 1101
    keep("hit_nn_params", n_params(hit_model))
    Xtr_t, Ytr_t, Xte_t, Yte_t = tensor(Xtr), tensor(Ytr), tensor(Xte), tensor(Yte)
    hist = fit(hit_model, Xtr_t, Ytr_t, mse, epochs=1500, batch_size=32, lr=0.001,
               monitor=lambda m: {"train_mae": float((m(Xtr_t).flatten() - Ytr_t).abs().mean()),
                                  "test_mae": float((m(Xte_t).flatten() - Yte_t).abs().mean())})
    with torch.no_grad():
        mae_nn = float((hit_model(Xte_t).flatten() - Yte_t).abs().mean())
    print("neural network: test MAE %.4f" % mae_nn)
    keep("hit_nn_mae", mae_nn)

    fig, ax = plt.subplots(figsize=(6.4, 4))
    ax.plot(hist.epoch, hist.train_mae, color="black", lw=1.4, label="training")
    ax.plot(hist.epoch, hist.test_mae, color="red", lw=1.4, label="test (validation)")
    ax.axhline(mae_lm, color="grey", ls="--", lw=1, label="least squares, test")
    ax.set(xlabel="Epoch", ylabel="Mean absolute error", ylim=(0, 700))
    ax.legend(frameon=False)
    savefig(fig, 10, "lab-py-hitters", formats=("svg",))

    # ================================================================== [2] MNIST
    step(2, "10.9.2 A multilayer network on the MNIST digits")
    x_train, g_train = common.mnist(train=True)          # 60000 x 784, pixel values scaled to [0, 1]
    x_test, g_test = common.mnist(train=False)
    print("MNIST:", x_train.shape, "training,", x_test.shape, "test")
    fit_rows = np.arange(48000)                          # the last 20% are the validation set
    val = np.arange(48000, 60000)
    Xm, Gm = tensor(x_train), torch.tensor(g_train)
    Xm_test, Gm_test = tensor(x_test), torch.tensor(g_test)

    def accuracy(model, X, g):
        with torch.no_grad():
            return float((model(X).argmax(1) == g).float().mean())

    torch.manual_seed(2)
    mnist_model = nn.Sequential(nn.Linear(784, 256), nn.ReLU(), nn.Dropout(0.4),
                                nn.Linear(256, 128), nn.ReLU(), nn.Dropout(0.3),
                                nn.Linear(128, 10))
    print("parameters:", n_params(mnist_model))         # 200960 + 32896 + 1290 = 235146
    keep("mnist_params", n_params(mnist_model))
    t0 = time.time()
    mhist = fit(mnist_model, Xm[fit_rows], Gm[fit_rows], nn.functional.cross_entropy, epochs=30, batch_size=128,
                lr=0.001, monitor=lambda m: {"train_acc": accuracy(m, Xm[:12000], Gm[:12000]),
                                             "valid_acc": accuracy(m, Xm[val], Gm[val])})
    print("fitting took %.0f seconds" % (time.time() - t0))
    acc_nn = accuracy(mnist_model, Xm_test, Gm_test)
    print("two hidden layers with dropout: test accuracy %.4f" % acc_nn)
    keep("mnist_nn_acc", acc_nn)

    torch.manual_seed(3)
    mlr_model = nn.Linear(784, 10)                       # no hidden layer: multinomial logistic regression
    print("parameters:", n_params(mlr_model))           # 785 * 10 = 7850
    fit(mlr_model, Xm[fit_rows], Gm[fit_rows], nn.functional.cross_entropy, epochs=30, batch_size=128, lr=0.001)
    acc_lr = accuracy(mlr_model, Xm_test, Gm_test)
    print("multinomial logistic regression: test accuracy %.4f" % acc_lr)
    keep("mnist_mlr_acc", acc_lr)

    fig, ax = plt.subplots(figsize=(6.4, 4))
    ax.plot(mhist.epoch, mhist.train_acc, "o-", ms=3, color="black", label="training (first 12000 images)")
    ax.plot(mhist.epoch, mhist.valid_acc, "o-", ms=3, color="red", label="validation (last 12000)")
    ax.set(xlabel="Epoch", ylabel="Accuracy", ylim=(0.9, 1))
    ax.legend(frameon=False, loc="lower right")
    savefig(fig, 10, "lab-py-mnist", formats=("svg",))

    # ================================================================== [3] convolutional networks
    step(3, "10.9.3 Convolutional networks")

    class BuildingBlock(nn.Module):
        """3 x 3 convolution with 'same' padding, ReLU, then 2 x 2 max pooling."""
        def __init__(self, in_channels, out_channels):
            super().__init__()
            self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding="same")
            self.pool = nn.MaxPool2d(2)

        def forward(self, x):
            return self.pool(torch.relu(self.conv(x)))

    cifar_model = nn.Sequential(BuildingBlock(3, 32), BuildingBlock(32, 64), BuildingBlock(64, 128),
                                BuildingBlock(128, 256), nn.Flatten(), nn.Dropout(0.5),
                                nn.Linear(2 * 2 * 256, 512), nn.ReLU(), nn.Linear(512, 100))
    shape = torch.zeros(1, 3, 32, 32)
    for layer in cifar_model:
        shape = layer(shape)
        print("  %-14s output %-18s parameters %d" % (type(layer).__name__, tuple(shape.shape[1:]), n_params(layer)))
    print("CIFAR100 network, total parameters:", n_params(cifar_model))   # 964516
    keep("cifar_params", n_params(cifar_model))

    torch.manual_seed(4)
    cnn = nn.Sequential(BuildingBlock(1, 16), BuildingBlock(16, 32), nn.Flatten(), nn.Dropout(0.5),
                        nn.Linear(32 * 7 * 7, 128), nn.ReLU(), nn.Linear(128, 10))
    print("small MNIST CNN, parameters:", n_params(cnn))
    keep("cnn_params", n_params(cnn))
    Xc, Xc_test = Xm.reshape(-1, 1, 28, 28), Xm_test.reshape(-1, 1, 28, 28)
    t0 = time.time()
    fit(cnn, Xc[fit_rows], Gm[fit_rows], nn.functional.cross_entropy, epochs=3, batch_size=128, lr=0.001)
    print("fitting took %.0f seconds" % (time.time() - t0))
    acc_cnn = accuracy(cnn, Xc_test, Gm_test)
    print("small CNN: test accuracy %.4f" % acc_cnn)
    keep("cnn_acc", acc_cnn)

    # ================================================================== [4] NYSE
    step(4, "10.9.6 Recurrent networks: the NYSE data")
    NYSE = load_nyse()
    cols = ["DJ_return", "log_volume", "log_volatility"]
    xdata = (NYSE[cols] - NYSE[cols].mean()) / NYSE[cols].std()    # R's scale()
    istrain = NYSE["train"].to_numpy(bool)
    arframe = pd.DataFrame({"log_volume": xdata["log_volume"]})
    for lag in range(1, 6):
        for c in cols:
            arframe["L%d.%s" % (lag, c)] = xdata[c].shift(lag)
    arframe, istrain = arframe.iloc[5:], istrain[5:]
    print("training days", istrain.sum(), " test days", (~istrain).sum())
    yv = arframe["log_volume"].to_numpy()
    V0 = yv[~istrain].var(ddof=1)                         # R's var(); sklearn's score() divides by n instead

    def r2(pred):
        return 1 - np.mean((pred - yv[~istrain]) ** 2) / V0

    def ols_pred(Xd):
        Xd = np.column_stack([np.ones(len(Xd)), Xd])
        b = np.linalg.lstsq(Xd[istrain], yv[istrain], rcond=None)[0]
        return Xd[~istrain] @ b

    Xlag = arframe.drop(columns="log_volume").to_numpy()
    r2_ar = r2(ols_pred(Xlag))
    sst = np.sum((yv[~istrain] - yv[~istrain].mean()) ** 2)
    print("AR(5): test R2 %.4f (with the n-denominator of sklearn's score(): %.4f)"
          % (r2_ar, 1 - np.sum((ols_pred(Xlag) - yv[~istrain]) ** 2) / sst))
    keep("ar_r2", r2_ar)
    day = pd.get_dummies(NYSE["day_of_week"].iloc[5:], dtype=float).to_numpy()   # fri, mon, thur, tues, wed
    r2_ard = r2(ols_pred(np.column_stack([day[:, 1:], Xlag])))
    print("AR(5) + day of week: test R2 %.4f" % r2_ard)
    keep("ard_r2", r2_ard)

    X_rnn = Xlag.reshape(-1, 5, 3)[:, ::-1, :].copy()     # n x 5 (time, oldest first) x 3 (variables)
    print("RNN input:", X_rnn.shape)

    class NYSEModel(nn.Module):
        def __init__(self, p=3, K=12):
            super().__init__()
            self.rnn = nn.RNN(p, K, batch_first=True)      # A_l = tanh(W x_l + U A_{l-1} + b), (10.16)
            self.dropout = nn.Dropout(0.1)
            self.dense = nn.Linear(K, 1)

        def forward(self, x):
            val, _ = self.rnn(x)
            return self.dense(self.dropout(val[:, -1])).flatten()

    torch.manual_seed(4)
    nyse_model = NYSEModel()
    print("RNN parameters:", n_params(nyse_model))       # 217: torch's RNN has two bias vectors
    keep("rnn_params", n_params(nyse_model))
    Xr_tr, Xr_te, Yr_tr = tensor(X_rnn[istrain]), tensor(X_rnn[~istrain]), tensor(yv[istrain])
    t0 = time.time()
    fit(nyse_model, Xr_tr, Yr_tr, mse, epochs=200, batch_size=64, lr=0.001)
    print("fitting took %.0f seconds" % (time.time() - t0))
    with torch.no_grad():
        kpred = nyse_model(Xr_te).numpy()
    r2_rnn = r2(kpred)
    print("RNN: test R2 %.4f" % r2_rnn)
    keep("rnn_r2", r2_rnn)

    Xday = np.column_stack([day, Xlag])                   # 5 day dummies + 15 lags, as model.matrix(~ . - 1)
    torch.manual_seed(5)
    nl_model = nn.Sequential(nn.Linear(20, 32), nn.ReLU(), nn.Dropout(0.5), nn.Linear(32, 1))
    fit(nl_model, tensor(Xday[istrain]), Yr_tr, mse, epochs=100, batch_size=32, lr=0.001)
    with torch.no_grad():
        r2_nar = r2(nl_model(tensor(Xday[~istrain])).flatten().numpy())
    print("nonlinear AR + day of week: test R2 %.4f" % r2_nar)
    keep("nard_r2", r2_nar)

    fig, ax = plt.subplots(figsize=(7, 3.6))
    dates = pd.to_datetime(NYSE["date"].iloc[5:][~istrain])
    ax.plot(dates, yv[~istrain], color="0.3", lw=0.6, label="observed")
    ax.plot(dates, kpred, color="orange", lw=0.8, label="RNN forecast")
    ax.set(ylabel="log(Trading Volume)")
    ax.legend(frameon=False, loc="lower left")
    savefig(fig, 10, "lab-py-nyse", formats=("svg",))

    # ================================================================== [5] exercises
    step(5, "Exercises 6, 7, 9-12")

    def gd(b, rho=0.1, tol=1e-10, maxit=10000):          # Exercise 6: R(beta) = sin(beta) + beta / 10
        for it in range(1, maxit + 1):
            b_new = b - rho * (np.cos(b) + 0.1)
            if abs(b_new - b) < tol:
                break
            b = b_new
        return b_new, np.sin(b_new) + b_new / 10, it

    ex6a, ex6b = gd(2.3), gd(1.4)
    print("Exercise 6: from 2.3 -> beta %.6f, R %.6f (%d steps); from 1.4 -> beta %.6f, R %.6f (%d steps)"
          % (ex6a + ex6b))
    keep("ex6_a", ex6a[:2])
    keep("ex6_b", ex6b[:2])

    Default = common.isl_data("Default")                  # Exercise 7
    Xd7 = np.column_stack([(Default["student"] == "Yes").astype(float), Default["balance"], Default["income"]])
    Xd7 = (Xd7 - Xd7.mean(0)) / Xd7.std(0, ddof=1)
    yd7 = (Default["default"] == "Yes").to_numpy(float)
    te7 = np.arange(len(yd7)) % 3 == 0                    # every third row is a test row
    glm7 = sm.GLM(yd7[~te7], sm.add_constant(Xd7[~te7]), family=sm.families.Binomial()).fit()
    err_glm = np.mean((glm7.predict(sm.add_constant(Xd7[te7])) > 0.5) != yd7[te7])
    torch.manual_seed(7)
    nn7 = nn.Sequential(nn.Linear(3, 10), nn.ReLU(), nn.Dropout(0.4), nn.Linear(10, 1))
    fit(nn7, tensor(Xd7[~te7]), tensor(yd7[~te7]), lambda o, t: nn.functional.binary_cross_entropy_with_logits(o.flatten(), t),
        epochs=30, batch_size=32, lr=0.001)
    with torch.no_grad():
        err_nn = np.mean((nn7(tensor(Xd7[te7])).flatten().numpy() > 0) != yd7[te7])
    print("Exercise 7: test error, logistic regression %.4f  network %.4f" % (err_glm, err_nn))
    keep("ex7_glm_err", err_glm)
    keep("ex7_nn_err", err_nn)

    month = pd.get_dummies(pd.to_datetime(NYSE["date"].iloc[5:]).dt.month, dtype=float).to_numpy()   # Exercise 9
    r2_m = r2(ols_pred(np.column_stack([month[:, 1:], Xlag])))
    print("Exercise 9: AR(5) + month: test R2 %.4f (without month %.4f)" % (r2_m, r2_ar))
    keep("ex9_r2", r2_m)

    Xflat = X_rnn.reshape(len(yv), 15)                    # Exercise 10: the flattened RNN input
    r2_flat = r2(ols_pred(Xflat))
    print("Exercise 10: flattened linear AR: test R2 %.4f" % r2_flat)
    keep("ex10_r2", r2_flat)

    torch.manual_seed(11)                                 # Exercise 11
    nn11 = nn.Sequential(nn.Linear(15, 32), nn.ReLU(), nn.Dropout(0.5), nn.Linear(32, 1))
    fit(nn11, tensor(Xflat[istrain]), Yr_tr, mse, epochs=100, batch_size=32, lr=0.001)
    with torch.no_grad():
        r2_11 = r2(nn11(tensor(Xflat[~istrain])).flatten().numpy())
    print("Exercise 11: flattened nonlinear AR: test R2 %.4f" % r2_11)
    keep("ex11_r2", r2_11)

    X_rnn_d = np.concatenate([X_rnn, np.repeat(day[:, None, :], 5, axis=1)], axis=2)   # Exercise 12
    torch.manual_seed(12)
    rnn12 = NYSEModel(p=8)
    fit(rnn12, tensor(X_rnn_d[istrain]), Yr_tr, mse, epochs=200, batch_size=64, lr=0.001)
    with torch.no_grad():
        r2_12 = r2(rnn12(tensor(X_rnn_d[~istrain])).numpy())
    print("Exercise 12: RNN + day of week: test R2 %.4f" % r2_12)
    keep("ex12_r2", r2_12)

    names, vals = [], []
    for k, v in KEY.items():
        names += [k] if len(v) == 1 else ["%s%d" % (k, i + 1) for i in range(len(v))]
        vals += list(v)
    os.makedirs(CACHE, exist_ok=True)
    pd.DataFrame({"name": names, "value": vals}).to_csv(os.path.join(CACHE, "ch10_lab_py.csv"), index=False,
                                                        float_format="%.10g")
    print("wrote %d numbers to cache/ch10_lab_py.csv" % len(vals))
