"""Chapter 13 lab: multiple testing (Python version).

Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie, Tibshirani and
Taylor (2023), An Introduction to Statistical Learning with Applications in Python, Section 13.6, and
the R lab of James et al. (2021), Section 13.6.

The lab does what ch13_lab.R does, with numpy, pandas, scipy and statsmodels:
    13.6.1  one-sample t-tests on simulated data, and the 2 x 2 table of Table 13.2;
    13.6.2  the FWER curves (13.5), t-tests for five fund managers, Bonferroni and Holm
            (statsmodels multipletests, and by hand), a paired t-test, Tukey's method
            (pairwise_tukeyhsd), and Scheffe's threshold for the contrast (13.8);
    13.6.3  t-tests for all 2,000 managers and Benjamini-Hochberg (multipletests, and by hand);
    13.6.4  a permutation p-value (Algorithm 13.3) and the plug-in FDR (Algorithm 13.4) on Khan;
and the applied Exercises 7 (Carseats) and 8 (simulated fund managers).

The random numbers follow the Python lab of the book: one generator, numpy default_rng(12), is used
in the same order for the simulations, the 10,000 shuffles of gene 11, the choice of 100 genes and
their 10,000 shuffles each, so the printed numbers can be compared with that edition. The shuffles
are recorded and the t-statistics of all of them are then computed in one vectorised step.

Khan is an R data set (package ISLR2); it is read from cache/ch13-Khan.csv, which
ch13_multiple_testing.khan() writes once with Rscript.

Run from the code/ folder:  python3 ch13_lab.py      (about 15 seconds)
Figures go to ../img/ch13/lab-py-*.svg (and .pdf) and the key numbers to cache/ch13_lab_py.csv.
"""
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import numpy as np
import pandas as pd
import statsmodels.api as sm
from scipy.stats import f as f_dbn, studentized_range, t as t_dbn, ttest_1samp, ttest_ind, ttest_rel
from statsmodels.stats.multicomp import pairwise_tukeyhsd
from statsmodels.stats.multitest import multipletests as mult_test

import common

HERE = os.path.dirname(os.path.abspath(__file__))
key = {}


def keep(name, value):
    key[name] = np.atleast_1d(np.asarray(value, float)).ravel()


def show(label, value):
    print(label, value)


def save(fig, name):
    common.savefig(fig, 13, name, formats=("svg", "pdf"))


def tstat_rows(D, n1):
    """Pooled two-sample t-statistic (13.11) for every row of D: the first n1 entries against the rest."""
    a, b = D[:, :n1], D[:, n1:]
    na, nb = a.shape[1], b.shape[1]
    s2 = ((na - 1) * a.var(1, ddof=1) + (nb - 1) * b.var(1, ddof=1)) / (na + nb - 2)
    return (a.mean(1) - b.mean(1)) / np.sqrt(s2 * (1 / na + 1 / nb))


def main():
    import matplotlib.pyplot as plt

    # ============================================================== 13.6.1 review of hypothesis tests
    rng = np.random.default_rng(12)
    X = rng.standard_normal((10, 100))            # 100 variables, 10 observations each
    true_mean = np.array([0.5] * 50 + [0] * 50)
    X += true_mean[None, :]
    result = ttest_1samp(X[:, 0], 0)
    show("first variable: t, p =", (round(result.statistic, 4), round(result.pvalue, 4)))   # p = 0.931
    keep("sim1_p1", result.pvalue)

    def crosstab(p_values, true_mean):
        decision = pd.Categorical(np.where(p_values <= 0.05, "Reject H0", "Do not reject H0"),
                                  categories=["Do not reject H0", "Reject H0"])
        truth = pd.Categorical(np.where(true_mean == 0, "H0 is True", "H0 is False"),
                               categories=["H0 is False", "H0 is True"])
        return pd.crosstab(decision, truth, rownames=["decision"], colnames=["truth"], dropna=False)

    p_values = ttest_1samp(X, 0).pvalue           # all 100 tests at once (one per column)
    tab1 = crosstab(p_values, true_mean)
    print(tab1)                                   # S = 15, V = 5 in this draw
    keep("sim1_table", tab1.to_numpy().T)         # column-major, as R: W, S, U, V

    true_mean = np.array([1] * 50 + [0] * 50)     # a stronger signal
    X = rng.standard_normal((10, 100))
    X += true_mean[None, :]
    p_values = ttest_1samp(X, 0).pvalue
    tab2 = crosstab(p_values, true_mean)
    print(tab2)                                   # S = 40, V = 2
    keep("sim2_table", tab2.to_numpy().T)

    # ============================================================== 13.6.2 the family-wise error rate
    m = np.arange(1, 501)
    fig, ax = plt.subplots(figsize=(6, 4.5))
    for alpha in (0.05, 0.01, 0.001):
        ax.plot(m, 1 - (1 - alpha) ** m, label=r"$\alpha=%s$" % alpha)      # (13.5)
    ax.set_xscale("log")
    ax.set(xlabel="Number of Hypotheses", ylabel="Family-Wise Error Rate", ylim=(0, 1))
    ax.legend()
    ax.axhline(0.05, c="k", ls="--")
    save(fig, "lab-py-fwer")
    keep("fwer_100_005", 1 - 0.95 ** 100)

    Fund = common.isl_data("Fund", index_col=0)  # 50 months x 2,000 managers
    fund_mini = Fund.iloc[:, :5]
    res = ttest_1samp(fund_mini, 0)               # one test per column
    fund_mini_pvals = res.pvalue
    show("five managers, p-values:", np.round(fund_mini_pvals, 4))           # 0.0062 0.9183 0.0116 0.6005 0.7558
    keep("mini_t", res.statistic)
    keep("mini_p", fund_mini_pvals)
    keep("mini_mean", fund_mini.mean())
    keep("mini_sd", fund_mini.std(ddof=1))

    reject, bonf = mult_test(fund_mini_pvals, method="bonferroni")[:2]
    show("Bonferroni:", (reject, np.round(bonf, 4)))                       # only Manager One
    show("  = min(5 p, 1):", np.allclose(bonf, np.minimum(fund_mini_pvals * 5, 1)))
    reject, holm = mult_test(fund_mini_pvals, method="holm", alpha=0.05)[:2]
    show("Holm:", (reject, np.round(holm, 4)))                             # Managers One and Three
    keep("mini_bonf", bonf)
    keep("mini_holm", holm)

    # Holm's adjusted p-values by hand: the j-th smallest times m + 1 - j, running maximum, cap at 1.
    def holm_by_hand(p):
        m = len(p)
        o = np.argsort(p)
        adj = np.minimum(1, np.maximum.accumulate((m - np.arange(m)) * p[o]))
        out = np.empty(m)
        out[o] = adj
        return out

    show("  Holm by hand agrees:", np.allclose(holm_by_hand(fund_mini_pvals), holm))

    show("means:", fund_mini.mean().to_numpy())                               # 3.0 -0.1 2.8 0.5 0.3
    pair_p = ttest_rel(fund_mini["Manager1"], fund_mini["Manager2"]).pvalue
    two_p = ttest_ind(fund_mini["Manager1"], fund_mini["Manager2"], equal_var=True).pvalue
    show("paired t-test p:", round(pair_p, 4))                                 # 0.038
    show("unpaired two-sample t-test p:", round(two_p, 4))                     # 0.0324
    keep("pair_p", pair_p)
    keep("two_p", two_p)

    returns = np.hstack([fund_mini.iloc[:, i] for i in range(5)])
    managers = np.hstack([[i + 1] * 50 for i in range(5)])
    tukey = pairwise_tukeyhsd(returns, managers)
    print(tukey.summary())                        # 1-2: meandiff -3.1, p-adj 0.1862
    keep("tukey_diff", tukey.meandiffs)
    keep("tukey_lwr", tukey.confint[:, 0])
    keep("tukey_padj", tukey.pvalues)
    fig, ax = plt.subplots(figsize=(6, 5))
    tukey.plot_simultaneous(ax=ax)
    save(fig, "lab-py-tukey")

    # Beyond the book's lab: Tukey's p-value threshold, and Scheffe's test of the contrast (13.8).
    df_res = len(returns) - 5
    s2 = sum(((fund_mini.iloc[:, g] - fund_mini.iloc[:, g].mean()) ** 2).sum() for g in range(5)) / df_res
    alpha_T = 2 * t_dbn.sf(studentized_range.ppf(0.95, 5, df_res) / np.sqrt(2), df_res)
    c = np.array([1 / 2, -1 / 3, 1 / 2, -1 / 3, -1 / 3])
    est = c @ fund_mini.mean().to_numpy()
    t_contrast = est / np.sqrt(s2 * np.sum(c ** 2) / 50)
    p_contrast = 2 * t_dbn.sf(abs(t_contrast), df_res)
    alpha_S = 2 * t_dbn.sf(np.sqrt(4 * f_dbn.ppf(0.95, 4, df_res)), df_res)
    show("alpha_T, contrast t and p, alpha_S:", np.round([alpha_T, t_contrast, p_contrast, alpha_S], 5))
    keep("alpha_T", alpha_T)
    keep("contrast_t", t_contrast)
    keep("contrast_p", p_contrast)
    keep("alpha_S", alpha_S)

    # ============================================================== 13.6.3 the false discovery rate
    fund_pvalues = ttest_1samp(Fund, 0).pvalue    # 2,000 one-sample tests
    fund_qvalues = mult_test(fund_pvalues, method="fdr_bh")[1]
    show("first ten q-values:", np.round(fund_qvalues[:10], 4))
    show("q <= 0.1:", (fund_qvalues <= 0.1).sum())                             # 146
    show("Bonferroni at 0.1:", (fund_pvalues <= 0.1 / 2000).sum())             # 0
    keep("fund_p", fund_pvalues[np.r_[0:10, 999, 1999]])
    keep("fund_q", fund_qvalues[:10])
    keep("bh_counts", [(fund_qvalues <= q).sum() for q in (0.05, 0.1, 0.2, 0.3)])
    keep("bonf_counts", [(fund_pvalues <= a / 2000).sum() for a in (0.05, 0.1, 0.3)])
    keep("holm_counts", [(mult_test(fund_pvalues, method="holm")[1] <= a).sum() for a in (0.05, 0.1, 0.3)])
    keep("p_below_001", (fund_pvalues < 0.001).sum())

    # Benjamini-Hochberg by hand (Algorithm 13.2)
    sorted_ = np.sort(fund_pvalues)
    m = fund_pvalues.shape[0]
    q = 0.1
    passing = np.nonzero(sorted_ <= q * np.arange(1, m + 1) / m)[0]
    L = passing.max() + 1 if len(passing) else 0          # number of rejections
    selected_ = fund_pvalues <= sorted_[L - 1] if L else np.zeros(m, bool)
    show("BH by hand:", (L, selected_.sum()))                                  # 146 146
    keep("bh_by_hand", selected_.sum())
    fig, ax = plt.subplots(figsize=(6, 4.5))
    j = np.arange(1, m + 1)
    ax.scatter(j, sorted_, s=6, c="grey")
    ax.scatter(j[:L], sorted_[:L], s=8, c="b")
    ax.plot(j, q * j / m, c="r")                  # the line p = q j / m, drawn point by point
    ax.axhline(q / m, c="g")
    ax.set(xscale="log", yscale="log", xlabel="Index", ylabel="P-Value", ylim=(4e-6, 1.2))
    save(fig, "lab-py-bh")

    # ============================================================== 13.6.4 a re-sampling approach
    Kd = pd.read_csv(os.path.join(HERE, "cache", "ch13-Khan.csv"))   # 83 patients: training rows, then test rows
    D = Kd.drop(columns="y")
    D["Y"] = Kd["y"]
    print(D["Y"].value_counts())                  # 2: 29, 4: 25, 3: 18, 1: 11
    D2 = D[lambda df: df["Y"] == 2]
    D4 = D[lambda df: df["Y"] == 4]
    gene_11 = "G0011"
    observedT, pvalue = ttest_ind(D2[gene_11], D4[gene_11], equal_var=True)
    show("gene 11: T, p =", (round(observedT, 4), round(pvalue, 4)))          # -2.0936, 0.0412
    keep("g11_T", observedT)
    keep("g11_p", pvalue)

    B = 10000
    D_ = np.hstack([D2[gene_11], D4[gene_11]])
    n_ = D2[gene_11].shape[0]
    D_null = D_.copy()
    shuffled = np.empty((B, len(D_)))
    for b in range(B):                            # Algorithm 13.3: B random relabellings
        rng.shuffle(D_null)
        shuffled[b] = D_null
    Tnull = tstat_rows(shuffled, n_)
    perm_p = (np.abs(Tnull) >= np.abs(observedT)).mean()
    show("re-sampling p-value:", perm_p)          # 0.0398 (also with > instead of >=)
    keep("g11_perm", perm_p)
    keep("g11_perm_strict", (np.abs(Tnull) > np.abs(observedT)).mean())
    fig, ax = plt.subplots(figsize=(6, 4.5))
    ax.hist(Tnull, bins=100, density=True, facecolor="y", label="Null")
    xval = np.linspace(-4.2, 4.2, 1001)
    ax.plot(xval, t_dbn.pdf(xval, D_.shape[0] - 2), c="r")
    ax.axvline(observedT, c="b", label="Observed")
    ax.legend()
    ax.set_xlabel("Null Distribution of Test Statistic")
    save(fig, "lab-py-perm11")

    # Algorithm 13.4 on 100 randomly chosen genes, B = 10,000 relabellings each
    m, B = 100, 10000
    idx = rng.choice(Kd.columns[1:], m, replace=False)
    T_vals = np.empty(m)
    Tnull_vals = np.empty((m, B))
    for j in range(m):
        col = idx[j]
        T_vals[j] = ttest_ind(D2[col], D4[col], equal_var=True).statistic
        D_ = np.hstack([D2[col], D4[col]])
        D_null = D_.copy()
        for b in range(B):
            rng.shuffle(D_null)
            shuffled[b] = D_null
        Tnull_vals[j] = tstat_rows(shuffled, n_)
    cutoffs = np.sort(np.abs(T_vals))
    FDRs, Rs, Vs = np.empty((3, m))
    for j in range(m):
        R = np.sum(np.abs(T_vals) >= cutoffs[j])
        V = np.sum(np.abs(Tnull_vals) >= cutoffs[j]) / B
        Rs[j], Vs[j], FDRs[j] = R, V, V / R
    show("rejections at estimated FDR 0.1 and 0.2:", (Rs[FDRs <= 0.1].max(), Rs[FDRs <= 0.2].max()))   # 15 28
    genes01 = sorted(idx[np.abs(T_vals) >= cutoffs[FDRs < 0.1].min()])
    show("genes at FDR 0.1:", genes01)
    keep("R_fdr", [Rs[FDRs <= 0.1].max(), Rs[FDRs <= 0.2].max()])
    keep("idx", [int(g[1:]) for g in idx])
    keep("fdr100", FDRs)
    fig, ax = plt.subplots(figsize=(6, 4.5))
    ax.plot(Rs, FDRs, "b", linewidth=3)
    ax.set_xlabel("Number of Rejections")
    ax.set_ylabel("False Discovery Rate")
    save(fig, "lab-py-fdr100")

    # Beyond the book's lab: gene 877 (Figure 13.8), whose null distribution is far from t_52.
    D_ = np.hstack([D2["G0877"], D4["G0877"]])
    T877 = ttest_ind(D2["G0877"], D4["G0877"], equal_var=True)
    D_null = D_.copy()
    for b in range(B):
        rng.shuffle(D_null)
        shuffled[b] = D_null
    perm877 = (np.abs(tstat_rows(shuffled, n_)) >= abs(T877.statistic)).mean()
    show("gene 877: T, t p-value, re-sampling p-value:", np.round([T877.statistic, T877.pvalue, perm877], 4))
    keep("g877_T", T877.statistic)
    keep("g877_perm", perm877)

    # ============================================================== exercises
    # Exercise 7: Sales on each quantitative variable of Carseats, one simple regression at a time
    Carseats = common.isl_data("Carseats")
    quant = ["CompPrice", "Income", "Advertising", "Population", "Price", "Age", "Education"]
    ex7_p = pd.Series({v: sm.OLS(Carseats["Sales"], sm.add_constant(Carseats[v])).fit().pvalues[v] for v in quant})
    print(ex7_p.map("{:.3g}".format))
    show("(b) p <= 0.05:", list(ex7_p.index[ex7_p <= 0.05]))
    show("(c) Bonferroni:", list(ex7_p.index[mult_test(ex7_p, 0.05, "bonferroni")[0]]))
    show("    Holm:", list(ex7_p.index[mult_test(ex7_p, 0.05, "holm")[0]]))
    show("(d) BH at 0.2:", list(ex7_p.index[mult_test(ex7_p, 0.2, "fdr_bh")[0]]))
    keep("ex7_p", ex7_p.to_numpy())

    # Exercise 8: 100 simulated managers with no skill, 20 months each
    rng = np.random.default_rng(1)
    n, m = 20, 100
    X = rng.normal(size=(n, m))
    ex8_p = ttest_1samp(X, 0).pvalue
    fig, ax = plt.subplots(figsize=(6, 4.5))
    ax.hist(ex8_p, bins=20, color="0.8", edgecolor="k")
    ax.set_xlabel("p-value")
    save(fig, "lab-py-ex8")
    ex8 = [(ex8_p <= 0.05).sum(), mult_test(ex8_p, 0.05, "holm")[0].sum(), mult_test(ex8_p, 0.05, "fdr_bh")[0].sum()]
    show("(b)-(d) raw, Holm, BH:", ex8)
    best = np.argsort(-X.mean(0))[:10]            # (e) cherry-pick the ten best managers
    ex8_cherry = [mult_test(ex8_p[best], 0.05, "holm")[0].sum(), mult_test(ex8_p[best], 0.05, "fdr_bh")[0].sum()]
    show("(e) cherry-picked ten, Holm and BH:", ex8_cherry)
    keep("ex8", ex8)
    keep("ex8_cherry", ex8_cherry)
    keep("ex8_pmin", ex8_p.min())

    names, vals = [], []
    for k, v in key.items():
        names += [k] if len(v) == 1 else ["%s_%d" % (k, i + 1) for i in range(len(v))]
        vals += list(v)
    os.makedirs(os.path.join(HERE, "cache"), exist_ok=True)
    pd.DataFrame({"name": names, "value": vals}).to_csv(os.path.join(HERE, "cache", "ch13_lab_py.csv"), index=False,
                                                        float_format="%.10g")
    print("wrote %d numbers to cache/ch13_lab_py.csv" % len(vals))


if __name__ == "__main__":
    main()
