"""Chapter 11 lab: survival analysis (Python version).

Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie, Tibshirani and
Taylor (2023), An Introduction to Statistical Learning with Applications in Python, Section 11.8, and
the R lab of James et al. (2021), Section 11.8.

The lab runs the analyses of ch11_lab.R with numpy, pandas, scipy and statsmodels only. The
Kaplan-Meier estimator and the log-rank test are written out below; Cox's proportional hazards model
is fitted with statsmodels' PHReg (Efron's method for tied times, R's default), and the survival
curves of a fitted Cox model are computed from the baseline cumulative hazard written out here:

    km_table          the Kaplan-Meier estimate (11.3) with Greenwood standard errors, as R's survfit()
    km_at             the Kaplan-Meier step function evaluated at given times
    logrank_test      the log-rank test (11.5)-(11.8) for two or more groups, as R's survdiff()
    model_matrix      treatment-coded dummies with R's baseline levels
    coxph             PHReg fit with R's summary table and the likelihood ratio, Wald and score tests
    cox_survival      S(t | x) = exp(-H0(t) exp(x'beta)) from the Breslow or Efron baseline hazard

Run from the code/ folder:  python3 ch11_lab.py
Figures go to ../img/ch11/lab-py-*.svg and the key numbers to cache/ch11_lab_py.csv.
"""
import os
import warnings
from types import SimpleNamespace

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import numpy as np
import pandas as pd
from scipy import stats
from statsmodels.duration.hazard_regression import PHReg

import common

HERE = os.path.dirname(os.path.abspath(__file__))
Z95 = stats.norm.ppf(0.975)

# the factor levels of the R data sets (the first level is the baseline of the dummies)
LEVELS = {"sex": ["Female", "Male"], "diagnosis": ["Meningioma", "LG glioma", "HG glioma", "Other"],
          "loc": ["Infratentorial", "Supratentorial"], "stereo": ["SRS", "SRT"]}


# ======================================================================== Kaplan-Meier and log-rank
def km_table(time, status):
    """Kaplan-Meier estimate at every distinct observed time, with the columns of R's survfit().

    surv is the product (11.3) of the conditional survival fractions (r_j - q_j) / r_j;
    se_logS is Greenwood's standard error of -log S(t), sqrt(sum q_j / (r_j (r_j - q_j)));
    se_S = S(t) * se_logS; lower and upper form R's default 95% interval, S(t) exp(-/+ 1.96 se_logS).
    """
    t, d = np.asarray(time, float), np.asarray(status, int)
    times = np.unique(t)
    n_risk = np.array([(t >= s).sum() for s in times])
    n_event = np.array([((t == s) & (d == 1)).sum() for s in times])
    n_censor = np.array([((t == s) & (d == 0)).sum() for s in times])
    surv = np.cumprod(1.0 - n_event / n_risk)
    with np.errstate(divide="ignore", invalid="ignore"):        # S(t) = 0 after a last death: se is undefined
        se_logS = np.sqrt(np.cumsum(np.where(n_event > 0, n_event / (n_risk * (n_risk - n_event)), 0.0)))
        return pd.DataFrame({"time": times, "n_risk": n_risk, "n_event": n_event, "n_censor": n_censor,
                             "surv": surv, "se_logS": se_logS, "se_S": surv * se_logS,
                             "lower": surv * np.exp(-Z95 * se_logS),
                             "upper": np.minimum(1.0, surv * np.exp(Z95 * se_logS))})


def km_at(table, t, column="surv"):
    """The Kaplan-Meier step function (right-continuous, 1 before the first time) at times t."""
    idx = np.searchsorted(table["time"].to_numpy(), np.atleast_1d(t), side="right") - 1
    vals = table[column].to_numpy()
    start = 1.0 if column in ("surv", "lower", "upper") else 0.0
    return np.where(idx >= 0, vals[np.maximum(idx, 0)], start)


def logrank_test(time, status, group):
    """The log-rank test of equal survival in K groups (R's survdiff).

    At each distinct death time d_k with r_k at risk and q_k deaths, the deaths in group g have
    (conditional) mean q_k r_gk / r_k and the hypergeometric covariance
    q_k (r_k - q_k) / (r_k - 1) * (diag(p) - p p'),  p = (r_gk / r_k)_g.
    Summing over death times gives observed O, expected E and V; the statistic is
    (O - E)' V^- (O - E), chi-squared with K - 1 df (for K = 2 it is W^2 of (11.8)).
    """
    t, d, g = np.asarray(time, float), np.asarray(status, int), np.asarray(group)
    levels = np.unique(g)
    K = len(levels)
    O, E, V = np.zeros(K), np.zeros(K), np.zeros((K, K))
    for s in np.unique(t[d == 1]):
        at_risk = t >= s
        r = np.array([(at_risk & (g == lv)).sum() for lv in levels], float)
        q = np.array([((t == s) & (d == 1) & (g == lv)).sum() for lv in levels], float)
        rk, qk = r.sum(), q.sum()
        p = r / rk
        O += q
        E += qk * p
        if rk > 1:
            V += qk * (rk - qk) / (rk - 1) * (np.diag(p) - np.outer(p, p))
    chisq = float((O - E)[:-1] @ np.linalg.solve(V[:-1, :-1], (O - E)[:-1]))
    table = pd.DataFrame({"N": [(g == lv).sum() for lv in levels], "Observed": O, "Expected": E,
                          "(O-E)^2/E": (O - E) ** 2 / E, "(O-E)^2/V": (O - E) ** 2 / np.diag(V)},
                         index=[str(lv) for lv in levels])
    return {"table": table, "O": O, "E": E, "V": V, "chisq": chisq, "df": K - 1,
            "p": stats.chi2.sf(chisq, K - 1)}


# ======================================================================== Cox's proportional hazards model
def model_matrix(df, terms, levels=LEVELS):
    """Columns for the terms: numbers as they are, text as dummies for every level but the first."""
    cols = {}
    for v in terms:
        x = df[v]
        if x.dtype == object or isinstance(x.dtype, pd.CategoricalDtype):
            lv = levels.get(v, sorted(x.dropna().unique()))
            for level in lv[1:]:
                cols["%s%s" % (v, level)] = (x == level).astype(float).where(x.notna())
        else:
            cols[v] = x.astype(float)
    return pd.DataFrame(cols, index=df.index)


def coxph(time, status, X, ties="efron"):
    """Fit h(t | x) = h0(t) exp(x'beta) by maximising the partial likelihood (11.16) (statsmodels PHReg).

    The fit runs on standardised columns, BFGS steps first and then Newton's method (plain Newton steps
    from 0 can overshoot and overflow exp(x'beta) when a column such as budget has large outliers); the
    estimates and their covariance are then put back on the original scale. Returns the fit (params,
    bse, cov, llf) and a dictionary with R's summary table, the log partial likelihood at 0 and at the
    estimate, and the three tests of beta = 0: likelihood ratio, Wald and score.
    """
    X = pd.DataFrame(X)
    x, t, d = X.to_numpy(float), np.asarray(time, float), np.asarray(status, int)
    centre, scale = x.mean(0), x.std(0)
    scale = np.where(scale > 0, scale, 1.0)
    model_s = PHReg(t, (x - centre) / scale, status=d, ties=ties)
    with warnings.catch_warnings():                              # quasi-Newton steps first (they cannot
        warnings.simplefilter("ignore")                          # overshoot), then Newton's method to
        start = model_s.fit(method="bfgs", maxiter=200).params   # full precision
    fit_s = model_s.fit(start_params=start, method="newton")
    b = fit_s.params / scale
    cov = fit_s.cov_params() / np.outer(scale, scale)
    fit = SimpleNamespace(params=b, bse=np.sqrt(np.diag(cov)), cov=cov, llf=fit_s.llf)
    model = PHReg(t, x, status=d, ties=ties)                     # the tests, on the original scale
    zero = np.zeros(len(b))
    U, Info = model.score(zero), -model.hessian(zero)            # score and information at beta = 0
    tests = {"LR": 2 * (fit.llf - model.loglike(zero)),
             "Wald": float(b @ np.linalg.solve(cov, b)),
             "score": float(U @ np.linalg.solve(Info, U))}
    table = pd.DataFrame({"coef": b, "exp(coef)": np.exp(b), "se(coef)": fit.bse, "z": b / fit.bse,
                          "p": 2 * stats.norm.sf(np.abs(b / fit.bse))}, index=X.columns)
    return fit, {"table": table, "loglik": (model.loglike(zero), fit.llf), "tests": tests, "df": len(b)}


def cox_survival(time, status, X, beta, Xnew, ties="efron"):
    """Survival curves S(t | x) = exp(-H0(t) exp(x'beta)) of a fitted Cox model, as R's survfit(fit, newdata).

    H0 is the baseline cumulative hazard: at a death time with q deaths it increases by
    q / sum_{risk set} exp(x'beta) (Breslow), or with Efron's correction for ties by
    sum_{l=0}^{q-1} 1 / (sum_{risk set} exp(x'beta) - (l / q) sum_{deaths} exp(x'beta)).
    Returns the death times and an array with one column per row of Xnew.
    """
    t, d = np.asarray(time, float), np.asarray(status, int)
    w = np.exp(np.asarray(X, float) @ beta)
    dtimes = np.unique(t[d == 1])
    inc = np.empty(len(dtimes))
    for k, s in enumerate(dtimes):
        risk, dead = w[t >= s].sum(), w[(t == s) & (d == 1)]
        q = len(dead)
        if ties == "efron":
            inc[k] = sum(1.0 / (risk - (l / q) * dead.sum()) for l in range(q))
        else:
            inc[k] = q / risk
    H0 = np.cumsum(inc)
    return dtimes, np.exp(-np.outer(H0, np.exp(np.asarray(Xnew, float) @ beta)))


def concordance(time, status, risk):
    """Harrell's C (Section 11.7.1): among comparable pairs (the shorter time is a death), the share in
    which the patient who died first has the larger risk score; ties in the score count 1/2."""
    t, d, r = np.asarray(time, float), np.asarray(status, int), np.asarray(risk, float)
    num = den = 0.0
    for i in np.flatnonzero(d == 1):
        later = (t > t[i]) | ((t == t[i]) & (d == 0))
        den += later.sum()
        num += (r[i] > r[later]).sum() + 0.5 * (r[i] == r[later]).sum()
    return num / den


def step_plot(ax, table, colour, label=None, band=False, marks=True):
    """Draw a Kaplan-Meier table as a step function, with + at the censored times."""
    x = np.r_[0.0, table["time"]]
    ax.step(x, np.r_[1.0, table["surv"]], where="post", color=colour, lw=1.4, label=label)
    if band:
        ax.step(x, np.r_[1.0, table["lower"]], where="post", color=colour, lw=0.9, ls="--")
        ax.step(x, np.r_[1.0, table["upper"]], where="post", color=colour, lw=0.9, ls="--")
    if marks:
        c = table["n_censor"].to_numpy() > 0
        ax.plot(table["time"][c], table["surv"][c], "+", color=colour, ms=6, mew=1)


def figure(width=6.0, height=4.6, ncols=1):
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots(1, ncols, figsize=(width, height))
    return fig, ax


# ======================================================================== applied exercises (Section 11.9)
def exercises(keep, BrainCancer, km_all, X_all, fit_all):
    # ================================================================ Exercise 10
    # (a) +/- 1 standard error bands
    fig, ax = figure()
    x = np.r_[0.0, km_all["time"]]
    ax.step(x, np.r_[1.0, km_all["surv"]], where="post", color="k", lw=1.4)
    for sign in (1, -1):
        ax.step(x, np.r_[1.0, km_all["surv"] + sign * km_all["se_S"]], where="post", color="k", lw=0.9, ls="--")
    ax.set(xlabel="Months", ylabel="Estimated Probability of Survival", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-ex10a", formats=("svg",))
    tt = np.array([10, 20, 40, 60])
    greenwood = km_at(km_all, tt, "se_S")
    keep("ex10_greenwood", greenwood)

    # (b) bootstrap standard errors of S(t), B = 200 resamples of the pairs (y_i, delta_i)
    rng = np.random.default_rng(1)
    y, delta = BrainCancer["time"].to_numpy(), BrainCancer["status"].to_numpy()
    boot = np.array([km_at(km_table(y[i], delta[i]), tt)
                     for i in (rng.integers(0, len(y), len(y)) for _ in range(200))])
    print(pd.DataFrame({"greenwood": greenwood, "bootstrap": boot.std(axis=0, ddof=1)}, index=tt).round(4))
    keep("ex10_boot", boot.std(axis=0, ddof=1))

    # (c) the Cox model with all predictors is fit_all of the lab

    # (d) strata of the Karnofsky index (40 merged with 60), adjusted for the other predictors
    cc = BrainCancer.dropna().copy()
    cc["ki5"] = np.where(cc["ki"] == 40, 60, cc["ki"])
    Xs = model_matrix(cc, ["sex", "diagnosis", "loc", "gtv", "stereo"])
    model = PHReg(cc["time"].to_numpy(), Xs.to_numpy(), status=cc["status"].to_numpy(),
                  strata=cc["ki5"].to_numpy(), ties="efron")
    fs = model.fit()
    print(pd.DataFrame({"coef": fs.params, "se(coef)": fs.bse}, index=Xs.columns).round(4))
    keep("ex10d_coef", fs.params)
    typical = np.zeros(Xs.shape[1])                     # female, meningioma, infratentorial, SRS
    typical[list(Xs.columns).index("locSupratentorial")] = 1.0
    typical[list(Xs.columns).index("stereoSRT")] = 1.0
    typical[list(Xs.columns).index("gtv")] = BrainCancer["gtv"].mean()
    fig, ax = figure()
    s30 = []
    colours = (common.RED, common.GREEN, common.BLUE, common.ORANGE, common.PURPLE)
    for level, colour in zip(sorted(cc["ki5"].unique()), colours):
        rows = (cc["ki5"] == level).to_numpy()          # each stratum has its own baseline hazard
        tk, Sk = cox_survival(cc["time"][rows], cc["status"][rows], Xs[rows], fs.params, typical[None, :])
        ax.step(np.r_[0.0, tk], np.r_[1.0, Sk[:, 0]], where="post", color=colour, label="ki = %d" % level)
        s30.append(Sk[np.searchsorted(tk, 30, side="right") - 1, 0] if (tk <= 30).any() else 1.0)
    ax.legend(loc="lower left")
    ax.set(xlabel="Months", ylabel="Survival Probability", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-ex10d", formats=("svg",))
    print("Exercise 10(d): S(30 months) by ki stratum", np.round(s30, 4))
    keep("ex10d_surv_end", s30)

    # ================================================================ Exercise 11: the data of Table 11.4
    ex11 = pd.DataFrame({"y": [26.5, 37.2, 57.3, 90.8, 20.2, 89.8], "delta": [1, 1, 1, 0, 0, 0],
                         "x": [0.1, 11, -0.3, 2.8, 1.8, 0.4]})
    ex11["group"] = np.where(ex11["x"] < 2, 1, 2)
    fig, ax = figure()
    for k, colour in ((1, common.RED), (2, common.BLUE)):
        part = ex11[ex11["group"] == k]
        tab = km_table(part["y"], part["delta"])
        print("group %d\n" % k, tab[["time", "n_risk", "n_event", "surv"]])
        step_plot(ax, tab, colour, label="Group %d (%s)" % (k, "X < 2" if k == 1 else "X >= 2"))
    ax.legend(loc="lower left")
    ax.set(xlabel="Time in Days", ylabel="Estimated Probability of Survival", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-ex11", formats=("svg",))
    fit11, res11 = coxph(ex11["y"], ex11["delta"], (ex11["group"] == 2).astype(float).to_frame("group2"))
    lr11 = logrank_test(ex11["y"], ex11["delta"], ex11["group"])
    print(res11["table"].round(4))
    print("score test %.6f, log-rank %.6f, p = %.4f" % (res11["tests"]["score"], lr11["chisq"], lr11["p"]))
    keep("ex11_coef", fit11.params); keep("ex11_se", fit11.bse)
    keep("ex11_score", res11["tests"]["score"]); keep("ex11_lr", lr11["chisq"])
    keep("ex11_km_all", km_table(ex11["y"], ex11["delta"])["surv"])


# ======================================================================== the lab
def main():
    import matplotlib.pyplot as plt  # noqa: F401  (common sets the style)

    key = {}

    def keep(name, value):
        key[name] = np.atleast_1d(np.asarray(value, float)).ravel()

    # ================================================================ 11.8.1 brain cancer data
    BrainCancer = common.isl_data("BrainCancer", index_col=0)
    print(list(BrainCancer.columns))
    print(BrainCancer["sex"].value_counts())
    print(BrainCancer["diagnosis"].value_counts())
    print(BrainCancer["status"].value_counts())      # 1 = died (uncensored), 0 = censored

    km_all = km_table(BrainCancer["time"], BrainCancer["status"])
    print(km_all.head(8).round(4))
    print("S(t) at 10, 20, 40, 60 months:", km_at(km_all, [10, 20, 40, 60]).round(4))
    fig, ax = figure()
    step_plot(ax, km_all, common.BLUE, band=True)
    ax.set(xlabel="Months", ylabel="Estimated Probability of Survival", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-km", formats=("svg",))
    keep("km_time", km_all["time"]); keep("km_surv", km_all["surv"]); keep("km_nrisk", km_all["n_risk"])
    keep("km_se_logS", km_all["se_logS"])
    keep("km_S20", km_at(km_all, 20)); keep("km_lower20", km_at(km_all, 20, "lower"))
    keep("km_upper20", km_at(km_all, 20, "upper"))

    fig, ax = figure()
    by_sex = {}
    for sex, colour in (("Female", common.RED), ("Male", common.BLUE)):
        by_sex[sex] = BrainCancer[BrainCancer["sex"] == sex]
        step_plot(ax, km_table(by_sex[sex]["time"], by_sex[sex]["status"]), colour, label=sex)
    ax.legend(loc="lower left")
    ax.set(xlabel="Months", ylabel="Estimated Probability of Survival", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-km-sex", formats=("svg",))

    lr = logrank_test(BrainCancer["time"], BrainCancer["status"], BrainCancer["sex"])
    print(lr["table"].round(3))
    print("Chisq = %.8f on %d df, p = %.4f" % (lr["chisq"], lr["df"], lr["p"]))
    keep("lr_sex_obs", lr["O"]); keep("lr_sex_exp", lr["E"]); keep("lr_sex_var", lr["V"][0, 0])
    keep("lr_sex_chisq", lr["chisq"])

    X_sex = model_matrix(BrainCancer, ["sex"])
    fit_sex, res_sex = coxph(BrainCancer["time"], BrainCancer["status"], X_sex)
    print(res_sex["table"].round(4))
    print({k: round(v, 8) for k, v in res_sex["tests"].items()})
    print("log-rank chi-squared: %.8f" % lr["chisq"])
    C_sex = concordance(BrainCancer["time"], BrainCancer["status"], X_sex.to_numpy() @ fit_sex.params)
    print("concordance %.4f" % C_sex)
    keep("cox_sex_coef", fit_sex.params); keep("cox_sex_se", fit_sex.bse)
    keep("cox_sex_tests", [res_sex["tests"][k] for k in ("LR", "Wald", "score")])
    keep("cox_sex_concordance", C_sex)

    cleaned = BrainCancer.dropna()                   # one patient has no diagnosis
    terms = ["sex", "diagnosis", "loc", "ki", "gtv", "stereo"]
    X_all = model_matrix(cleaned, terms)
    fit_all, res_all = coxph(cleaned["time"], cleaned["status"], X_all)
    print(res_all["table"].round(4))
    print("likelihood ratio test = %.2f on %d df" % (res_all["tests"]["LR"], res_all["df"]))
    keep("cox_all_coef", fit_all.params); keep("cox_all_se", fit_all.bse)
    keep("cox_all_LR", res_all["tests"]["LR"])

    # survival curves for each diagnosis, the other predictors at their mean (numeric) or modal value
    modal = pd.DataFrame({"diagnosis": LEVELS["diagnosis"], "sex": "Female", "loc": "Supratentorial",
                          "ki": BrainCancer["ki"].mean(), "gtv": BrainCancer["gtv"].mean(), "stereo": "SRT"})
    X_modal = model_matrix(modal, terms)
    times, S_modal = cox_survival(cleaned["time"], cleaned["status"], X_all, fit_all.params, X_modal)
    curves = pd.DataFrame(S_modal, index=times, columns=LEVELS["diagnosis"])
    print(curves.iloc[[0, 1, 2, -2, -1]].round(3))
    fig, ax = figure()
    for k, colour in enumerate((common.RED, common.GREEN, common.BLUE, common.CYAN)):
        ax.step(np.r_[0.0, times], np.r_[1.0, S_modal[:, k]], where="post", color=colour, label=LEVELS["diagnosis"][k])
    ax.legend(loc="lower left")
    ax.set(xlabel="Months", ylabel="Survival Probability", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-diagnosis", formats=("svg",))
    at = lambda t: S_modal[np.searchsorted(times, t, side="right") - 1]   # noqa: E731
    keep("modal_surv20", at(20)); keep("modal_surv60", at(60)); keep("modal_surv_last", S_modal[-1])

    # ================================================================ 11.8.2 publication data
    Publication = common.isl_data("Publication", index_col=0)
    fig, ax = figure()
    for result, colour, label in ((0, common.GREEN, "Negative Result"), (1, common.BLUE, "Positive Result")):
        part = Publication[Publication["posres"] == result]
        step_plot(ax, km_table(part["time"], part["status"]), colour, label=label)
    ax.legend(loc="upper right")
    ax.set(xlabel="Months", ylabel="Probability of Not Being Published", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-km-posres", formats=("svg",))

    fit_pub, res_pub = coxph(Publication["time"], Publication["status"], Publication[["posres"]])
    print(res_pub["table"].round(4))
    lr_pub = logrank_test(Publication["time"], Publication["status"], Publication["posres"])
    print(lr_pub["table"].round(3))
    print("Chisq = %.6f, p = %.4f; Cox score test = %.6f" % (lr_pub["chisq"], lr_pub["p"], res_pub["tests"]["score"]))
    keep("pub_cox_coef", fit_pub.params); keep("pub_cox_se", fit_pub.bse)
    keep("pub_cox_score", res_pub["tests"]["score"]); keep("pub_lr_chisq", lr_pub["chisq"])
    keep("pub_lr_obs", lr_pub["O"]); keep("pub_lr_exp", lr_pub["E"])

    X_pub2 = Publication.drop(columns=["mech", "time", "status"])
    fit_pub2, res_pub2 = coxph(Publication["time"], Publication["status"], X_pub2)
    print(res_pub2["table"].round(6))
    keep("pub2_coef", fit_pub2.params); keep("pub2_se", fit_pub2.bse); keep("pub2_LR", res_pub2["tests"]["LR"])

    # all predictors, funding mechanism included (Table 11.3). No trial funded through R42 or RC2 was
    # published, so their coefficients have no finite maximum (they tend to -infinity); in that limit
    # those two trials drop out of every risk set, so the other estimates are those of the fit without them.
    X_pub3 = model_matrix(Publication, ["posres", "multi", "clinend", "mech", "sampsize", "budget", "impact"])
    never = Publication["mech"].isin(["R42", "RC2"]).to_numpy()
    X_fin = X_pub3.loc[~never].drop(columns=["mechR42", "mechRC2"])
    fit_pub3, res_pub3 = coxph(Publication["time"][~never], Publication["status"][~never], X_fin)
    print(res_pub3["table"].round(3))
    keep("pub3_loglik", res_pub3["loglik"][1]); keep("pub3_coef_finite", fit_pub3.params)

    # ================================================================ 11.8.3 call centre data
    rng = np.random.default_rng(10)
    N = 2000
    Operators = rng.choice(np.arange(5, 16), N, replace=True)
    Center = rng.choice(["A", "B", "C"], N, replace=True)
    Time = rng.choice(["Morn.", "After.", "Even."], N, replace=True)
    D = pd.DataFrame({"Operators": Operators, "Center": Center, "Time": Time})
    X = model_matrix(D, ["Operators", "Center", "Time"])          # baselines: Center A, afternoon
    print(X[:5])

    true_beta = np.array([0.04, -0.3, 0, 0.2, -0.2])
    true_linpred = X.to_numpy() @ true_beta
    hazard = lambda t: 1e-5 * t                                   # noqa: E731  baseline hazard h0(t)
    cum_hazard = lambda t: 1e-5 * t ** 2 / 2                      # noqa: E731  H0(t), its integral
    # S(t | x) = exp(-H0(t) exp(x'beta)): with U uniform, H0^{-1}(-log(U) / exp(x'beta)) has this law
    U = rng.uniform(size=N)
    W = np.sqrt(2 * (-np.log(U)) / (1e-5 * np.exp(true_linpred)))
    assert np.allclose(cum_hazard(W) * np.exp(true_linpred), -np.log(U)) and hazard(1.0) == 1e-5
    D["Wait time"] = np.clip(W, 0, 1000)                          # waits are capped at 1000 seconds
    D["Failed"] = rng.choice([1, 0], N, p=[0.9, 0.1])             # 1 = answered, 0 = hung up (censored)
    print(D[:5])
    print("share answered: %.4f" % D["Failed"].mean())

    fig, axes = figure(width=9.0, height=4.2, ncols=2)
    for ax, var, labels in ((axes[0], "Center", ["A", "B", "C"]), (axes[1], "Time", ["After.", "Even.", "Morn."])):
        for lv, colour in zip(labels, (common.RED, common.BLUE, common.CYAN)):
            part = D[D[var] == lv]
            step_plot(ax, km_table(part["Wait time"], part["Failed"]), colour, label="%s = %s" % (var, lv), marks=False)
        ax.legend(loc="upper right")
        ax.set(xlabel="Seconds", ylabel="Probability of Still Being on Hold", ylim=(0, 1.02))
    common.savefig(fig, 11, "lab-py-callcenter", formats=("svg",))

    lr_center = logrank_test(D["Wait time"], D["Failed"], D["Center"])
    lr_time = logrank_test(D["Wait time"], D["Failed"], D["Time"])
    print(lr_center["table"].round(3)); print("Chisq = %.2f on 2 df, p = %.2g" % (lr_center["chisq"], lr_center["p"]))
    print(lr_time["table"].round(3)); print("Chisq = %.2f on 2 df, p = %.2g" % (lr_time["chisq"], lr_time["p"]))
    keep("cc_lr_center", lr_center["chisq"]); keep("cc_lr_time", lr_time["chisq"])

    LR_center = coxph(D["Wait time"], D["Failed"], model_matrix(D, ["Center"]))[1]["tests"]["LR"]
    LR_time = coxph(D["Wait time"], D["Failed"], model_matrix(D, ["Time"]))[1]["tests"]["LR"]
    print("LR tests: Center %.2f, Time %.2f" % (LR_center, LR_time))
    keep("cc_LR_center", LR_center); keep("cc_LR_time", LR_time)

    fit_queuing, res_queuing = coxph(D["Wait time"], D["Failed"], X)
    print(res_queuing["table"].round(5))
    keep("cc_cox_coef", fit_queuing.params); keep("cc_cox_se", fit_queuing.bse)

    exercises(keep, BrainCancer, km_all, X_all, fit_all)

    names, vals = [], []
    for k, v in key.items():
        names += [k] if len(v) == 1 else ["%s_%d" % (k, i + 1) for i in range(len(v))]
        vals += list(v)
    os.makedirs(os.path.join(HERE, "cache"), exist_ok=True)
    pd.DataFrame({"name": names, "value": vals}).to_csv(os.path.join(HERE, "cache", "ch11_lab_py.csv"), index=False, float_format="%.10g")
    print("wrote %d numbers to cache/ch11_lab_py.csv" % len(vals))


if __name__ == "__main__":
    main()
