"""James et al. (2021, 2023), Chapter 11: Survival Analysis and Censored Data.

Reproduces the figures of Chapter 11 of G. James, D. Witten, T. Hastie and R. Tibshirani, An Introduction
to Statistical Learning with Applications in R, 2nd edition (Springer, 2021), pp. 461-495 (the same
figures appear in the Python edition of 2023, with J. Taylor), written from scratch with numpy, scipy,
statsmodels and matplotlib. The Kaplan-Meier, log-rank and Cox helpers come from ch11_lab.py
(km_table, logrank_test, coxph, cox_survival, concordance); this file adds:

* cox_breslow: the log partial likelihood (11.16) with Breslow's handling of ties, its gradient and
  Hessian (the quantities behind Newton's method, the score test and the lasso);
* cox_efron_1d: the same for one covariate with Efron's handling of ties (the third demonstration);
* cox_lasso: the lasso-penalised Cox model (11.17) on standardised inputs, by proximal Newton steps
  with coordinate descent, minimising -(1/n) log PL + lambda * sum_j |beta_j| (glmnet's scaling);
* cv_cox_lasso: K-fold cross-validation with the partial likelihood deviance of Figure 11.7;
* auc_t: the time-dependent (cumulative/dynamic) AUC with inverse-probability-of-censoring weights.

Figures reproduced: 11.1-11.9 (all of them). Figure 11.1 is drawn for four fictitious patients, as in
the book; Figure 11.4 uses Weibull hazards chosen here. Two extra figures: fig11-auc (time-dependent
AUC on the Publication test set) and fig11-loglog (log(-log S) curves to check proportional hazards).
Run `python3 ch11_survival_analysis.py --list`, or `--fig 11.2`, or `--fig demos` to write the data of
the three demonstrations to ../data/ch11-*.json and print the numbers quoted in the notes.

Settings chosen where the book gives none:
* Figure 11.2: the band is R's default 95% interval S(t) exp(+/- 1.96 se), se the Greenwood standard
  error of -log S(t) (the book's caption calls it "standard error bands"); + marks censored times.
* Figure 11.6: Cox model with all predictors (Table 11.3), the others set to their means and mech to R01.
  No trial funded through R42 or RC2 was published, so those coefficients diverge to -infinity; the
  limit fit (the same model without those two trials) is used, which is what R's coxph() approaches.
* Figures 11.7-11.8: the 244 trials are split at random into 122 training and 122 test trials
  (numpy default_rng(1)); 10-fold cross-validation on the training set (default_rng(2)); all 19
  columns of the model matrix (mech as 13 dummies); Breslow's ties, as glmnet. The x-axis of Figure
  11.7 is lambda: the book's ||beta_lambda||_1 / ||beta_hat||_1 divides by the l1 norm of the
  unpenalised fit, which does not exist on these data (see the notes).
* Figure 11.4: h(t | x=0) = 0.4 t and h(t | x=1) = t (proportional); h(t | x=0) = 1.5 t^2 and
  h(t | x=1) = t (not proportional).
"""
import json
import os

import numpy as np
import pandas as pd

import common
from common import BLUE, GREEN, GREY, ORANGE, PURPLE, RED, savefig
from ch11_lab import (LEVELS, concordance, cox_survival, coxph, km_at, km_table, logrank_test,
                      model_matrix)

HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
CACHE = os.path.join(HERE, "cache")


# ======================================================================== data
def brain():
    return common.isl_data("BrainCancer", index_col=0)


def publication():
    return common.isl_data("Publication", index_col=0)


def pub_matrix(P):
    """All 19 columns of the Publication model matrix (mech as dummies, baseline Contract)."""
    return model_matrix(P, ["posres", "multi", "clinend", "mech", "sampsize", "budget", "impact"])


# ======================================================================== Cox partial likelihood
def cox_breslow(X, time, status, beta):
    """Log partial likelihood (11.16) with Breslow's ties, its gradient and its Hessian.

    log PL = sum over death times s of [ sum_{deaths at s} x_i'beta - q_s log sum_{risk set} exp(x'beta) ];
    gradient = sum_s [ sum_{deaths} x_i - q_s xbar_s ], xbar_s the exp(x'beta)-weighted mean of the risk set;
    Hessian = - sum_s q_s * (weighted covariance of x over the risk set), negative semi-definite.
    """
    X = np.asarray(X, float)
    t, d = np.asarray(time, float), np.asarray(status, int)
    eta = X @ beta
    w = np.exp(eta - eta.max())
    ll, g, H = 0.0, np.zeros(X.shape[1]), np.zeros((X.shape[1], X.shape[1]))
    for s in np.unique(t[d == 1]):
        R, D = t >= s, (t == s) & (d == 1)
        q = D.sum()
        wr = w[R]
        S0 = wr.sum()
        xbar = wr @ X[R] / S0
        Xc = X[R] - xbar
        ll += eta[D].sum() - q * (np.log(S0) + eta.max())
        g += X[D].sum(0) - q * xbar
        H -= q * (Xc.T * wr) @ Xc / S0
    return ll, g, H


def cox_efron_1d(x, time, status, beta):
    """Log partial likelihood, score and information for one covariate with Efron's ties (as R)."""
    x, t, d = np.asarray(x, float), np.asarray(time, float), np.asarray(status, int)
    w = np.exp(beta * x)
    ll = U = I = 0.0
    for s in np.unique(t[d == 1]):
        R, D = t >= s, (t == s) & (d == 1)
        q = D.sum()
        S0, S1, S2 = w[R].sum(), (w * x)[R].sum(), (w * x * x)[R].sum()
        D0, D1, D2 = w[D].sum(), (w * x)[D].sum(), (w * x * x)[D].sum()
        ll += beta * x[D].sum()
        U += x[D].sum()
        for l in range(q):
            a = l / q
            s0, s1, s2 = S0 - a * D0, S1 - a * D1, S2 - a * D2
            ll -= np.log(s0)
            U -= s1 / s0
            I += s2 / s0 - (s1 / s0) ** 2
    return ll, U, I


def newton_1d(x, time, status, ties="efron", beta=0.0, steps=50):
    """Newton's method for one covariate; returns beta-hat and the path of iterates."""
    path = [beta]
    for _ in range(steps):
        if ties == "efron":
            _, U, I = cox_efron_1d(x, time, status, beta)
        else:
            _, U, H = cox_breslow(np.asarray(x, float)[:, None], time, status, np.array([beta]))
            U, I = U[0], -H[0, 0]
        beta = beta + U / I
        path.append(beta)
        if abs(U / I) < 1e-12:
            break
    return beta, path


# ======================================================================== the lasso-penalised Cox model
def _soft(z, g):
    return np.sign(z) * np.maximum(np.abs(z) - g, 0.0)


def cox_lasso(Z, time, status, lam, b0=None, tol=1e-11, maxit=200):
    """Minimise F(b) = -(1/n) log PL_Breslow(b) + lam * ||b||_1 for standardised inputs Z.

    Proximal Newton: at b, the smooth part is replaced by its second-order Taylor expansion (exact
    gradient and Hessian), the resulting lasso problem is solved by coordinate descent, and a
    backtracking line search on F keeps every step a descent step.
    """
    n, p = Z.shape
    b = np.zeros(p) if b0 is None else b0.copy()

    def F(bb):
        return -cox_breslow(Z, time, status, bb)[0] / n + lam * np.abs(bb).sum()

    f = F(b)
    for _ in range(maxit):
        ll, g, H = cox_breslow(Z, time, status, b)
        g, H = -g / n, -H / n                            # gradient and Hessian of -(1/n) log PL
        bn = b.copy()
        for _ in range(500):                             # coordinate descent on the quadratic model
            old = bn.copy()
            for j in range(p):
                r = g[j] + H[j] @ (bn - b) - H[j, j] * (bn[j] - b[j])
                bn[j] = _soft(H[j, j] * b[j] - r, lam) / H[j, j] if H[j, j] > 0 else 0.0
            if np.max(np.abs(bn - old)) < 1e-13:
                break
        step, delta = 1.0, bn - b
        while True:
            fn = F(b + step * delta)
            if fn <= f + 1e-15 or step < 1e-8:
                break
            step /= 2
        b, f_old, f = b + step * delta, f, fn
        if np.max(np.abs(step * delta)) < tol or abs(f_old - f) < 1e-15:
            break
    return b


def lasso_path(X, time, status, lambdas=None, nlambda=100, ratio=1e-4, fdev=1e-5):
    """The lasso path on the standardised columns (mean 0, sd 1 with divisor n), back on the original scale.

    With lambdas=None the sequence is glmnet's: 100 values from lambda_max (the smallest lambda with all
    coefficients 0, max_j |d log PL / d b_j at 0| / n) down to ratio * lambda_max, evenly spaced on the log
    scale, and the path stops early, as glmnet's does, once the fraction of the null deviance explained
    grows by less than fdev (relative) from one lambda to the next, or exceeds 0.999.
    """
    X = np.asarray(X, float)
    t, d = np.asarray(time, float), np.asarray(status, int)
    n = len(X)
    mu, sd = X.mean(0), X.std(0)
    sd = np.where(sd > 0, sd, 1.0)                       # a constant column (e.g. a funding mechanism absent
    Z = (X - mu) / sd                                    # from the training set) stays at 0, as in glmnet
    stop = lambdas is None
    if stop:
        g0 = cox_breslow(Z, t, d, np.zeros(Z.shape[1]))[1]
        lmax = np.max(np.abs(g0)) / n
        lambdas = lmax * np.exp(np.linspace(0, np.log(ratio), nlambda))
    ll0 = cox_breslow(Z, t, d, np.zeros(Z.shape[1]))[0]
    q = pd.Series(t[d == 1]).value_counts().to_numpy()
    llsat = -np.sum(q * np.log(q))                       # the saturated log partial likelihood (Breslow)
    B, b, ratio_old = [], None, 0.0
    for k, lam in enumerate(lambdas):
        b = cox_lasso(Z, t, d, lam, b)
        B.append(b / sd)
        dev_ratio = (cox_breslow(Z, t, d, b)[0] - ll0) / (llsat - ll0)
        if stop and k > 0 and ((dev_ratio - ratio_old) < fdev * dev_ratio or dev_ratio > 0.999):
            break
        ratio_old = dev_ratio
    return np.asarray(lambdas)[:len(B)], np.array(B)


def cv_cox_lasso(X, time, status, foldid, lambdas):
    """K-fold CV with the partial likelihood deviance (glmnet's 'grouped' version, Figure 11.7).

    For fold k, beta_(-k) is fitted without the fold; the fold's contribution is
    D_k = -2 [ log PL_all(beta_(-k)) - log PL_(-k)(beta_(-k)) ], which uses the fold's patients inside the
    risk sets of all the others. The curve is sum_k D_k / (number of deaths), with a standard error from
    the spread of D_k / deaths_k across folds.
    """
    X = np.asarray(X, float)
    t, d = np.asarray(time, float), np.asarray(status, int)
    K = foldid.max() + 1
    raw, ev = np.zeros((K, len(lambdas))), np.zeros(K)
    for k in range(K):
        tr = foldid != k
        _, B = lasso_path(X[tr], t[tr], d[tr], lambdas)
        ev[k] = d[~tr].sum()
        for j, b in enumerate(B):
            raw[k, j] = -2 * (cox_breslow(X, t, d, b)[0] - cox_breslow(X[tr], t[tr], d[tr], b)[0])
    cvm = raw.sum(0) / ev.sum()
    per = raw / ev[:, None]
    cvsd = np.sqrt(((per - cvm) ** 2 * ev[:, None]).sum(0) / ev.sum() / (K - 1))
    return cvm, cvsd


def publication_split(seed=1):
    P = publication()
    rng = np.random.default_rng(seed)
    train = np.zeros(len(P), bool)
    train[rng.choice(len(P), len(P) // 2, replace=False)] = True
    return P, train


def lasso_study():
    """The lasso analysis of Section 11.6 on a random half of the Publication data (cached)."""
    path = os.path.join(CACHE, "ch11-lasso.npz")
    P, train = publication_split()
    X = pub_matrix(P)
    names = list(X.columns)
    if os.path.exists(path):
        z = np.load(path)
        return dict(P=P, train=train, names=names, X=X.to_numpy(float), lambdas=z["lambdas"], B=z["B"],
                    cvm=z["cvm"], cvsd=z["cvsd"])
    Xtr, t, d = X.to_numpy(float)[train], P["time"].to_numpy()[train], P["status"].to_numpy()[train]
    lambdas, B = lasso_path(Xtr, t, d)
    foldid = np.random.default_rng(2).permutation(np.arange(train.sum()) % 10)
    cvm, cvsd = cv_cox_lasso(Xtr, t, d, foldid, lambdas)
    os.makedirs(CACHE, exist_ok=True)
    np.savez(path, lambdas=lambdas, B=B, cvm=cvm, cvsd=cvsd)
    return dict(P=P, train=train, names=names, X=X.to_numpy(float), lambdas=lambdas, B=B, cvm=cvm, cvsd=cvsd)


# ======================================================================== time-dependent AUC
def auc_t(time, status, risk, t, time_c=None, status_c=None):
    """Cumulative/dynamic AUC at time t: the probability that a patient who dies by t has a larger risk
    score than one who survives past t. Deaths are weighted by 1 / G(y_i-), G the Kaplan-Meier curve of
    the censoring times (Uno et al. 2007), so that censoring does not bias the estimate."""
    y, d, r = np.asarray(time, float), np.asarray(status, int), np.asarray(risk, float)
    yc = y if time_c is None else np.asarray(time_c, float)
    dc = d if status_c is None else np.asarray(status_c, int)
    G = km_table(yc, 1 - dc)                             # censoring is the 'event' here
    Gm = np.array([km_at(G, yi - 1e-9)[0] for yi in y])
    cases = (y <= t) & (d == 1)
    ctrl = y > t
    num = den = 0.0
    for i in np.flatnonzero(cases):
        w = 1.0 / Gm[i]
        num += w * ((r[i] > r[ctrl]).sum() + 0.5 * (r[i] == r[ctrl]).sum())
        den += w * ctrl.sum()
    return num / den if den > 0 else np.nan


# ======================================================================== figures
def _km_steps(ax, tab, colour, label=None, lw=1.4, marks=True, ls="-"):
    x = np.r_[0.0, tab["time"]]
    ax.step(x, np.r_[1.0, tab["surv"]], where="post", color=colour, lw=lw, label=label, ls=ls)
    if marks:
        c = tab["n_censor"].to_numpy() > 0
        ax.plot(tab["time"][c], tab["surv"][c], "+", color=colour, ms=6, mew=1)


def fig_11_1(out=None):
    """Censored survival data for four fictitious patients."""
    import matplotlib.pyplot as plt
    y = [300, 365, 150, 250]
    event = [True, False, True, False]
    fig, ax = plt.subplots(figsize=(6.0, 3.4))
    for i, (yi, e) in enumerate(zip(y, event)):
        ax.plot([0, yi], [i + 1, i + 1], color="k", lw=1)
        ax.plot(yi, i + 1, "o", color=ORANGE if e else "white", mec=ORANGE if e else BLUE, ms=7, mew=1.5)
    ax.axvline(365, color=GREY, ls=":", lw=1.2)
    ax.set(xlabel="Time in Days", ylabel="Patient", yticks=[1, 2, 3, 4], ylim=(0.5, 4.5), xlim=(-15, 385))
    savefig(fig, 11, "fig11-1", out)


def fig_11_2(out=None):
    """Kaplan-Meier curve for BrainCancer with the 95% band."""
    import matplotlib.pyplot as plt
    B = brain()
    tab = km_table(B["time"], B["status"])
    fig, ax = plt.subplots(figsize=(5.6, 4.3))
    _km_steps(ax, tab, BLUE)
    x = np.r_[0.0, tab["time"]]
    ax.step(x, np.r_[1.0, tab["lower"]], where="post", color=BLUE, lw=0.9, ls="--")
    ax.step(x, np.r_[1.0, tab["upper"]], where="post", color=BLUE, lw=0.9, ls="--")
    ax.set(xlabel="Months", ylabel="Estimated Probability of Survival", ylim=(0, 1.03))
    savefig(fig, 11, "fig11-2", out)


def fig_11_3(out=None):
    """Kaplan-Meier curves by sex."""
    import matplotlib.pyplot as plt
    B = brain()
    fig, ax = plt.subplots(figsize=(5.6, 4.3))
    for sex, colour in (("Female", RED), ("Male", BLUE)):
        part = B[B["sex"] == sex]
        _km_steps(ax, km_table(part["time"], part["status"]), colour, label=sex)
    ax.legend(loc="lower left")
    ax.set(xlabel="Months", ylabel="Estimated Probability of Survival", ylim=(0, 1.03))
    savefig(fig, 11, "fig11-3", out)


def fig_11_4(out=None):
    """Log hazards and survival curves with and without proportional hazards."""
    import matplotlib.pyplot as plt
    t = np.linspace(0.1, 2, 300)
    cases = [((0.4 * t, 0.2 * t ** 2), (t, 0.5 * t ** 2)),          # (h, H) for x = 0 and x = 1
             ((1.5 * t ** 2, 0.5 * t ** 3), (t, 0.5 * t ** 2))]
    fig, axes = plt.subplots(2, 2, figsize=(8.4, 5.6))
    for row, ((h0, H0), (h1, H1)) in enumerate(cases):
        axes[row, 0].plot(t, np.log(h0), color=GREEN, label="$x_i=0$")
        axes[row, 0].plot(t, np.log(h1), color="k", label="$x_i=1$")
        axes[row, 1].plot(t, np.exp(-H0), color=GREEN)
        axes[row, 1].plot(t, np.exp(-H1), color="k")
        axes[row, 0].set(xlabel="Time", ylabel="Log Hazard")
        axes[row, 1].set(xlabel="Time", ylabel="Survival Probability", ylim=(0, 1.02))
    axes[0, 0].legend(loc="lower right")
    axes[0, 0].set_title("proportional hazards")
    axes[1, 0].set_title("hazards not proportional")
    fig.tight_layout()
    savefig(fig, 11, "fig11-4", out)


def fig_11_5(out=None):
    """Kaplan-Meier curves for time to publication by result."""
    import matplotlib.pyplot as plt
    P = publication()
    fig, ax = plt.subplots(figsize=(5.6, 4.3))
    for res, colour, lab in ((0, GREEN, "Negative Result"), (1, BLUE, "Positive Result")):
        part = P[P["posres"] == res]
        _km_steps(ax, km_table(part["time"], part["status"]), colour, label=lab)
    ax.legend(loc="upper right")
    ax.set(xlabel="Months", ylabel="Probability of Not Being Published", ylim=(0, 1.03))
    savefig(fig, 11, "fig11-5", out)


def pub_full_fit():
    """Cox model with all predictors (Table 11.3) in its finite limit: the R42 and RC2 trials removed."""
    P = publication()
    keep = ~P["mech"].isin(["R42", "RC2"]).to_numpy()
    X = pub_matrix(P).loc[keep].drop(columns=["mechR42", "mechRC2"])
    fit, res = coxph(P["time"][keep], P["status"][keep], X)
    return P, keep, X, fit, res


def adjusted_posres_curves():
    P, keep, X, fit, res = pub_full_fit()
    x = X.mean().to_numpy() * 0
    for j, c in enumerate(X.columns):
        x[j] = 1.0 if c == "mechR01" else (0.0 if c.startswith("mech") else P[c].mean())
    rows = []
    for v in (0, 1):
        xv = x.copy()
        xv[list(X.columns).index("posres")] = v
        rows.append(xv)
    times, S = cox_survival(P["time"][keep], P["status"][keep], X, fit.params, np.array(rows))
    return times, S


def fig_11_6(out=None):
    """Survival curves by result, adjusted for the other predictors."""
    import matplotlib.pyplot as plt
    times, S = adjusted_posres_curves()
    fig, ax = plt.subplots(figsize=(5.6, 4.3))
    for k, (colour, lab) in enumerate(((GREEN, "Negative Result"), (BLUE, "Positive Result"))):
        ax.step(np.r_[0.0, times], np.r_[1.0, S[:, k]], where="post", color=colour, label=lab)
    ax.legend(loc="upper right")
    ax.set(xlabel="Months", ylabel="Probability of Not Being Published", ylim=(0, 1.03))
    savefig(fig, 11, "fig11-6", out)


def fig_11_7(out=None):
    """Cross-validated partial likelihood deviance of the lasso-penalised Cox model."""
    import matplotlib.pyplot as plt
    L = lasso_study()
    lam, cvm, cvsd, B = L["lambdas"], L["cvm"], L["cvsd"], L["B"]
    j = int(np.argmin(cvm))
    nz = (np.abs(B) > 0).sum(1)
    fig, ax = plt.subplots(figsize=(6.4, 4.0))
    ax.plot(lam, cvm, color=BLUE)
    ax.axvline(lam[j], color="k", ls="--", lw=1)
    ax.set_xscale("log")
    ax.invert_xaxis()
    ax.set(xlabel=r"$\lambda$ (decreasing: less shrinkage to the right)", ylabel="Partial Likelihood Deviance")
    top = ax.secondary_xaxis("top")
    ticks = [lam[np.argmax(nz >= k)] for k in (1, 2, 3, 5, 8, 12, 16, 19) if (nz >= k).any()]
    top.set_xticks(ticks, [str(nz[np.argmax(lam == tk)]) for tk in ticks])
    top.set_xlabel("non-zero coefficients", fontsize=9)
    savefig(fig, 11, "fig11-7", out)


def lasso_risk():
    """Risk scores of the test trials from the coefficients at the CV minimum."""
    L = lasso_study()
    j = int(np.argmin(L["cvm"]))
    b = L["B"][j]
    test = ~L["train"]
    eta = L["X"][test] @ b
    return L, j, b, test, eta


def tertiles(eta):
    """0, 1, 2 for the lowest, middle and highest third of the risk scores, by rank: many test trials have
    the same score (impact = 0 for unpublished trials), so ties are broken by their order in the data."""
    rank = np.argsort(np.argsort(eta, kind="stable"), kind="stable")
    return (3 * rank) // len(eta)


def fig_11_8(out=None):
    """Kaplan-Meier curves of the test trials in tertiles of predicted risk."""
    import matplotlib.pyplot as plt
    L, j, b, test, eta = lasso_risk()
    P = L["P"][test]
    grp = tertiles(eta)
    fig, ax = plt.subplots(figsize=(5.6, 4.3))
    for k, (colour, lab) in enumerate(((ORANGE, "Low Risk"), (BLUE, "Medium Risk"), (GREEN, "High Risk"))):
        part = P[grp == k]
        _km_steps(ax, km_table(part["time"], part["status"]), colour, label=lab)
    ax.legend(loc="upper right")
    ax.set(xlabel="Months", ylabel="Probability of Not Being Published", ylim=(0, 1.03))
    savefig(fig, 11, "fig11-8", out)


def fig_11_9(out=None):
    """Kaplan-Meier curve of the six observations of Table 11.4 (Exercise 4)."""
    import matplotlib.pyplot as plt
    tab = km_table([26.5, 37.2, 57.3, 90.8, 20.2, 89.8], [1, 1, 1, 0, 0, 0])
    fig, ax = plt.subplots(figsize=(5.0, 3.8))
    _km_steps(ax, tab, "k")
    ax.set(xlabel="Time in Days", ylabel="Estimated Probability of Survival", ylim=(0, 1.03), xlim=(0, 95))
    savefig(fig, 11, "fig11-9", out)


def fig_auc(out=None):
    """Time-dependent AUC of the lasso risk score on the test trials (extra figure)."""
    import matplotlib.pyplot as plt
    L, j, b, test, eta = lasso_risk()
    P = L["P"]
    tt = np.arange(6, 61, 2)
    auc = [auc_t(P["time"][test], P["status"][test], eta, t) for t in tt]
    fig, ax = plt.subplots(figsize=(5.6, 3.6))
    ax.plot(tt, auc, color=BLUE, marker="o", ms=3)
    C = concordance(P["time"][test], P["status"][test], eta)
    ax.axhline(C, color=GREY, ls="--", lw=1)
    ax.text(7, C - 0.015, "Harrell's C = %.3f" % C, ha="left", va="top", fontsize=9, color=GREY)
    ax.set(xlabel="Months t", ylabel="AUC(t)", ylim=(0.5, 1.0))
    savefig(fig, 11, "fig11-auc", out)


def fig_loglog(out=None):
    """log(-log S(t)) against log t, by sex and by diagnosis (extra figure)."""
    import matplotlib.pyplot as plt
    B = brain()
    fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.6))
    for ax, var, colours in ((axes[0], "sex", (RED, BLUE)), (axes[1], "diagnosis", (RED, GREEN, BLUE, PURPLE))):
        for lv, colour in zip(LEVELS[var], colours):
            part = B[B[var] == lv]
            tab = km_table(part["time"], part["status"])
            ok = (tab["surv"] < 1) & (tab["surv"] > 0) & (tab["n_event"] > 0)
            ax.step(np.log(tab["time"][ok]), np.log(-np.log(tab["surv"][ok])), where="post", color=colour, label=lv)
        ax.legend(loc="upper left", fontsize=8)
        ax.set(xlabel="log(months)", ylabel="log(-log S(t))", title=var)
    fig.tight_layout()
    savefig(fig, 11, "fig11-loglog", out)


# ======================================================================== numbers quoted in the notes
def best_logrank_split(B, var):
    """Survival-tree style search: the cut of a numeric variable with the largest log-rank statistic."""
    x = B[var].to_numpy(float)
    best = (0, None)
    for c in np.unique(x)[:-1]:
        g = (x <= c).astype(int)
        if min(g.sum(), (1 - g).sum()) < 5:
            continue
        chi = logrank_test(B["time"], B["status"], g)["chisq"]
        if chi > best[0]:
            best = (chi, c)
    return best


def permutation_logrank(B, B_perm=1000, seed=0):
    rng = np.random.default_rng(seed)
    obs = logrank_test(B["time"], B["status"], B["sex"])
    W = (obs["O"][0] - obs["E"][0]) / np.sqrt(obs["V"][0, 0])
    sex = B["sex"].to_numpy()
    Wp = []
    for _ in range(B_perm):
        r = logrank_test(B["time"], B["status"], rng.permutation(sex))
        Wp.append((r["O"][0] - r["E"][0]) / np.sqrt(r["V"][0, 0]))
    Wp = np.array(Wp)
    return W, np.mean(np.abs(Wp) >= abs(W) - 1e-12)


def call_centre_python():
    """The simulated call-centre data of the Python lab (the same draws, numpy default_rng(10))."""
    rng = np.random.default_rng(10)
    N = 2000
    Operators = rng.choice(np.arange(5, 16), N, replace=True)
    Center = rng.choice(["A", "B", "C"], N, replace=True)
    Time = rng.choice(["Morn.", "After.", "Even."], N, replace=True)
    D = pd.DataFrame({"Operators": Operators, "Center": Center, "Time": Time})
    X = model_matrix(D, ["Operators", "Center", "Time"])
    lp = X.to_numpy() @ np.array([0.04, -0.3, 0, 0.2, -0.2])
    W = np.sqrt(2 * (-np.log(rng.uniform(size=N))) / (1e-5 * np.exp(lp)))
    D["Wait time"] = np.clip(W, 0, 1000)
    D["Failed"] = rng.choice([1, 0], N, p=[0.9, 0.1])
    return D, lp


def hangup_bias(ts=(200, 300, 400)):
    """Both labs mark 10% of the calls as hang-ups at random but keep their answer time as the observed
    time. The Kaplan-Meier curve then estimates exp(-0.9 H(t)) = S(t)^0.9, not S(t)."""
    D, lp = call_centre_python()
    tab = km_table(D["Wait time"], D["Failed"])
    full = km_table(D["Wait time"], np.ones(len(D), int))           # every call's true wait
    true_S = [np.mean(np.exp(-1e-5 * t ** 2 / 2 * np.exp(lp))) for t in ts]
    return {"t": ts, "km": km_at(tab, ts), "all_waits": km_at(full, ts), "true_S": true_S,
            "true_S_pow_0.9": [np.mean(np.exp(-0.9 * 1e-5 * t ** 2 / 2 * np.exp(lp))) for t in ts]}


def numbers():
    B = brain()
    y, d = B["time"].to_numpy(), B["status"].to_numpy()
    out = {}
    out["n_Y_gt_20"] = int((y > 20).sum())
    out["n_cens_before_20"] = int(((y <= 20) & (d == 0)).sum())
    out["naive1"] = (y > 20).mean()
    out["naive2"] = (y > 20).sum() / (len(y) - out["n_cens_before_20"])
    tab = km_table(y, d)
    out["S20"] = km_at(tab, 20)[0]
    out["band20"] = (km_at(tab, 20, "lower")[0], km_at(tab, 20, "upper")[0])
    below = tab["surv"].to_numpy() <= 0.5
    out["median"] = tab["time"][below].iloc[0] if below.any() else np.nan
    out["S_end"] = tab["surv"].iloc[-1]
    out["n_deaths"] = int(d.sum())
    out["exp_lambda"] = d.sum() / y.sum()
    out["exp_mean"] = y.sum() / d.sum()
    out["exp_S20"] = np.exp(-out["exp_lambda"] * 20)
    lr = logrank_test(y, d, B["sex"])
    out["W_sex"] = (lr["O"][0] - lr["E"][0]) / np.sqrt(lr["V"][0, 0])
    out["W_p"] = lr["p"]
    out["perm"] = permutation_logrank(B)
    # Cox with sex: Newton path, tests
    x = (B["sex"] == "Male").astype(float).to_numpy()
    bhat, path = newton_1d(x, y, d)
    out["sex_newton_path"] = path[:6]
    ll0, U0, I0 = cox_efron_1d(x, y, d, 0.0)
    out["sex_score_U_I"] = (U0, I0, U0 ** 2 / I0)
    # log-rank as O - E for the males
    out["O_minus_E_male"] = lr["O"][1] - lr["E"][1]
    # Publication: ties, score tests
    P = publication()
    tp, dp, xp = P["time"].to_numpy(), P["status"].to_numpy(), P["posres"].to_numpy(float)
    lrp = logrank_test(tp, dp, xp)
    _, Ub, Hb = cox_breslow(xp[:, None], tp, dp, np.zeros(1))
    _, Ue, Ie = cox_efron_1d(xp, tp, dp, 0.0)
    out["pub_scores"] = {"logrank": lrp["chisq"], "breslow": Ub[0] ** 2 / -Hb[0, 0], "efron": Ue ** 2 / Ie}
    out["pub_tied_deaths"] = int(pd.Series(tp[dp == 1]).duplicated(keep=False).sum())
    out["pub_events"] = int(dp.sum())
    # Table 11.3 in its finite limit
    _, _, Xf, fit3, res3 = pub_full_fit()
    out["table113_posres"] = (fit3.params[0], fit3.bse[0])
    times, S = adjusted_posres_curves()
    out["fig116_S24"] = S[np.searchsorted(times, 24, side="right") - 1]
    out["fig116_S48"] = S[np.searchsorted(times, 48, side="right") - 1]
    # lasso and C-index
    L, j, b, test, eta = lasso_risk()
    out["lasso_lambda_min"] = L["lambdas"][j]
    out["lasso_nonzero"] = [n for n, v in zip(L["names"], b) if v != 0]
    out["lasso_coef"] = dict(zip(out["lasso_nonzero"], b[b != 0]))
    out["lasso_cvm_min"] = L["cvm"][j]
    Pt = L["P"][test]
    out["C_lasso_test"] = concordance(Pt["time"], Pt["status"], eta)
    tr = L["train"]
    X2 = L["P"].drop(columns=["mech", "time", "status"])
    fit2, _ = coxph(L["P"]["time"][tr], L["P"]["status"][tr], X2[tr])
    out["C_cox_nomech_test"] = concordance(Pt["time"], Pt["status"], X2[test].to_numpy() @ fit2.params)
    grp = tertiles(eta)
    out["tertile_logrank"] = logrank_test(Pt["time"], Pt["status"], grp)["chisq"]
    out["auc_12_24_36"] = [auc_t(Pt["time"], Pt["status"], eta, t) for t in (12, 24, 36)]
    # survival tree stumps
    out["split_ki"] = best_logrank_split(B, "ki")
    out["split_gtv"] = best_logrank_split(B, "gtv")
    out["hangup"] = hangup_bias()
    impact0 = P["impact"] == 0
    out["impact0_by_status"] = (int((impact0 & (P["status"] == 0)).sum()), int((P["status"] == 0).sum()),
                                int((impact0 & (P["status"] == 1)).sum()), int((P["status"] == 1).sum()))
    fit_ni, res_ni = coxph(P["time"], P["status"], P[["posres", "multi", "clinend", "sampsize", "budget"]])
    out["posres_without_impact"] = (fit_ni.params[0], res_ni["table"]["p"].iloc[0])
    for k, v in out.items():
        print("%-22s %s" % (k, v))
    return out


# ======================================================================== demonstrations
def demos(out=None):
    """Write the data of the three demonstrations and print their reference numbers."""
    os.makedirs(DATA_OUT, exist_ok=True)
    B = brain()
    rows = B.reset_index(drop=True)
    jb = {"time": rows["time"].tolist(), "status": rows["status"].astype(int).tolist(),
          "sex": rows["sex"].tolist(), "diagnosis": rows["diagnosis"].fillna("NA").tolist(),
          "loc": rows["loc"].tolist(), "stereo": rows["stereo"].tolist(), "ki": rows["ki"].astype(int).tolist(),
          "gtv": rows["gtv"].tolist()}
    P = publication().reset_index(drop=True)
    jp = {"time": [round(v, 6) for v in P["time"]], "status": P["status"].astype(int).tolist(),
          "posres": P["posres"].astype(int).tolist(), "multi": P["multi"].astype(int).tolist(),
          "clinend": P["clinend"].astype(int).tolist(), "impact": P["impact"].tolist(),
          "budget": P["budget"].tolist()}
    with open(os.path.join(DATA_OUT, "ch11-survival.json"), "w") as f:
        json.dump({"brain": jb, "publication": jp}, f, separators=(",", ":"))
    print("wrote ../data/ch11-survival.json")
    ref = {}
    tab = km_table(B["time"], B["status"])
    ref["km_S"] = {t: km_at(tab, t)[0] for t in (10, 20, 40, 60)}
    for var in ("sex", "loc", "stereo"):
        g = {lv: km_at(km_table(B[B[var] == lv]["time"], B[B[var] == lv]["status"]), 20)[0] for lv in LEVELS[var]}
        lr = logrank_test(B["time"], B["status"], B[var])
        ref["km20_" + var] = g
        ref["lr_" + var] = (lr["chisq"], lr["p"], lr["O"].tolist(), lr["E"].tolist(), lr["V"][0, 0])
    Bd = B.dropna()
    lrd = logrank_test(Bd["time"], Bd["status"], Bd["diagnosis"])
    ref["lr_diagnosis4"] = (lrd["chisq"], lrd["p"])
    hg = (Bd["diagnosis"] == "HG glioma").astype(int)
    lrh = logrank_test(Bd["time"], Bd["status"], hg)
    ref["lr_hg"] = (lrh["chisq"], lrh["p"])
    lrp = logrank_test(P["time"], P["status"], P["posres"])
    ref["lr_posres"] = (lrp["chisq"], lrp["p"])
    for name, x, t, d in (("sex", (B["sex"] == "Male").astype(float), B["time"], B["status"]),
                          ("ki", B["ki"].astype(float), B["time"], B["status"]),
                          ("loc", (B["loc"] == "Supratentorial").astype(float), B["time"], B["status"]),
                          ("posres", P["posres"].astype(float), P["time"], P["status"]),
                          ("impact", P["impact"].astype(float), P["time"], P["status"])):
        for ties in ("efron", "breslow"):
            fit, res = coxph(t, d, x.to_frame(name), ties=ties)
            ref["cox_%s_%s" % (name, ties)] = (fit.params[0], fit.bse[0], res["tests"]["LR"], res["tests"]["Wald"], res["tests"]["score"])
    for k, v in ref.items():
        print("%-20s %s" % (k, v))
    return ref


FIGURES = {
    "11.1": fig_11_1, "11.2": fig_11_2, "11.3": fig_11_3, "11.4": fig_11_4, "11.5": fig_11_5, "11.6": fig_11_6,
    "11.7": fig_11_7, "11.8": fig_11_8, "11.9": fig_11_9, "auc": fig_auc, "loglog": fig_loglog,
    "numbers": lambda out=None: numbers(), "demos": demos,
}

if __name__ == "__main__":
    common.run(FIGURES, 11)
