# Chapter 11 lab: survival analysis (R version).
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 11.8, and the Python lab of James et al. (2023), Section 11.8.
#
# The lab analyses three data sets with the survival package:
#   11.8.1  BrainCancer (88 patients): Kaplan-Meier curves, the log-rank test, Cox models, and
#           survival curves for each diagnosis adjusted for the other predictors;
#   11.8.2  Publication (244 clinical trials): Kaplan-Meier curves by result, the log-rank test and
#           Cox models with one and with all predictors;
#   11.8.3  a simulated call centre (2000 calls): Kaplan-Meier curves, log-rank tests for three
#           groups, and a Cox model whose true coefficients are known.
# The book's R lab simulates the call centre with sim.survdata() from the coxed package; that package
# is not used here: the waiting times are drawn directly from the same proportional-hazards model by
# inverting its cumulative hazard. The last part solves applied Exercises 10 and 11.
# The Python version (ch11_lab.py) runs the same analyses with its own Kaplan-Meier and log-rank code
# and statsmodels' PHReg. Run from the code/ folder:
#   Rscript ch11_lab.R
# Figures go to ../img/ch11/lab-r-*.svg and the key numbers to cache/ch11_lab_R.csv.

suppressPackageStartupMessages({
  library(ISLR2)
  library(survival)
})
options(digits = 6, width = 100)

out_dir <- file.path("..", "img", "ch11")
dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
dir.create("cache", showWarnings = FALSE)
key <- list()                                 # key numbers, written to cache/ch11_lab_R.csv at the end
keep <- function(name, value) key[[name]] <<- unname(value)
open_svg <- function(name, width = 6, height = 4.6) {
  path <- file.path(out_dir, paste0(name, ".svg"))
  if (requireNamespace("svglite", quietly = TRUE)) svglite::svglite(path, width = width, height = height)
  else grDevices::svg(path, width = width, height = height)
}

# ================================================================== 11.8.1 brain cancer data
print(names(BrainCancer))
print(table(BrainCancer$sex))
print(table(BrainCancer$diagnosis))
print(table(BrainCancer$status))              # 1 = died (uncensored), 0 = censored

fit.surv <- survfit(Surv(time, status) ~ 1, data = BrainCancer)
open_svg("lab-r-km")
plot(fit.surv, xlab = "Months", ylab = "Estimated Probability of Survival")
dev.off()
print(summary(fit.surv, times = c(10, 20, 40, 60)))
keep("km_time", fit.surv$time); keep("km_surv", fit.surv$surv); keep("km_nrisk", fit.surv$n.risk)
keep("km_se_logS", fit.surv$std.err)          # survfit stores the standard error of -log S(t)
keep("km_S20", summary(fit.surv, times = 20)$surv)
keep("km_lower20", summary(fit.surv, times = 20)$lower)
keep("km_upper20", summary(fit.surv, times = 20)$upper)

fit.sex <- survfit(Surv(time, status) ~ sex, data = BrainCancer)
open_svg("lab-r-km-sex")
plot(fit.sex, xlab = "Months", ylab = "Estimated Probability of Survival", col = c(2, 4))
legend("bottomleft", levels(BrainCancer$sex), col = c(2, 4), lty = 1)
dev.off()

logrank.test <- survdiff(Surv(time, status) ~ sex, data = BrainCancer)
print(logrank.test)
keep("lr_sex_obs", logrank.test$obs); keep("lr_sex_exp", logrank.test$exp)
keep("lr_sex_var", logrank.test$var[1, 1]); keep("lr_sex_chisq", logrank.test$chisq)

fit.cox <- coxph(Surv(time, status) ~ sex, data = BrainCancer)
print(summary(fit.cox))
tests <- c(LR = summary(fit.cox)$logtest[1], Wald = summary(fit.cox)$waldtest[1],
           score = summary(fit.cox)$sctest[1])
print(tests)
cat("log-rank chi-squared:", format(logrank.test$chisq, digits = 9), "\n")
keep("cox_sex_coef", coef(fit.cox)); keep("cox_sex_se", sqrt(diag(vcov(fit.cox))))
keep("cox_sex_tests", tests); keep("cox_sex_concordance", fit.cox$concordance["concordance"])

fit.all <- coxph(Surv(time, status) ~ sex + diagnosis + loc + ki + gtv + stereo,
                 data = BrainCancer)
print(fit.all)
keep("cox_all_coef", coef(fit.all)); keep("cox_all_se", sqrt(diag(vcov(fit.all))))
keep("cox_all_LR", 2 * diff(fit.all$loglik))

# survival curves for each diagnosis, the other predictors at their mean (numeric) or modal value
modaldata <- data.frame(
  diagnosis = levels(BrainCancer$diagnosis),
  sex = rep("Female", 4),
  loc = rep("Supratentorial", 4),
  ki = rep(mean(BrainCancer$ki), 4),
  gtv = rep(mean(BrainCancer$gtv), 4),
  stereo = rep("SRT", 4)
)
survplots <- survfit(fit.all, newdata = modaldata)
open_svg("lab-r-diagnosis")
plot(survplots, xlab = "Months", ylab = "Survival Probability", col = 2:5)
legend("bottomleft", levels(BrainCancer$diagnosis), col = 2:5, lty = 1)
dev.off()
print(summary(survplots, times = c(20, 60))$surv)
keep("modal_surv20", summary(survplots, times = 20)$surv)
keep("modal_surv60", summary(survplots, times = 60)$surv)
keep("modal_surv_last", survplots$surv[length(survplots$time), ])

# ================================================================== 11.8.2 publication data
fit.posres <- survfit(Surv(time, status) ~ posres, data = Publication)
open_svg("lab-r-km-posres")
plot(fit.posres, xlab = "Months", ylab = "Probability of Not Being Published", col = 3:4)
legend("topright", c("Negative Result", "Positive Result"), col = 3:4, lty = 1)
dev.off()

fit.pub <- coxph(Surv(time, status) ~ posres, data = Publication)
print(fit.pub)
logrank.test <- survdiff(Surv(time, status) ~ posres, data = Publication)
print(logrank.test)
keep("pub_cox_coef", coef(fit.pub)); keep("pub_cox_se", sqrt(diag(vcov(fit.pub))))
keep("pub_cox_score", summary(fit.pub)$sctest[1]); keep("pub_lr_chisq", logrank.test$chisq)
keep("pub_lr_obs", logrank.test$obs); keep("pub_lr_exp", logrank.test$exp)

fit.pub2 <- coxph(Surv(time, status) ~ . - mech, data = Publication)
print(fit.pub2)
keep("pub2_coef", coef(fit.pub2)); keep("pub2_se", sqrt(diag(vcov(fit.pub2))))
keep("pub2_LR", 2 * diff(fit.pub2$loglik))

# all predictors, funding mechanism included (Table 11.3 of the text)
fit.pub3 <- coxph(Surv(time, status) ~ ., data = Publication)
print(round(coef(summary(fit.pub3)), 3))
keep("pub3_loglik", fit.pub3$loglik[2])
keep("pub3_coef_finite", coef(fit.pub3)[!names(coef(fit.pub3)) %in% c("mechR42", "mechRC2")])

# ================================================================== 11.8.3 call centre data
set.seed(4)
N <- 2000
Operators <- sample(5:15, N, replace = TRUE)
Center <- sample(c("A", "B", "C"), N, replace = TRUE)
Time <- sample(c("Morn.", "After.", "Even."), N, replace = TRUE)
X <- model.matrix(~ Operators + Center + Time)[, -1]
print(X[1:5, ])

true.beta <- c(0.04, -0.3, 0, 0.2, -0.2)
h.fn <- function(x) 0.00001 * x               # the baseline hazard h0(t)
H.fn <- function(x) 0.00001 * x^2 / 2         # its integral, the cumulative baseline hazard H0(t)
# Under proportional hazards S(t | x) = exp(-H0(t) exp(x'beta)), so with U uniform the time
# H0^{-1}(-log(U) / exp(x'beta)) has survival function S(t | x)  (inverse-transform sampling)
lin.pred <- drop(X %*% true.beta)
wait <- sqrt(2 * (-log(runif(N))) / (0.00001 * exp(lin.pred)))
y <- pmin(wait, 1000)                         # waits are capped at 1000 seconds
failed <- rbinom(N, 1, 0.9) == 1              # 90% of calls answered, 10% hang-ups (censored)
queuing <- data.frame(X, y = y, failed = failed)
print(head(queuing))
print(mean(queuing$failed))
write.csv(queuing, file.path("cache", "ch11_callcenter_R.csv"), row.names = FALSE)

queuing$Center <- Center                      # the group labels, for the stratified curves and tests
queuing$Time <- Time
fit.Center <- survfit(Surv(y, failed) ~ Center, data = queuing)
fit.Time <- survfit(Surv(y, failed) ~ Time, data = queuing)
open_svg("lab-r-callcenter", width = 9, height = 4.2)
par(mfrow = c(1, 2))
plot(fit.Center, xlab = "Seconds", ylab = "Probability of Still Being on Hold", col = c(2, 4, 5))
legend("topright", c("Call Center A", "Call Center B", "Call Center C"), col = c(2, 4, 5), lty = 1)
plot(fit.Time, xlab = "Seconds", ylab = "Probability of Still Being on Hold", col = c(2, 4, 5))
legend("topright", c("Afternoon", "Evening", "Morning"), col = c(2, 4, 5), lty = 1)
dev.off()

lr.center <- survdiff(Surv(y, failed) ~ Center, data = queuing)
lr.time <- survdiff(Surv(y, failed) ~ Time, data = queuing)
print(lr.center)
print(lr.time)
keep("cc_lr_center", lr.center$chisq); keep("cc_lr_time", lr.time$chisq)
keep("cc_lr_center_exp", lr.center$exp); keep("cc_lr_time_exp", lr.time$exp)

# likelihood ratio tests of the Cox models with Center alone and with Time alone
lrt <- function(fit) 2 * diff(fit$loglik)
cat("LR tests: Center", lrt(coxph(Surv(y, failed) ~ Center, data = queuing)),
    " Time", lrt(coxph(Surv(y, failed) ~ Time, data = queuing)), "\n")
keep("cc_LR_center", lrt(coxph(Surv(y, failed) ~ Center, data = queuing)))
keep("cc_LR_time", lrt(coxph(Surv(y, failed) ~ Time, data = queuing)))

fit.queuing <- coxph(Surv(y, failed) ~ Operators + CenterB + CenterC + TimeEven. + TimeMorn.,
                     data = queuing)
print(fit.queuing)
keep("cc_cox_coef", coef(fit.queuing)); keep("cc_cox_se", sqrt(diag(vcov(fit.queuing))))

# ================================================================== Exercise 10: brain cancer data
# (a) the Kaplan-Meier curve with +/- 1 standard error bands (Greenwood standard error of S(t))
km.se <- summary(fit.surv)$std.err
open_svg("lab-r-ex10a")
plot(fit.surv, conf.int = FALSE, xlab = "Months", ylab = "Estimated Probability of Survival")
lines(stepfun(fit.surv$time, c(1, fit.surv$surv + fit.surv$surv * fit.surv$std.err)), lty = 2,
      do.points = FALSE)
lines(stepfun(fit.surv$time, c(1, fit.surv$surv - fit.surv$surv * fit.surv$std.err)), lty = 2,
      do.points = FALSE)
dev.off()
keep("ex10_greenwood", summary(fit.surv, times = c(10, 20, 40, 60))$std.err)

# (b) bootstrap standard errors of S(t), B = 200 resamples of the pairs (y_i, delta_i)
km_at <- function(time, status, t) {
  f <- survfit(Surv(time, status) ~ 1)
  stepfun(f$time, c(1, f$surv))(t)
}
set.seed(1)
tt <- c(10, 20, 40, 60)
boot <- replicate(200, {
  i <- sample(nrow(BrainCancer), replace = TRUE)
  km_at(BrainCancer$time[i], BrainCancer$status[i], tt)
})
print(rbind(greenwood = summary(fit.surv, times = tt)$std.err, bootstrap = apply(boot, 1, sd)))
keep("ex10_boot", apply(boot, 1, sd))

# (c) the Cox model with all predictors is fit.all above; its main findings are in the notes

# (d) strata of the Karnofsky index (40 merged with 60), adjusted for the other predictors
BrainCancer$ki5 <- ifelse(BrainCancer$ki == 40, 60, BrainCancer$ki)
fit.ki <- coxph(Surv(time, status) ~ sex + diagnosis + loc + gtv + stereo + strata(ki5),
                data = BrainCancer)
print(fit.ki)
typical <- data.frame(sex = "Female", diagnosis = "Meningioma", loc = "Supratentorial",
                      gtv = mean(BrainCancer$gtv), stereo = "SRT")
surv.ki <- survfit(fit.ki, newdata = typical)
open_svg("lab-r-ex10d")
plot(surv.ki, col = 2:6, xlab = "Months", ylab = "Survival Probability")
legend("bottomleft", paste("ki =", sort(unique(BrainCancer$ki5))), col = 2:6, lty = 1)
dev.off()
keep("ex10d_coef", coef(fit.ki))
keep("ex10d_surv_end", summary(surv.ki, times = 30, extend = TRUE)$surv)

# ================================================================== Exercise 11: the data of Table 11.4
ex11 <- data.frame(y = c(26.5, 37.2, 57.3, 90.8, 20.2, 89.8),
                   delta = c(1, 1, 1, 0, 0, 0),
                   x = c(0.1, 11, -0.3, 2.8, 1.8, 0.4))
ex11$group <- ifelse(ex11$x < 2, 1, 2)
km11 <- survfit(Surv(y, delta) ~ group, data = ex11)
print(summary(km11))
open_svg("lab-r-ex11")
plot(km11, col = c(2, 4), xlab = "Time in Days", ylab = "Estimated Probability of Survival")
legend("bottomleft", c("Group 1 (X < 2)", "Group 2 (X >= 2)"), col = c(2, 4), lty = 1)
dev.off()
cox11 <- coxph(Surv(y, delta) ~ I(group == 2), data = ex11)
print(summary(cox11))
lr11 <- survdiff(Surv(y, delta) ~ group, data = ex11)
print(lr11)
keep("ex11_coef", coef(cox11)); keep("ex11_se", sqrt(vcov(cox11)[1, 1]))
keep("ex11_score", summary(cox11)$sctest[1]); keep("ex11_lr", lr11$chisq)
keep("ex11_km_all", survfit(Surv(y, delta) ~ 1, data = ex11)$surv)

# ================================================================== save the key numbers
flat <- do.call(rbind, lapply(names(key), function(k) {
  v <- key[[k]]
  data.frame(name = if (length(v) == 1) k else paste0(k, "_", seq_along(v)), value = as.numeric(v))
}))
write.csv(flat, file.path("cache", "ch11_lab_R.csv"), row.names = FALSE)
cat("wrote", nrow(flat), "numbers to cache/ch11_lab_R.csv\n")
