# Chapter 9 lab: support vector machines (R version).
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 9.6, and the Python lab of James et al. (2023), Section 9.6.
#
# The lab follows the R lab of the book, with the same random seeds, so the printed numbers are the book's:
#   9.6.1  the support vector classifier (e1071::svm with a linear kernel) on 20 simulated points,
#          cost = 10 and 0.1, tune() by ten-fold cross-validation, test predictions, separable data;
#   9.6.2  the support vector machine with a radial kernel, tuned over cost and gamma;
#   9.6.3  ROC curves from the decision values (ROCR);
#   9.6.4  three classes (one-versus-one);
#   9.6.5  the Khan gene-expression data.
# It also computes what svm() does not print (the coefficients beta and the margin), writes its
# simulated data and the Khan data to cache/ for the Python version (ch09_lab.py), and solves the
# applied exercises 4-8 on data that both languages generate identically.
# Run from the code/ folder:
#   Rscript ch09_lab.R
# Figures go to ../img/ch09/lab-r-*.svg and the key numbers to cache/ch09_lab_R.csv.

suppressPackageStartupMessages({
  library(ISLR2)
  library(e1071)
  library(ROCR)
})
options(digits = 6, width = 100)

out_dir <- file.path("..", "img", "ch09")
dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
dir.create("cache", showWarnings = FALSE)
key <- list()                                 # key numbers, written to cache/ch09_lab_R.csv at the end
keep <- function(name, value) key[[name]] <<- unname(as.numeric(value))
open_svg <- function(name, width, height) {
  path <- file.path(out_dir, paste0(name, ".svg"))
  if (requireNamespace("svglite", quietly = TRUE)) svglite::svglite(path, width = width, height = height)
  else grDevices::svg(path, width = width, height = height)
}

# CSV with all 17 significant digits, so that the Python version reads exactly the same numbers
write_full <- function(df, path) {
  df[] <- lapply(df, function(v) sprintf("%.17g", v))
  write.csv(df, path, row.names = FALSE, quote = FALSE)
}
# a plot of a two-dimensional fit: shaded regions, the margins f = -1, 0, 1 (two classes), support vectors as +
plot_fit <- function(fit, x, y, main = "", pal = c("#7b4fbf", "#1f5fbf", "#2ca02c")) {
  g1 <- seq(min(x[, 1]) - 0.5, max(x[, 1]) + 0.5, length.out = 200)
  g2 <- seq(min(x[, 2]) - 0.5, max(x[, 2]) + 0.5, length.out = 200)
  G <- expand.grid(x.1 = g1, x.2 = g2)
  pr <- predict(fit, G, decision.values = TRUE)
  cls <- matrix(as.integer(pr), length(g1))
  image(g1, g2, cls, col = adjustcolor(pal[seq_along(levels(y))], 0.15),
        xlab = "x.1", ylab = "x.2", main = main, useRaster = TRUE)
  if (nlevels(y) == 2) contour(g1, g2, matrix(attr(pr, "decision.values"), length(g1)), levels = c(-1, 0, 1),
                               lty = c(2, 1, 2), drawlabels = FALSE, add = TRUE)
  col <- pal[as.integer(y)]
  sv <- seq_len(nrow(x)) %in% fit$index
  points(x[!sv, ], col = col[!sv], pch = 19, cex = 0.8)
  points(x[sv, ], col = col[sv], pch = 3, cex = 1.3, lwd = 2)
}

# ================================================================== 9.6.1 support vector classifier
set.seed(1)
x <- matrix(rnorm(20 * 2), ncol = 2)
y <- c(rep(-1, 10), rep(1, 10))
x[y == 1, ] <- x[y == 1, ] + 1
dat <- data.frame(x = x, y = as.factor(y))
write_full(data.frame(x1 = x[, 1], x2 = x[, 2], y = y), "cache/ch09_rlab_linear.csv")

svmfit <- svm(y ~ ., data = dat, kernel = "linear", cost = 10, scale = FALSE)
print(svmfit$index)
summary(svmfit)
b10 <- drop(t(svmfit$coefs) %*% svmfit$SV)              # what svm() does not print: beta ...
cat("beta =", -b10, " beta_0 =", svmfit$rho, " margin 1/||beta|| =", 1 / sqrt(sum(b10^2)), "\n")
keep("rlab_sv10", svmfit$index); keep("rlab_coef10", c(svmfit$rho, -b10))
svmfit01 <- svm(y ~ ., data = dat, kernel = "linear", cost = 0.1, scale = FALSE)
print(svmfit01$index)
keep("rlab_sv01", svmfit01$index)
open_svg("lab-r-svc", 9, 4.3); par(mfrow = c(1, 2), mar = c(4, 4, 2, 1))
plot_fit(svmfit, x, dat$y, "cost = 10"); plot_fit(svmfit01, x, dat$y, "cost = 0.1")
invisible(dev.off())

set.seed(1)
tune.out <- tune(svm, y ~ ., data = dat, kernel = "linear",
                 ranges = list(cost = c(0.001, 0.01, 0.1, 1, 5, 10, 100)))
summary(tune.out)
bestmod <- tune.out$best.model
keep("rlab_tune_err", tune.out$performances$error)

xtest <- matrix(rnorm(20 * 2), ncol = 2)
ytest <- sample(c(-1, 1), 20, rep = TRUE)
xtest[ytest == 1, ] <- xtest[ytest == 1, ] + 1
testdat <- data.frame(x = xtest, y = as.factor(ytest))
write_full(data.frame(x1 = xtest[, 1], x2 = xtest[, 2], y = ytest), "cache/ch09_rlab_lintest.csv")
ypred <- predict(bestmod, testdat)
print(table(predict = ypred, truth = testdat$y))
svmfit <- svm(y ~ ., data = dat, kernel = "linear", cost = .01, scale = FALSE)
ypred01 <- predict(svmfit, testdat)
print(table(predict = ypred01, truth = testdat$y))
keep("rlab_test_right", c(sum(ypred == testdat$y), sum(ypred01 == testdat$y)))

x[y == 1, ] <- x[y == 1, ] + 0.5                       # now the classes are barely separable
dat <- data.frame(x = x, y = as.factor(y))
svmfit <- svm(y ~ ., data = dat, kernel = "linear", cost = 1e5)
summary(svmfit)
svmfit1 <- svm(y ~ ., data = dat, kernel = "linear", cost = 1)
summary(svmfit1)
keep("rlab_sep_nsv", c(sum(svmfit$nSV), sum(svmfit1$nSV)))
keep("rlab_sep_err", c(sum(svmfit$fitted != dat$y), sum(svmfit1$fitted != dat$y)))

# ================================================================== 9.6.2 support vector machine
set.seed(1)
x <- matrix(rnorm(200 * 2), ncol = 2)
x[1:100, ] <- x[1:100, ] + 2
x[101:150, ] <- x[101:150, ] - 2
y <- c(rep(1, 150), rep(2, 50))
dat <- data.frame(x = x, y = as.factor(y))
train <- sample(200, 100)
write_full(data.frame(x1 = x[, 1], x2 = x[, 2], y = y, train = match(seq_len(200), train, nomatch = 0)),
           "cache/ch09_rlab_radial.csv")              # train = position in the sample (0: test), as LIBSVM sees it
svmfit <- svm(y ~ ., data = dat[train, ], kernel = "radial", gamma = 1, cost = 1)
summary(svmfit)
keep("rlab_rbf_nsv", svmfit$nSV[order(svmfit$labels)])   # per class, in the order of the levels
svmfit.big <- svm(y ~ ., data = dat[train, ], kernel = "radial", gamma = 1, cost = 1e5)
open_svg("lab-r-radial", 9, 4.3); par(mfrow = c(1, 2), mar = c(4, 4, 2, 1))
plot_fit(svmfit, x[train, ], dat$y[train], "gamma = 1, cost = 1")
plot_fit(svmfit.big, x[train, ], dat$y[train], "gamma = 1, cost = 1e5")
invisible(dev.off())

set.seed(1)
tune.out <- tune(svm, y ~ ., data = dat[train, ], kernel = "radial",
                 ranges = list(cost = c(0.1, 1, 10, 100, 1000), gamma = c(0.5, 1, 2, 3, 4)))
summary(tune.out)
tab <- table(true = dat[-train, "y"], pred = predict(tune.out$best.model, newdata = dat[-train, ]))
print(tab)
keep("rlab_rbf_tune_err", tune.out$performances$error)
keep("rlab_rbf_test_table", tab)                      # true-by-predicted, read row-wise = Python's table

# ================================================================== 9.6.3 ROC curves
rocplot <- function(pred, truth, ...) {
  predob <- prediction(pred, truth)
  perf <- performance(predob, "tpr", "fpr")
  plot(perf, ...)
}
auc <- function(pred, truth) performance(prediction(pred, truth), "auc")@y.values[[1]]
svmfit.opt <- svm(y ~ ., data = dat[train, ], kernel = "radial", gamma = 2, cost = 1, decision.values = TRUE)
svmfit.flex <- svm(y ~ ., data = dat[train, ], kernel = "radial", gamma = 50, cost = 1, decision.values = TRUE)
dv <- function(fit, newdata) attributes(predict(fit, newdata, decision.values = TRUE))$decision.values
open_svg("lab-r-roc", 9, 4.3); par(mfrow = c(1, 2), mar = c(4, 4, 2, 1))
rocplot(-dv(svmfit.opt, dat[train, ]), dat[train, "y"], main = "Training Data")
rocplot(-dv(svmfit.flex, dat[train, ]), dat[train, "y"], add = TRUE, col = "red")
rocplot(-dv(svmfit.opt, dat[-train, ]), dat[-train, "y"], main = "Test Data")
rocplot(-dv(svmfit.flex, dat[-train, ]), dat[-train, "y"], add = TRUE, col = "red")
invisible(dev.off())
aucs <- c(auc(-dv(svmfit.opt, dat[train, ]), dat[train, "y"]), auc(-dv(svmfit.opt, dat[-train, ]), dat[-train, "y"]),
          auc(-dv(svmfit.flex, dat[train, ]), dat[train, "y"]), auc(-dv(svmfit.flex, dat[-train, ]), dat[-train, "y"]))
cat("AUC train/test, gamma 2:", aucs[1:2], " gamma 50:", aucs[3:4], "\n")
keep("rlab_roc_auc", aucs)

# ================================================================== 9.6.4 more than two classes
set.seed(1)
x <- rbind(x, matrix(rnorm(50 * 2), ncol = 2))
y <- c(y, rep(0, 50))
x[y == 0, 2] <- x[y == 0, 2] + 2
dat <- data.frame(x = x, y = as.factor(y))
write_full(data.frame(x1 = x[, 1], x2 = x[, 2], y = y), "cache/ch09_rlab_three.csv")
svmfit <- svm(y ~ ., data = dat, kernel = "radial", cost = 10, gamma = 1)
cat("three classes: support vectors per class", svmfit$nSV, " training errors", sum(svmfit$fitted != dat$y), "\n")
keep("rlab_three_nsv", svmfit$nSV[order(svmfit$labels)]); keep("rlab_three_err", sum(svmfit$fitted != dat$y))
open_svg("lab-r-three", 4.8, 4.3); par(mar = c(4, 4, 2, 1))
plot_fit(svmfit, x, dat$y, "three classes, one-versus-one", pal = c("#2ca02c", "#7b4fbf", "#1f5fbf"))
invisible(dev.off())

# ================================================================== 9.6.5 gene expression data
names(Khan)
dim(Khan$xtrain); dim(Khan$xtest)
table(Khan$ytrain); table(Khan$ytest)
if (!file.exists("cache/ch09_khan_xtrain.csv"))          # for the Python version (no ISLR2 there)
  for (k in c("xtrain", "xtest", "ytrain", "ytest"))
    write_full(as.data.frame(Khan[[k]]), sprintf("cache/ch09_khan_%s.csv", k))
dat <- data.frame(x = Khan$xtrain, y = as.factor(Khan$ytrain))
out <- svm(y ~ ., data = dat, kernel = "linear", cost = 10)
summary(out)
table(out$fitted, dat$y)
dat.te <- data.frame(x = Khan$xtest, y = as.factor(Khan$ytest))
pred.te <- predict(out, newdata = dat.te)
table(pred.te, dat.te$y)
keep("khan_nsv", out$nSV[order(out$labels)]); keep("khan_test_table", t(table(pred.te, dat.te$y)))
raw <- svm(Khan$xtrain, as.factor(Khan$ytrain), kernel = "linear", cost = 10, scale = FALSE)
cat("unscaled genes (as in the Python lab): support vectors", raw$nSV, " test errors",
    sum(predict(raw, Khan$xtest) != Khan$ytest), "\n")
keep("khan_raw_nsv", raw$nSV[order(raw$labels)]); keep("khan_raw_test_err", sum(predict(raw, Khan$xtest) != Khan$ytest))
# one-versus-all (Section 9.4.2), which svm() does not offer: four "class k against the rest" fits
ova <- sapply(1:4, function(k) {
  f <- svm(Khan$xtrain, factor(Khan$ytrain == k), kernel = "linear", cost = 10, scale = FALSE)
  dv <- attr(predict(f, Khan$xtest, decision.values = TRUE), "decision.values")
  if (startsWith(colnames(dv), "TRUE")) dv else -dv        # orient so that f > 0 means "class k"
})
pred.ova <- max.col(ova)
cat("one-versus-all: test errors", sum(pred.ova != Khan$ytest), "\n")
keep("khan_ova_pred", pred.ova)

# ================================================================== exercises 4-8
# frac(i * sqrt(p_k)): the same "random" numbers in R and Python, so both give the same answers
qrand <- function(n, k, start = 1) {
  a <- sqrt(c(2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37)[k]) %% 1
  (seq(start, length.out = n) * a) %% 1
}
cv_error <- function(x, y, ..., K = 10) {               # K folds, row i in fold (i - 1) mod K
  fold <- (seq_along(y) - 1) %% K
  wrong <- 0
  for (k in 0:(K - 1)) {
    fit <- svm(x[fold != k, , drop = FALSE], y[fold != k], scale = FALSE, ...)
    wrong <- wrong + sum(predict(fit, x[fold == k, , drop = FALSE]) != y[fold == k])
  }
  wrong / length(y)
}
err <- function(fit, x, y) mean(predict(fit, x) != y)

# Exercise 4
n <- 200
x1 <- 4 * qrand(n, 1) - 2; x2 <- 4 * qrand(n, 2) - 2; z <- qnorm(qrand(n, 3))
y4 <- as.factor(ifelse(x2 - (x1^2 - 1.5) + 0.3 * z > 0, 1, -1))
X4 <- cbind(x1, x2); tr <- seq_len(n) <= 100
f_lin <- svm(X4[tr, ], y4[tr], kernel = "linear", cost = 1, scale = FALSE)
f_pol <- svm(X4[tr, ], y4[tr], kernel = "polynomial", degree = 2, gamma = 1, coef0 = 1, cost = 1, scale = FALSE)
f_rbf <- svm(X4[tr, ], y4[tr], kernel = "radial", gamma = 1, cost = 1, scale = FALSE)
ex4 <- sapply(list(f_lin, f_pol, f_rbf), function(f) c(err(f, X4[tr, ], y4[tr]), err(f, X4[!tr, ], y4[!tr])))
print(round(ex4, 2))                                    # columns: linear, polynomial, radial; rows: training, test
keep("ex4_err", ex4)

# Exercise 5
n <- 500
x1 <- qrand(n, 4) - 0.5; x2 <- qrand(n, 5) - 0.5; z <- qnorm(qrand(n, 6))
y5 <- as.integer(x1^2 - x2^2 + 0.02 * z > 0)
lin <- glm(y5 ~ x1 + x2, family = binomial)
quad <- glm(y5 ~ x1 + x2 + I(x1^2) + I(x2^2) + I(x1 * x2), family = binomial)
svc5 <- svm(cbind(x1, x2), as.factor(y5), kernel = "linear", cost = 1, scale = FALSE)
rbf5 <- svm(cbind(x1, x2), as.factor(y5), kernel = "radial", gamma = 10, cost = 10, scale = FALSE)
ex5 <- c(mean((fitted(lin) > 0.5) != y5), mean((fitted(quad) > 0.5) != y5),
         err(svc5, cbind(x1, x2), y5), err(rbf5, cbind(x1, x2), y5))
print(round(ex5, 3)); print(round(coef(quad), 3))
keep("ex5_err", ex5); keep("ex5_quad_coef", coef(quad))

# Exercise 6
x1 <- 4 * qrand(200, 7) - 2; x2 <- 4 * qrand(200, 8) - 2
X6 <- cbind(x1, x2)[abs(x2 - x1) > 0.3, ][1:100, ]
X6 <- rbind(X6, c(1.2, 1.05), c(1.5, 1.38))
y6 <- as.factor(c(ifelse(X6[1:100, 2] > X6[1:100, 1], 1, -1), 1, 1))
xt1 <- 4 * qrand(2000, 7, start = 1001) - 2; xt2 <- 4 * qrand(2000, 8, start = 1001) - 2
Xt6 <- cbind(xt1, xt2); yt6 <- ifelse(xt2 > xt1, 1, -1)
ex6 <- t(sapply(c(0.01, 0.1, 1, 10, 100, 1000, 10000), function(cost) {
  f <- svm(X6, y6, kernel = "linear", cost = cost, scale = FALSE)
  c(cost = cost, train_errors = sum(predict(f, X6) != y6), cv = cv_error(X6, y6, kernel = "linear", cost = cost),
    test = err(f, Xt6, yt6))
}))
print(ex6)
keep("ex6_table", t(ex6[, -1]))

# Exercise 7
Auto7 <- na.omit(Auto)
y7 <- as.factor(as.integer(Auto7$mpg > median(Auto7$mpg)))
X7 <- scale(as.matrix(Auto7[, c("cylinders", "displacement", "horsepower", "weight", "acceleration", "year", "origin")]))
cv_lin <- sapply(c(0.01, 0.1, 1, 10, 100), function(cost) cv_error(X7, y7, kernel = "linear", cost = cost))
cv_rbf <- t(sapply(c(0.01, 0.1, 1), function(g) sapply(c(0.1, 1, 10, 100), function(cost)
  cv_error(X7, y7, kernel = "radial", gamma = g, cost = cost))))
cv_pol <- t(sapply(2:3, function(d) sapply(c(0.01, 0.1, 1, 10), function(cost)
  cv_error(X7, y7, kernel = "polynomial", degree = d, gamma = 1, coef0 = 1, cost = cost))))
print(round(cv_lin, 4)); print(round(cv_rbf, 4)); print(round(cv_pol, 4))
keep("ex7_lin", cv_lin); keep("ex7_rbf", t(cv_rbf)); keep("ex7_pol", t(cv_pol))

# Exercise 8
X8 <- model.matrix(Purchase ~ ., data = OJ)[, -1]
perm <- (0:(nrow(OJ) - 1) * 377) %% nrow(OJ)
tr <- seq_len(nrow(OJ)) %in% (perm[1:800] + 1)
Xtr <- scale(X8[tr, ])
Xte <- scale(X8[!tr, ], center = attr(Xtr, "scaled:center"), scale = attr(Xtr, "scaled:scale"))
ytr <- OJ$Purchase[tr]; yte <- OJ$Purchase[!tr]
grid8 <- c(0.01, 0.0316, 0.1, 0.316, 1, 3.16, 10)
p8 <- ncol(X8)
kernels <- list(linear = list(kernel = "linear"), radial = list(kernel = "radial", gamma = 1 / p8),
                polynomial = list(kernel = "polynomial", degree = 2, gamma = 1 / p8, coef0 = 0))
ex8 <- c()
for (nm in names(kernels)) {
  fit <- function(cost) do.call(svm, c(list(x = Xtr, y = ytr, cost = cost, scale = FALSE), kernels[[nm]]))
  f0 <- fit(0.01)
  cv <- sapply(grid8, function(cost) do.call(cv_error, c(list(x = Xtr, y = ytr, cost = cost), kernels[[nm]])))
  cbest <- grid8[which.min(cv)]
  fb <- fit(cbest)
  row <- c(sum(f0$nSV), err(f0, Xtr, ytr), err(f0, Xte, yte), cbest, err(fb, Xtr, ytr), err(fb, Xte, yte))
  cat(sprintf("Exercise 8, %-10s cost 0.01: %d SVs, train %.4f test %.4f; best cost %g: train %.4f test %.4f\n",
              nm, row[1], row[2], row[3], row[4], row[5], row[6]))
  ex8 <- c(ex8, row)
  keep(paste0("ex8_cv_", nm), cv)
}
keep("ex8", ex8)

# ================================================================== key numbers
flat <- unlist(lapply(names(key), function(k) {
  v <- key[[k]]; nm <- if (length(v) > 1) paste0(k, "_", seq_along(v)) else k
  setNames(as.numeric(v), nm)
}))
write.csv(data.frame(name = names(flat), value = sprintf("%.10g", flat)), "cache/ch09_lab_R.csv", row.names = FALSE)
cat("wrote", length(flat), "numbers to cache/ch09_lab_R.csv\n")
