# Chapter 4 lab: classification methods, in R.
# Course: Applied Statistical Learning in R and Python. The lab follows the steps of the R lab of
# James, Witten, Hastie and Tibshirani (2021), An Introduction to Statistical Learning, 2nd edition,
# Section 4.7 (same data, same models, same questions); the code is written for this course.
# The Python version, ch04_lab.py, does the same steps and prints the same labels, so the two
# outputs can be compared line by line.
#
# Packages: ISLR2 (data), MASS (lda, qda), e1071 (naiveBayes), class (knn); all base R otherwise.
# Run:  Rscript ch04_lab.R        (writes two figures to ../img/ch04/lab4-r-*.png)

suppressPackageStartupMessages({
  library(ISLR2)
  library(MASS)
  suppressWarnings(library(e1071))   # (silences a "built under R 4.4.3" note)
  library(class)
})

this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
img_dir <- file.path(this_dir, "..", "img", "ch04")
dir.create(img_dir, showWarnings = FALSE, recursive = TRUE)

show <- function(label, value, digits = 4) {
  if (is.numeric(value)) value <- paste(formatC(value, digits = digits, format = "f"), collapse = " ")
  cat(label, ": ", value, "\n", sep = "")
}
show_table <- function(label, tab) {       # rows = predicted, columns = truth
  cat(label, ":\n", sep = "")
  print(tab)
}

# ================================================================ 4.7.1 the stock market data
cat("\n== 4.7.1 Smarket\n")
show("dim", dim(Smarket), 0)
show("Direction counts (Down, Up)", as.numeric(table(Smarket$Direction)), 0)
cors <- cor(Smarket[, names(Smarket) != "Direction"])
show("cor(Year, Volume)", cors["Year", "Volume"], 3)
lags <- c("Lag1", "Lag2", "Lag3", "Lag4", "Lag5", "Today")
lag.cor <- cors[lags, lags]
show("largest |cor| among Lag1-Lag5 and Today", max(abs(lag.cor[upper.tri(lag.cor)])), 3)
png(file.path(img_dir, "lab4-r-volume.png"), width = 640, height = 380, res = 110)
plot(Smarket$Volume, pch = 20, cex = 0.5, col = "grey30", xlab = "Day (2001-2005)", ylab = "Volume")
invisible(dev.off())

# ================================================================ 4.7.2 logistic regression
cat("\n== 4.7.2 Logistic regression\n")
glm.fits <- glm(Direction ~ Lag1 + Lag2 + Lag3 + Lag4 + Lag5 + Volume,
                data = Smarket, family = binomial)
print(round(summary(glm.fits)$coef, 4))
show("residual deviance", deviance(glm.fits), 2)
glm.probs <- predict(glm.fits, type = "response")      # Pr(Up | x): R codes Up as 1
show("first five fitted probabilities", glm.probs[1:5])
glm.pred <- ifelse(glm.probs > 0.5, "Up", "Down")
show_table("training confusion (all 1250 days)", table(glm.pred, Smarket$Direction))
show("training accuracy", mean(glm.pred == Smarket$Direction))

train <- Smarket$Year < 2005
Smarket.2005 <- Smarket[!train, ]
Direction.2005 <- Smarket$Direction[!train]
show("test days in 2005", nrow(Smarket.2005), 0)

glm.fits <- glm(Direction ~ Lag1 + Lag2 + Lag3 + Lag4 + Lag5 + Volume,
                data = Smarket, family = binomial, subset = train)
glm.probs <- predict(glm.fits, Smarket.2005, type = "response")
glm.pred <- ifelse(glm.probs > 0.5, "Up", "Down")
show_table("2005 confusion, six predictors", table(glm.pred, Direction.2005))
show("2005 accuracy, six predictors", mean(glm.pred == Direction.2005))

glm.fits <- glm(Direction ~ Lag1 + Lag2, data = Smarket, family = binomial, subset = train)
glm.probs <- predict(glm.fits, Smarket.2005, type = "response")
glm.pred <- ifelse(glm.probs > 0.5, "Up", "Down")
show_table("2005 confusion, Lag1 + Lag2", table(glm.pred, Direction.2005))
show("2005 accuracy, Lag1 + Lag2", mean(glm.pred == Direction.2005))
show("precision on predicted Up days", mean(Direction.2005[glm.pred == "Up"] == "Up"))
show("always-Up accuracy in 2005", mean(Direction.2005 == "Up"))
show("Pr(Up) at (1.2, 1.1) and (1.5, -0.8)",
     predict(glm.fits, newdata = data.frame(Lag1 = c(1.2, 1.5), Lag2 = c(1.1, -0.8)), type = "response"))

# ================================================================ 4.7.3 LDA
cat("\n== 4.7.3 Linear discriminant analysis\n")
lda.fit <- lda(Direction ~ Lag1 + Lag2, data = Smarket, subset = train)
show("priors (Down, Up)", lda.fit$prior)
show("group means Down (Lag1, Lag2)", lda.fit$means["Down", ])
show("group means Up (Lag1, Lag2)", lda.fit$means["Up", ])
show("LD1 coefficients (Lag1, Lag2)", lda.fit$scaling[, 1])
lda.pred <- predict(lda.fit, Smarket.2005)
show_table("2005 confusion, LDA", table(lda.pred$class, Direction.2005))
show("2005 accuracy, LDA", mean(lda.pred$class == Direction.2005))
show("days with Pr(Down) >= 0.5 and < 0.5",
     c(sum(lda.pred$posterior[, "Down"] >= 0.5), sum(lda.pred$posterior[, "Down"] < 0.5)), 0)
show("days with Pr(Down) > 0.9", sum(lda.pred$posterior[, "Down"] > 0.9), 0)
show("largest Pr(Down) in 2005", max(lda.pred$posterior[, "Down"]))

# ================================================================ 4.7.4 QDA
cat("\n== 4.7.4 Quadratic discriminant analysis\n")
qda.fit <- qda(Direction ~ Lag1 + Lag2, data = Smarket, subset = train)
show("QDA covariance of class Down",
     as.numeric(cov(Smarket[train & Smarket$Direction == "Down", c("Lag1", "Lag2")])))
qda.class <- predict(qda.fit, Smarket.2005)$class
show_table("2005 confusion, QDA", table(qda.class, Direction.2005))
show("2005 accuracy, QDA", mean(qda.class == Direction.2005))

# ================================================================ 4.7.5 naive Bayes
cat("\n== 4.7.5 Naive Bayes\n")
nb.fit <- naiveBayes(Direction ~ Lag1 + Lag2, data = Smarket, subset = train)
show("NB mean and sd of Lag1 in class Down", nb.fit$tables$Lag1["Down", ])
x.down <- Smarket$Lag1[train & Smarket$Direction == "Down"]
show("by hand: mean, sd (divisor n - 1)", c(mean(x.down), sd(x.down)))
nb.class <- predict(nb.fit, Smarket.2005)
show_table("2005 confusion, naive Bayes", table(nb.class, Direction.2005))
show("2005 accuracy, naive Bayes", mean(nb.class == Direction.2005))
nb.preds <- predict(nb.fit, Smarket.2005, type = "raw")
show("first three Pr(Up), naive Bayes", nb.preds[1:3, "Up"])

# ================================================================ 4.7.6 K-nearest neighbours
cat("\n== 4.7.6 K-nearest neighbours\n")
train.X <- cbind(Smarket$Lag1, Smarket$Lag2)[train, ]
test.X <- cbind(Smarket$Lag1, Smarket$Lag2)[!train, ]
train.Direction <- Smarket$Direction[train]
set.seed(1)                       # knn() breaks ties at random
knn.pred <- knn(train.X, test.X, train.Direction, k = 1)
show_table("2005 confusion, KNN K = 1", table(knn.pred, Direction.2005))
show("2005 accuracy, KNN K = 1", mean(knn.pred == Direction.2005))
knn.pred <- knn(train.X, test.X, train.Direction, k = 3)
show("2005 accuracy, KNN K = 3", mean(knn.pred == Direction.2005))

# Caravan insurance: 85 predictors, 5,822 customers
show("Caravan dim", dim(Caravan), 0)
show("purchase rate", mean(Caravan$Purchase == "Yes"))
standardized.X <- scale(Caravan[, -86])
show("variance of columns 1 and 2 before / after scaling",
     c(var(Caravan[, 1]), var(Caravan[, 2]), var(standardized.X[, 1]), var(standardized.X[, 2])))
test <- 1:1000                    # the first 1,000 customers form the test set
train.X <- standardized.X[-test, ]
test.X <- standardized.X[test, ]
train.Y <- Caravan$Purchase[-test]
test.Y <- Caravan$Purchase[test]
set.seed(1)
for (k in c(1, 3, 5)) {
  knn.pred <- knn(train.X, test.X, train.Y, k = k)
  tab <- table(knn.pred, test.Y)
  show(sprintf("Caravan KNN K = %d: error, predicted Yes, correct Yes, success rate", k),
       c(mean(knn.pred != test.Y), sum(tab["Yes", ]), tab["Yes", "Yes"], tab["Yes", "Yes"] / sum(tab["Yes", ])))
}
show("Caravan null error (always No)", mean(test.Y != "No"))

glm.fits <- suppressWarnings(glm(Purchase ~ ., data = Caravan, family = binomial, subset = -test))
glm.probs <- predict(glm.fits, Caravan[test, ], type = "response")
for (cut in c(0.5, 0.25)) {
  glm.pred <- factor(ifelse(glm.probs > cut, "Yes", "No"), levels = c("No", "Yes"))
  tab <- table(glm.pred, test.Y)
  show(sprintf("Caravan logistic, threshold %.2f: predicted Yes, correct Yes", cut),
       c(sum(tab["Yes", ]), tab["Yes", "Yes"]), 0)
}

# ================================================================ 4.7.7 Poisson regression
cat("\n== 4.7.7 Linear and Poisson regression on Bikeshare\n")
show("Bikeshare dim", dim(Bikeshare), 0)
mod.lm <- lm(bikers ~ mnth + hr + workingday + temp + weathersit, data = Bikeshare)
show("lm (treatment coding): intercept, mnthFeb, mnthMarch", coef(mod.lm)[1:3], 3)
contrasts(Bikeshare$hr) <- contr.sum(24)
contrasts(Bikeshare$mnth) <- contr.sum(12)
mod.lm2 <- lm(bikers ~ mnth + hr + workingday + temp + weathersit, data = Bikeshare)
show("lm (sum coding): intercept, Jan, Feb", coef(mod.lm2)[1:3], 3)
show("max |difference| of fitted values between the codings",
     max(abs(fitted(mod.lm) - fitted(mod.lm2))), 10)
coef.months <- c(coef(mod.lm2)[2:12], -sum(coef(mod.lm2)[2:12]))
coef.hours <- c(coef(mod.lm2)[13:35], -sum(coef(mod.lm2)[13:35]))
show("lm December and hour-23 effects", c(coef.months[12], coef.hours[24]), 3)
show("share of negative fitted values", mean(fitted(mod.lm2) < 0))

mod.pois <- glm(bikers ~ mnth + hr + workingday + temp + weathersit,
                data = Bikeshare, family = poisson)
print(round(summary(mod.pois)$coef[c(1, 36:40), ], 4))
coef.mnth <- c(coef(mod.pois)[2:12], -sum(coef(mod.pois)[2:12]))
coef.hours <- c(coef(mod.pois)[13:35], -sum(coef(mod.pois)[13:35]))
show("Poisson month effects (Jan..Dec)", coef.mnth, 3)
show("Poisson December and hour-23 effects", c(coef.mnth[12], coef.hours[24]), 4)
show("Poisson deviance", deviance(mod.pois), 1)
show("smallest Poisson fitted value", min(fitted(mod.pois)), 3)

png(file.path(img_dir, "lab4-r-poisson-coef.png"), width = 900, height = 380, res = 110)
par(mfrow = c(1, 2), mar = c(4, 4, 1, 1))
plot(coef.mnth, xlab = "Month", ylab = "Coefficient", xaxt = "n", col = "blue", pch = 19, type = "o")
axis(side = 1, at = 1:12, labels = c("J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D"))
plot(0:23, coef.hours, xlab = "Hour", ylab = "Coefficient", col = "blue", pch = 19, type = "o")
invisible(dev.off())

# ================================================================ applied exercises 13, 14 and 16
# (Section 4.8 of the book; the notes give the questions and discuss the answers.)
acc <- function(pred, truth) mean(as.character(pred) == as.character(truth))

cat("\n== Exercise 13 (Weekly)\n")
show("Weekly dim", dim(Weekly), 0)
fit13 <- glm(Direction ~ Lag1 + Lag2 + Lag3 + Lag4 + Lag5 + Volume, data = Weekly, family = binomial)
show("Ex13b p-values (Lag1..Lag5, Volume)", summary(fit13)$coef[-1, 4])
pred13 <- ifelse(fitted(fit13) > 0.5, "Up", "Down")
show_table("Ex13c confusion (all weeks)", table(pred13, Weekly$Direction))
show("Ex13c accuracy", acc(pred13, Weekly$Direction))
tr13 <- Weekly$Year <= 2008
te13 <- Weekly[!tr13, ]
fit13d <- glm(Direction ~ Lag2, data = Weekly, family = binomial, subset = tr13)
pred13d <- ifelse(predict(fit13d, te13, type = "response") > 0.5, "Up", "Down")
show_table("Ex13d confusion, logistic on Lag2 (2009-2010)", table(pred13d, te13$Direction))
show("Ex13d-h accuracy: logistic, LDA, QDA, KNN-1, naive Bayes", c(
  acc(pred13d, te13$Direction),
  acc(predict(lda(Direction ~ Lag2, data = Weekly, subset = tr13), te13)$class, te13$Direction),
  acc(predict(qda(Direction ~ Lag2, data = Weekly, subset = tr13), te13)$class, te13$Direction),
  { set.seed(1); acc(knn(cbind(Weekly$Lag2[tr13]), cbind(te13$Lag2), Weekly$Direction[tr13], k = 1), te13$Direction) },
  acc(predict(naiveBayes(Direction ~ Lag2, data = Weekly, subset = tr13), te13), te13$Direction)))
set.seed(1)
show("Ex13j KNN on Lag2, K = 1..15", sapply(1:15, function(k)
  acc(knn(cbind(Weekly$Lag2[tr13]), cbind(te13$Lag2), Weekly$Direction[tr13], k = k), te13$Direction)), 3)
show("Ex13j LDA on Lag1 + Lag2, logistic on Lag2 + Lag2^2", c(
  acc(predict(lda(Direction ~ Lag1 + Lag2, data = Weekly, subset = tr13), te13)$class, te13$Direction),
  acc(ifelse(predict(glm(Direction ~ Lag2 + I(Lag2^2), data = Weekly, family = binomial, subset = tr13),
                     te13, type = "response") > 0.5, "Up", "Down"), te13$Direction)))

cat("\n== Exercise 14 (Auto)\n")
Auto14 <- Auto
Auto14$mpg01 <- factor(as.integer(Auto14$mpg > median(Auto14$mpg)))
show("Ex14a median mpg, share of 1s", c(median(Auto$mpg), mean(Auto14$mpg01 == "1")))
show("Ex14b cor(mpg01, cylinders, displacement, horsepower, weight, acceleration, year)",
     cor(as.integer(Auto14$mpg01 == "1"), Auto14[, c("cylinders", "displacement", "horsepower", "weight",
                                                     "acceleration", "year")]), 3)
tr14 <- seq_len(nrow(Auto14)) %% 2 == 1        # odd rows train, even rows test
te14 <- Auto14[!tr14, ]
f14 <- mpg01 ~ cylinders + displacement + horsepower + weight
p14 <- predict(glm(f14, data = Auto14, family = binomial, subset = tr14), te14, type = "response")
show("Ex14d-g test error: LDA, QDA, logistic, naive Bayes", c(
  1 - acc(predict(lda(f14, data = Auto14, subset = tr14), te14)$class, te14$mpg01),
  1 - acc(predict(qda(f14, data = Auto14, subset = tr14), te14)$class, te14$mpg01),
  1 - acc(as.integer(p14 > 0.5), te14$mpg01),
  1 - acc(predict(naiveBayes(f14, data = Auto14, subset = tr14), te14), te14$mpg01)))
v14 <- c("cylinders", "displacement", "horsepower", "weight")
m14 <- colMeans(Auto14[tr14, v14]); s14 <- apply(Auto14[tr14, v14], 2, sd)
Z14 <- scale(Auto14[, v14], center = m14, scale = s14)
set.seed(1)
show("Ex14h KNN test error, K = 1..10 (standardised)", sapply(1:10, function(k)
  1 - acc(knn(Z14[tr14, ], Z14[!tr14, ], Auto14$mpg01[tr14], k = k), te14$mpg01)), 3)

cat("\n== Exercise 16 (Boston)\n")
Boston16 <- ISLR2::Boston          # MASS (loaded above) has an older Boston with 14 columns
Boston16$crim01 <- factor(as.integer(Boston16$crim > median(Boston16$crim)))
tr16 <- seq_len(nrow(Boston16)) %% 2 == 1
te16 <- Boston16[!tr16, ]
for (vars in list(c("nox", "rad", "dis", "age", "tax", "indus"),
                  setdiff(names(ISLR2::Boston), "crim"))) {
  f16 <- reformulate(vars, "crim01")
  p16 <- predict(glm(f16, data = Boston16, family = binomial, subset = tr16), te16, type = "response")
  Z16 <- scale(Boston16[, vars], center = colMeans(Boston16[tr16, vars]), scale = apply(Boston16[tr16, vars], 2, sd))
  set.seed(1)
  show(sprintf("Ex16 test error with %d predictors: logistic, LDA, naive Bayes, KNN-1, KNN-5, KNN-10", length(vars)), c(
    1 - acc(as.integer(p16 > 0.5), te16$crim01),
    1 - acc(predict(lda(f16, data = Boston16, subset = tr16), te16)$class, te16$crim01),
    1 - acc(predict(naiveBayes(f16, data = Boston16, subset = tr16), te16), te16$crim01),
    sapply(c(1, 5, 10), function(k) 1 - acc(knn(Z16[tr16, ], Z16[!tr16, ], Boston16$crim01[tr16], k = k), te16$crim01))))
}
