# Chapter 2 lab: Introduction to R
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Springer, Section 2.3, and the applied exercises 8-10 of Section 2.4.
#
# The same steps are written in Python in ch02_lab.py; each block below starts with section("...")
# and the Python file has a block with the same title, so the two can be read side by side.
# The code is my own; it follows the steps of the book's lab.
#
# Data: Auto.data, Auto.csv and College.csv are read from the folder of the book's CSV files
# (set ISL_DATA, or put the folder or a link to it at code/isl-data); Boston comes from the ISLR2
# package. Figures are written to ../img/ch02/ (lab-r-*.svg, .png, .pdf).
#
# Run: Rscript ch02_lab.R

.this_dir <- local({
  a <- grep("^--file=", commandArgs(FALSE), value = TRUE)
  if (length(a)) dirname(normalizePath(sub("^--file=", "", a[1]))) else getwd()
})
DATA <- Sys.getenv("ISL_DATA", file.path(.this_dir, "isl-data"))
IMG <- file.path(.this_dir, "..", "img", "ch02")
dir.create(IMG, showWarnings = FALSE, recursive = TRUE)
if (!interactive()) pdf(NULL)   # run as a script: plots meant for the screen are discarded

section <- function(title) cat("\n==== ", title, " ====\n", sep = "")
# print a labelled line of numbers (whole numbers, or 4 decimals), in the same format as the Python version
report <- function(label, values) {
  fmt <- if (all(values == round(values))) "%.0f" else "%.4f"
  cat(label, ": ", paste(sprintf(fmt, values), collapse = " "), "\n", sep = "")
}
# SVG files for the web page (svglite; R's own svg() needs X11 on some systems)
open_svg <- function(name, width, height) {
  svglite::svglite(file.path(IMG, paste0(name, ".svg")), width = width, height = height)
}
open_png <- function(name, width, height) {
  ragg::agg_png(file.path(IMG, paste0(name, ".png")), width = width, height = height, units = "in", res = 110)
}

section("Basic commands: vectors and functions")
x <- c(1, 3, 2, 5)
x
x = c(1, 6, 2)          # = also assigns, but <- is the usual R style
x
y = c(1, 4, 3)
length(x)
length(y)

section("Adding two vectors")
x + y                   # elementwise: R vectors are numeric by nature

section("Listing and removing objects")
ls()
rm(x, y)
exists("x")

section("Matrices and arrays")
x <- matrix(data = c(1, 2, 3, 4), nrow = 2, ncol = 2)
x
dim(x)
typeof(x)

section("Filling order and reshaping")
matrix(c(1, 2, 3, 4), 2, 2, byrow = TRUE)   # R fills by column unless byrow = TRUE
v <- 1:6
dim(v) <- c(2, 3)                           # reshape in place: still column-major
v

section("Views and copies")
w <- v                  # R copies on modification: changing w leaves v alone
w[1, 1] <- 5
v[1, 1]
w[1, 1]

section("Elementwise functions")
sqrt(x)
x^2
x^0.5

section("Random numbers and correlation")
set.seed(1)
x <- rnorm(50)
y <- x + rnorm(50, mean = 50, sd = 0.1)
cor(x, y)

section("Reproducible random numbers: seeds")
set.seed(1303)
round(rnorm(50), 4)[1:7]
set.seed(1303)
round(rnorm(50), 4)[1:7]  # the same numbers again

section("Mean, variance and standard deviation")
set.seed(3)
y <- rnorm(100)
mean(y)
var(y)                  # divisor n - 1
sqrt(var(y))
sd(y)
report("variance with divisor n", mean((y - mean(y))^2))

section("Column means of a matrix")
X <- matrix(rnorm(30), 10, 3)
colMeans(X)
apply(X, 2, mean)       # the same, by applying mean() to each column

section("Scatterplots and labels")
x <- rnorm(100)
y <- rnorm(100)
plot(x, y, xlab = "this is the x-axis", ylab = "this is the y-axis", main = "Plot of X vs Y")

section("Several plots in one figure and saving figures")
pdf(file.path(IMG, "lab-r-scatter.pdf"), width = 9, height = 4)
par(mfrow = c(2, 3))    # a 2 x 3 grid, filled row by row
plot(x, y, col = "green")
plot(x, y, pch = 3)
invisible(dev.off())

section("Sequences")
seq(1, 10)
1:10
seq(0, 1, length = 11)
x <- seq(-pi, pi, length = 50)

section("Contour plots, heatmaps and perspective plots")
y <- x
f <- outer(x, y, function(x, y) cos(y) / (1 + x^2))   # f[i, j] is the value at (x[i], y[j])
fa <- (f - t(f)) / 2                                  # the antisymmetric part
report("f at (x[2], y[5])", f[2, 5])
report("largest |fa|", max(abs(fa)))
open_svg("lab-r-contour", 10, 3.6)
par(mfrow = c(1, 3), mar = c(4, 4, 2, 1))
contour(x, y, f, main = "contour(x, y, f)")
contour(x, y, f, nlevels = 45, add = TRUE)
image(x, y, fa, main = "image(x, y, fa)")
persp(x, y, fa, theta = 30, phi = 20, main = "persp(x, y, fa)")
invisible(dev.off())

section("Slices")
substr("hello world", 4, 6)        # characters 4 to 6 (both ends included)
(0:9)[4:6]                         # R positions start at 1

section("Indexing a matrix")
A <- matrix(1:16, 4, 4)
A
A[2, 3]
B <- matrix(0:15, 4, 4, byrow = TRUE)   # the array np.arange(16).reshape(4, 4) of the Python lab
B[2, 3]

section("Selecting submatrices")
A[c(1, 3), c(2, 4)]
A[1:3, 2:4]
A[1:2, ]
A[, 1:2]
A[1, ]
A[-c(1, 3), ]
A[-c(1, 3), -c(1, 3, 4)]
dim(A)

section("Logical (Boolean) indexing")
keep_rows <- c(FALSE, TRUE, FALSE, TRUE)
A[keep_rows, ]
A[c(1, 2, 1, 2), ]                     # integers select (and repeat) rows
A[keep_rows, c(TRUE, FALSE, TRUE, TRUE)]

section("Loading the Auto data")
Auto <- read.table(file.path(DATA, "Auto.data"))
head(Auto, 3)                           # wrong: the header line was read as data
Auto <- read.table(file.path(DATA, "Auto.data"), header = TRUE, na.strings = "?", stringsAsFactors = TRUE)
Auto <- read.csv(file.path(DATA, "Auto.csv"), na.strings = "?", stringsAsFactors = TRUE)
dim(Auto)
Auto[1:4, ]
report("horsepower total (missing values skipped)", sum(Auto$horsepower, na.rm = TRUE))

section("Missing values")
colSums(is.na(Auto))
Auto <- na.omit(Auto)
dim(Auto)
names(Auto)

section("Selecting rows and columns of a data frame")
Auto[1:3, ]
idx_80 <- Auto$year > 80
report("cars with year > 80", sum(idx_80))
head(Auto[, c("mpg", "horsepower")])

section("Rows by name")
Auto[Auto$name %in% c("amc rebel sst", "ford torino"), c("mpg", "displacement", "horsepower")]
Auto[Auto$name == "ford galaxie 500", c("mpg", "origin")]

section("Rows by condition")
sel1 <- subset(Auto, year > 80, select = c(weight, origin))
sel2 <- subset(Auto, year > 80 & mpg > 30, select = c(weight, origin))
sel3 <- subset(Auto, displacement < 300 & grepl("ford|datsun", name), select = c(weight, origin))
report("rows selected (year > 80; and mpg > 30; ford or datsun with displacement < 300)",
       c(nrow(sel1), nrow(sel2), nrow(sel3)))
head(sel3, 4)

section("For loops")
total <- 0
for (value in c(3, 2, 19)) total <- total + value
cat("Total is:", total, "\n")
total <- 0
for (value in c(2, 3, 19)) for (weight in c(3, 2, 1)) total <- total + value * weight
cat("Total is:", total, "\n")
values <- c(2, 3, 19)
weights <- c(0.2, 0.3, 0.5)
total <- 0
for (i in seq_along(values)) total <- total + weights[i] * values[i]
cat("Weighted average is:", total, "\n")

section("String formatting and missing values by column")
set.seed(1)
A <- matrix(rnorm(127 * 5), 127, 5)
M <- matrix(sample(c(0, NA), 127 * 5, replace = TRUE, prob = c(0.8, 0.2)), 127, 5)
D <- as.data.frame(A + M)
names(D) <- c("food", "bar", "pickle", "snack", "popcorn")
for (col in names(D)) cat(sprintf('Column "%s" has %.2f%% missing values\n', col, 100 * mean(is.na(D[[col]]))))

section("Scatterplots of the Auto data")
plot(Auto$horsepower, Auto$mpg, xlab = "horsepower", ylab = "mpg", main = "Horsepower vs. MPG")
report("correlation of horsepower and mpg", cor(Auto$horsepower, Auto$mpg))

section("Qualitative variables and boxplots")
Auto$cylinders <- as.factor(Auto$cylinders)
table(Auto$cylinders)
report("median mpg for 3, 4, 5, 6, 8 cylinders", tapply(Auto$mpg, Auto$cylinders, median))
open_svg("lab-r-boxplot", 5, 4)
plot(Auto$cylinders, Auto$mpg, col = "red", varwidth = TRUE, xlab = "cylinders", ylab = "MPG")
invisible(dev.off())

section("Histograms")
h <- hist(Auto$mpg, breaks = 15, plot = FALSE)
report("histogram break points", h$breaks)
open_svg("lab-r-hist", 5, 4)
hist(Auto$mpg, col = 2, breaks = 15, main = "Histogram of mpg", xlab = "mpg")
invisible(dev.off())

section("Scatterplot matrices")
open_png("lab-r-pairs", 7, 7)
pairs(~ mpg + displacement + horsepower + weight + acceleration, data = Auto, cex = 0.4)
invisible(dev.off())
report("correlations of mpg with displacement, horsepower, weight, acceleration",
       cor(Auto[, c("displacement", "horsepower", "weight", "acceleration")], Auto$mpg))

section("Numerical summaries")
summary(Auto)
summary(Auto$mpg)
report("mpg: min, 1st quartile, median, mean, 3rd quartile, max", summary(Auto$mpg))
report("weight: mean, standard deviation", c(mean(Auto$weight), sd(Auto$weight)))

# ============================================================================== applied exercises

section("Exercise 8: the College data")
college <- read.csv(file.path(DATA, "College.csv"), stringsAsFactors = TRUE)
rownames(college) <- college[, 1]
college <- college[, -1]
summary(college[, 1:5])
open_png("lab-r-college-pairs", 8, 8)
pairs(college[, 1:10], cex = 0.3)
invisible(dev.off())
Elite <- rep("No", nrow(college))
Elite[college$Top10perc > 50] <- "Yes"
college <- data.frame(college, Elite = as.factor(Elite))
summary(college$Elite)
report("median out-of-state tuition: public, private", tapply(college$Outstate, college$Private, median))
report("median out-of-state tuition: not elite, elite", tapply(college$Outstate, college$Elite, median))
open_svg("lab-r-college", 8, 3.6)
par(mfrow = c(1, 2))
plot(college$Private, college$Outstate, xlab = "Private", ylab = "Outstate")
plot(college$Elite, college$Outstate, xlab = "Elite", ylab = "Outstate")
invisible(dev.off())
open_svg("lab-r-college-hist", 8, 6)
par(mfrow = c(2, 2))
hist(college$Apps, breaks = 50, main = "Apps (50 bins)", xlab = "")
hist(college$Outstate, breaks = 20, main = "Outstate (20 bins)", xlab = "")
hist(college$Grad.Rate, breaks = 10, main = "Grad.Rate (10 bins)", xlab = "")
hist(college$S.F.Ratio, breaks = 25, main = "S.F.Ratio (25 bins)", xlab = "")
invisible(dev.off())
college$AcceptRate <- college$Accept / college$Apps
report("acceptance rate: median for public, private", tapply(college$AcceptRate, college$Private, median))
report("correlation of Expend and Grad.Rate", cor(college$Expend, college$Grad.Rate))
college[college$Grad.Rate > 100, c("Private", "Grad.Rate")]   # a graduation rate above 100%

section("Exercise 9: the Auto data")
Auto <- na.omit(read.csv(file.path(DATA, "Auto.csv"), na.strings = "?", stringsAsFactors = TRUE))
quant <- c("mpg", "cylinders", "displacement", "horsepower", "weight", "acceleration", "year")
sapply(Auto[, quant], range)
for (v in quant) report(paste(v, "min, max, mean, sd"),
                        c(range(Auto[[v]]), mean(Auto[[v]]), sd(Auto[[v]])))
sub <- Auto[-(10:85), ]
report("rows left after removing observations 10 to 85", nrow(sub))
for (v in quant) report(paste("subset", v, "min, max, mean, sd"),
                        c(range(sub[[v]]), mean(sub[[v]]), sd(sub[[v]])))
report("correlation of mpg with cylinders, displacement, horsepower, weight, acceleration, year",
       cor(Auto[, quant])[1, -1])
report("mean mpg by origin (1 America, 2 Europe, 3 Japan)", tapply(Auto$mpg, Auto$origin, mean))

section("Exercise 10: the Boston data")
library(ISLR2)
dim(Boston)
report("correlation of crim with the other variables", cor(Boston)[1, -1])
sapply(Boston[, c("crim", "tax", "ptratio")], range)
report("tracts with crim > 20, tax > 600, ptratio > 20",
       c(sum(Boston$crim > 20), sum(Boston$tax > 600), sum(Boston$ptratio > 20)))
report("tracts bounding the Charles river", sum(Boston$chas))
report("median pupil-teacher ratio", median(Boston$ptratio))
i <- which.min(Boston$medv)
report("tract with the lowest median value (row), and how many share it", c(i, sum(Boston$medv == min(Boston$medv))))
Boston[Boston$medv == min(Boston$medv), ]
report("percentile of that tract for crim, lstat, ptratio",
       sapply(c("crim", "lstat", "ptratio"), function(v) mean(Boston[[v]] <= Boston[i, v])))
report("tracts averaging more than 7 and more than 8 rooms", c(sum(Boston$rm > 7), sum(Boston$rm > 8)))
report("tracts with more than 8 rooms: mean crim, lstat, medv",
       colMeans(Boston[Boston$rm > 8, c("crim", "lstat", "medv")]))
