# Chapter 10 lab: deep learning (R version).
# Course: Applied Statistical Learning in R and Python, following James, Witten, Hastie and
# Tibshirani (2021), An Introduction to Statistical Learning with Applications in R, 2nd edition,
# Section 10.9, and the Python lab of James et al. (2023), Section 10.9.
#
# The R lab of the book uses the keras package, which needs a Python installation of TensorFlow.
# This version needs nothing beyond base R, ISLR2 and glmnet: the networks are written out in a
# few short functions (forward pass, backpropagation by the chain rule, dropout and RMSprop), so
# every step of Section 10.7 is visible. It covers
#   10.9.1  a single-layer network on the Hitters data, against least squares and the lasso;
#   10.9.2  a two-hidden-layer network and multinomial logistic regression on the MNIST digits;
#   10.9.6  the NYSE trading-volume forecasts: linear AR models, a recurrent network (written out,
#           with backpropagation through time) and a nonlinear AR model;
# and the applied exercises 6, 7 and 9-12. The convolutional network (10.9.3), the pretrained
# ResNet50 (10.9.4) and the IMDb models (10.9.5) are in the Python lab only.
# Run from the code/ folder:
#   Rscript ch10_lab.R
# The MNIST files are read from the same cache as the Python code (~/.cache/prml-repro/mnist, or
# $ISL_CACHE/mnist), and downloaded there if missing. Figures go to ../img/ch10/lab-r-*.svg and the
# key numbers to cache/ch10_lab_R.csv. It takes about three minutes.

suppressPackageStartupMessages({
  library(ISLR2)
  library(glmnet)
})
options(digits = 6, width = 100)

out_dir <- file.path("..", "img", "ch10")
dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
dir.create("cache", showWarnings = FALSE)
key <- list()                                 # key numbers, written to cache/ch10_lab_R.csv at the end
keep <- function(name, value) key[[name]] <<- unname(value)
open_svg <- function(name, width, height) {
  path <- file.path(out_dir, paste0(name, ".svg"))
  if (requireNamespace("svglite", quietly = TRUE)) svglite::svglite(path, width = width, height = height)
  else grDevices::svg(path, width = width, height = height)
}

# ================================================================== a small neural-network toolkit
# A network is a list of layers, each with a weight matrix W (inputs x units) and a bias vector b.
# Hidden layers use ReLU; the output layer is linear (a prediction, or the logits Z_m of (10.12)).
relu <- function(z) pmax(z, 0)
init_layer <- function(fan_in, fan_out) {     # the default of torch's nn.Linear: U(-1/sqrt(fan_in), +)
  s <- 1 / sqrt(fan_in)
  list(W = matrix(runif(fan_in * fan_out, -s, s), fan_in, fan_out), b = runif(fan_out, -s, s))
}
mlp_init <- function(sizes) lapply(seq_len(length(sizes) - 1), function(l) init_layer(sizes[l], sizes[l + 1]))
n_params <- function(net) sum(sapply(net, function(layer) length(layer$W) + length(layer$b)))

mlp_forward <- function(net, X, dropout = 0, train = FALSE) {
  L <- length(net); A <- vector("list", L + 1); M <- vector("list", L); A[[1]] <- X
  dropout <- rep_len(dropout, L)
  for (l in 1:L) {
    Z <- A[[l]] %*% net[[l]]$W + rep(net[[l]]$b, each = nrow(A[[l]]))
    if (l < L) {
      Z <- relu(Z)
      if (train && dropout[l] > 0) {          # dropout (Section 10.7.3): zero a fraction phi, scale by 1/(1-phi)
        M[[l]] <- (runif(length(Z)) > dropout[l]) / (1 - dropout[l])
        Z <- Z * M[[l]]
      }
    }
    A[[l + 1]] <- Z
  }
  list(A = A, M = M)
}

# Backpropagation: dOut is the derivative of the loss with respect to the network's output.
# Layer by layer this is the chain rule of (10.29)-(10.30): the gradient of W is
# (inputs of the layer)' x (derivative at its output), and the derivative is passed back through W'
# and multiplied by g'(z), which for ReLU is 1 where the unit was active and 0 elsewhere.
mlp_backward <- function(net, fw, dOut) {
  L <- length(net); G <- vector("list", L); D <- dOut
  for (l in L:1) {
    G[[l]] <- list(W = crossprod(fw$A[[l]], D), b = colSums(D))
    if (l > 1) {
      D <- D %*% t(net[[l]]$W)
      if (!is.null(fw$M[[l - 1]])) D <- D * fw$M[[l - 1]]
      D <- D * (fw$A[[l]] > 0)
    }
  }
  G
}

softmax <- function(Z) { E <- exp(Z - apply(Z, 1, max)); E / rowSums(E) }

# RMSprop with torch's defaults (alpha = 0.99, eps = 1e-8): each weight's step is the gradient divided
# by a running root-mean-square of its past gradients.
rmsprop_step <- function(net, G, V, lr, alpha = 0.99, eps = 1e-8) {
  for (l in seq_along(net)) for (nm in c("W", "b")) {
    V[[l]][[nm]] <- alpha * V[[l]][[nm]] + (1 - alpha) * G[[l]][[nm]]^2
    net[[l]][[nm]] <- net[[l]][[nm]] - lr * G[[l]][[nm]] / (sqrt(V[[l]][[nm]]) + eps)
  }
  list(net = net, V = V)
}

# Minibatch training. loss = "mse" (quantitative y), "xent" (y a 0/1 matrix, softmax output) or
# "logit" (binary y, one logit output). monitor(net) is called after every epoch.
fit_mlp <- function(net, X, y, loss, epochs, batch_size, lr = 0.001, dropout = 0, monitor = NULL) {
  y <- as.matrix(y); n <- nrow(X)
  V <- lapply(net, function(layer) list(W = 0 * layer$W, b = 0 * layer$b))
  hist <- NULL
  for (ep in 1:epochs) {
    perm <- sample.int(n)
    for (s in seq(1, n, by = batch_size)) {
      idx <- perm[s:min(s + batch_size - 1, n)]
      fw <- mlp_forward(net, X[idx, , drop = FALSE], dropout, train = TRUE)
      out <- fw$A[[length(net) + 1]]
      dOut <- switch(loss,
        mse = 2 * (out - y[idx, , drop = FALSE]) / length(idx),
        xent = (softmax(out) - y[idx, , drop = FALSE]) / length(idx),
        logit = (1 / (1 + exp(-out)) - y[idx, , drop = FALSE]) / length(idx))
      st <- rmsprop_step(net, mlp_backward(net, fw, dOut), V, lr)
      net <- st$net; V <- st$V
    }
    if (!is.null(monitor)) hist <- rbind(hist, c(epoch = ep, monitor(net)))
  }
  list(net = net, history = as.data.frame(hist))
}
predict_mlp <- function(net, X) mlp_forward(net, X)$A[[length(net) + 1]]

# ================================================================== 10.9.1 Hitters: one hidden layer
Gitters <- na.omit(Hitters)
n <- nrow(Gitters)
set.seed(13)
ntest <- trunc(n / 3)
testid <- sample(1:n, ntest)
cat("test players:", head(testid, 10), "...\n")
keep("testid", testid)

lfit <- lm(Salary ~ ., data = Gitters[-testid, ])
lpred <- predict(lfit, Gitters[testid, ])
mae_lm <- with(Gitters[testid, ], mean(abs(lpred - Salary)))
cat("least squares: test MAE", mae_lm, "\n")
keep("hit_lm_mae", mae_lm)

x <- scale(model.matrix(Salary ~ . - 1, data = Gitters))   # 20 columns: both League dummies
y <- Gitters$Salary
cvfit <- cv.glmnet(x[-testid, ], y[-testid], type.measure = "mae")   # random folds, as in the book
cpred <- predict(cvfit, x[testid, ], s = "lambda.min")
cat("lasso, random folds: test MAE", mean(abs(y[testid] - cpred)), "\n")
keep("hit_lasso_mae_book", mean(abs(y[testid] - cpred)))
foldid <- (seq_len(n - ntest) - 1) %% 10 + 1                    # fixed folds: training row i in fold i mod 10
cvfit2 <- cv.glmnet(x[-testid, ], y[-testid], type.measure = "mae", foldid = foldid,
                    control = list(thresh = 1e-12))            # converge fully, to compare with Python
cpred2 <- predict(cvfit2, x[testid, ], s = "lambda.min")
cat("lasso, fixed folds: lambda.min", cvfit2$lambda.min, " test MAE", mean(abs(y[testid] - cpred2)),
    " nonzero coefficients", sum(coef(cvfit2, s = "lambda.min")[-1] != 0), "\n")
keep("hit_lasso_lambda", cvfit2$lambda.min)
keep("hit_lasso_mae", mean(abs(y[testid] - cpred2)))
keep("hit_lasso_nz", sum(coef(cvfit2, s = "lambda.min")[-1] != 0))

set.seed(1)
modnn <- mlp_init(c(ncol(x), 50, 1))                 # 20 inputs -> 50 ReLU units -> 1 output
cat("parameters:", n_params(modnn), "\n")            # 50 * 21 + 51 = 1101
xtr <- x[-testid, ]; ytr <- y[-testid]; xte <- x[testid, ]; yte <- y[testid]
hit <- fit_mlp(modnn, xtr, ytr, loss = "mse", epochs = 1500, batch_size = 32, lr = 0.001, dropout = 0.4,
               monitor = function(net) c(train_mae = mean(abs(predict_mlp(net, xtr) - ytr)),
                                         test_mae = mean(abs(predict_mlp(net, xte) - yte))))
npred <- predict_mlp(hit$net, xte)
cat("neural network: test MAE", mean(abs(yte - npred)), "\n")
keep("hit_nn_params", n_params(modnn))
keep("hit_nn_mae", mean(abs(yte - npred)))

open_svg("lab-r-hitters", 6.4, 4)
plot(hit$history$epoch, hit$history$train_mae, type = "l", ylim = c(0, 700), xlab = "Epoch",
     ylab = "Mean absolute error", lwd = 1.5)
lines(hit$history$epoch, hit$history$test_mae, col = "red", lwd = 1.5)
abline(h = mae_lm, lty = 2, col = "grey40")
legend("topright", c("training", "test (validation)", "least squares, test"), col = c("black", "red", "grey40"),
       lty = c(1, 1, 2), bty = "n")
invisible(dev.off())

# ================================================================== 10.9.2 MNIST: a multilayer network
read_idx <- function(path) {
  con <- gzfile(path, "rb"); on.exit(close(con))
  magic <- readBin(con, "integer", 1, size = 4, endian = "big")
  dims <- readBin(con, "integer", magic %% 256, size = 4, endian = "big")
  v <- readBin(con, "integer", prod(dims), size = 1, signed = FALSE)
  if (length(dims) == 1) v else matrix(v, dims[1], prod(dims[-1]), byrow = TRUE)
}
mnist_file <- function(name) {
  dir <- file.path(Sys.getenv("ISL_CACHE", file.path(Sys.getenv("HOME"), ".cache", "prml-repro")), "mnist")
  path <- file.path(dir, name)
  if (!file.exists(path)) {
    dir.create(dir, showWarnings = FALSE, recursive = TRUE)
    download.file(paste0("https://ossci-datasets.s3.amazonaws.com/mnist/", name), path, mode = "wb")
  }
  path
}
x_train <- read_idx(mnist_file("train-images-idx3-ubyte.gz")) / 255   # 60000 x 784, rows are images
g_train <- read_idx(mnist_file("train-labels-idx1-ubyte.gz"))
x_test <- read_idx(mnist_file("t10k-images-idx3-ubyte.gz")) / 255
g_test <- read_idx(mnist_file("t10k-labels-idx1-ubyte.gz"))
cat("MNIST:", dim(x_train), "training,", dim(x_test), "test\n")
one_hot <- function(g, K = 10) { Y <- matrix(0, length(g), K); Y[cbind(seq_along(g), g + 1)] <- 1; Y }
y_train <- one_hot(g_train)

val <- 48001:60000                         # the last 20% of the training images, as validation_split = 0.2
accuracy <- function(pred, truth) mean(drop(pred) == drop(truth))
classes <- function(net, X) max.col(predict_mlp(net, X), ties.method = "first") - 1
set.seed(2)
modelnn <- mlp_init(c(784, 256, 128, 10))
cat("parameters:", n_params(modelnn), "\n")         # 200960 + 32896 + 1290 = 235146
keep("mnist_params", n_params(modelnn))
t0 <- proc.time()[["elapsed"]]
mn <- fit_mlp(modelnn, x_train[-val, ], y_train[-val, ], loss = "xent", epochs = 30, batch_size = 128,
              lr = 0.001, dropout = c(0.4, 0.3),
              monitor = function(net) c(train_acc = accuracy(classes(net, x_train[1:12000, ]), g_train[1:12000]),
                                        valid_acc = accuracy(classes(net, x_train[val, ]), g_train[val])))
cat("fitting took", round(proc.time()[["elapsed"]] - t0), "seconds\n")
acc_nn <- accuracy(classes(mn$net, x_test), g_test)
cat("two hidden layers with dropout: test accuracy", acc_nn, "\n")
keep("mnist_nn_acc", acc_nn)

set.seed(3)
modellr <- mlp_init(c(784, 10))                     # no hidden layer: multinomial logistic regression
cat("parameters:", n_params(modellr), "\n")         # 785 * 10 = 7850
lr_fit <- fit_mlp(modellr, x_train[-val, ], y_train[-val, ], loss = "xent", epochs = 30, batch_size = 128, lr = 0.001)
acc_lr <- accuracy(classes(lr_fit$net, x_test), g_test)
cat("multinomial logistic regression: test accuracy", acc_lr, "\n")
keep("mnist_mlr_acc", acc_lr)

open_svg("lab-r-mnist", 6.4, 4)
plot(mn$history$epoch, mn$history$train_acc, type = "b", pch = 19, cex = 0.6, ylim = c(0.9, 1), xlab = "Epoch",
     ylab = "Accuracy")
lines(mn$history$epoch, mn$history$valid_acc, type = "b", pch = 19, cex = 0.6, col = "red")
legend("bottomright", c("training (first 12000 images)", "validation (last 12000)"), col = c("black", "red"),
       lty = 1, pch = 19, bty = "n")
invisible(dev.off())

# ================================================================== 10.9.6 NYSE: time series
xdata <- data.matrix(NYSE[, c("DJ_return", "log_volume", "log_volatility")])
istrain <- NYSE[, "train"]
xdata <- scale(xdata)
lagm <- function(x, k = 1) {
  n <- nrow(x)
  pad <- matrix(NA, k, ncol(x))
  rbind(pad, x[1:(n - k), ])
}
arframe <- data.frame(log_volume = xdata[, "log_volume"],
                      L1 = lagm(xdata, 1), L2 = lagm(xdata, 2),
                      L3 = lagm(xdata, 3), L4 = lagm(xdata, 4),
                      L5 = lagm(xdata, 5))
arframe <- arframe[-(1:5), ]
istrain <- istrain[-(1:5)]
cat("training days", sum(istrain), " test days", sum(!istrain), "\n")

arfit <- lm(log_volume ~ ., data = arframe[istrain, ])
arpred <- predict(arfit, arframe[!istrain, ])
V0 <- var(arframe[!istrain, "log_volume"])
r2_ar <- 1 - mean((arpred - arframe[!istrain, "log_volume"])^2) / V0
cat("AR(5): test R2", r2_ar, "\n")
keep("ar_r2", r2_ar)

arframed <- data.frame(day = NYSE[-(1:5), "day_of_week"], arframe)
arfitd <- lm(log_volume ~ ., data = arframed[istrain, ])
arpredd <- predict(arfitd, arframed[!istrain, ])
r2_ard <- 1 - mean((arpredd - arframe[!istrain, "log_volume"])^2) / V0
cat("AR(5) + day of week: test R2", r2_ard, "\n")
keep("ard_r2", r2_ard)

n <- nrow(arframe)
xrnn <- data.matrix(arframe[, -1])
xrnn <- array(xrnn, c(n, 3, 5))
xrnn <- xrnn[, , 5:1]                     # index 1 is furthest back in time
xrnn <- aperm(xrnn, c(1, 3, 2))           # n x 5 (time) x 3 (variables)
cat("RNN input:", dim(xrnn), "\n")

# A simple recurrent network, (10.16)-(10.17): A_l = tanh(W x_l + U A_{l-1} + b), output beta0 + beta' A_L,
# with dropout on A_L during training. Gradients by backpropagation through time.
rnn_init <- function(p, K) {
  s <- 1 / sqrt(K)
  list(W = matrix(runif(K * p, -s, s), p, K), U = matrix(runif(K * K, -s, s), K, K), b = runif(K, -s, s),
       beta = runif(K, -s, s), beta0 = runif(1, -s, s))
}
rnn_forward <- function(par, X, mask = NULL) {
  m <- dim(X)[1]; L <- dim(X)[2]; K <- length(par$b)
  A <- vector("list", L + 1); A[[1]] <- matrix(0, m, K)
  for (l in 1:L) A[[l + 1]] <- tanh(X[, l, ] %*% par$W + A[[l]] %*% par$U + rep(par$b, each = m))
  AL <- if (is.null(mask)) A[[L + 1]] else A[[L + 1]] * mask
  list(A = A, AL = AL, out = drop(AL %*% par$beta) + par$beta0)
}
rnn_grad <- function(par, X, y, mask) {
  fw <- rnn_forward(par, X, mask); m <- length(y); L <- dim(X)[2]
  dout <- 2 * (fw$out - y) / m
  g <- list(W = 0 * par$W, U = 0 * par$U, b = 0 * par$b, beta = drop(crossprod(fw$AL, dout)), beta0 = sum(dout))
  dA <- outer(dout, par$beta) * mask
  for (l in L:1) {
    dZ <- dA * (1 - fw$A[[l + 1]]^2)      # tanh'(z) = 1 - tanh(z)^2
    g$W <- g$W + crossprod(X[, l, ], dZ)
    g$U <- g$U + crossprod(fw$A[[l]], dZ)
    g$b <- g$b + colSums(dZ)
    dA <- dZ %*% t(par$U)
  }
  g
}
fit_rnn <- function(par, X, y, epochs, batch_size, lr = 0.001, dropout = 0.1, alpha = 0.99, eps = 1e-8) {
  V <- lapply(par, function(p) 0 * p); n <- length(y); K <- length(par$b)
  for (ep in 1:epochs) {
    perm <- sample.int(n)
    for (s in seq(1, n, by = batch_size)) {
      idx <- perm[s:min(s + batch_size - 1, n)]
      mask <- matrix((runif(length(idx) * K) > dropout) / (1 - dropout), length(idx), K)
      g <- rnn_grad(par, X[idx, , , drop = FALSE], y[idx], mask)
      for (nm in names(par)) {
        V[[nm]] <- alpha * V[[nm]] + (1 - alpha) * g[[nm]]^2
        par[[nm]] <- par[[nm]] - lr * g[[nm]] / (sqrt(V[[nm]]) + eps)
      }
    }
  }
  par
}
set.seed(4)
rnn <- rnn_init(3, 12)
cat("RNN parameters:", sum(lengths(rnn)), "\n")     # 12 * (3 + 12 + 1) + 13 = 205
keep("rnn_params", sum(lengths(rnn)))
t0 <- proc.time()[["elapsed"]]
rnn <- fit_rnn(rnn, xrnn[istrain, , ], arframe[istrain, "log_volume"], epochs = 200, batch_size = 64)
cat("fitting took", round(proc.time()[["elapsed"]] - t0), "seconds\n")
kpred <- rnn_forward(rnn, xrnn[!istrain, , ])$out
r2_rnn <- 1 - mean((kpred - arframe[!istrain, "log_volume"])^2) / V0
cat("RNN: test R2", r2_rnn, "\n")
keep("rnn_r2", r2_rnn)

x <- model.matrix(log_volume ~ . - 1, data = arframed)   # 5 day dummies + 15 lags
cat("nonlinear AR inputs:", colnames(x)[1:7], "...\n")
set.seed(5)
arnnd <- mlp_init(c(ncol(x), 32, 1))
arnn <- fit_mlp(arnnd, x[istrain, ], arframe[istrain, "log_volume"], loss = "mse", epochs = 100,
                batch_size = 32, lr = 0.001, dropout = 0.5)
npred <- predict_mlp(arnn$net, x[!istrain, ])
r2_nar <- 1 - mean((arframe[!istrain, "log_volume"] - npred)^2) / V0
cat("nonlinear AR + day of week: test R2", r2_nar, "\n")
keep("nard_r2", r2_nar)

open_svg("lab-r-nyse", 7, 3.6)
test_dates <- as.Date(NYSE$date[-(1:5)][!istrain])
plot(test_dates, arframe[!istrain, "log_volume"], type = "l", col = "grey30", xlab = "", ylab = "log(Trading Volume)")
lines(test_dates, kpred, col = "orange", lwd = 1.2)
legend("bottomleft", c("observed", "RNN forecast"), col = c("grey30", "orange"), lty = 1, bty = "n")
invisible(dev.off())

# ================================================================== exercises 6, 7, 9-12
# Exercise 6: gradient descent on R(beta) = sin(beta) + beta / 10 with learning rate 0.1
gd <- function(b0, rho = 0.1, tol = 1e-10, maxit = 10000) {
  b <- b0
  for (it in 1:maxit) {
    b_new <- b - rho * (cos(b) + 1 / 10)
    if (abs(b_new - b) < tol) break
    b <- b_new
  }
  c(beta = b_new, R = sin(b_new) + b_new / 10, steps = it)
}
ex6a <- gd(2.3); ex6b <- gd(1.4)
cat("Exercise 6: from 2.3 ->", round(ex6a, 6), "; from 1.4 ->", round(ex6b, 6), "\n")
keep("ex6_a", ex6a[1:2]); keep("ex6_b", ex6b[1:2])

# Exercise 7: Default data, one hidden layer of 10 units with dropout, against logistic regression
Xd <- scale(cbind(student = as.numeric(Default$student == "Yes"), balance = Default$balance, income = Default$income))
yd <- as.numeric(Default$default == "Yes")
te7 <- (seq_len(nrow(Default)) - 1) %% 3 == 0          # every third row is a test row
glm7 <- glm(yd ~ Xd, family = binomial, subset = !te7)
p_glm <- 1 / (1 + exp(-cbind(1, Xd[te7, ]) %*% coef(glm7)))
err_glm <- mean((p_glm > 0.5) != yd[te7])
set.seed(7)
nn7 <- fit_mlp(mlp_init(c(3, 10, 1)), Xd[!te7, ], yd[!te7], loss = "logit", epochs = 30, batch_size = 32,
               lr = 0.001, dropout = 0.4)$net
err_nn <- mean((predict_mlp(nn7, Xd[te7, ]) > 0) != yd[te7])
cat("Exercise 7: test error, logistic regression", err_glm, " network", err_nn, "\n")
keep("ex7_glm_err", err_glm); keep("ex7_nn_err", err_nn)

# Exercise 9: AR(5) with a 12-level month factor
arframem <- data.frame(month = factor(format(as.Date(NYSE$date[-(1:5)]), "%m")), arframe)
r2_m <- 1 - mean((predict(lm(log_volume ~ ., data = arframem[istrain, ]), arframem[!istrain, ]) -
                  arframe[!istrain, "log_volume"])^2) / V0
cat("Exercise 9: AR(5) + month: test R2", r2_m, " (without month", r2_ar, ")\n")
keep("ex9_r2", r2_m)

# Exercise 10: a linear AR model from the flattened RNN input
xflat <- matrix(xrnn, n, 15)
r2_flat <- 1 - mean((cbind(1, xflat[!istrain, ]) %*% coef(lm(arframe[istrain, "log_volume"] ~ xflat[istrain, ])) -
                     arframe[!istrain, "log_volume"])^2) / V0
cat("Exercise 10: flattened linear AR: test R2", r2_flat, "\n")
keep("ex10_r2", r2_flat)

# Exercise 11: a nonlinear AR model from the flattened input (no day of week)
set.seed(11)
nn11 <- fit_mlp(mlp_init(c(15, 32, 1)), xflat[istrain, ], arframe[istrain, "log_volume"], loss = "mse",
                epochs = 100, batch_size = 32, lr = 0.001, dropout = 0.5)$net
r2_11 <- 1 - mean((predict_mlp(nn11, xflat[!istrain, ]) - arframe[!istrain, "log_volume"])^2) / V0
cat("Exercise 11: flattened nonlinear AR: test R2", r2_11, "\n")
keep("ex11_r2", r2_11)

# Exercise 12: the RNN with the day of week of the target day added to every step of the sequence
day <- model.matrix(~ day - 1, data = arframed)            # 5 dummies
xrnn_d <- array(0, c(n, 5, 8)); xrnn_d[, , 1:3] <- xrnn
for (l in 1:5) xrnn_d[, l, 4:8] <- day
set.seed(12)
rnn12 <- fit_rnn(rnn_init(8, 12), xrnn_d[istrain, , ], arframe[istrain, "log_volume"], epochs = 200, batch_size = 64)
r2_12 <- 1 - mean((rnn_forward(rnn12, xrnn_d[!istrain, , ])$out - arframe[!istrain, "log_volume"])^2) / V0
cat("Exercise 12: RNN + day of week: test R2", r2_12, "\n")
keep("ex12_r2", r2_12)

flat <- unlist(key)
write.csv(data.frame(name = names(flat), value = sprintf("%.10g", flat)), "cache/ch10_lab_R.csv", row.names = FALSE)
cat("wrote", length(flat), "numbers to cache/ch10_lab_R.csv\n")
