"""Shared helpers for the course "Applied Statistical Learning in R and Python", which follows
James, G., Witten, D., Hastie, T. and Tibshirani, R. (2021), An Introduction to Statistical Learning
with Applications in R, 2nd edition, Springer, and James, G., Witten, D., Hastie, T., Tibshirani, R.
and Taylor, J. (2023), An Introduction to Statistical Learning with Applications in Python, Springer.

Every chapter module (chNN_<topic>.py) imports this file for

* data loaders: isl_data() for the book's data sets (the CSV files distributed with the book;
  set ISL_DATA to their folder, or place it or a link to it at code/isl-data), plus MNIST;
* a common plotting style and colour palette;
* savefig(), which writes ../img/chNN/figN-k.{png,svg,pdf} next to the lecture notes;
* torch_setup(), which fixes PyTorch's seed and thread count so runs are reproducible;
* run(), a small command-line driver: python ch03_linear_regression.py --fig 3.4

Data are cached in ~/.cache/prml-repro (set ESL_DATA to change this) and figures go to
../img (set ESL_OUT, or pass --out, to change this).
"""
import argparse
import gzip
import logging
import os
import struct
import sys
import time
import urllib.request

import numpy as np
import matplotlib

matplotlib.use("Agg")
# fontTools (used to embed fonts in the PDFs) logs a harmless note about font timestamps
logging.getLogger("fontTools").setLevel(logging.ERROR)
import matplotlib.pyplot as plt  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
DATA_DIR = os.path.join(HERE, "data")
CACHE = os.environ.get("ISL_CACHE", os.path.join(os.path.expanduser("~"), ".cache", "prml-repro"))
OUT_DIR = os.environ.get("ISL_OUT", os.path.normpath(os.path.join(HERE, "..", "img")))
USER_AGENT = "course-repro/1.0 (teaching code)"

# Colours used throughout, chosen to match the book's red / blue / green figures.
RED = "#d62728"
BLUE = "#1f5fbf"
GREEN = "#2ca02c"
MAGENTA = "#c2185b"
CYAN = "#17becf"
ORANGE = "#ff7f0e"
PURPLE = "#7b4fbf"
GREY = "#7f7f7f"
CLASS_COLOURS = (RED, BLUE, GREEN)


# --------------------------------------------------------------------------- style
def setup_style():
    """A clean, book-like matplotlib style (white background, boxed axes)."""
    plt.rcParams.update({
        "figure.dpi": 100,
        "savefig.dpi": 130,
        "font.size": 10,
        "axes.titlesize": 10.5,
        "axes.labelsize": 10,
        "legend.fontsize": 9,
        "xtick.labelsize": 9,
        "ytick.labelsize": 9,
        "axes.linewidth": 0.8,
        "lines.linewidth": 1.6,
        "figure.facecolor": "white",
        "axes.facecolor": "white",
        "savefig.facecolor": "white",
        "savefig.bbox": "tight",
        "savefig.pad_inches": 0.06,
        "mathtext.fontset": "cm",
        "svg.hashsalt": "isl-repro",   # stable ids, so re-running gives identical SVG files
        "pdf.fonttype": 42,             # embed TrueType fonts: the PDFs stay editable in Illustrator/Inkscape
    })


setup_style()


def panel_grid(nrows, ncols, size=(2.6, 2.4), **kw):
    """fig, axes for a grid of equally sized panels (axes is always 2-D)."""
    fig, axes = plt.subplots(nrows, ncols, figsize=(size[0] * ncols, size[1] * nrows), squeeze=False, **kw)
    return fig, axes


def panel_label(ax, label, loc="upper left"):
    """Write (a), (b), ... inside a panel, as in the book."""
    x, ha = (0.03, "left") if "left" in loc else (0.97, "right")
    y, va = (0.97, "top") if "upper" in loc else (0.03, "bottom")
    ax.text(x, y, "(%s)" % label, transform=ax.transAxes, ha=ha, va=va, fontsize=10,
            bbox=dict(boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.8))


def savefig(fig, chapter, name, out_dir=None, formats=("png", "svg", "pdf")):
    """Save fig as <out_dir>/chNN/<name>.png, .svg and .pdf and return the PNG path.

    name is the figure id in the book with a dash, e.g. 'fig3-8' for Figure 3.8.
    The SVG is what the web pages show (sharp at any zoom); the PDF is for slides
    and papers; the PNG is a light fallback and the thumbnail in the code gallery.
    """
    out = os.path.join(out_dir or OUT_DIR, "ch%02d" % int(chapter))
    os.makedirs(out, exist_ok=True)
    paths = {}
    for ext in formats:
        path = os.path.join(out, name + "." + ext)
        meta = {"Date": None} if ext in ("svg", "pdf") else None   # no timestamps: reproducible files
        if ext == "pdf":
            meta["CreationDate"] = None
            del meta["Date"]
        fig.savefig(path, metadata=meta)
        paths[ext] = path
    plt.close(fig)
    print("wrote", os.path.relpath(paths.get("png", next(iter(paths.values()))), os.getcwd()), "(+ " + ", ".join(e for e in formats if e != "png") + ")")
    return paths.get("png")


# --------------------------------------------------------------------------- downloads
def _download(urls, dest):
    """Download the first URL that works to dest (atomic rename)."""
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    last = None
    for url in urls:
        try:
            req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
            with urllib.request.urlopen(req, timeout=120) as r:
                data = r.read()
            tmp = dest + ".part"
            with open(tmp, "wb") as f:
                f.write(data)
            os.replace(tmp, dest)
            return dest
        except Exception as e:  # try the next mirror
            last = e
            time.sleep(1)
    raise RuntimeError("could not download %s (last error: %s)" % (os.path.basename(dest), last))


# --------------------------------------------------------------------------- data sets
def old_faithful(standardize=False):
    """Old Faithful geyser data (Bishop 2006, Appendix A): 272 x 2 array.

    Column 0 is the eruption duration (minutes) and column 1 the waiting time to
    the next eruption (minutes). The same 272 observations ship with R as
    datasets::faithful (Azzalini and Bowman 1990; Haerdle 1991).
    With standardize=True each column has mean 0 and standard deviation 1, the
    're-scaled' version used in Figures 9.1, 9.8 and 10.6.
    """
    X = np.loadtxt(os.path.join(DATA_DIR, "faithful.csv"), delimiter=",", skiprows=1)
    if standardize:
        X = (X - X.mean(0)) / X.std(0)
    return X


_MNIST_MIRRORS = [
    "https://ossci-datasets.s3.amazonaws.com/mnist/",
    "https://storage.googleapis.com/cvdf-datasets/mnist/",
]


def _mnist_file(name):
    path = os.path.join(CACHE, "mnist", name)
    if not os.path.exists(path):
        _download([m + name for m in _MNIST_MIRRORS], path)
    return path


def mnist(train=True, digits=None):
    """MNIST digits (LeCun et al. 1998), the 'off-line digits' of Appendix A.

    Returns images as a float array in [0, 1] of shape (N, 784) and integer labels.
    digits, e.g. (2, 3, 4), keeps only those classes.
    """
    pre = "train" if train else "t10k"
    with gzip.open(_mnist_file(pre + "-images-idx3-ubyte.gz"), "rb") as f:
        _, n, r, c = struct.unpack(">IIII", f.read(16))
        X = np.frombuffer(f.read(), dtype=np.uint8).reshape(n, r * c)
    with gzip.open(_mnist_file(pre + "-labels-idx1-ubyte.gz"), "rb") as f:
        f.read(8)
        y = np.frombuffer(f.read(), dtype=np.uint8)
    if digits is not None:
        keep = np.isin(y, digits)
        X, y = X[keep], y[keep]
    return X.astype(float) / 255.0, y.astype(int)



# --------------------------------------------------------------------------- the book's data sets
ISL_DATA = os.environ.get("ISL_DATA", os.path.join(HERE, "isl-data"))


def isl_data(name, **read_kw):
    """Read one of the book's data sets by name, e.g. isl_data("Advertising"), from the CSV files
    distributed with the book (folder ISL_DATA). Extra keyword arguments go to pandas.read_csv."""
    import pandas as pd
    path = os.path.join(ISL_DATA, name if name.endswith(".csv") else name + ".csv")
    if not os.path.exists(path):
        raise FileNotFoundError("%s not found; set ISL_DATA to the folder of the book's CSV files" % path)
    return pd.read_csv(path, **read_kw)


# --------------------------------------------------------------------------- maths helpers
def logsumexp(a, axis=None, keepdims=False):
    a = np.asarray(a, dtype=float)
    m = np.max(a, axis=axis, keepdims=True)
    m = np.where(np.isfinite(m), m, 0.0)
    s = np.log(np.sum(np.exp(a - m), axis=axis, keepdims=True)) + m
    return s if keepdims else np.squeeze(s, axis=axis) if axis is not None else s.item()


def gauss_logpdf(X, mu, Sigma):
    """log N(x | mu, Sigma) for each row of X."""
    X = np.atleast_2d(X)
    D = X.shape[1]
    L = np.linalg.cholesky(Sigma)
    Z = np.linalg.solve(L, (X - mu).T)
    return -0.5 * (D * np.log(2 * np.pi) + 2 * np.log(np.diag(L)).sum() + (Z ** 2).sum(0))


def ellipse_xy(mu, Sigma, nsd=1.0, n=200):
    """Points on the nsd-standard-deviation contour of N(mu, Sigma) in 2-D."""
    vals, vecs = np.linalg.eigh(Sigma)
    t = np.linspace(0, 2 * np.pi, n)
    circ = np.stack([np.cos(t), np.sin(t)])
    pts = vecs @ (np.sqrt(np.maximum(vals, 0))[:, None] * circ) * nsd
    return pts[0] + mu[0], pts[1] + mu[1]


def sigmoid(a):
    return 0.5 * (1.0 + np.tanh(0.5 * np.asarray(a, dtype=float)))



# --------------------------------------------------------------------------- PyTorch
def torch_setup(seed=0, threads=1):
    """Seed numpy and PyTorch and limit PyTorch to `threads` CPU threads; returns torch.

    All training in this course runs on the CPU in double or single precision and is sized
    to finish in seconds to a few minutes, so results are reproducible run to run.
    """
    import torch
    torch.manual_seed(seed)
    np.random.seed(seed)
    torch.set_num_threads(threads)
    try:
        torch.use_deterministic_algorithms(True)
    except Exception:
        pass
    return torch

# --------------------------------------------------------------------------- driver
def run(figures, chapter, argv=None):
    """Command-line driver shared by the chapter modules.

    figures maps the book's figure number ('3.8') to a function f(out_dir) that
    builds and saves that figure. With no --fig option every figure is made.
    """
    p = argparse.ArgumentParser(description="Reproduce James et al. (2021, 2023) Chapter %d figures." % chapter)
    p.add_argument("--fig", nargs="*", help="figure numbers, e.g. 3.8 3.10 (default: all)")
    p.add_argument("--out", default=None, help="output folder (default: ../img)")
    p.add_argument("--list", action="store_true", help="list the reproducible figures")
    a = p.parse_args(argv)
    if a.list:
        for k, f in figures.items():
            print(k, "-", (f.__doc__ or "").strip().split("\n")[0])
        return
    keys = a.fig or list(figures)
    for k in keys:
        if k not in figures:
            sys.exit("unknown figure %s; use --list" % k)
        t0 = time.time()
        figures[k](a.out)
        print("  Figure %s done in %.1f s" % (k, time.time() - t0))
