"""Murphy (2022), Chapter 12: Generalized Linear Models.

Companion code for Chapter 12 of K. P. Murphy, Probabilistic Machine Learning: An Introduction
(MIT Press, 2022; April 2025 draft), pp. 415-421, written from scratch with numpy, scipy and
matplotlib, to be read next to the lecture notes (../ch12-generalized-linear-models.html).
scikit-learn and statsmodels are only used to confirm the implementations.

Methods
* Section 12.1-12.2, the exponential dispersion family: the log normaliser A, the mean A' and the
  variance function A'' for the Gaussian, binomial and Poisson models (FAMILIES).
* Section 12.3, links: identity, logit, probit, complementary log-log, log (LINKS).
* Section 12.4, maximum likelihood by Fisher scoring / iteratively reweighted least squares
  (glm_fit), which is Newton's method when the link is canonical (12.31)-(12.34), with optional
  sample weights, offsets and a ridge penalty; the weighted negative log likelihood and its gradient
  and Hessian (glm_nll, glm_grad, glm_hessian); plain gradient descent for comparison.
* Section 12.5, predicting insurance claims: the data generator (simulate_claims), the three
  predictors (constant, ridge, Poisson regression), the metrics MSE, MAE and Poisson deviance
  (12.35)-(12.37), and the calibration curve.

Data. The book's example uses the French motor claims data (freMTPL2, 678 thousand policies, 9
features), which is not available offline. simulate_claims() makes a stand-in with the same
structure: policy exposures, nine risk factors (driver age, vehicle age, vehicle power, bonus-malus,
density, area, brand, fuel, region), about 94 percent of policies without a claim, a mean claim
frequency near 10 percent, and a driver-age effect that is not linear. The numbers in Table 12.1
therefore differ from the book's; the ordering of the three predictors is the point.

Figures. Reproduced here: 12.1 and 12.2 (the book's two figures, on the simulated data). Table 12.1
is printed and written to ../data/ch12-claims-table.json. Supplementary figures for the gaps filled in
the notes:
  12.S1 three links for a binary response: logit, probit and complementary log-log fitted to data
        generated with an asymmetric (complementary log-log) truth;
  12.S2 the iterates of IRLS (Newton) and of gradient descent on the negative log likelihood of a
        Poisson regression, drawn over its contours.
No figure is skipped (the chapter has no drawn diagrams).
Run `python3 ch12_generalized_linear_models.py --list`, `python3 ch12_generalized_linear_models.py`
(all figures, about a minute) or `python3 ch12_generalized_linear_models.py --fig 12.1`. Files go to
../img/ch12/; the data used by the browser demonstrations go to ../data/ch12-*.json.
"""
import json
import os
import sys

os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")

import numpy as np
from scipy import special, stats

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common  # noqa: E402
from common import BLUE, CYAN, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, plt, savefig  # noqa: E402

CH = 12
HERE = os.path.dirname(os.path.abspath(__file__))
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))


# ----------------------------------------------------------------------------- families and links
class Family:
    """Exponential dispersion family p(y|eta) = exp((y eta - A(eta))/phi + log h(y, phi))."""

    def __init__(self, name, A, dA, d2A, var, logh, mean_range):
        self.name, self.A, self.dA, self.d2A, self.var, self.logh, self.mean_range = name, A, dA, d2A, var, logh, mean_range


def _xlogy(x, y):
    return special.xlogy(x, y)


FAMILIES = {
    # N(eta, phi): A = eta^2/2
    "gaussian": Family("gaussian", lambda e: e ** 2 / 2, lambda e: e, lambda e: np.ones_like(e, dtype=float),
                       lambda m: np.ones_like(m, dtype=float), lambda y, phi=1.0: -0.5 * (y ** 2 / phi + np.log(2 * np.pi * phi)),
                       (-np.inf, np.inf)),
    # Bernoulli (one trial): A = log(1 + e^eta)
    "bernoulli": Family("bernoulli", lambda e: np.logaddexp(0, e), lambda e: common.sigmoid(e),
                        lambda e: common.sigmoid(e) * (1 - common.sigmoid(e)), lambda m: m * (1 - m),
                        lambda y, phi=1.0: np.zeros_like(y, dtype=float), (0, 1)),
    # Poisson: A = e^eta, h = 1/y!
    "poisson": Family("poisson", lambda e: np.exp(e), lambda e: np.exp(e), lambda e: np.exp(e), lambda m: m,
                      lambda y, phi=1.0: -special.gammaln(np.asarray(y, dtype=float) + 1), (0, np.inf)),
}


class Link:
    """eta = g(mu); inv is the mean function g^-1; dmu is d mu / d eta as a function of eta."""

    def __init__(self, name, g, inv, dmu):
        self.name, self.g, self.inv, self.dmu = name, g, inv, dmu


def _probit_inv(e):
    return stats.norm.cdf(e)


LINKS = {
    "identity": Link("identity", lambda m: m, lambda e: e, lambda e: np.ones_like(e, dtype=float)),
    "logit": Link("logit", lambda m: np.log(m / (1 - m)), common.sigmoid, lambda e: common.sigmoid(e) * (1 - common.sigmoid(e))),
    "probit": Link("probit", stats.norm.ppf, _probit_inv, stats.norm.pdf),
    "cloglog": Link("cloglog", lambda m: np.log(-np.log1p(-m)), lambda e: -np.expm1(-np.exp(e)),
                    lambda e: np.exp(e - np.exp(e))),
    "log": Link("log", np.log, np.exp, np.exp),
}
CANONICAL = {"gaussian": "identity", "bernoulli": "logit", "poisson": "log"}


# ----------------------------------------------------------------------------- likelihood
def glm_nll(w, X, y, family="poisson", link=None, weights=None, offset=0.0, phi=1.0):
    """Negative log likelihood -sum_n weights_n log p(y_n | x_n, w) with the dispersion phi."""
    fam = FAMILIES[family]
    lk = LINKS[link or CANONICAL[family]]
    eta = X @ w + offset
    mu = lk.inv(eta)
    sw = np.ones(len(y)) if weights is None else weights
    if (link or CANONICAL[family]) == CANONICAL[family]:
        ll = (y * eta - fam.A(eta)) / phi + fam.logh(y, phi)
    elif family == "bernoulli":
        ll = _xlogy(y, mu) + _xlogy(1 - y, 1 - mu)
    elif family == "poisson":
        ll = _xlogy(y, mu) - mu + fam.logh(y)
    else:
        ll = -0.5 * ((y - mu) ** 2 / phi + np.log(2 * np.pi * phi))
    return -np.sum(sw * ll)


def glm_grad(w, X, y, family="poisson", weights=None, offset=0.0, phi=1.0):
    """Gradient of the NLL for the canonical link: -X^T diag(weights) (y - mu) / phi, (12.31)."""
    fam = FAMILIES[family]
    mu = fam.dA(X @ w + offset)
    sw = np.ones(len(y)) if weights is None else weights
    return -(X.T @ (sw * (y - mu))) / phi


def glm_hessian(w, X, family="poisson", weights=None, offset=0.0, phi=1.0):
    """Hessian of the NLL for the canonical link: X^T diag(weights A''(eta)) X / phi, (12.34)."""
    fam = FAMILIES[family]
    d2 = fam.d2A(X @ w + offset)
    sw = np.ones(X.shape[0]) if weights is None else weights
    return (X.T * (sw * d2)) @ X / phi


def glm_fit(X, y, family="poisson", link=None, weights=None, offset=0.0, ridge=0.0, penalise=None,
            max_iter=50, tol=1e-10, w0=None, return_path=False):
    """Maximum likelihood (or ridge-penalised) fit of a GLM by Fisher scoring (IRLS).

    At each step the working response z = eta + (y - mu)/(dmu/deta) is regressed on X by weighted
    least squares with weights  s_n (dmu/deta)^2 / V(mu), where s_n are the sample weights and V the
    variance function. For the canonical link this equals Newton's method, since the observed and
    expected Hessians coincide (Filling the gap 12.3).
    ridge adds ridge * ||w||^2 / 2 with the coordinates flagged False in `penalise` left free.
    Returns w (and the list of iterates, starting from w0, when return_path is True).
    """
    fam = FAMILIES[family]
    lk = LINKS[link or CANONICAL[family]]
    n, d = X.shape
    sw = np.ones(n) if weights is None else np.asarray(weights, dtype=float)
    pen = np.ones(d) if penalise is None else np.asarray(penalise, dtype=float)
    off = np.zeros(n) + offset
    if w0 is None:
        w = np.zeros(d)
        # a starting value: regress the link of a shrunk response on X (standard GLM start)
        if family == "gaussian":
            mu0 = y.astype(float)
        elif family == "bernoulli":
            mu0 = (y + 0.5) / 2.0
        else:
            mu0 = y + 0.5
        if link is not None or family != "gaussian":
            eta0 = lk.g(mu0) - off
            A = (X.T * sw) @ X + ridge * np.diag(pen) + 1e-10 * np.eye(d)
            w = np.linalg.solve(A, (X.T * sw) @ eta0)
    else:
        w = np.asarray(w0, dtype=float).copy()
    path = [w.copy()]
    prev = np.inf
    for _ in range(max_iter):
        eta = X @ w + off
        mu = lk.inv(eta)
        dm = lk.dmu(eta)
        V = fam.var(mu)
        W = sw * dm ** 2 / V
        z = eta - off + (y - mu) / dm
        A = (X.T * W) @ X + ridge * np.diag(pen)
        wn = np.linalg.solve(A + 1e-12 * np.eye(d), (X.T * W) @ z)
        path.append(wn.copy())
        done = np.max(np.abs(wn - w)) < tol
        w = wn
        if done:
            break
    return (w, path) if return_path else w


def gradient_descent(X, y, family, lr, steps, w0, weights=None):
    """Gradient descent on the mean negative log likelihood with a fixed step size lr."""
    w = np.asarray(w0, dtype=float).copy()
    path = [w.copy()]
    n = len(y)
    for _ in range(steps):
        w = w - lr * glm_grad(w, X, y, family, weights) / n
        path.append(w.copy())
    return path


# ----------------------------------------------------------------------------- deviance
def poisson_deviance(y, mu, weights=None):
    """D(y, mu) = 2 sum w (y log(y/mu) - y + mu), (12.37); the term y log(y/mu) is 0 when y = 0."""
    sw = np.ones(len(y)) if weights is None else weights
    return 2 * np.sum(sw * (_xlogy(y, y / mu) - y + mu))


def gaussian_deviance(y, mu, weights=None):
    sw = np.ones(len(y)) if weights is None else weights
    return np.sum(sw * (y - mu) ** 2)


def bernoulli_deviance(y, mu, weights=None):
    sw = np.ones(len(y)) if weights is None else weights
    return 2 * np.sum(sw * (_xlogy(y, y / mu) + _xlogy(1 - y, (1 - y) / (1 - mu))))


def deviance_from_definition(y, mu, family, phi=1.0):
    """The definition (12.35): 2 sum (log p(y | mu*) - log p(y | mu)) with mu* = y, times phi."""
    fam = FAMILIES[family]
    lk_eta = {"gaussian": lambda m: m, "bernoulli": lambda m: np.log(np.clip(m, 1e-300, None) / np.clip(1 - m, 1e-300, None)),
              "poisson": lambda m: np.log(np.clip(m, 1e-300, None))}[family]
    eta_sat = lk_eta(np.asarray(y, dtype=float))
    eta = lk_eta(mu)
    if family == "bernoulli":
        sat = np.where((y == 0) | (y == 1), 0.0, 0.0)  # A(eta_sat) - y eta_sat = 0 at the boundary
        ll_sat = sat
    else:
        ll_sat = y * eta_sat - fam.A(eta_sat)
    ll = y * eta - fam.A(eta)
    return 2 * np.sum(ll_sat - ll) * phi / phi


# ----------------------------------------------------------------------------- insurance data
AREAS = list("ABCDEF")
BRANDS = ["B1", "B2", "B3", "B4", "B5", "B6"]
REGIONS = ["R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8"]
AGE_EDGES = [18, 21, 25, 30, 40, 50, 60, 70, 100]
VEHAGE_EDGES = [0, 1, 3, 6, 10, 15, 100]
POWER_EDGES = [4, 6, 7, 8, 9, 11, 16]
BM_EDGES = [50, 60, 70, 80, 100, 130, 230]
DENS_EDGES = [0, 3, 5, 6.5, 8, 12]


def simulate_claims(n, seed=0):
    """Stand-in for the French motor claims data: policies with exposure (years) and nine features.

    The claim count of policy i is Poisson(exposure_i * rate_i), with
    log rate = -2.4 + young-driver term + 0.034 (bonus-malus - 70) + vehicle age, power, fuel, area, brand
    and region effects, so a log-linear model is correctly specified in the binned features but
    a linear model is not. Returns a dict of arrays: the raw features, exposure, claims, y = claims/exposure.
    """
    rng = np.random.default_rng(seed)
    age = np.clip(np.round(rng.gamma(9.0, 4.6, n) + 12), 18, 95)
    vehage = np.clip(np.floor(rng.gamma(1.8, 3.6, n)), 0, 30)
    power = np.clip(np.round(rng.gamma(9.0, 0.9, n) + 1), 4, 15)
    bm = np.clip(np.round(50 + rng.gamma(1.0, 14.0, n) * (1.0 + 0.9 * (age < 25))), 50, 220)
    density = rng.normal(5.6, 1.9, n)
    density = np.clip(density, 0, 11)
    area = rng.choice(len(AREAS), n, p=[.1, .14, .2, .22, .2, .14])
    brand = rng.choice(len(BRANDS), n, p=[.38, .18, .17, .12, .09, .06])
    gas = rng.integers(0, 2, n)
    region = rng.choice(len(REGIONS), n)
    exposure = np.where(rng.random(n) < 0.45, 1.0, rng.beta(1.3, 1.1, n))
    exposure = np.clip(exposure, 0.02, 1.0)
    lograte = (-2.4 + 2.6 * np.exp(-(age - 18) / 8.0) + 0.012 * np.clip(age - 60, 0, None) / 1.0
               + 0.034 * (bm - 70) - 0.09 * np.minimum(vehage, 14) + 0.07 * (power - 7) * (power > 7) - 0.06 * (power < 6)
               + 0.14 * gas + 0.09 * (area - 2.5) / 2.5 + np.array([0, .08, -.05, .12, -.1, .05])[brand]
               + np.array([0.1, 0, -.12, .08, -.05, .15, -.2, 0.04])[region] + 0.06 * (density - 5.6) / 1.9)
    rate = np.exp(lograte)
    claims = rng.poisson(exposure * rate)
    y = claims / exposure
    return dict(age=age, vehage=vehage, power=power, bm=bm, density=density, area=area, brand=brand, gas=gas,
                region=region, exposure=exposure, claims=claims, y=y)


def _bin_onehot(v, edges):
    idx = np.clip(np.digitize(v, edges[1:-1]), 0, len(edges) - 2)
    return np.eye(len(edges) - 1)[idx][:, 1:]


def claims_design(D):
    """Feature engineering as in the book: bin the continuous features, one-hot encode the categorical
    ones (the first level of each is the baseline); an intercept column comes first."""
    cols = [np.ones((len(D["y"]), 1)),
            _bin_onehot(D["age"], AGE_EDGES), _bin_onehot(D["vehage"], VEHAGE_EDGES), _bin_onehot(D["power"], POWER_EDGES),
            _bin_onehot(D["bm"], BM_EDGES), _bin_onehot(D["density"], DENS_EDGES),
            np.eye(len(AREAS))[D["area"]][:, 1:], np.eye(len(BRANDS))[D["brand"]][:, 1:],
            D["gas"][:, None].astype(float), np.eye(len(REGIONS))[D["region"]][:, 1:]]
    return np.hstack(cols)


def split(D, frac_test=0.25, seed=1):
    rng = np.random.default_rng(seed)
    n = len(D["y"])
    perm = rng.permutation(n)
    nt = int(n * frac_test)
    te, tr = perm[:nt], perm[nt:]
    return {k: v[tr] for k, v in D.items()}, {k: v[te] for k, v in D.items()}


def weighted_ridge(X, y, w, alpha):
    """Weighted ridge regression with an unpenalised intercept (column 0)."""
    pen = np.eye(X.shape[1]) * alpha
    pen[0, 0] = 0
    A = (X.T * w) @ X + pen + 1e-10 * np.eye(X.shape[1])
    return np.linalg.solve(A, (X.T * w) @ y)


_CLAIMS = {}


def claims_experiment(n=160000, seed=0):
    """Fit the constant, ridge and Poisson predictors on the training part, evaluate on the test part."""
    key = (n, seed)
    if key in _CLAIMS:
        return _CLAIMS[key]
    D = simulate_claims(n, seed)
    tr, te = split(D)
    Xtr, Xte = claims_design(tr), claims_design(te)
    wtr, wte = tr["exposure"], te["exposure"]
    const = np.sum(wtr * tr["y"]) / np.sum(wtr)
    w_ridge = weighted_ridge(Xtr, tr["y"], wtr, 1e-6)
    w_pois = glm_fit(Xtr, tr["y"], "poisson", weights=wtr, max_iter=60, tol=1e-9)
    preds = {"Dummy": np.full(len(te["y"]), const), "Ridge": Xte @ w_ridge, "Poisson": np.exp(Xte @ w_pois)}
    res = dict(train=tr, test=te, Xtr=Xtr, Xte=Xte, w_ridge=w_ridge, w_pois=w_pois, preds=preds, const=const)
    _CLAIMS[key] = res
    return res


def claim_metrics(y, mu, w):
    """Exposure-weighted MSE, MAE and mean Poisson deviance (a negative prediction gets a floor 1e-3 in the deviance)."""
    sw = w / w.sum()
    mu_dev = np.clip(mu, 1e-3, None)
    return dict(mse=float(np.sum(sw * (y - mu) ** 2)), mae=float(np.sum(sw * np.abs(y - mu))),
                dev=float(poisson_deviance(y, mu_dev, w) / w.sum()))


def calibration(y, mu, w, bins=10):
    """Sort by the prediction and split into `bins` groups of equal exposure; return the mean
    predicted and the mean observed frequency in each group (exposure weighted)."""
    o = np.argsort(mu, kind="stable")
    cw = np.cumsum(w[o]) / w.sum()
    grp = np.minimum((cw * bins - 1e-12).astype(int), bins - 1)
    pm, ob = [], []
    for g in range(bins):
        k = o[grp == g]
        pm.append(np.sum(w[k] * mu[k]) / np.sum(w[k]))
        ob.append(np.sum(w[k] * y[k]) / np.sum(w[k]))
    return np.array(pm), np.array(ob)


# ----------------------------------------------------------------------------- figures
def fig_12_1(out_dir=None):
    """Figure 12.1: test-set claim frequencies and the predicted frequencies of three models."""
    E = claims_experiment()
    te = E["test"]
    fig, axes = plt.subplots(1, 4, figsize=(12.4, 2.9))
    bins_y = np.linspace(0, 30, 31)
    ax = axes[0]
    ax.hist(np.clip(te["y"], 0, 30), bins=bins_y, color=GREY, weights=te["exposure"] * 0 + 1)
    ax.set_yscale("log")
    ax.set_xlabel("observed frequency y")
    ax.set_ylabel("number of policies")
    ax.set_title("(a) data")
    names = ["Dummy", "Ridge", "Poisson"]
    labels = ["(b) constant", "(c) ridge regression", "(d) Poisson regression"]
    for ax, nm, lab, col in zip(axes[1:], names, labels, [CYAN, BLUE, RED]):
        mu = E["preds"][nm]
        lo, hi = -1.0, 4.0
        ax.hist(np.clip(mu, lo, hi), bins=np.linspace(lo, hi, 51), color=col)
        ax.set_yscale("log")
        ax.set_xlabel("predicted frequency")
        ax.set_title(lab)
        ax.axvline(0, color="k", lw=0.6, ls=":")
        ax.set_ylim(0.8, 1e5)
        ax.text(0.97, 0.95, "negative: %.1f%%" % (100 * np.mean(mu < 0)), transform=ax.transAxes, ha="right", va="top", fontsize=8.5)
    axes[0].set_ylim(0.8, 1e5)
    fig.tight_layout()
    return savefig(fig, CH, "fig12-1", out_dir)


def fig_12_2(out_dir=None):
    """Figure 12.2: calibration of the constant, ridge and Poisson predictors."""
    E = claims_experiment()
    te = E["test"]
    fig, axes = plt.subplots(1, 3, figsize=(10.2, 3.1), sharey=True)
    tot_true = float(te["claims"].sum())
    for ax, nm, lab in zip(axes, ["Dummy", "Ridge", "Poisson"], ["(a) constant", "(b) ridge regression", "(c) Poisson regression"]):
        mu = E["preds"][nm]
        pm, ob = calibration(te["y"], mu, te["exposure"])
        x = (np.arange(10) + 0.5) / 10
        ax.plot(x, pm, "o-", color=RED, label="predictions", ms=4)
        ax.plot(x, ob, "s-", color=BLUE, label="observations", ms=4)
        tot_pred = float(np.sum(te["exposure"] * mu))
        ax.set_title(lab)
        ax.set_xlabel("fraction of exposure, sorted by prediction")
        ax.text(0.03, 0.97, "total claims: predicted %.0f\n(observed %.0f)" % (tot_pred, tot_true), transform=ax.transAxes,
                va="top", fontsize=8.5)
    axes[0].set_ylabel("mean claim frequency")
    axes[0].legend(loc="lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig12-2", out_dir)


def link_data(n=600, seed=3):
    """Binary data whose true link is the complementary log-log: P(y=1|x) = 1 - exp(-exp(a + b x))."""
    rng = np.random.default_rng(seed)
    x = np.sort(rng.uniform(-3.2, 2.4, n))
    rng.shuffle(x)
    a, b = -0.4, 0.95
    p = LINKS["cloglog"].inv(a + b * x)
    y = (rng.random(n) < p).astype(float)
    return x, y, (a, b)


def fig_12_S1(out_dir=None):
    """Figure 12.S1: logit, probit and complementary log-log links fitted to the same binary data."""
    x, y, (a, b) = link_data()
    X = np.column_stack([np.ones_like(x), x])
    fig, axes = plt.subplots(1, 2, figsize=(9.2, 3.4))
    ax = axes[0]
    ax.scatter(x, y + 0.03 * np.sin(97 * x), s=7, color=GREY, alpha=0.6)
    xs = np.linspace(-3.3, 2.5, 300)
    Xs = np.column_stack([np.ones_like(xs), xs])
    ax.plot(xs, LINKS["cloglog"].inv(a + b * xs), "k--", lw=1.2, label="truth")
    rows = []
    for lk, col in [("logit", BLUE), ("probit", GREEN), ("cloglog", RED)]:
        w = glm_fit(X, y, "bernoulli", link=lk)
        dev = bernoulli_deviance(y, LINKS[lk].inv(X @ w))
        rows.append((lk, w, dev))
        ax.plot(xs, LINKS[lk].inv(Xs @ w), color=col, label="%s (deviance %.1f)" % (lk, dev))
    ax.set_xlabel("x")
    ax.set_ylabel("P(y = 1 | x)")
    ax.set_title("(a) fitted mean functions")
    ax.legend(fontsize=8, loc="upper left")
    ax = axes[1]
    p = np.linspace(0.002, 0.998, 400)
    for lk, col in [("logit", BLUE), ("probit", GREEN), ("cloglog", RED)]:
        eta = LINKS[lk].g(p)
        # standardise so that the three links have slope 1 at p = 1/2 for the comparison
        ax.plot(p, eta, color=col, label=lk)
    ax.set_ylim(-4, 4)
    ax.set_xlabel("mean mu")
    ax.set_ylabel("linear predictor eta = g(mu)")
    ax.set_title("(b) the link functions")
    ax.legend(fontsize=8)
    fig.tight_layout()
    return savefig(fig, CH, "fig12-S1", out_dir)


def poisson_toy(n=40, seed=5):
    rng = np.random.default_rng(seed)
    x = rng.uniform(-1.5, 1.5, n)
    X = np.column_stack([np.ones(n), x])
    y = rng.poisson(np.exp(0.6 + 0.9 * x)).astype(float)
    return x, X, y


def fig_12_S2(out_dir=None):
    """Figure 12.S2: IRLS (Newton) and gradient descent paths over the Poisson negative log likelihood."""
    x, X, y = poisson_toy()
    w_hat, newton = glm_fit(X, y, "poisson", w0=np.zeros(2), return_path=True, max_iter=8)
    gd = gradient_descent(X, y, "poisson", 0.2, 60, np.zeros(2))
    g0 = np.linspace(-0.6, 1.6, 120)
    g1 = np.linspace(-0.6, 2.0, 120)
    Z = np.array([[glm_nll(np.array([a, b]), X, y) for a in g0] for b in g1])
    fig, ax = plt.subplots(figsize=(5.2, 4.2))
    ax.contour(g0, g1, Z, levels=np.sort(Z.min() + np.array([0.5, 1.5, 3, 6, 12, 25, 50, 100, 200])), colors="0.55", linewidths=0.8)
    P = np.array(newton)
    ax.plot(P[:, 0], P[:, 1], "o-", color=RED, label="IRLS / Newton (%d steps)" % (len(P) - 1), ms=4)
    Q = np.array(gd)
    ax.plot(Q[:, 0], Q[:, 1], ".-", color=BLUE, label="gradient descent, step 0.2 (60 steps)", ms=3, lw=0.9)
    ax.plot(*w_hat, "k*", ms=10, label="maximum likelihood")
    ax.set_xlabel("intercept $w_0$")
    ax.set_ylabel("slope $w_1$")
    ax.legend(fontsize=8, loc="upper left")
    ax.set_title("Poisson regression, N = 40")
    fig.tight_layout()
    return savefig(fig, CH, "fig12-S2", out_dir)


# ----------------------------------------------------------------------------- data for the browser demos
def export_demo_data():
    os.makedirs(DATA_OUT, exist_ok=True)
    rnd = lambda a, k=6: [round(float(v), k) for v in np.ravel(a)]
    # demo 1: three small data sets and the IRLS iterates from several starts
    rng = np.random.default_rng(11)
    n = 40
    x = np.sort(rng.uniform(-1.5, 1.5, n))
    X = np.column_stack([np.ones(n), x])
    yg = 0.5 + 1.2 * x + rng.normal(0, 0.6, n)
    yb = (rng.random(n) < common.sigmoid(0.3 + 1.6 * x)).astype(float)
    yp = rng.poisson(np.exp(0.6 + 0.9 * x)).astype(float)
    out = {"x": rnd(x), "sets": {}}
    for fam, y in [("gaussian", yg), ("bernoulli", yb), ("poisson", yp)]:
        w, path = glm_fit(X, y, fam, w0=np.zeros(2), return_path=True, max_iter=12, tol=1e-13)
        gd = gradient_descent(X, y, fam, 0.3 if fam != "poisson" else 0.1, 100, np.zeros(2))
        out["sets"][fam] = {"y": rnd(y, 4), "mle": rnd(w), "newton_path": [rnd(p) for p in path[:9]],
                            "gd_lr": 0.3 if fam != "poisson" else 0.1, "gd_w100": rnd(gd[-1]),
                            "nll_mle": float(glm_nll(w, X, y, fam))}
    json.dump(out, open(os.path.join(DATA_OUT, "ch12-irls.json"), "w"), separators=(",", ":"))
    # demo 2: binary data with an asymmetric truth, fitted with three links
    xl, yl, (a, b) = link_data()
    Xl = np.column_stack([np.ones_like(xl), xl])
    ref = {}
    for lk in ["logit", "probit", "cloglog"]:
        w = glm_fit(Xl, yl, "bernoulli", link=lk, tol=1e-13)
        ref[lk] = {"w": rnd(w), "deviance": float(bernoulli_deviance(yl, LINKS[lk].inv(Xl @ w))),
                   "nll": float(glm_nll(w, Xl, yl, "bernoulli", link=lk))}
    json.dump({"x": rnd(xl, 4), "y": rnd(yl, 0), "truth": [a, b], "ref": ref}, open(os.path.join(DATA_OUT, "ch12-links.json"), "w"), separators=(",", ":"))
    # demo 3: a small claims data set with three covariates
    D = simulate_claims(24000, seed=21)
    tr, te = split(D, 0.5, seed=2)
    def feats(S):
        age_b = np.digitize(S["age"], [21, 25, 30, 40, 50, 60, 70])          # 8 bins
        veh_b = np.digitize(S["vehage"], [1, 3, 6, 10, 15])                  # 6 bins
        bm_b = np.digitize(S["bm"], [60, 70, 80, 100, 130])                  # 6 bins
        return np.column_stack([age_b, veh_b, bm_b]).astype(int)
    ftr, fte = feats(tr), feats(te)
    json.dump({"levels": [8, 6, 6], "names": ["driver age (8 bins)", "vehicle age (6 bins)", "bonus-malus (6 bins)"],
               "train": {"f": ftr.tolist(), "exposure": rnd(tr["exposure"], 4), "claims": tr["claims"].astype(int).tolist()},
               "test": {"f": fte.tolist(), "exposure": rnd(te["exposure"], 4), "claims": te["claims"].astype(int).tolist()}},
              open(os.path.join(DATA_OUT, "ch12-claims.json"), "w"), separators=(",", ":"))


def claims_table():
    E = claims_experiment()
    te = E["test"]
    rows = {}
    for nm in ["Dummy", "Ridge", "Poisson"]:
        m = claim_metrics(te["y"], E["preds"][nm], te["exposure"])
        m["total_claims_pred"] = float(np.sum(te["exposure"] * E["preds"][nm]))
        rows[nm] = m
    rows["_truth"] = {"total_claims": float(te["claims"].sum()), "zero_share": float(np.mean(te["claims"] == 0)),
                      "mean_freq": float(np.sum(te["claims"]) / np.sum(te["exposure"])), "n_test": int(len(te["y"]))}
    os.makedirs(DATA_OUT, exist_ok=True)
    json.dump(rows, open(os.path.join(DATA_OUT, "ch12-claims-table.json"), "w"), indent=1)
    return rows


def table_12_1(out_dir=None):
    """Table 12.1: test-set MSE, MAE and Poisson deviance of the three predictors (printed, saved as JSON)."""
    rows = claims_table()
    print("%-9s %7s %7s %9s %12s" % ("Name", "MSE", "MAE", "Deviance", "total claims"))
    for nm in ["Dummy", "Ridge", "Poisson"]:
        r = rows[nm]
        print("%-9s %7.3f %7.3f %9.3f %12.0f" % (nm, r["mse"], r["mae"], r["dev"], r["total_claims_pred"]))
    print("observed total claims %.0f; share of policies without claim %.3f; mean frequency %.3f" % (
        rows["_truth"]["total_claims"], rows["_truth"]["zero_share"], rows["_truth"]["mean_freq"]))
    export_demo_data()
    return rows


FIGURES = {"12.1": fig_12_1, "12.2": fig_12_2, "12.S1": fig_12_S1, "12.S2": fig_12_S2, "12.T1": table_12_1}

if __name__ == "__main__":
    common.run(FIGURES, CH)
