"""Murphy (2023), Chapter 16: Deep neural networks (with Supplement 16).

Companion code for Chapter 16 of K. P. Murphy, Probabilistic Machine Learning: Advanced Topics
(MIT Press, 2023; December 2025 draft), pp. 631-646 and supplement pp. 173-180, written from
scratch with numpy (PyTorch is used only to train the convolutional autoencoder of Figure 16.14 and
to confirm hand-written gradients), to be read next to ../ch16-deep-neural-networks.html.

Building blocks (numpy, explicit formulas)
* Section 16.2.2 activation functions and their derivatives (ACTIVATIONS, Figure 16.2).
* Section 16.2.3 convolution as cross-correlation, multi-channel (conv2d), pooling, shift equivariance.
* Section 16.2.4 residual connections; 16.2.5 batch and layer normalisation (layer_norm);
  16.2.6 dropout (dropout_mask, dropout_ridge_penalty); 16.2.7 attention (attention, multi_head,
  masked attention, the kernel-regression view); 16.2.9 multiplicative layers (bilinear, film);
  16.2.10 implicit layers (the implicit-function-theorem gradient).
* Section 16.3.1 the heteroskedastic MLP with a shared backbone and two heads (HeteroMLP, trained by
  full-batch Adam; export_json writes data, initial weights and reference losses for the browser demo);
  16.3.3 autoencoders (linear autoencoder = PCA; Figure 16.14 with a convolutional autoencoder);
  16.3.4 RNN factorisation of p(y_1:T).
* Supplement 16.1: sinusoidal positional encodings, message passing on graphs (message_passing,
  propagation matrix), graph attention, transformers as fully connected GNNs.

Figures reproduced: 16.2 (activations and gradients) and 16.14 (autoencoder on MNIST digits:
originals, reconstructions, t-SNE of the 20-d codes; 8000 training images and 8 epochs instead of the
book's full training, so the numbers differ slightly). Own figure: own16.1 (heteroskedastic regression).

Skipped figures, with reasons: 16.1, 16.3 to 16.13, 16.15, 16.16 (main text) and Figures 16.1 to 16.12 of the
supplement are drawings (neurons, convolution diagrams, residual and dropout sketches, attention and
transformer block diagrams, graph illustrations, recurrent units) with nothing computed; the numerical
facts they illustrate are confirmed in code instead (shapes of convolution outputs, equivariance, the
transformer = fully connected GNN identity).

Data: MNIST (cached by common.mnist) for Figure 16.14; simulation for everything else (seeds fixed).
Browser demos read ../data/ch16-*.json, written by `python3 ch16_deep_neural_networks.py --json`.
"""
import json
import os
import sys

import numpy as np
from scipy import special

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, mnist, plt, run, savefig  # noqa: E402

CH = 16
DATA_OUT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data"))


# ================================================================== 16.2.2 activation functions
def sigmoid(a):
    return special.expit(a)


def softplus(a):
    return np.logaddexp(0.0, a)


def relu(a):
    return np.maximum(a, 0.0)


def leaky_relu(a, alpha=0.01):
    return np.maximum(a, 0.0) + alpha * np.minimum(a, 0.0)


def elu(a, alpha=1.0):
    return np.maximum(a, 0.0) + np.minimum(alpha * np.expm1(np.minimum(a, 0.0)), 0.0)


def swish(a):
    return a * sigmoid(a)


def gelu(a):
    return a * special.ndtr(a)


def d_sigmoid(a):
    s = sigmoid(a)
    return s * (1 - s)


def d_tanh(a):
    return 1 - np.tanh(a) ** 2


def d_softplus(a):
    return sigmoid(a)


def d_relu(a):
    return (a > 0).astype(float)


def d_leaky_relu(a, alpha=0.01):
    return np.where(a > 0, 1.0, alpha)


def d_elu(a, alpha=1.0):
    return np.where(a > 0, 1.0, alpha * np.exp(np.minimum(a, 0.0)))


def d_swish(a):
    s = sigmoid(a)
    return s + a * s * (1 - s)


def d_gelu(a):
    return special.ndtr(a) + a * np.exp(-0.5 * a * a) / np.sqrt(2 * np.pi)


ACTIVATIONS = {
    "sigmoid": (sigmoid, d_sigmoid), "tanh": (np.tanh, d_tanh), "softplus": (softplus, d_softplus),
    "relu": (relu, d_relu), "leaky-relu": (leaky_relu, d_leaky_relu), "elu": (elu, d_elu),
    "swish": (swish, d_swish), "gelu": (gelu, d_gelu),
}


# ================================================================== 16.2.3 convolution
def conv2d(X, Wk):
    """Valid cross-correlation (16.4)/(16.5). X: (H, W, C); Wk: (h, w, C, D); returns (H-h+1, W-w+1, D)."""
    H, W, C = X.shape
    h, w, C2, D = Wk.shape
    assert C == C2
    out = np.zeros((H - h + 1, W - w + 1, D))
    for u in range(h):
        for v in range(w):
            patch = X[u:u + H - h + 1, v:v + W - w + 1, :]          # (H', W', C)
            out += np.tensordot(patch, Wk[u, v], axes=([2], [0]))   # (H', W', D)
    return out


def conv2d_circular(X, k):
    """Single-channel cross-correlation with circular boundary, same size output."""
    H, W = X.shape
    h, w = k.shape
    out = np.zeros_like(X, dtype=float)
    for u in range(h):
        for v in range(w):
            out += k[u, v] * np.roll(X, shift=(-u, -v), axis=(0, 1))
    return out


def conv_as_matrix(k, H, W):
    """The (H*W) x (H*W) matrix of the circular cross-correlation with kernel k (doubly block circulant)."""
    M = np.zeros((H * W, H * W))
    for i in range(H * W):
        e = np.zeros(H * W)
        e[i] = 1.0
        M[:, i] = conv2d_circular(e.reshape(H, W), k).ravel()
    return M


def max_pool(X, s=2):
    H, W = X.shape
    return X[:H // s * s, :W // s * s].reshape(H // s, s, W // s, s).max(axis=(1, 3))


# ================================================================== 16.2.5 normalisation
def layer_norm(x, eps=0.0):
    """y = (x - mean) / sqrt(var + eps) along the last axis (no scale and bias)."""
    mu = x.mean(-1, keepdims=True)
    var = ((x - mu) ** 2).mean(-1, keepdims=True)
    return (x - mu) / np.sqrt(var + eps)


# ================================================================== 16.2.6 dropout
def dropout_mask(rng, shape, p):
    """eps ~ Ber(1 - p) (keep probability 1 - p)."""
    return (rng.uniform(size=shape) >= p).astype(float)


def dropout_ridge_penalty(w, x, p):
    """E over the inverted-dropout noise of the squared error equals (y - w.x)^2 + this penalty (gap 16.6)."""
    return p / (1 - p) * np.sum((w * x) ** 2)


# ================================================================== 16.2.7 attention
def softmax(a, axis=-1):
    return special.softmax(a, axis=axis)


def attention(Q, K, V, mask=None, scale=None):
    """Z = softmax(Q K^T / sqrt(d_k)) V (16.12). mask: boolean (n_q, n_k), True = allowed. Returns (Z, A)."""
    dk = K.shape[-1]
    S = Q @ K.T / (np.sqrt(dk) if scale is None else scale)
    if mask is not None:
        S = np.where(mask, S, -np.inf)
    A = softmax(S, axis=-1)
    return A @ V, A


def multi_head(Q, K, V, WQ, WK, WV, WO):
    """(16.13)-(16.14): WQ, WK, WV are lists of per-head projections, WO the output matrix."""
    heads = [attention(Q @ wq, K @ wk, V @ wv)[0] for wq, wk, wv in zip(WQ, WK, WV)]
    return np.concatenate(heads, axis=-1) @ WO


def softmax_jacobian(a):
    al = softmax(a)
    return np.diag(al) - np.outer(al, al)


def circle_attention_curve(n, beta, phis, mask_causal=False):
    """Demo reference: keys k_i = (cos t_i, sin t_i), t_i = 2 pi (i + 1/2) / n, values y_i = sin(2 t_i) + 0.3 cos(5 t_i),
    query q = (cos phi, sin phi); scores beta * q.k (beta plays the role of 1 / temperature).
    Returns the attention output at every phi, the weights, and the Gaussian-kernel regression estimate."""
    t = 2 * np.pi * (np.arange(n) + 0.5) / n
    y = np.sin(2 * t) + 0.3 * np.cos(5 * t)
    K = np.stack([np.cos(t), np.sin(t)], 1)
    out, nw = [], []
    for ph in phis:
        q = np.array([np.cos(ph), np.sin(ph)])
        s = beta * (K @ q)
        allowed = np.ones(n, bool)
        if mask_causal:
            allowed = t <= ph + 1e-12         # keys are ordered by angle (time): a query sees only keys at or before it
            if not allowed.any():
                allowed[0] = True
        a = np.where(allowed, s, -np.inf)
        w = softmax(a)
        out.append(w @ y)
        # kernel regression with Gaussian kernel exp(-beta ||q - k||^2 / 2) on the same allowed keys
        kk = np.where(allowed, -0.5 * beta * np.sum((K - q) ** 2, 1), -np.inf)
        nw.append(softmax(kk) @ y)
    return np.array(out), np.array(nw)


# ================================================================== 16.2.9 multiplicative layers, 16.2.10 implicit layers
def bilinear(z, x, W, U, V, b):
    """f(x, z) = z^T W x + U z + V x + b (16.16), W: (m, n, k)."""
    return np.einsum("i,ijk,j->k", z, W, x) + U @ z + V @ x + b


def hypernet_affine(z, x, W, U, V, b):
    """Same function through a hypernetwork that generates W' = z^T W + V and b' = U z + b."""
    Wp = np.einsum("i,ijk->kj", z, W) + V
    bp = U @ z + b
    return Wp @ x + bp


def film(x, a, b):
    """(16.18): feature-wise linear modulation."""
    return a * x + b


def implicit_argmin_grad(x, y_star, hess_yy, d2f_dydx):
    """dy*/dx = -H^{-1} d^2 f / dy dx at a solution y*(x) of grad_y f(x, y) = 0 (implicit function theorem)."""
    return -np.linalg.solve(hess_yy, d2f_dydx)


# ================================================================== 16.3.1 heteroskedastic MLP
def hetero_data(n=150, seed=3):
    rng = np.random.default_rng(seed)
    x = np.sort(rng.uniform(-1, 1, n))
    mu = np.sin(3 * x)
    sd = 0.05 + 0.35 * x ** 2
    y = mu + sd * rng.standard_normal(n)
    return x, y


def true_mu_sd(x):
    return np.sin(3 * x), 0.05 + 0.35 * x ** 2


class HeteroMLP:
    """x -> h = tanh(W1 x + b1) (H units) -> mu = wm.h + bm, sigma = softplus(ws.h + bs) + 1e-3. (16.22)
    hetero=False freezes ws = 0 so that sigma is one constant (trained through bs)."""

    def __init__(self, H, seed=0, hetero=True):
        r = np.random.default_rng(seed)
        self.H = H
        self.p = {"W1": 1.5 * r.standard_normal(H), "b1": r.uniform(-1, 1, H), "wm": r.standard_normal(H) / np.sqrt(H),
                  "bm": 0.0, "ws": r.standard_normal(H) / np.sqrt(H), "bs": -1.0}
        self.hetero = hetero
        if not hetero:
            self.p["ws"] = np.zeros(H)

    def forward(self, x):
        p = self.p
        h = np.tanh(np.outer(x, p["W1"]) + p["b1"])
        mu = h @ p["wm"] + p["bm"]
        s = h @ p["ws"] + p["bs"]
        return h, mu, s, softplus(s) + 1e-3

    def loss(self, x, y):
        _, mu, _, sg = self.forward(x)
        return float(np.mean(0.5 * np.log(2 * np.pi * sg ** 2) + (y - mu) ** 2 / (2 * sg ** 2)))

    def grads(self, x, y):
        p = self.p
        h, mu, s, sg = self.forward(x)
        n = len(x)
        dmu = -(y - mu) / sg ** 2 / n
        dsg = (1 / sg - (y - mu) ** 2 / sg ** 3) / n
        ds = dsg * sigmoid(s)
        dh = np.outer(dmu, p["wm"]) + np.outer(ds, p["ws"])
        da = dh * (1 - h ** 2)
        g = {"wm": h.T @ dmu, "bm": dmu.sum(), "ws": h.T @ ds, "bs": ds.sum(),
             "W1": da.T @ x, "b1": da.sum(0)}
        if not self.hetero:
            g["ws"] = np.zeros(self.H)
        return g

    def fit(self, x, y, steps, lr=0.01, state=None, b1=0.9, b2=0.999, eps=1e-8):
        """Full-batch Adam; returns the loss after every step (index 0 = before training); state carries m, v, t."""
        if state is None:
            state = {"m": {k: np.zeros_like(v, dtype=float) for k, v in self.p.items()},
                     "v": {k: np.zeros_like(v, dtype=float) for k, v in self.p.items()}, "t": 0}
        losses = [self.loss(x, y)]
        for _ in range(steps):
            g = self.grads(x, y)
            state["t"] += 1
            t = state["t"]
            for k in self.p:
                state["m"][k] = b1 * state["m"][k] + (1 - b1) * g[k]
                state["v"][k] = b2 * state["v"][k] + (1 - b2) * g[k] ** 2
                mh = state["m"][k] / (1 - b1 ** t)
                vh = state["v"][k] / (1 - b2 ** t)
                self.p[k] = self.p[k] - lr * mh / (np.sqrt(vh) + eps)
            losses.append(self.loss(x, y))
        return np.array(losses), state


# ================================================================== 16.3.3 autoencoders
def linear_autoencoder_subspace_error(X, k):
    """Reconstruction error of the best rank-k linear map (Eckart-Young): sum of the smallest singular values squared."""
    Xc = X - X.mean(0)
    s = np.linalg.svd(Xc, compute_uv=False)
    return float(np.sum(s[k:] ** 2))


# ================================================================== 16.3.4 RNN
def rnn_logprob(Wh, Wy, Wo, y, h0):
    """log p(y_1:T) of a tiny RNN with deterministic state h_t = tanh(Wh h_{t-1} + Wy y_{t-1}) and
    p(y_t | h_t) = N(Wo.h_t, 1); y_0 = 0. This is the autoregressive factorisation (16.24)."""
    h, ylast, lp = h0, 0.0, 0.0
    for t in range(len(y)):
        h = np.tanh(Wh @ h + Wy * ylast)
        mu = Wo @ h
        lp += -0.5 * np.log(2 * np.pi) - 0.5 * (y[t] - mu) ** 2
        ylast = y[t]
    return lp


# ================================================================== supplement: positional encoding and graphs
def sinusoidal_pe(T, d, base=10000.0):
    """u_{t,2j} = sin(t / base^{2j/d}), u_{t,2j+1} = cos(t / base^{2j/d})."""
    pe = np.zeros((T, d))
    t = np.arange(T)[:, None]
    w = 1.0 / base ** (2 * np.arange(d // 2) / d)
    pe[:, 0::2] = np.sin(t * w)
    pe[:, 1::2] = np.cos(t * w)
    return pe


def pe_shift_matrix(k, d, base=10000.0):
    """Block-diagonal rotation R_k with PE(t + k) = R_k PE(t) (gap 16.13)."""
    w = 1.0 / base ** (2 * np.arange(d // 2) / d)
    R = np.zeros((d, d))
    for j, wj in enumerate(w):
        c, s = np.cos(k * wj), np.sin(k * wj)
        R[2 * j:2 * j + 2, 2 * j:2 * j + 2] = [[c, s], [-s, c]]
    return R


def graph_adj(kind, n=12):
    """Adjacency matrices of the demo topologies (no self loops)."""
    A = np.zeros((n, n))
    if kind == "path":
        for i in range(n - 1):
            A[i, i + 1] = A[i + 1, i] = 1
    elif kind == "ring":
        for i in range(n):
            A[i, (i + 1) % n] = A[(i + 1) % n, i] = 1
    elif kind == "grid":
        s = int(round(np.sqrt(n)))
        assert s * s == n
        for r in range(s):
            for c in range(s):
                i = r * s + c
                if c + 1 < s:
                    A[i, i + 1] = A[i + 1, i] = 1
                if r + 1 < s:
                    A[i, i + s] = A[i + s, i] = 1
    elif kind == "complete":
        A = np.ones((n, n)) - np.eye(n)
    elif kind == "global":      # a ring of n-1 nodes plus one global node joined to all of them
        m = n - 1
        for i in range(m):
            A[i, (i + 1) % m] = A[(i + 1) % m, i] = 1
            A[i, m] = A[m, i] = 1
    else:
        raise ValueError(kind)
    return A


def message_passing(H, A, U, V, act=relu):
    """h_i' = act(U h_i + V sum_{j in nbr(i)} h_j) (supplement 16.2), written for the rows of H."""
    return act(H @ U.T + (A @ H) @ V.T)


def propagation_matrix(A, a):
    """P = (1 - a) I + a D^{-1} A: each node keeps a fraction 1 - a of its feature and replaces a by its neighbour mean."""
    deg = A.sum(1)
    return (1 - a) * np.eye(len(A)) + a * (A / np.maximum(deg, 1)[:, None])


def dirichlet_energy(h, A):
    """sum over undirected edges of (h_i - h_j)^2."""
    d = h[:, None] - h[None, :]
    return float(0.5 * np.sum(A * d ** 2))


def gnn_init_features(n):
    return np.cos(2.399963229728653 * np.arange(n))


def gat_layer(H, A_mask, Wq, Wk, Wv):
    """Graph attention with scaled dot-product scores restricted to the neighbours in A_mask (incl. self loops)."""
    return attention(H @ Wq, H @ Wk, H @ Wv, mask=A_mask > 0)[0]


# ================================================================== figures
def fig_16_2(out=None):
    """Figure 16.2: activation functions and their gradients."""
    a = np.linspace(-5, 5, 801)
    names = ["sigmoid", "leaky-relu", "elu", "swish", "gelu"]
    cols = [BLUE, RED, GREEN, ORANGE, PURPLE]
    fig, ax = plt.subplots(1, 2, figsize=(7.8, 2.9))
    for nm, c in zip(names, cols):
        f, d = ACTIVATIONS[nm]
        ls = "--" if nm == "leaky-relu" else "-"
        ax[0].plot(a, f(a), color=c, ls=ls, label=nm)
        ax[1].plot(a, d(a), color=c, ls=ls, label=nm)
    for k, t in enumerate(["Activation function", "Gradient of activation function"]):
        ax[k].set_title(t); ax[k].set_xlim(-5, 5); ax[k].set_ylim(-0.5, 2.0); ax[k].set_xlabel("$a$")
        ax[k].grid(alpha=0.25)
    ax[0].legend(fontsize=7.5, loc="upper left")
    fig.tight_layout()
    savefig(fig, CH, "fig16-2", out)


def _train_conv_ae(Xtr, epochs=8, seed=0, zdim=20):
    import torch
    from torch import nn
    from common import torch_setup
    torch_setup(seed)
    torch.use_deterministic_algorithms(False)
    enc = nn.Sequential(nn.Conv2d(1, 16, 3, 2, 1), nn.ReLU(), nn.Conv2d(16, 32, 3, 2, 1), nn.ReLU(), nn.Flatten(),
                        nn.Linear(32 * 49, zdim))
    dec_lin = nn.Linear(zdim, 32 * 49)
    dec = nn.Sequential(nn.ReLU(), nn.Unflatten(1, (32, 7, 7)), nn.ConvTranspose2d(32, 16, 3, 2, 1, output_padding=1), nn.ReLU(),
                        nn.ConvTranspose2d(16, 1, 3, 2, 1, output_padding=1), nn.Sigmoid())
    params = list(enc.parameters()) + list(dec_lin.parameters()) + list(dec.parameters())
    opt = torch.optim.Adam(params, lr=2e-3)
    X = torch.tensor(Xtr, dtype=torch.float32).reshape(-1, 1, 28, 28)
    g = torch.Generator().manual_seed(seed)
    hist = []
    for ep in range(epochs):
        perm = torch.randperm(len(X), generator=g)
        tot = 0.0
        for i in range(0, len(X), 128):
            xb = X[perm[i:i + 128]]
            loss = ((dec(dec_lin(enc(xb))) - xb) ** 2).mean()
            opt.zero_grad(); loss.backward(); opt.step()
            tot += loss.item() * len(xb)
        hist.append(tot / len(X))
    f_enc = lambda A: enc(torch.tensor(A, dtype=torch.float32).reshape(-1, 1, 28, 28)).detach()
    f_dec = lambda Z: dec(dec_lin(Z)).detach().reshape(-1, 784).numpy()
    return f_enc, f_dec, hist


def fig_16_14(out=None):
    """Figure 16.14: MNIST digits, their reconstruction by a convolutional autoencoder, and a t-SNE map of the 20-d codes."""
    from sklearn.manifold import TSNE
    Xtr, _ = mnist(True)
    Xte, yte = mnist(False)
    f_enc, f_dec, hist = _train_conv_ae(Xtr[:8000], epochs=8)
    Z = f_enc(Xte[:1500])
    rec = f_dec(Z[:10])
    emb = TSNE(n_components=2, perplexity=30, init="pca", random_state=0).fit_transform(Z.numpy())
    fig = plt.figure(figsize=(8.2, 3.0))
    gs = fig.add_gridspec(2, 2, width_ratios=[1.0, 1.1], hspace=0.05, wspace=0.12)
    for row, (imgs, title) in enumerate([(Xte[:10], "(a) test digits"), (rec, "(b) reconstructions")]):
        ax = fig.add_subplot(gs[row, 0])
        ax.imshow(np.hstack([im.reshape(28, 28) for im in imgs]), cmap="gray_r")
        ax.set_xticks([]); ax.set_yticks([]); ax.set_title(title, fontsize=9, loc="left")
    ax = fig.add_subplot(gs[:, 1])
    sc = ax.scatter(emb[:, 0], emb[:, 1], c=yte[:1500], cmap="tab10", s=5)
    ax.set_xticks([]); ax.set_yticks([]); ax.set_title("(c) t-SNE of the 20-d codes", fontsize=9)
    ax.legend(*sc.legend_elements(), fontsize=6, ncol=2, loc="center left", bbox_to_anchor=(1.0, 0.5), title="digit", title_fontsize=7)
    print("  final training mse %.4f" % hist[-1])
    savefig(fig, CH, "fig16-14", out)


def fig_own_16_1(out=None):
    """Own figure: heteroskedastic regression with a shared backbone and two heads, against a homoskedastic head."""
    x, y = hetero_data()
    fig, ax = plt.subplots(1, 2, figsize=(7.8, 2.9), sharey=True)
    g = np.linspace(-1, 1, 300)
    mu0, sd0 = true_mu_sd(g)
    for a, hetero, ttl in [(ax[0], False, "(a) one constant $\\sigma$"), (ax[1], True, "(b) input-dependent $\\sigma(x)$")]:
        m = HeteroMLP(16, seed=1, hetero=hetero)
        L, _ = m.fit(x, y, 2000)
        _, mu, _, sg = m.forward(g)
        a.plot(x, y, ".", color=GREY, ms=4)
        a.plot(g, mu, color=BLUE, label="$\\mu(x)$")
        a.fill_between(g, mu - 2 * sg, mu + 2 * sg, color=BLUE, alpha=0.2, label="$\\pm2\\sigma(x)$")
        a.plot(g, mu0 + 2 * sd0, "--", color=RED, lw=1); a.plot(g, mu0 - 2 * sd0, "--", color=RED, lw=1, label="true $\\pm2$ sd")
        a.set_title(ttl + ", NLL %.3f" % L[-1], fontsize=9); a.set_xlabel("$x$")
    ax[0].legend(fontsize=7.5, loc="lower right")
    fig.tight_layout()
    savefig(fig, CH, "own16-1", out)


# ================================================================== data for the browser demos
def export_json():
    os.makedirs(DATA_OUT, exist_ok=True)
    x, y = (np.round(v, 6) for v in hetero_data())
    d = {"x": x.tolist(), "y": y.tolist(), "inits": {}, "ref": {}}
    for H in (4, 8, 16):
        m = HeteroMLP(H, seed=1)
        d["inits"][str(H)] = {k: (np.asarray(v).tolist() if np.ndim(v) else float(v)) for k, v in m.p.items()}
    steps = [0, 100, 300, 1000, 2000]
    for H in (4, 8, 16):
        for het in (True, False):
            m = HeteroMLP(H, seed=1, hetero=het)
            # the browser starts from the stored (rounded) initial weights, so do the same here
            for k in m.p:
                m.p[k] = np.array(d["inits"][str(H)][k]) if np.ndim(m.p[k]) else float(d["inits"][str(H)][k])
            if not het:
                m.p["ws"] = np.zeros(H)
            L, _ = m.fit(x, y, 2000, lr=0.01)
            d["ref"]["%d-%s" % (H, "het" if het else "hom")] = {str(s): float(L[s]) for s in steps}
    with open(os.path.join(DATA_OUT, "ch16-hetero.json"), "w") as f:
        json.dump(d, f, separators=(",", ":"))
    print("wrote ../data/ch16-hetero.json")


FIGURES = {"16.2": fig_16_2, "16.14": fig_16_14, "own16.1": fig_own_16_1}

if __name__ == "__main__":
    if "--json" in sys.argv:
        export_json()
    else:
        run(FIGURES, CH)
        if "--list" not in sys.argv:
            export_json()
