"""Murphy (2023), Chapter 32: Representation learning (and Supplement Chapter 32).

Companion code for Chapter 32 of K. P. Murphy, Probabilistic Machine Learning: Advanced Topics
(MIT Press, 2023; December 2025 draft), pp. 1049-1072. The supplement chapter S32 (pages 301-302 of the
supplement) holds only a chapter heading and no material. Written from scratch with numpy, scipy,
scikit-learn and PyTorch (CPU, one thread), to be read next to the lecture notes
(../ch32-representation-learning.html).

Scale. Every network is a small multilayer perceptron (32 to 128 hidden units) trained on two-dimensional
points or on the 8 x 8 scikit-learn digits in seconds to a minute; the book's ImageNet-scale encoders are
not reproduced. All training is seeded and results are cached in ./cache/ch32.

Methods
* Section 32.1: a supervised bottleneck network and a variational information bottleneck (VIB) turn two
  entangled rings into a representation that a linear classifier separates (fig_32_1).
* Section 32.2.1: regularised linear evaluation is not invariant to an invertible linear change of the
  features (fig_x32_a, panel c; ridge_scale_experiment).
* Section 32.2.2: representational similarity. Linear and kernel centred kernel alignment (linear_cka,
  kernel_cka), canonical correlation analysis by the QR/SVD algorithm (cca), the mean squared canonical
  correlation, the mean canonical correlation, SVCCA, projection-weighted CCA, the variance-weighted
  variant R2_LR and the regression R2 (r2_cca, mean_rho, svcca, pwcca, r2_lr_cca, r2_lr), the
  principal-component forms (32.10)-(32.12) (r2_cca_pc, r2_lr_pc, cka_pc), the invariance table, the curse
  of dimensionality (fig_x32_a) and the layer-matching sanity check between two networks (fig_x32_b).
* Section 32.3.2.3: a linear autoencoder trained by gradient descent finds the principal subspace but not the
  eigenvectors; a linear denoising autoencoder learns the shrinkage lambda / (lambda + sigma^2)
  (fig_x32_c).
* Section 32.3.3 and 32.3.4: a comparison of pretext tasks and multiview losses (autoencoder, denoising,
  masked prediction, rotation prediction, SimCLR/InfoNCE, Barlow Twins, BYOL) by linear evaluation on the
  digits (fig_x32_e); the InfoNCE bound (32.17) against the number of candidates (fig_x32_d); a one-mode
  linear model of BYOL and SimSiam with its conserved quantity (fig_x32_g).
* Section 32.4: the entangling measure-preserving map that makes a factorial prior non-identifiable
  (fig_x32_f), and the saturation of mutual-information estimators for deterministic encoders (fig_x32_d,
  panel b).

Skipped figures, with reasons: 32.2 (diagram of the three families of approaches), 32.3 (the masked
autoencoder diagram of He et al., with photographs), 32.4 (diagram of transformation-prediction tasks) and
32.5 (diagram of positive and negative pairs).  They contain nothing computed.

Own figures (not in the book), named as in the notes: x32.a to x32.g.

Data: scikit-learn digits (8 x 8 images, bundled), two-dimensional toy data, simulation.
"""
import json
import os
import pickle
import sys
import time

import numpy as np
from scipy import stats
from scipy.special import logsumexp

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import common  # noqa: E402
from common import BLUE, CYAN, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, plt, savefig  # noqa: E402

CH = 32
CACHE_DIR = os.path.join(HERE, "cache", "ch32")
DATA_OUT = os.path.normpath(os.path.join(HERE, "..", "data"))
os.makedirs(CACHE_DIR, exist_ok=True)


def _cached(name, fn, force=False):
    path = os.path.join(CACHE_DIR, name + ".pkl")
    if os.path.exists(path) and not force:
        with open(path, "rb") as f:
            return pickle.load(f)
    out = fn()
    with open(path, "wb") as f:
        pickle.dump(out, f)
    return out


# =========================================================================== Section 32.2.2: similarity measures
def center(X):
    X = np.asarray(X, dtype=float)
    return X - X.mean(0, keepdims=True)


def _basis(Xc, tol=1e-7):
    """Thin SVD of a centred matrix: left vectors U, singular values s and right vectors V (all cut at the
    numerical rank)."""
    U, s, Vt = np.linalg.svd(Xc, full_matrices=False)
    r = int((s > tol * s[0]).sum()) if s[0] > 0 else 0
    return U[:, :r], s[:r], Vt[:r].T


def linear_cka(X, Y):
    """Linear centred kernel alignment, the right-hand side of (32.2): ||Xt^T Yt||_F^2 / (||Xt^T Xt||_F ||Yt^T Yt||_F)."""
    Xc, Yc = center(X), center(Y)
    return np.linalg.norm(Xc.T @ Yc) ** 2 / (np.linalg.norm(Xc.T @ Xc) * np.linalg.norm(Yc.T @ Yc))


def cka_gram(X, Y):
    """Linear CKA through the n x n similarity matrices, the left-hand side of (32.2): <HKH, HK'H>_F / (||HKH|| ||HK'H||)."""
    n = len(X)
    H = np.eye(n) - np.ones((n, n)) / n
    K, L = H @ (X @ X.T) @ H, H @ (Y @ Y.T) @ H
    return (K * L).sum() / (np.linalg.norm(K) * np.linalg.norm(L))


def kernel_cka(K, L):
    """Centred kernel alignment (32.1) of two kernel matrices."""
    n = len(K)
    H = np.eye(n) - np.ones((n, n)) / n
    Kc, Lc = H @ K @ H, H @ L @ H
    return (Kc * Lc).sum() / (np.linalg.norm(Kc) * np.linalg.norm(Lc))


def rbf_gram(X, frac=0.5):
    """Gaussian kernel matrix with bandwidth frac times the median pairwise distance."""
    D2 = ((X[:, None, :] - X[None, :, :]) ** 2).sum(-1)
    s = frac * np.sqrt(np.median(D2[D2 > 0]))
    return np.exp(-D2 / (2 * s * s))


def cca(X, Y):
    """Canonical correlation analysis by the QR/SVD algorithm of Bjorck and Golub (Section 32.2.2.2).

    Returns the canonical correlations rho (descending), orthonormal bases Q, Qp of the column spaces of the
    centred matrices, the singular vectors A, B of Q^T Qp (so the canonical vectors are Q A and Qp B)."""
    Q, _, _ = _basis(center(X))
    Qp, _, _ = _basis(center(Y))
    A, rho, Bt = np.linalg.svd(Q.T @ Qp, full_matrices=False)
    return dict(rho=np.clip(rho, 0, 1), A=A, B=Bt.T, Q=Q, Qp=Qp)


def r2_cca(X, Y):
    """Mean squared canonical correlation (32.4)."""
    return float(np.mean(cca(X, Y)["rho"] ** 2))


def mean_rho(X, Y):
    """Mean canonical correlation (32.5)."""
    return float(np.mean(cca(X, Y)["rho"]))


def svcca(X, Y, k):
    """SVCCA (32.6): CCA after keeping the k leading principal components of each representation; returns the mean correlation."""
    Xc, Yc = center(X), center(Y)
    U, _, _ = _basis(Xc)
    Up, _, _ = _basis(Yc)
    k1, k2 = min(k, U.shape[1]), min(k, Up.shape[1])
    rho = np.linalg.svd(U[:, :k1].T @ Up[:, :k2], compute_uv=False)
    return float(np.mean(np.clip(rho, 0, 1)))


def pwcca(X, Y):
    """Projection-weighted CCA (32.7): weights alpha_i = ||(Y w'_i)^T Y||_1; asymmetric in (X, Y)."""
    c = cca(X, Y)
    Yc = center(Y)
    H = c["Qp"] @ c["B"]                      # canonical vectors of Y, unit length
    alpha = np.abs(H.T @ Yc).sum(1)
    p = len(c["rho"])
    return float((alpha[:p] * c["rho"]).sum() / alpha[:p].sum())


def r2_lr_cca(X, Y):
    """The variance-weighted CCA similarity (32.8): weights alpha'_i = sum_j (h_i^T y_j)^2, h_i the i-th canonical vector of Y."""
    c = cca(X, Y)
    Yc = center(Y)
    H = c["Qp"] @ c["B"]
    alpha = ((H.T @ Yc) ** 2).sum(1)
    p = len(c["rho"])
    return float((alpha[:p] * c["rho"] ** 2).sum() / alpha[:p].sum())


def r2_lr(X, Y):
    """Variance of Y explained by regressing each column of Y on X (32.9): 1 - ||Yc - Xc B||_F^2 / ||Yc||_F^2."""
    Xc, Yc = center(X), center(Y)
    B = np.linalg.lstsq(Xc, Yc, rcond=None)[0]
    return float(1 - np.linalg.norm(Yc - Xc @ B) ** 2 / np.linalg.norm(Yc) ** 2)


def _pcs(X):
    Xc = center(X)
    U, s, _ = _basis(Xc)
    return U, s ** 2


def r2_cca_pc(X, Y):
    """(32.10) with the division by p = min(rank X, rank Y)."""
    U, _ = _pcs(X)
    Up, _ = _pcs(Y)
    return float(((U.T @ Up) ** 2).sum() / min(U.shape[1], Up.shape[1]))


def r2_lr_pc(X, Y):
    """(32.11): sum_ij lambda'_j (u_i^T u'_j)^2 / sum_j lambda'_j."""
    U, _ = _pcs(X)
    Up, lam_p = _pcs(Y)
    return float((((U.T @ Up) ** 2) * lam_p[None, :]).sum() / lam_p.sum())


def cka_pc(X, Y):
    """(32.12): sum_ij lambda_i lambda'_j (u_i^T u'_j)^2 / (sqrt(sum lambda_i^2) sqrt(sum lambda'_j^2))."""
    U, lam = _pcs(X)
    Up, lam_p = _pcs(Y)
    num = (((U.T @ Up) ** 2) * lam[:, None] * lam_p[None, :]).sum()
    return float(num / (np.sqrt((lam ** 2).sum()) * np.sqrt((lam_p ** 2).sum())))


MEASURES = [("linear CKA", linear_cka), ("mean sq. CCA", r2_cca), ("mean CCA", mean_rho),
            ("SVCCA (k=10)", lambda X, Y: svcca(X, Y, 10)), ("PWCCA", pwcca), ("R2_LR", r2_lr_cca)]


# =========================================================================== data for the similarity experiments
def structured_reps(n, p, rng, latent=8, noise=0.5):
    """n x p representation with a decaying spectrum: a random linear image of a few latent factors plus noise."""
    Z = rng.standard_normal((n, latent)) * np.linspace(2.0, 0.5, latent)
    A = rng.standard_normal((latent, p)) / np.sqrt(latent)
    return Z @ A + noise * rng.standard_normal((n, p))


def random_orthogonal(p, rng):
    Q, R = np.linalg.qr(rng.standard_normal((p, p)))
    return Q * np.sign(np.diag(R))


def random_invertible(p, rng, cond=100.0):
    U, V = random_orthogonal(p, rng), random_orthogonal(p, rng)
    return U @ np.diag(np.logspace(0, -np.log10(cond), p)) @ V.T


TRANSFORMS = ["none (Y = X + noise)", "permute neurons", "rotate", "scale by 10", "scale neurons unevenly", "invertible linear map", "independent"]


def apply_transform(X, name, rng):
    n, p = X.shape
    if name.startswith("none"):
        return X
    if name == "permute neurons":
        return X[:, rng.permutation(p)]
    if name == "rotate":
        return X @ random_orthogonal(p, rng)
    if name == "scale by 10":
        return 10.0 * X
    if name == "scale neurons unevenly":
        return X * np.logspace(-1.5, 1.5, p)[None, :]
    if name == "invertible linear map":
        return X @ random_invertible(p, rng, 100.0)
    if name == "independent":
        return structured_reps(n, p, rng)
    raise ValueError(name)


def invariance_table(n=200, p=20, seed=0):
    rng = np.random.default_rng(seed)
    X = structured_reps(n, p, rng)
    Y0 = X + 0.8 * X.std() * rng.standard_normal(X.shape)     # a noisy copy: similarity is below 1, so invariances can be seen
    M = np.zeros((len(TRANSFORMS), len(MEASURES)))
    for i, t in enumerate(TRANSFORMS):
        Y = apply_transform(Y0, t, np.random.default_rng(100 + i))
        for j, (_, f) in enumerate(MEASURES):
            M[i, j] = f(X, Y)
    return M


def curse_curve(n=100, ps=None, reps=4, seed=0):
    """Similarity of two independent Gaussian representations as a function of the width p (n fixed)."""
    ps = ps if ps is not None else [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 98, 100, 110, 130, 160, 200]
    rng = np.random.default_rng(seed)
    out = {k: [] for k in ("R2cca", "meanrho", "cka", "R2lr")}
    for p in ps:
        v = {k: [] for k in out}
        for _ in range(reps):
            X, Y = rng.standard_normal((n, p)), rng.standard_normal((n, p))
            v["R2cca"].append(r2_cca(X, Y)); v["meanrho"].append(mean_rho(X, Y))
            v["cka"].append(linear_cka(X, Y)); v["R2lr"].append(r2_lr(X, Y))
        for k in out:
            out[k].append(np.mean(v[k]))
    return ps, {k: np.array(v) for k, v in out.items()}


def ridge_scale_experiment(scales, C_list=(1.0, 1e4), n_train=60, d=10, reps=20, seed=0):
    """Regularised logistic regression on features whose informative coordinate is multiplied by s
    (an invertible linear map): test accuracy as a function of s for each inverse regularisation C."""
    from sklearn.linear_model import LogisticRegression
    rng = np.random.default_rng(seed)
    acc = np.zeros((len(C_list), len(scales)))
    for r in range(reps):
        def draw(n):
            y = rng.integers(0, 2, n)
            X = rng.standard_normal((n, d))
            X[:, 0] += 1.2 * (2 * y - 1)
            return X, y
        Xtr, ytr = draw(n_train)
        Xte, yte = draw(4000)
        for i, C in enumerate(C_list):
            for j, s in enumerate(scales):
                sc = np.ones(d); sc[0] = s
                m = LogisticRegression(C=C, max_iter=5000).fit(Xtr * sc, ytr)
                acc[i, j] += m.score(Xte * sc, yte) / reps
    return acc


# =========================================================================== Section 32.3.2.3: autoencoders
def aeData(n=2000, d=10, seed=0):
    rng = np.random.default_rng(seed)
    lam = np.array([4.0, 2.5, 1.5, 0.8, 0.4, 0.2, 0.1, 0.08, 0.05, 0.03])[:d]
    Q = random_orthogonal(d, rng)
    X = (rng.standard_normal((n, d)) * np.sqrt(lam)) @ Q.T
    return X


def linear_ae(X, k, steps=4000, lr=0.02, seed=0, init=0.1, init_d=0.6):
    """Full-batch gradient descent on L = (1/n)||X - X We^T Wd^T||_F^2 using the sample covariance S.
    Returns the loss curve, the maximal sine of the principal angle to the top-k eigenspace, We and Wd."""
    n, d = X.shape
    S = X.T @ X / n
    ev, V = np.linalg.eigh(S)
    ev, V = ev[::-1], V[:, ::-1]
    P = V[:, :k] @ V[:, :k].T
    rng = np.random.default_rng(seed)
    We, Wd = init * rng.standard_normal((k, d)), init_d * rng.standard_normal((d, k))
    loss, ang = [], []
    I = np.eye(d)
    for t in range(steps + 1):
        M = Wd @ We
        if t % 20 == 0 or t == steps:
            loss.append((t, np.trace(S) - 2 * np.trace(M @ S) + np.trace(M @ S @ M.T)))
            Qd, _ = np.linalg.qr(Wd)
            ang.append((t, np.linalg.norm((I - P) @ Qd, 2)))
        G = (M - I) @ S
        gd, ge = 2 * G @ We.T, 2 * Wd.T @ G
        Wd = Wd - lr * gd
        We = We - lr * ge
    return np.array(loss), np.array(ang), We, Wd, ev, V


def dae_gains(X, sigma, rng, n_noisy=200000):
    """Least-squares linear denoiser from x + sigma*eps to x, returned in the principal basis: its diagonal gains."""
    n, d = X.shape
    S = X.T @ X / n
    ev, V = np.linalg.eigh(S)
    ev, V = ev[::-1], V[:, ::-1]
    idx = rng.integers(0, n, n_noisy)
    Xc = X[idx]
    Xn = Xc + sigma * rng.standard_normal(Xc.shape)
    A = np.linalg.lstsq(Xn, Xc, rcond=None)[0].T          # x ~ A xn
    G = V.T @ A @ V
    return ev, np.diag(G), np.abs(G - np.diag(np.diag(G))).max()


# =========================================================================== Section 32.3.4: contrastive objectives
def gauss_critic(x, y, rho):
    """Optimal critic log p(y|x)/p(y) for a standard bivariate Gaussian pair with correlation rho."""
    return -0.5 * np.log(1 - rho ** 2) - (y - rho * x) ** 2 / (2 * (1 - rho ** 2)) + y ** 2 / 2


def infonce_bound(f, K):
    """log K - L_InfoNCE for a critic matrix f[n, K] whose column 0 holds the positive."""
    return np.log(K) + np.mean(f[:, 0] - logsumexp(f, axis=1))


def infonce_gauss(rho, K, N, rng):
    x = rng.standard_normal(N)
    y1 = rho * x + np.sqrt(1 - rho ** 2) * rng.standard_normal(N)
    cand = np.concatenate([y1[:, None], rng.standard_normal((N, K - 1))], 1)
    return infonce_bound(gauss_critic(x[:, None], cand, rho), K)


def infonce_noisy_encoder(sig, K, N, rng):
    """x ~ N(0,1), z = x + sig*eps (a deterministic encoder plus a little noise); the optimal critic is log N(z; x, sig^2) - log N(z; 0, 1+sig^2)."""
    x = rng.standard_normal(N)
    z1 = x + sig * rng.standard_normal(N)
    cand = np.concatenate([z1[:, None], np.sqrt(1 + sig ** 2) * rng.standard_normal((N, K - 1))], 1)
    f = (stats.norm.logpdf(cand, x[:, None], sig) - stats.norm.logpdf(cand, 0, np.sqrt(1 + sig ** 2)))
    return infonce_bound(f, K)


def nt_xent_loss_np(Z1, Z2, tau):
    """SimCLR loss for L2-normalised views Z1, Z2 (n x d): the average of the InfoNCE losses over the 2n anchors."""
    n = len(Z1)
    Z = np.concatenate([Z1, Z2])
    S = Z @ Z.T / tau
    np.fill_diagonal(S, -np.inf)
    pos = np.concatenate([np.arange(n, 2 * n), np.arange(0, n)])
    return float(np.mean(logsumexp(S, axis=1) - S[np.arange(2 * n), pos]))


def barlow_loss_np(Z1, Z2, lam):
    """Barlow Twins loss (32.18) from two batches of features."""
    a = (Z1 - Z1.mean(0)) / Z1.std(0)
    b = (Z2 - Z2.mean(0)) / Z2.std(0)
    C = a.T @ b / len(a)
    off = C - np.diag(np.diag(C))
    return float(((1 - np.diag(C)) ** 2).sum() + lam * (off ** 2).sum())


def dino_loss_np(S, T, tau_s, tau_t, center=True):
    """DINO-type loss (32.20) in numpy. S: student logits, T: teacher (target network) logits, both (batch, K).
    The teacher output is centred by the batch mean (optional) and sharpened with the lower temperature tau_t;
    the loss is the cross-entropy H(q_teacher, p_student) averaged over the batch."""
    Tc = T - T.mean(0, keepdims=True) if center else T
    q = np.exp(Tc / tau_t - logsumexp(Tc / tau_t, axis=1, keepdims=True))
    logp = S / tau_s - logsumexp(S / tau_s, axis=1, keepdims=True)
    return float(-(q * logp).sum(1).mean())


# ----- one-mode linear model of BYOL / SimSiam (Section 32.3.4.3; gap 32.11 of the notes)
def byol_rhs(u, mode, a, b, eta):
    """Gradient-flow vector field of the one-mode model. u = (w, p, wt); a = E x1^2, b = E x1 x2."""
    w, p, wt = u
    if mode == "sg":                  # stop-gradient (SimSiam) with a predictor
        e = p * a - b
        return np.array([-2 * p * w * e, -2 * w * w * e, 0.0])
    if mode == "nopred":              # stop-gradient, no predictor (p = 1)
        return np.array([-2 * w * (a - b), 0.0, 0.0])
    if mode == "nosg":                # no stop-gradient, with predictor
        return np.array([-2 * w * (p * p * a - 2 * p * b + a), -2 * w * w * (p * a - b), 0.0])
    if mode == "ema":                 # moving-average target (BYOL) with a predictor
        e = p * w * a - wt * b
        return np.array([-2 * p * e, -2 * w * e, eta * (w - wt)])
    if mode == "ema_nopred":
        e = w * a - wt * b
        return np.array([-2 * e, 0.0, eta * (w - wt)])
    raise ValueError(mode)


def byol_run(mode, w0, p0, a, b, eta=0.1, T=20.0, dt=0.01):
    """RK4 integration; returns the times and the trajectory (w, p, wt)."""
    n = int(round(T / dt))
    u = np.array([w0, p0, w0], dtype=float)
    out = np.zeros((n + 1, 3))
    out[0] = u
    for i in range(n):
        k1 = byol_rhs(u, mode, a, b, eta)
        k2 = byol_rhs(u + 0.5 * dt * k1, mode, a, b, eta)
        k3 = byol_rhs(u + 0.5 * dt * k2, mode, a, b, eta)
        k4 = byol_rhs(u + dt * k3, mode, a, b, eta)
        u = u + dt / 6 * (k1 + 2 * k2 + 2 * k3 + k4)
        out[i + 1] = u
    return np.arange(n + 1) * dt, out


# =========================================================================== Section 32.4.1: identifiability
def entangle(Z, theta=np.pi / 4, dist=None):
    """The measure-preserving map of a factorial prior with CDF F that mixes the coordinates:
    z -> F^-1(Phi(R Phi^-1(F(z)))), R a rotation by theta. Returns z' and the Jacobian dz'/dz."""
    dist = dist or stats.laplace
    U = np.clip(dist.cdf(Z), 1e-12, 1 - 1e-12)
    G = stats.norm.ppf(U)
    c, s = np.cos(theta), np.sin(theta)
    R = np.array([[c, -s], [s, c]])
    Gp = G @ R.T
    Zp = dist.ppf(np.clip(stats.norm.cdf(Gp), 1e-12, 1 - 1e-12))
    J = np.empty((len(Z), 2, 2))
    for i in range(2):
        for j in range(2):
            J[:, i, j] = (stats.norm.pdf(Gp[:, i]) / dist.pdf(Zp[:, i])) * R[i, j] * (dist.pdf(Z[:, j]) / stats.norm.pdf(G[:, j]))
    return Zp, J


# =========================================================================== torch experiments
def _torch():
    return common.torch_setup(0)


def digits_split(seed=0, n_train=1200):
    from sklearn.datasets import load_digits
    X, y = load_digits(return_X_y=True)
    X = X / 16.0
    perm = np.random.default_rng(seed).permutation(len(X))
    tr, te = perm[:n_train], perm[n_train:]
    return X[tr], y[tr], X[te], y[te]


def mlp(torch, sizes, last_act=False):
    layers = []
    for i in range(len(sizes) - 1):
        layers.append(torch.nn.Linear(sizes[i], sizes[i + 1]))
        if i < len(sizes) - 2 or last_act:
            layers.append(torch.nn.ReLU())
    return torch.nn.Sequential(*layers)


# ---- Figure 32.1: representation learning makes the classes linearly separable
def train_bottleneck(X, y, vib=False, beta=1e-2, epochs=1500, seed=0):
    torch = common.torch_setup(seed)
    Xt, yt = torch.tensor(X, dtype=torch.float32), torch.tensor(y)
    if vib:
        enc = mlp(torch, [2, 64, 64, 4])
        clf = torch.nn.Linear(2, 2)
        params = list(enc.parameters()) + list(clf.parameters())
    else:
        enc = mlp(torch, [2, 64, 64, 2])
        clf = torch.nn.Linear(2, 2)
        params = list(enc.parameters()) + list(clf.parameters())
    opt = torch.optim.Adam(params, lr=3e-3)
    for ep in range(epochs):
        opt.zero_grad()
        h = enc(Xt)
        if vib:
            mu, logvar = h[:, :2], h[:, 2:]
            z = mu + torch.exp(0.5 * logvar) * torch.randn_like(mu)
            kl = 0.5 * (mu ** 2 + logvar.exp() - 1 - logvar).sum(1).mean()
            loss = torch.nn.functional.cross_entropy(clf(z), yt) + beta * kl
        else:
            loss = torch.nn.functional.cross_entropy(clf(h), yt)
        loss.backward()
        opt.step()
    with torch.no_grad():
        h = enc(Xt)
        rep = h[:, :2] if vib else h
    return rep.numpy()


def fig_32_1(out_dir=None):
    """Figure 32.1: data whose classes are mixed (left) and learned representations in which they are not."""
    from sklearn.datasets import make_circles
    from sklearn.linear_model import LogisticRegression
    X, y = make_circles(n_samples=500, noise=0.06, factor=0.45, random_state=0)
    reps = _cached("fig32-1", lambda: dict(sup=train_bottleneck(X, y, False), vib=train_bottleneck(X, y, True, beta=1e-2)))
    fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.1))
    cols = np.array([RED, BLUE])
    for ax, (title, Z) in zip(axes, [("input data", X), ("supervised bottleneck", reps["sup"]), ("variational bottleneck (beta=0.01)", reps["vib"])]):
        acc = LogisticRegression(max_iter=3000).fit(Z, y).score(Z, y)
        ax.scatter(Z[:, 0], Z[:, 1], c=cols[y], s=8, alpha=0.8, linewidths=0)
        ax.set_title("%s\nlinear accuracy %.3f" % (title, acc), fontsize=9.5)
        ax.set_xticks([]); ax.set_yticks([])
        ax.set_aspect("equal", adjustable="datalim")
    fig.tight_layout()
    savefig(fig, CH, "fig32-1", out_dir)


# ---- Figure x32.a: invariances, curse of dimensionality, regularised linear evaluation
def fig_x32_a(out_dir=None):
    """(a) similarity measures under transformations of a representation; (b) independent representations of width p, n = 100; (c) regularised linear evaluation after an invertible rescaling."""
    M = invariance_table()
    ps, cur = curse_curve()
    scales = np.logspace(-2, 2, 13)
    acc = _cached("figx32-a-ridge", lambda: ridge_scale_experiment(scales))
    fig = plt.figure(figsize=(12.5, 3.7))
    gs = fig.add_gridspec(1, 3, width_ratios=[1.35, 1, 1], wspace=0.35)
    ax = fig.add_subplot(gs[0])
    im = ax.imshow(M, vmin=0, vmax=1, cmap="viridis", aspect="auto")
    ax.set_xticks(range(len(MEASURES))); ax.set_xticklabels([m[0] for m in MEASURES], rotation=35, ha="right", fontsize=8)
    ax.set_yticks(range(len(TRANSFORMS))); ax.set_yticklabels(TRANSFORMS, fontsize=8.5)
    for i in range(M.shape[0]):
        for j in range(M.shape[1]):
            ax.text(j, i, "%.2f" % M[i, j], ha="center", va="center", fontsize=8, color="white" if M[i, j] < 0.6 else "black")
    ax.set_title("(a) similarity of X with T(Y), Y = noisy copy of X, n = 200, p = 20", fontsize=9.5)
    ax = fig.add_subplot(gs[1])
    ax.plot(ps, cur["R2cca"], "-o", ms=3, color=RED, label="mean squared CCA")
    ax.plot(ps, cur["meanrho"], "-o", ms=3, color=ORANGE, label="mean CCA")
    ax.plot(ps, cur["R2lr"], "-o", ms=3, color=PURPLE, label="$R^2_{LR}$")
    ax.plot(ps, cur["cka"], "-o", ms=3, color=BLUE, label="linear CKA")
    ax.axvline(99, color=GREY, ls=":"); ax.text(101, 0.05, "p = n-1", fontsize=8, color=GREY)
    ax.set_xlabel("width $p$ of both representations"); ax.set_ylabel("similarity of independent data")
    ax.set_title("(b) the curse of dimensionality, n = 100", fontsize=9.5); ax.legend(fontsize=7.5, loc="center right")
    ax = fig.add_subplot(gs[2])
    for a, C, c in zip(acc, (1.0, 1e4), (RED, BLUE)):
        ax.semilogx(scales, a, "-o", ms=3, color=c, label="C = %g" % C)
    ax.set_xlabel("scale $s$ of the informative feature"); ax.set_ylabel("test accuracy")
    ax.set_title("(c) linear evaluation is not invariant", fontsize=9.5); ax.legend(fontsize=8, title="inverse penalty", title_fontsize=8)
    savefig(fig, CH, "figx32-a", out_dir)


# ---- Figure x32.b: matching layers of two networks (the sanity check of Section 32.2.2.3)
def layer_activations(seed, X, y, Xeval, width=100, depth=5, epochs=60):
    torch = common.torch_setup(seed)
    sizes = [64] + [width] * depth + [10]
    layers = []
    for i in range(len(sizes) - 1):
        layers.append(torch.nn.Linear(sizes[i], sizes[i + 1]))
        if i < len(sizes) - 2:
            layers.append(torch.nn.ReLU())
    net = torch.nn.Sequential(*layers)
    opt = torch.optim.Adam(net.parameters(), lr=2e-3)
    Xt, yt = torch.tensor(X, dtype=torch.float32), torch.tensor(y)
    for ep in range(epochs):
        perm = torch.randperm(len(Xt))
        for i in range(0, len(Xt), 64):
            b = perm[i:i + 64]
            opt.zero_grad()
            torch.nn.functional.cross_entropy(net(Xt[b]), yt[b]).backward()
            opt.step()
    acts, h = [], torch.tensor(Xeval, dtype=torch.float32)
    with torch.no_grad():
        for m in net[:-1]:
            h = m(h)
            if isinstance(m, torch.nn.ReLU):
                acts.append(h.numpy().copy())
    return acts


def layer_similarity(actsA, actsB, f):
    return np.array([[f(a, b) for b in actsB] for a in actsA])


def sanity_scores(S):
    """Fraction of layers of network A whose most similar layer of network B is the corresponding one."""
    return float(np.mean(np.argmax(S, axis=1) == np.arange(len(S))))


def layer_experiment(n_eval, pairs=5):
    Xtr, ytr, Xte, yte = digits_split(0)
    rng = np.random.default_rng(1)
    idx = rng.permutation(len(Xte))[:n_eval]
    Xev = Xte[idx] if n_eval <= len(Xte) else Xtr[:n_eval]
    fs = {"linear CKA": linear_cka, "mean sq. CCA": r2_cca, "SVCCA (k=20)": lambda a, b: svcca(a, b, 20), "PWCCA": pwcca}
    mats = {k: [] for k in fs}
    for pr in range(pairs):
        A = layer_activations(10 + 2 * pr, Xtr, ytr, Xev)
        B = layer_activations(11 + 2 * pr, Xtr, ytr, Xev)
        for k, f in fs.items():
            mats[k].append(layer_similarity(A, B, f))
    return {k: (np.mean(v, 0), np.mean([sanity_scores(s) for s in v])) for k, v in mats.items()}


def fig_x32_b(out_dir=None):
    """Layer-by-layer similarity of two identical networks trained from different initialisations, with n = 397 and n = 60 examples."""
    res = _cached("figx32-b", lambda: {n: layer_experiment(n) for n in (397, 60)})
    names = ["linear CKA", "mean sq. CCA", "SVCCA (k=20)", "PWCCA"]
    fig, axes = plt.subplots(2, 4, figsize=(11.5, 5.4))
    for r, n in enumerate((397, 60)):
        for c, nm in enumerate(names):
            S, sc = res[n][nm]
            ax = axes[r, c]
            ax.imshow(S, vmin=0, vmax=1, cmap="viridis")
            for i in range(S.shape[0]):
                for j in range(S.shape[1]):
                    ax.text(j, i, "%.2f" % S[i, j], ha="center", va="center", fontsize=6.5, color="white" if S[i, j] < 0.6 else "black")
            ax.set_xticks(range(5)); ax.set_xticklabels(range(1, 6)); ax.set_yticks(range(5)); ax.set_yticklabels(range(1, 6))
            ax.set_title("%s, n = %d\nlayers matched %.2f" % (nm, n, sc), fontsize=8.5)
            if c == 0:
                ax.set_ylabel("layer of network A")
            if r == 1:
                ax.set_xlabel("layer of network B")
    fig.tight_layout()
    savefig(fig, CH, "figx32-b", out_dir)


# ---- Figure x32.c: linear autoencoder, denoising autoencoder
def fig_x32_c(out_dir=None):
    """(a) loss of a linear autoencoder with k = 3 against the PCA optimum; (b) angle to the principal subspace and the singular values of the decoder; (c) gains of a linear denoising autoencoder."""
    X = aeData()
    k = 3
    loss, ang, We, Wd, ev, V = linear_ae(X, k)
    floor = ev[k:].sum()
    rng = np.random.default_rng(0)
    sigs = (0.3, 0.7, 1.4)
    gains = [dae_gains(X, s, rng) for s in sigs]
    fig, axes = plt.subplots(1, 3, figsize=(12.0, 3.4))
    ax = axes[0]
    ax.semilogy(loss[:, 0], loss[:, 1] - floor + 1e-12, color=BLUE)
    ax.set_xlabel("gradient step"); ax.set_ylabel("loss minus PCA optimum")
    ax.set_title("(a) linear autoencoder, k = 3\noptimum = sum of the discarded eigenvalues = %.3f" % floor, fontsize=9)
    ax = axes[1]
    ax.semilogy(ang[:, 0], ang[:, 1], color=RED, label="sin of largest principal angle")
    ax.set_xlabel("gradient step"); ax.set_ylabel("angle to top-3 eigenspace")
    cosv = np.abs(np.sum((Wd / np.linalg.norm(Wd, axis=0)) * V[:, :k], axis=0))
    ax.set_title("(b) the subspace converges, but decoder columns vs. eigenvectors:\n|cos| = %s (weights are not the eigenvectors)" % np.array2string(cosv, precision=2), fontsize=9)
    ax = axes[2]
    cs = (ORANGE, GREEN, PURPLE)
    lam = np.linspace(0, ev.max(), 200)
    for s, (e, g, off), c in zip(sigs, gains, cs):
        ax.plot(lam, lam / (lam + s * s), color=c, lw=1.2)
        ax.plot(e, g, "o", color=c, ms=5, label=r"$\sigma$ = %.1f" % s)
    ax.set_xlabel(r"eigenvalue $\lambda_i$ of the data"); ax.set_ylabel("gain of the denoiser")
    ax.set_title(r"(c) learned gain vs. $\lambda/(\lambda+\sigma^2)$ (curves)", fontsize=9); ax.legend(fontsize=8)
    fig.tight_layout()
    savefig(fig, CH, "figx32-c", out_dir)


# ---- Figure x32.d: InfoNCE bound
def fig_x32_d(out_dir=None):
    """(a) The InfoNCE bound log K - L against K for Gaussian pairs with correlation rho; (b) a deterministic encoder plus noise sigma: true information against estimates."""
    rng = np.random.default_rng(0)
    Ks = np.array([2, 4, 8, 16, 32, 64, 128, 256, 512, 1024])
    rhos = (0.5, 0.9, 0.99)
    fig, axes = plt.subplots(1, 2, figsize=(9.6, 3.6))
    ax = axes[0]
    for rho, c in zip(rhos, (BLUE, GREEN, RED)):
        est = [infonce_gauss(rho, int(K), 20000, rng) for K in Ks]
        ax.semilogx(Ks, est, "-o", ms=3.5, color=c, label=r"$\rho$ = %g" % rho)
        ax.axhline(-0.5 * np.log(1 - rho ** 2), color=c, ls=":", lw=1)
    ax.semilogx(Ks, np.log(Ks), color=GREY, ls="--", label=r"$\log K$")
    ax.set_xlabel("number of candidates $K$ (one positive, $K-1$ negatives)"); ax.set_ylabel("nats")
    ax.set_ylim(0, 4.2); ax.legend(fontsize=8)
    ax.set_title("(a) InfoNCE bound with the optimal critic\n(dotted: true information)", fontsize=9.5)
    ax = axes[1]
    sig = np.logspace(0, -2, 15)
    true = 0.5 * np.log(1 + 1 / sig ** 2)
    ax.semilogx(sig, true, color="k", lw=1.6, label="true $I(X;Z)$")
    for K, c in zip((8, 64, 512), (BLUE, GREEN, RED)):
        est = [infonce_noisy_encoder(s, K, 20000, rng) for s in sig]
        ax.semilogx(sig, est, "-o", ms=3, color=c, label="InfoNCE, K = %d" % K)
    ax.invert_xaxis(); ax.set_xlabel(r"encoder noise $\sigma$ (decreasing to the right)"); ax.set_ylabel("nats")
    ax.legend(fontsize=8); ax.set_title(r"(b) $z = x + \sigma\varepsilon$: the estimate stops at $\log K$", fontsize=9.5)
    fig.tight_layout()
    savefig(fig, CH, "figx32-d", out_dir)


# ---- Figure x32.e: self-supervised methods on the digits
AUG_SHIFTS = [(dy, dx) for dy in (-1, 0, 1) for dx in (-1, 0, 1)]


def _shift_table(torch, X):
    """All nine one-pixel shifts of every image (zero filled): (9, N, 64)."""
    N = len(X)
    img = X.view(N, 8, 8)
    pad = torch.nn.functional.pad(img, (1, 1, 1, 1))
    return torch.stack([pad[:, 1 + dy:9 + dy, 1 + dx:9 + dx].reshape(N, 64) for dy, dx in AUG_SHIFTS])


def _augment(torch, table, idx, g):
    n = len(idx)
    s = torch.randint(0, 9, (n,), generator=g)
    x = table[s, idx]
    x = x * (0.8 + 0.4 * torch.rand(n, 1, generator=g)) + 0.08 * torch.randn(n, 64, generator=g)
    return x * (torch.rand(n, 64, generator=g) > 0.12).float()


def pretrain(method, Xtr, ytr, seed, steps=700, bs=256, dim=32):
    """Pretrain an MLP encoder 64-256-dim on the unlabelled digits (labels only for the supervised reference) and return it."""
    torch = common.torch_setup(seed)
    F = torch.nn.functional
    X = torch.tensor(Xtr, dtype=torch.float32)
    y = torch.tensor(ytr)
    N = len(X)
    enc = mlp(torch, [64, 256, dim])
    if method in ("random", "pixels", "pca"):
        return enc
    g = torch.Generator().manual_seed(seed)
    table = _shift_table(torch, X)
    extra = []
    if method in ("ae", "dae", "mae"):
        dec = mlp(torch, [dim, 256, 64]); extra = list(dec.parameters())
    elif method == "rotation":
        head = mlp(torch, [dim, 64, 4]); extra = list(head.parameters())
    elif method == "supervised":
        head = torch.nn.Linear(dim, 10); extra = list(head.parameters())
    elif method in ("simclr", "barlow"):
        proj = mlp(torch, [dim, 128, 64 if method == "barlow" else 32]); extra = list(proj.parameters())
    elif method in ("byol", "byol_nobn"):
        def head(i, h, o):          # projector and predictor; batch normalisation in the hidden layer unless method is byol_nobn
            L = [torch.nn.Linear(i, h)]
            if method == "byol":
                L.append(torch.nn.BatchNorm1d(h))
            return torch.nn.Sequential(*(L + [torch.nn.ReLU(), torch.nn.Linear(h, o)]))
        proj = head(dim, 128, 32); pred = head(32, 64, 32)
        import copy
        enc_t, proj_t = copy.deepcopy(enc), copy.deepcopy(proj)
        extra = list(proj.parameters()) + list(pred.parameters())
    else:
        raise ValueError(method)
    opt = torch.optim.Adam(list(enc.parameters()) + extra, lr=2e-3)
    for it in range(steps):
        idx = torch.randint(0, N, (bs,), generator=g)
        xb = X[idx]
        opt.zero_grad()
        if method == "ae":
            loss = F.mse_loss(dec(enc(xb)), xb)
        elif method == "dae":
            loss = F.mse_loss(dec(enc(xb + 0.4 * torch.randn(xb.shape, generator=g))), xb)
        elif method == "mae":
            m = (torch.rand(xb.shape, generator=g) < 0.5).float()          # 1 = masked
            loss = (((dec(enc(xb * (1 - m))) - xb) ** 2) * m).sum() / m.sum()
        elif method == "rotation":
            k = torch.randint(0, 4, (bs,), generator=g)
            xr = torch.stack([torch.rot90(xb[i].view(8, 8), int(k[i])).reshape(64) for i in range(bs)])
            loss = F.cross_entropy(head(enc(xr)), k)
        elif method == "supervised":
            loss = F.cross_entropy(head(enc(xb)), y[idx])
        elif method == "simclr":
            z1 = F.normalize(proj(enc(_augment(torch, table, idx, g))), dim=1)
            z2 = F.normalize(proj(enc(_augment(torch, table, idx, g))), dim=1)
            Z = torch.cat([z1, z2])
            S = Z @ Z.T / 0.2
            S = S.masked_fill(torch.eye(2 * bs, dtype=torch.bool), float("-inf"))
            tgt = torch.cat([torch.arange(bs, 2 * bs), torch.arange(0, bs)])
            loss = F.cross_entropy(S, tgt)
        elif method == "barlow":
            a = proj(enc(_augment(torch, table, idx, g))); b = proj(enc(_augment(torch, table, idx, g)))
            a = (a - a.mean(0)) / (a.std(0) + 1e-6); b = (b - b.mean(0)) / (b.std(0) + 1e-6)
            C = a.T @ b / bs
            on = ((1 - torch.diagonal(C)) ** 2).sum()
            off = (C ** 2).sum() - (torch.diagonal(C) ** 2).sum()
            loss = on + 5e-3 * off * 4
        elif method in ("byol", "byol_nobn"):
            v1, v2 = _augment(torch, table, idx, g), _augment(torch, table, idx, g)
            with torch.no_grad():
                t1, t2 = F.normalize(proj_t(enc_t(v1)), dim=1), F.normalize(proj_t(enc_t(v2)), dim=1)
            p1 = F.normalize(pred(proj(enc(v1))), dim=1); p2 = F.normalize(pred(proj(enc(v2))), dim=1)
            loss = (2 - 2 * (p1 * t2).sum(1)).mean() + (2 - 2 * (p2 * t1).sum(1)).mean()
        loss.backward()
        opt.step()
        if method in ("byol", "byol_nobn"):
            with torch.no_grad():
                for net, tgt_net in ((enc, enc_t), (proj, proj_t)):
                    for p, pt in zip(net.parameters(), tgt_net.parameters()):
                        pt.mul_(0.99).add_(0.01 * p)
    return enc


def represent(method, enc, Xtr, X):
    if method == "pixels":
        return X
    if method == "pca":
        from sklearn.decomposition import PCA
        return PCA(32, random_state=0).fit(Xtr).transform(X)
    torch = common.torch_setup(0)
    with torch.no_grad():
        return enc(torch.tensor(X, dtype=torch.float32)).numpy()


def linear_eval(Ztr, ytr, Zte, yte, per_class, rng, reps=5):
    from sklearn.linear_model import LogisticRegression
    mu, sd = Ztr.mean(0), Ztr.std(0) + 1e-6
    A, B = (Ztr - mu) / sd, (Zte - mu) / sd
    acc = []
    for _ in range(reps):
        idx = np.concatenate([rng.choice(np.where(ytr == c)[0], per_class, replace=False) for c in range(10)])
        acc.append(LogisticRegression(C=1.0, max_iter=3000).fit(A[idx], ytr[idx]).score(B, yte))
    return float(np.mean(acc))


SSL_METHODS = ["pixels", "random", "pca", "ae", "dae", "mae", "rotation", "simclr", "barlow", "byol", "byol_nobn", "supervised"]
SSL_NAMES = {"pixels": "raw pixels", "random": "random encoder", "pca": "PCA (32)", "ae": "autoencoder", "dae": "denoising AE",
             "mae": "masked prediction", "rotation": "rotation prediction", "simclr": "SimCLR (InfoNCE)", "barlow": "Barlow Twins",
             "byol": "BYOL", "byol_nobn": "BYOL without batch norm", "supervised": "supervised (all labels)"}


def ssl_experiment(seeds=(0, 1, 2), per_class=(2, 10, 50)):
    res = {m: np.zeros((len(seeds), len(per_class))) for m in SSL_METHODS}
    for si, seed in enumerate(seeds):
        Xtr, ytr, Xte, yte = digits_split(seed)
        for m in SSL_METHODS:
            t0 = time.time()
            enc = pretrain(m, Xtr, ytr, seed)
            Ztr, Zte = represent(m, enc, Xtr, Xtr), represent(m, enc, Xtr, Xte)
            for pi, pc in enumerate(per_class):
                res[m][si, pi] = linear_eval(Ztr, ytr, Zte, yte, pc, np.random.default_rng(100 + seed))
            print("   seed %d %-11s %s  (%.1f s)" % (seed, m, np.round(res[m][si], 3), time.time() - t0), flush=True)
    return res


def fig_x32_e(out_dir=None):
    """Linear-evaluation accuracy of encoders pretrained with different tasks on 1200 unlabelled digits, for 2, 10 and 50 labelled examples per class."""
    res = _cached("figx32-e", ssl_experiment)
    per_class = (2, 10, 50)
    fig, ax = plt.subplots(figsize=(9.6, 4.0))
    w = 0.26
    cols = (CYAN, BLUE, PURPLE)
    x = np.arange(len(SSL_METHODS))
    for j, (pc, c) in enumerate(zip(per_class, cols)):
        m = np.array([res[k][:, j].mean() for k in SSL_METHODS]); s = np.array([res[k][:, j].std() for k in SSL_METHODS])
        ax.bar(x + (j - 1) * w, m, w, yerr=s, color=c, label="%d labels per class" % pc, error_kw=dict(lw=0.8))
    ax.set_xticks(x); ax.set_xticklabels([SSL_NAMES[k] for k in SSL_METHODS], rotation=30, ha="right", fontsize=8.5)
    ax.set_ylabel("linear evaluation accuracy (test)"); ax.set_ylim(0.3, 1.0); ax.legend(fontsize=8.5, loc="lower center", bbox_to_anchor=(0.5, 1.0), ncol=3, frameon=False)
    fig.tight_layout()
    savefig(fig, CH, "figx32-e", out_dir)


# ---- Figure x32.f: non-identifiability of a factorial latent model
def fig_x32_f(out_dir=None):
    """A measure-preserving map of a factorial Laplace prior that mixes both coordinates."""
    rng = np.random.default_rng(0)
    Z = rng.laplace(size=(3000, 2))
    Zp, J = entangle(Z)
    fig, axes = plt.subplots(1, 4, figsize=(12.6, 3.2))
    ax = axes[0]
    ax.scatter(Z[:, 0], Z[:, 1], c=Z[:, 0], cmap="coolwarm", s=4, vmin=-3, vmax=3, rasterized=True)
    ax.set_title(r"(a) $z$ (colour: $z_1$)", fontsize=9.5); ax.set_xlim(-6, 6); ax.set_ylim(-6, 6)
    ax = axes[1]
    ax.scatter(Zp[:, 0], Zp[:, 1], c=Z[:, 0], cmap="coolwarm", s=4, vmin=-3, vmax=3, rasterized=True)
    ax.set_title(r"(b) $z'=T(z)$ (colour: $z_1$)", fontsize=9.5); ax.set_xlim(-6, 6); ax.set_ylim(-6, 6)
    ax = axes[2]
    xs = np.linspace(-6, 6, 300)
    for i, c in enumerate((BLUE, RED)):
        ax.hist(Zp[:, i], bins=60, range=(-6, 6), density=True, histtype="step", color=c, label="$z'_%d$" % (i + 1))
    ax.plot(xs, stats.laplace.pdf(xs), "k--", lw=1, label="Laplace(0,1)")
    ax.set_title("(c) same factorial marginals", fontsize=9.5); ax.legend(fontsize=8)
    ax = axes[3]
    ax.scatter(Z[:, 0], Zp[:, 0], s=3, color=BLUE, alpha=0.5, label=r"$z'_1$", rasterized=True)
    ax.scatter(Z[:, 0], Zp[:, 1], s=3, color=RED, alpha=0.5, label=r"$z'_2$", rasterized=True)
    ax.set_xlabel("$z_1$"); ax.set_title(r"(d) both $z'_1$ and $z'_2$ depend on $z_1$", fontsize=9.5); ax.legend(fontsize=8, markerscale=3)
    for a in axes[:2]:
        a.set_aspect("equal")
    fig.tight_layout()
    savefig(fig, CH, "figx32-f", out_dir)


# ---- Figure x32.g: one-mode model of BYOL / SimSiam
def fig_x32_g(out_dir=None):
    """(a) phase plane of the stop-gradient model with a predictor; (b) five variants from one starting point."""
    s, sig2 = 1.0, 0.5
    a, b = s + sig2, s
    q = b / a
    fig, axes = plt.subplots(1, 2, figsize=(10.4, 3.9))
    ax = axes[0]
    w0s = np.linspace(0.15, 1.6, 7)
    p0s = (0.25, 0.7, 1.1, 1.5)
    xs = np.linspace(0, 1.7, 200)
    ax.fill_between(xs, np.sqrt(xs ** 2 + q * q), 1.9, color=RED, alpha=0.10)
    ax.plot(xs, np.sqrt(xs ** 2 + q * q), color=RED, lw=1, ls="--")
    ax.text(0.05, 1.75, "collapse: $p_0^2\\geq w_0^2+(b/a)^2$", color=RED, fontsize=8.5)
    ax.axhline(q, color=GREY, ls=":"); ax.text(1.3, q - 0.09, "$p^*=b/a$", color=GREY, fontsize=8.5)
    for w0 in w0s:
        for p0 in p0s:
            t, u = byol_run("sg", w0, p0, a, b, T=30, dt=0.02)
            collapse = p0 ** 2 >= w0 ** 2 + q * q
            ax.plot(u[:, 0], u[:, 1], color=RED if collapse else BLUE, lw=0.8, alpha=0.8)
            ax.plot(u[0, 0], u[0, 1], "o", ms=2.5, color=RED if collapse else BLUE)
    ax.set_xlim(0, 1.7); ax.set_ylim(0, 1.9)
    ax.set_xlabel("encoder weight $w$"); ax.set_ylabel("predictor weight $p$")
    ax.set_title("(a) stop-gradient + predictor: $w^2-p^2$ is conserved\nand $p\\to b/a$ (here %.3f)" % q, fontsize=9)
    ax = axes[1]
    modes = [("sg", "stop-gradient + predictor", BLUE), ("ema", "moving-average target + predictor", GREEN), ("nopred", "stop-gradient, no predictor", RED),
             ("nosg", "no stop-gradient + predictor", ORANGE), ("ema_nopred", "moving-average target, no predictor", PURPLE)]
    for mname, lab, c in modes:
        t, u = byol_run(mname, 1.0, 0.5, a, b, eta=0.2, T=30, dt=0.02)
        ax.plot(t, u[:, 0], color=c, label=lab)
    ax.set_xlabel("time"); ax.set_ylabel("encoder weight $w(t)$"); ax.set_title("(b) $w_0=1$, $p_0=0.5$, $E x^2=1$, view noise 0.5", fontsize=9)
    ax.legend(fontsize=7.5)
    fig.tight_layout()
    savefig(fig, CH, "figx32-g", out_dir)


# =========================================================================== JSON for the browser demonstrations
def demo_reps(n=80, p=60, seed=0):
    """Base matrices of the similarity demonstration: X0 (n x p, structured), E (n x p, independent N(0,1), the
    noise added to X) and E2 (n x p, independent N(0,1), the 'independent' representation)."""
    rng = np.random.default_rng(seed)
    X0 = structured_reps(n, p, rng, latent=8, noise=0.5)
    E = rng.standard_normal((n, p))
    E2 = rng.standard_normal((n, p))
    return X0, E, E2


DEMO_TRANSFORMS = ["none", "permute", "rotate", "scale10", "uneven", "invertible", "independent"]


def demo_Y(X0, E, E2, n, p, noise, transform):
    """The second representation of the browser demonstration, built by deterministic formulas (no random draws)
    from the first n rows and p columns: Y0 = X + noise * std(X) * E, then one transformation of Y0."""
    X = X0[:n, :p]
    Y0 = X + noise * X.std() * E[:n, :p]
    if transform == "none":
        return X, Y0
    if transform == "permute":
        return X, Y0[:, ::-1]
    if transform == "rotate":
        Y = Y0.copy()
        for j in range(p - 1):
            th = 0.7 + 0.37 * j
            c, s_ = np.cos(th), np.sin(th)
            a, b = Y[:, j].copy(), Y[:, j + 1].copy()
            Y[:, j], Y[:, j + 1] = c * a - s_ * b, s_ * a + c * b
        return X, Y
    if transform == "scale10":
        return X, 10.0 * Y0
    if transform == "uneven":
        return X, Y0 * (10.0 ** (-1.5 + 3.0 * np.arange(p) / max(p - 1, 1)))[None, :]
    if transform == "invertible":
        T = np.zeros((p, p))
        for i in range(p):
            T[i, i] = 10.0 ** (-i / max(p - 1, 1))
            for j in range(i + 1, p):
                T[i, j] = 0.5 * np.sin(1.7 * i + 2.3 * j + 1.0) / np.sqrt(p)
        return X, Y0 @ T
    if transform == "independent":
        return X, E2[:n, :p]
    raise ValueError(transform)


def export_json():
    os.makedirs(DATA_OUT, exist_ok=True)
    X0, E, E2 = demo_reps()
    with open(os.path.join(DATA_OUT, "ch32-reps.json"), "w") as f:
        json.dump(dict(X0=np.round(X0, 4).tolist(), E=np.round(E, 4).tolist(), E2=np.round(E2, 4).tolist()), f, separators=(",", ":"))
    rng = np.random.default_rng(7)
    N, Kmax = 200, 128
    d = dict(x=rng.standard_normal(N), e=rng.standard_normal(N), neg=rng.standard_normal((N, Kmax - 1)), eps=rng.standard_normal(N))
    with open(os.path.join(DATA_OUT, "ch32-infonce.json"), "w") as f:
        json.dump({k: np.round(v, 4).tolist() for k, v in d.items()}, f, separators=(",", ":"))
    for nm in ("ch32-reps.json", "ch32-infonce.json"):
        print("wrote", nm, "%.0f kB" % (os.path.getsize(os.path.join(DATA_OUT, nm)) / 1024))


FIGURES = {
    "32.1": fig_32_1,
    "x32.a": fig_x32_a, "x32.b": fig_x32_b, "x32.c": fig_x32_c, "x32.d": fig_x32_d, "x32.e": fig_x32_e, "x32.f": fig_x32_f, "x32.g": fig_x32_g,
}

if __name__ == "__main__":
    common.run(FIGURES, CH)
    if "--fig" not in sys.argv:
        export_json()
