"""Bishop and Bishop (2024), Chapter 17: Generative Adversarial Networks.

Reproduces the figures of Chapter 17 of C. M. Bishop and H. Bishop, Deep Learning: Foundations and
Concepts (Springer, 2024), pp. 533-545, written from scratch with numpy, scipy and matplotlib (and
PyTorch for the MNIST and CycleGAN models), and supplies the reference numbers of the three browser
demonstrations. Every algorithm is a short function meant to be read next to the lecture notes
(ch17-generative-adversarial-networks.html):

* One-dimensional densities on a grid (log_mix, grid_divergences): the optimal discriminator (17.15),
  the Jensen-Shannon divergence and the error at the optimal discriminator, ln 4 - 2 JS (17.16),
  both Kullback-Leibler divergences, and the Wasserstein-1 distance as the area between the two
  cumulative distribution functions; the gradients of the minimax and non-saturating generator
  objectives against a frozen optimal discriminator (ideal_gradients, ideal_descent), with optional
  instance noise; the translation experiment of the Wasserstein-versus-JS demo (shift_curves,
  shift_descent).
* Exercise 17.2: the bilinear game E(a, b) = ab under continuous-time gradient play, simultaneous and
  alternating gradient descent-ascent, and the extragradient method (bilinear_paths).
* A GAN on a ring of eight Gaussians written in plain numpy with hand-coded forward and backward
  passes (mlp_forward, mlp_backward), including the double backpropagation needed by the gradient
  penalty (critic_gp), Adam, and five losses: minimax (17.6)-(17.8), non-saturating (17.10), least
  squares (Mao et al. 2016), Wasserstein with weight clipping (Arjovsky et al. 2017) and with a
  gradient penalty (Gulrajani et al. 2017) (ring_gan). It uses only the seeded generator of the
  browser demos (JSRandom), so gan-ring-demo.html reproduces it number for number.
* Small PyTorch models: an MLP GAN and a conditional GAN on MNIST with an exponential moving average
  of the generator weights (mnist_gan), a digit classifier used to evaluate them (Frechet distance
  and inception score in its feature space), and a two-dimensional CycleGAN (cyclegan_2d).

Figures reproduced: 17.2 (computed from two Gaussian densities, their optimal discriminator and its
instance-noise smoothed version), 17.3 (the two generator losses, with a second panel for their
gradients with respect to the logit), and, at small scale, 17.6 (a CycleGAN between two
two-dimensional domains instead of photographs and paintings), 17.9 (latent-space walks of an MNIST
GAN instead of a bedroom DCGAN) and 17.10 (latent vector arithmetic on MNIST digits, stroke
thickness instead of glasses and gender).
Skipped: 17.1 (schematic of the two networks), 17.4 (DCGAN architecture diagram), 17.5 (BigGAN architecture diagram), 17.7 and 17.8 (diagrams of the
cycle-consistency error and of the CycleGAN information flow; they have no data).

Supplementary figures (our own, numbered 17.S1-17.S8 in the order they appear in the notes; they
illustrate the gaps filled there):
17.S1 the optimal discriminator, the Jensen-Shannon objective and the reverse-KL objective of the
non-saturating loss on a bimodal density; 17.S2 the bilinear game of Exercise 17.2 under four
dynamics; 17.S3 a conditional GAN on MNIST; 17.S4 GAN training on a ring of Gaussians with five
losses (snapshots); 17.S5 mode coverage and losses during that training; 17.S6 why a least-squares
discriminator does not saturate on separable data; 17.S7 Wasserstein distance versus Jensen-Shannon
divergence, the optimal critic and learned critics; 17.S8 evaluating the MNIST GAN (class histogram,
Frechet distance and inception score during training).

Run `python3 ch17_generative_adversarial_networks.py --list`, `python3
ch17_generative_adversarial_networks.py` (all figures; about twelve minutes on one CPU thread the first
time, when the MNIST models are trained and cached in cache/, then about a minute) or
`python3 ch17_generative_adversarial_networks.py --fig 17.2 17.S4`. Files go to ../img/ch17/.
"""
import json
import math
import os

for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"):
    os.environ.setdefault(_v, "1")

import numpy as np  # noqa: E402
import matplotlib.pyplot as plt  # noqa: E402
from scipy import linalg  # noqa: E402

import common  # noqa: E402
from common import savefig, run, RED, BLUE, GREEN, GREY, ORANGE, PURPLE  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = os.path.join(HERE, "cache")
CH = 17
M32 = 0xFFFFFFFF
LN2 = math.log(2.0)


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js).

    Uniforms come from the mulberry32 generator and normals from the Box-Muller transform, the
    second normal of each pair being cached, exactly as in JavaScript.
    """

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


# =========================================================================== 1-D densities on a grid
GRID = (-15.0, 15.0, 6001)       # the grid of optimal-discriminator-demo.html and wasserstein-js-demo.html


def grid_x(grid=GRID):
    lo, hi, n = grid
    h = (hi - lo) / (n - 1)
    return lo + h * np.arange(n), h


@np.errstate(divide="ignore", invalid="ignore")
def log_mix(x, comps):
    """log density of a mixture on the grid x. comps: list of ["g", w, mean, sd] or ["u", w, a, b]."""
    terms = []
    for c in comps:
        if c[0] == "g":
            z = (x - c[2]) / c[3]
            terms.append(math.log(c[1]) - math.log(c[3]) - 0.5 * math.log(2 * math.pi) - 0.5 * z * z)
        else:
            # uniform on [a, b]: the exact average of the density over each grid cell [x - h/2, x + h/2],
            # so that the discretised density moves continuously when a and b move
            h = x[1] - x[0]
            hi = np.clip((x + 0.5 * h - c[2]) / (c[3] - c[2]), 0.0, 1.0)
            lo = np.clip((x - 0.5 * h - c[2]) / (c[3] - c[2]), 0.0, 1.0)
            terms.append(np.log(c[1] * (hi - lo) / h))
    T = np.stack(terms)
    m = T.max(0)
    mm = np.where(np.isfinite(m), m, 0.0)
    return np.where(np.isfinite(m), mm + np.log(np.exp(T - mm).sum(0)), -np.inf)


def smooth(comps, tau):
    """Instance noise: a Gaussian mixture convolved with N(0, tau^2) (sd -> sqrt(sd^2 + tau^2))."""
    return [[c[0], c[1], c[2], math.sqrt(c[3] ** 2 + tau ** 2)] for c in comps]


def trap(f, h):
    return float(h * (f.sum() - 0.5 * (f[0] + f[-1])))


def cumtrap(f, h):
    F = np.zeros_like(f)
    F[1:] = np.cumsum(0.5 * h * (f[1:] + f[:-1]))
    return F


@np.errstate(divide="ignore", invalid="ignore")
def _lse2(a, b):
    m = np.maximum(a, b)
    mm = np.where(np.isfinite(m), m, 0.0)
    return np.where(np.isfinite(m), mm + np.log(np.exp(a - mm) + np.exp(b - mm)), -np.inf)


@np.errstate(divide="ignore", invalid="ignore")
def grid_divergences(lp, lq, h):
    """Divergences between p (data) and q (generator) given log densities on a grid of step h.

    Returns dict with JS = (KL(p||m) + KL(q||m))/2, m = (p + q)/2, C = 2 JS - ln 4 = -E(p_G, d*) (the
    expression printed as (17.17); the error (17.16) at the optimal discriminator is ln 4 - 2 JS = -C),
    KL(p||q), KL(q||p) and W1 = integral |F_p - F_q|.
    """
    p, q = np.exp(lp), np.exp(lq)
    lm = _lse2(lp, lq) - LN2
    jp = np.where(p > 0, p * (lp - lm), 0.0)
    jq = np.where(q > 0, q * (lq - lm), 0.0)
    js = 0.5 * trap(jp, h) + 0.5 * trap(jq, h)
    with np.errstate(invalid="ignore"):
        kpq = trap(np.where(p > 0, p * (lp - lq), 0.0), h) if np.all(np.isfinite(lq[p > 0])) else math.inf
        kqp = trap(np.where(q > 0, q * (lq - lp), 0.0), h) if np.all(np.isfinite(lp[q > 0])) else math.inf
    W1 = trap(np.abs(cumtrap(p, h) - cumtrap(q, h)), h)
    return {"JS": js, "C": 2 * js - math.log(4.0), "KLpq": kpq, "KLqp": kqp, "W1": W1}


@np.errstate(divide="ignore", invalid="ignore", over="ignore")
def d_star(lp, lq):
    """Optimal discriminator (17.15), d* = p / (p + q), computed from log densities (1/2 where both vanish)."""
    l = _lse2(lp, lq)
    return np.where(np.isfinite(l), np.exp(lp - np.where(np.isfinite(l), l, 0.0)), 0.5)


def ideal_gradients(data, mu, sigma, tau=0.0, grid=GRID):
    """Gradients with respect to (mu, sigma) of a Gaussian generator N(mu, sigma^2) trained against the
    optimal discriminator, held fixed (Result 17.8): minimax E_q[ln(1 - d*)] and non-saturating
    E_q[-ln d*]; with instance noise tau both densities are smoothed first."""
    x, h = grid_x(grid)
    lp = log_mix(x, smooth(data, tau))
    se = math.sqrt(sigma ** 2 + tau ** 2)
    lq = log_mix(x, [["g", 1.0, mu, se]])
    l = _lse2(lp, lq)
    ln1md = lq - l                    # ln(1 - d*) = ln q - ln(p + q)
    mlnd = l - lp                      # -ln d*
    q = np.exp(lq)
    dmu = q * (x - mu) / se ** 2
    dsg = q * ((x - mu) ** 2 / se ** 3 - 1.0 / se) * (sigma / se)
    fin = np.isfinite(mlnd)
    mlnd_f = np.where(fin, mlnd, 0.0)
    gmm = (trap(dmu * ln1md, h), trap(dsg * ln1md, h))
    gns = (trap(np.where(fin, dmu * mlnd_f, 0.0), h), trap(np.where(fin, dsg * mlnd_f, 0.0), h))
    return gmm, gns


def ideal_descent(data, mu0, sigma0, loss="ns", lr=0.5, steps=100, tau=0.0, grid=GRID, smin=0.05):
    """Gradient descent of (mu, sigma) on the minimax or non-saturating objective with d = d* recomputed
    after every step (the idealised inner loop of Exercise 17.1); the path of optimal-discriminator-demo.html."""
    mu, sg = mu0, sigma0
    path = [(mu, sg)]
    for _ in range(steps):
        gmm, gns = ideal_gradients(data, mu, sg, tau, grid)
        g = gmm if loss == "mm" else gns
        mu, sg = mu - lr * g[0], max(smin, sg - lr * g[1])
        path.append((mu, sg))
    return np.array(path)


def shape_comps(shape, shift=0.0, width=1.0):
    """The distributions of wasserstein-js-demo.html; Q is the same shape moved by `shift` and scaled by `width`."""
    if shape == "uniform":
        return [["u", 1.0, shift - 0.5 * width, shift + 0.5 * width]]
    if shape == "narrow":
        return [["g", 1.0, shift, 0.1 * width]]
    if shape == "gauss":
        return [["g", 1.0, shift, 1.0 * width]]
    if shape == "bimodal":
        return [["g", 0.5, shift - 1.5 * width, 0.3 * width], ["g", 0.5, shift + 1.5 * width, 0.3 * width]]
    raise ValueError(shape)


def shift_curves(shape, width_q=1.0, thetas=None, grid=GRID):
    """JS and W1 between P = shape at 0 and Q = shape moved by theta (with width ratio width_q)."""
    x, h = grid_x(grid)
    thetas = np.linspace(-6, 6, 121) if thetas is None else thetas
    lp = log_mix(x, shape_comps(shape))
    js, w1 = [], []
    for t in thetas:
        d = grid_divergences(lp, log_mix(x, shape_comps(shape, t, width_q)), h)
        js.append(d["JS"])
        w1.append(d["W1"])
    return np.asarray(thetas), np.array(js), np.array(w1)


def shift_descent(shape, theta0, dist="W1", lr=0.5, steps=60, width_q=1.0, delta=1e-3, grid=GRID):
    """Gradient descent on theta for JS or W1 with central finite differences (the demo's animation)."""
    x, h = grid_x(grid)
    lp = log_mix(x, shape_comps(shape))

    def D(t):
        return grid_divergences(lp, log_mix(x, shape_comps(shape, t, width_q)), h)[dist]

    th = theta0
    path = [th]
    for _ in range(steps):
        g = (D(th + delta) - D(th - delta)) / (2 * delta)
        th = th - lr * g
        path.append(th)
    return np.array(path)


def optimal_critic(lp, lq, h):
    """Kantorovich potential in 1-D: f' = sign(F_q - F_p), f = 0 at the left end; maximises E_p f - E_q f."""
    s = np.sign(cumtrap(np.exp(lq), h) - cumtrap(np.exp(lp), h))
    return cumtrap(s, h)


# =========================================================================== Exercise 17.2: bilinear game
def bilinear_paths(eta=0.2, steps=60, a0=1.0, b0=0.0):
    """E(a, b) = ab, a ascends and b descends. Returns dict of (steps+1, 2) paths for the exact flow
    a = cos(eta t), b = -sin(eta t) sampled at t = k, and for simultaneous GDA, alternating GDA and
    extragradient with step eta."""
    out = {}
    k = np.arange(steps + 1)
    out["flow"] = np.stack([a0 * np.cos(eta * k) + b0 * np.sin(eta * k), b0 * np.cos(eta * k) - a0 * np.sin(eta * k)], 1)
    for name in ("sim", "alt", "extra"):
        a, b = a0, b0
        P = [(a, b)]
        for _ in range(steps):
            if name == "sim":
                a, b = a + eta * b, b - eta * a
            elif name == "alt":
                a = a + eta * b
                b = b - eta * a
            else:
                ah, bh = a + eta * b, b - eta * a
                a, b = a + eta * bh, b - eta * ah
            P.append((a, b))
        out[name] = np.array(P)
    return out


# =========================================================================== numpy GAN on a ring of Gaussians
RING_K, RING_R, RING_S = 8, 2.0, 0.05
RING_LOSSES = ("mm", "ns", "ls", "wgan-clip", "wgan-gp")
RING_LABEL = {"mm": "minimax (17.6)", "ns": "non-saturating (17.10)", "ls": "least squares",
              "wgan-clip": "WGAN, weight clipping", "wgan-gp": "WGAN-GP"}


def ring_centres():
    a = 2 * np.pi * np.arange(RING_K) / RING_K
    return RING_R * np.stack([np.cos(a), np.sin(a)], 1)


def ring_batch(rng, B):
    """B points from the ring: mode k = floor(8u), then two normals for the offset (the demo's order)."""
    X = np.empty((B, 2))
    for i in range(B):
        k = int(rng.u() * RING_K)
        a = 2 * math.pi * k / RING_K
        X[i, 0] = RING_R * math.cos(a) + RING_S * rng.norm()
        X[i, 1] = RING_R * math.sin(a) + RING_S * rng.norm()
    return X


def latent_batch(rng, B):
    Z = np.empty((B, 2))
    for i in range(B):
        Z[i, 0] = rng.norm()
        Z[i, 1] = rng.norm()
    return Z


def mlp_init(rng, sizes):
    """Weights N(0, 1/fan_in) drawn row by row, biases 0; parameters [W1, b1, W2, b2, ...]."""
    P = []
    for i in range(len(sizes) - 1):
        sc = math.sqrt(1.0 / sizes[i])
        W = np.empty((sizes[i + 1], sizes[i]))
        for r in range(sizes[i + 1]):
            for c in range(sizes[i]):
                W[r, c] = rng.norm() * sc
        P += [W, np.zeros(sizes[i + 1])]
    return P


def mlp_forward(P, X):
    """tanh hidden layers and a linear output; returns the list of activations [X, h1, ..., out]."""
    hs = [X]
    h = X
    L = len(P) // 2
    for l in range(L):
        a = h @ P[2 * l].T + P[2 * l + 1]
        h = np.tanh(a) if l < L - 1 else a
        hs.append(h)
    return hs


def mlp_backward(P, hs, gout):
    """Backpropagation: gradients of sum(gout * out) with respect to the parameters and to the input."""
    L = len(P) // 2
    G = [None] * len(P)
    g = gout
    for l in range(L - 1, -1, -1):
        if l < L - 1:
            g = g * (1.0 - hs[l + 1] ** 2)
        G[2 * l] = g.T @ hs[l]
        G[2 * l + 1] = g.sum(0)
        g = g @ P[2 * l]
    return G, g


def critic_gp(P, X, lam):
    """lam * mean_n (||grad_x f(x_n)|| - 1)^2 for the two-hidden-layer tanh critic f, its gradient with
    respect to the parameters (double backpropagation written out), and the input-gradient norms."""
    W1, b1, W2, b2, w3, b3 = P
    B = X.shape[0]
    h1 = np.tanh(X @ W1.T + b1)
    h2 = np.tanh(h1 @ W2.T + b2)
    s1, s2 = 1 - h1 ** 2, 1 - h2 ** 2
    u2 = s2 * w3[0]                    # df/da2
    v1 = u2 @ W2                       # df/dh1
    u1 = s1 * v1                       # df/da1
    g = u1 @ W1                        # df/dx, one row per point
    nrm = np.sqrt((g ** 2).sum(1))
    val = lam * float(np.mean((nrm - 1) ** 2))
    r = (lam * 2.0 / B) * ((nrm - 1) / np.maximum(nrm, 1e-12))[:, None] * g
    dW1 = u1.T @ r
    du1 = r @ W1.T
    dv1 = s1 * du1
    ds1 = v1 * du1
    dW2 = u2.T @ dv1
    du2 = dv1 @ W2.T
    ds2 = w3[0] * du2
    dw3 = (s2 * du2).sum(0)[None, :]
    da2 = -2 * h2 * ds2 * s2
    dW2 = dW2 + da2.T @ h1
    db2 = da2.sum(0)
    da1 = (-2 * h1 * ds1 + da2 @ W2) * s1
    dW1 = dW1 + da1.T @ X
    db1 = da1.sum(0)
    return val, [dW1, db1, dW2, db2, dw3, np.zeros(1)], nrm


def softplus(a):
    return np.maximum(a, 0) + np.log1p(np.exp(-np.abs(a)))


def sigm(a):
    return 1.0 / (1.0 + np.exp(-a))


class Adam:
    """Adam (Section 7.3.3) with beta1 = 0.5, the usual choice for GANs."""

    def __init__(self, P, lr, b1=0.5, b2=0.999, eps=1e-8):
        self.m = [np.zeros_like(p) for p in P]
        self.v = [np.zeros_like(p) for p in P]
        self.t = 0
        self.lr, self.b1, self.b2, self.eps = lr, b1, b2, eps

    def step(self, P, G):
        self.t += 1
        c1 = 1 - self.b1 ** self.t
        c2 = 1 - self.b2 ** self.t
        for p, g, m, v in zip(P, G, self.m, self.v):
            m *= self.b1
            m += (1 - self.b1) * g
            v *= self.b2
            v += (1 - self.b2) * g * g
            p -= self.lr * (m / c1) / (np.sqrt(v / c2) + self.eps)


def eval_latents(n=1000):
    """The fixed latent points used to measure mode coverage (JSRandom(12345), as in the demo)."""
    return latent_batch(JSRandom(12345), n)


def ring_quality(F, dist=False):
    """(number of modes hit, fraction of high-quality samples[, mean distance to the nearest mode centre]):
    a sample is high quality if it lies within three standard deviations of a mode centre."""
    C = ring_centres()
    d = np.sqrt(((F[:, None, :] - C[None]) ** 2).sum(-1))
    near = d.min(1) < 3 * RING_S
    out = (len(set(d.argmin(1)[near].tolist())), float(near.mean()))
    return out + (float(d.min(1).mean()),) if dist else out


def ring_mode_fractions(F):
    C = ring_centres()
    d = np.sqrt(((F[:, None, :] - C[None]) ** 2).sum(-1))
    near = d.min(1) < 3 * RING_S
    return np.bincount(d.argmin(1)[near], minlength=RING_K) / len(F)


RING_DEFAULT = dict(B=64, HG=32, HD=64, lrG=1e-3, lrD=1e-3, nd=1, clip=0.05, lam=0.1, seed=5, g0=0.0)


def ring_gan(loss="ns", steps=2000, B=64, HG=32, HD=64, lrG=1e-3, lrD=1e-3, nd=1, clip=0.05, lam=0.1,
             seed=5, g0=0.0, every=0, snap=()):
    """Train the ring GAN; the browser demo gan-ring-demo.html runs exactly this loop.

    Generator 2 -> HG -> HG -> 2, discriminator (critic) 2 -> HD -> HD -> 1, tanh, Adam(0.5, 0.999).
    One iteration = nd discriminator steps (each: B real points, B latents, and for WGAN-GP B uniforms
    for the interpolates, in that order) and one generator step (B latents).
    g0 sets the initial output bias of the generator to (g0, g0), so that it starts away from the data.
    Returns PG, PD, history (every `every` iterations), snapshots of PG at the iterations in snap, and
    the last (discriminator loss, generator loss).
    """
    rng = JSRandom(seed)
    PG = mlp_init(rng, [2, HG, HG, 2])
    PG[-1][:] = g0
    PD = mlp_init(rng, [2, HD, HD, 1])
    oG, oD = Adam(PG, lrG), Adam(PD, lrD)
    Ze = eval_latents()
    hist = {"it": [], "modes": [], "hq": [], "dist": [], "lD": [], "lG": []}
    snaps = {}
    if 0 in snap:
        snaps[0] = ([p.copy() for p in PG], [p.copy() for p in PD])
    lD = lG = float("nan")
    for t in range(1, steps + 1):
        for _ in range(nd):
            X = ring_batch(rng, B)
            Z = latent_batch(rng, B)
            eps = np.array([rng.u() for _ in range(B)]) if loss == "wgan-gp" else None
            F = mlp_forward(PG, Z)[-1]
            hr = mlp_forward(PD, X)
            hf = mlp_forward(PD, F)
            ar, af = hr[-1][:, 0], hf[-1][:, 0]
            if loss in ("ns", "mm"):
                lD = float(np.mean(softplus(-ar)) + np.mean(softplus(af)))
                gr, gf = -(1 - sigm(ar)) / B, sigm(af) / B
            elif loss == "ls":
                lD = float(0.5 * np.mean((ar - 1) ** 2) + 0.5 * np.mean(af ** 2))
                gr, gf = (ar - 1) / B, af / B
            else:
                lD = float(np.mean(af) - np.mean(ar))
                gr, gf = -np.ones(B) / B, np.ones(B) / B
            G1, _ = mlp_backward(PD, hr, gr[:, None])
            G2, _ = mlp_backward(PD, hf, gf[:, None])
            GD = [a + b for a, b in zip(G1, G2)]
            if loss == "wgan-gp":
                Xh = eps[:, None] * X + (1 - eps[:, None]) * F
                pv, GP, _ = critic_gp(PD, Xh, lam)
                lD += pv
                GD = [a + b for a, b in zip(GD, GP)]
            oD.step(PD, GD)
            if loss == "wgan-clip":
                for p in PD:
                    np.clip(p, -clip, clip, out=p)
        Z = latent_batch(rng, B)
        hg = mlp_forward(PG, Z)
        hf = mlp_forward(PD, hg[-1])
        af = hf[-1][:, 0]
        if loss == "ns":
            lG, gf = float(np.mean(softplus(-af))), -(1 - sigm(af)) / B
        elif loss == "mm":
            lG, gf = float(-np.mean(softplus(af))), -sigm(af) / B
        elif loss == "ls":
            lG, gf = float(0.5 * np.mean((af - 1) ** 2)), (af - 1) / B
        else:
            lG, gf = float(-np.mean(af)), -np.ones(B) / B
        _, gx = mlp_backward(PD, hf, gf[:, None])
        GG, _ = mlp_backward(PG, hg, gx)
        oG.step(PG, GG)
        if every and t % every == 0:
            m, q, dd = ring_quality(mlp_forward(PG, Ze)[-1], True)
            for k, v in zip(("it", "modes", "hq", "dist", "lD", "lG"), (t, m, q, dd, lD, lG)):
                hist[k].append(v)
        if t in snap:
            snaps[t] = ([p.copy() for p in PG], [p.copy() for p in PD])
    return PG, PD, hist, snaps, (lD, lG)


RING_RUNS = {   # the six rows of Figures 17.S4-17.S5 (and presets of gan-ring-demo.html)
    "ns": dict(loss="ns", steps=5000),
    "collapse": dict(loss="ns", steps=5000, lrD=1e-4),
    "mm-far": dict(loss="mm", steps=5000, g0=4.0, lrG=2e-4, lrD=2e-3),
    "ns-far": dict(loss="ns", steps=5000, g0=4.0, lrG=2e-4, lrD=2e-3),
    "ls": dict(loss="ls", steps=5000),
    "wgan-gp": dict(loss="wgan-gp", steps=2500, nd=5),
}
RING_SNAPS = (0, 250, 750, 1500, 2500, 5000)


def ring_runs():
    """Train (or load from cache/ch17_ring.npz) the five runs of Figures 17.S4 and 17.S5."""
    path = os.path.join(CACHE_DIR, "ch17_ring.npz")
    if os.path.exists(path):
        z = np.load(path, allow_pickle=True)
        return z["res"].item()
    res = {}
    Ze = eval_latents()
    for key, kw in RING_RUNS.items():
        snaps = tuple(s for s in RING_SNAPS if s <= kw["steps"])
        if kw["steps"] not in snaps:
            snaps = snaps + (kw["steps"],)
        PG, PD, hist, sn, _ = ring_gan(every=50, snap=snaps, **kw)
        res[key] = {"hist": hist, "snaps": {t: mlp_forward(g, Ze)[-1] for t, (g, d) in sn.items()},
                    "D": [p for p in PD], "G": [p for p in PG], "kw": kw}
        print("  ring run", key, "modes", hist["modes"][-1], "hq %.2f" % hist["hq"][-1])
    os.makedirs(CACHE_DIR, exist_ok=True)
    np.savez(path, res=np.array(res, dtype=object))
    return res


# =========================================================================== PyTorch models
def _torch():
    return common.torch_setup(0, threads=1)


def mnist_tensors():
    torch = _torch()
    X, y = common.mnist(True)
    return torch.tensor(X * 2 - 1, dtype=torch.float32), torch.tensor(y)


DZ = 32


def _mnist_nets(cond):
    import torch.nn as nn
    c = 10 if cond else 0
    G = nn.Sequential(nn.Linear(DZ + c, 256), nn.LeakyReLU(0.2), nn.Linear(256, 512), nn.LeakyReLU(0.2),
                      nn.Linear(512, 784), nn.Tanh())
    D = nn.Sequential(nn.Linear(784 + c, 512), nn.LeakyReLU(0.2), nn.Linear(512, 256), nn.LeakyReLU(0.2),
                      nn.Linear(256, 1))
    return G, D


MNIST_STEPS = 20000
MNIST_CKPT = (1000, 2500, 5000, 10000, 20000)


def mnist_gan(cond=False):
    """MLP GAN (non-saturating loss, Adam 2e-4, beta1 0.5, batch 64, 20 000 steps) on MNIST scaled to
    [-1, 1]; the generator used for sampling is an exponential moving average (decay 0.999) of the
    trained one. cond=True gives the conditional GAN: the one-hot label is appended to z and to x.
    Cached in cache/ch17_mnist_gan[_cond].pt with checkpoints of the averaged generator."""
    torch = _torch()
    import copy
    path = os.path.join(CACHE_DIR, "ch17_mnist_gan%s.pt" % ("_cond" if cond else ""))
    G, D = _mnist_nets(cond)
    if os.path.exists(path):
        st = torch.load(path)
        G.load_state_dict(st["G"])
        return G.eval(), st
    torch.manual_seed(1 if cond else 0)
    X, y = mnist_tensors()
    Y1 = torch.nn.functional.one_hot(y, 10).float()
    Gema = copy.deepcopy(G)
    oG = torch.optim.Adam(G.parameters(), 2e-4, betas=(0.5, 0.999))
    oD = torch.optim.Adam(D.parameters(), 2e-4, betas=(0.5, 0.999))
    bce = torch.nn.BCEWithLogitsLoss()
    B = 64
    ones, zeros = torch.ones(B, 1), torch.zeros(B, 1)
    ckpt, losses = {}, []
    for t in range(1, MNIST_STEPS + 1):
        idx = torch.randint(0, len(X), (B,))
        xr = X[idx]
        z = torch.randn(B, DZ)
        if cond:
            c = Y1[idx]
            xf = G(torch.cat([z, c], 1))
            dr, df = D(torch.cat([xr, c], 1)), D(torch.cat([xf.detach(), c], 1))
        else:
            xf = G(z)
            dr, df = D(xr), D(xf.detach())
        lD = bce(dr, ones) + bce(df, zeros)
        oD.zero_grad()
        lD.backward()
        oD.step()
        lG = bce(D(torch.cat([xf, c], 1)) if cond else D(xf), ones)
        oG.zero_grad()
        lG.backward()
        oG.step()
        with torch.no_grad():
            for pe, p in zip(Gema.parameters(), G.parameters()):
                pe.mul_(0.999).add_(p, alpha=0.001)
        if t % 100 == 0:
            losses.append((t, lD.item(), lG.item()))
        if t in MNIST_CKPT:
            ckpt[t] = copy.deepcopy(Gema.state_dict())
            print("  MNIST GAN%s step %d: D loss %.3f, G loss %.3f" % (" (conditional)" if cond else "", t, lD.item(), lG.item()))
    st = {"G": Gema.state_dict(), "ckpt": ckpt, "losses": losses}
    os.makedirs(CACHE_DIR, exist_ok=True)
    torch.save(st, path)
    G.load_state_dict(st["G"])
    return G.eval(), st


def mnist_classifier():
    """784-256-10 ReLU classifier (two epochs of Adam on MNIST in [-1, 1]); its 256 hidden units are the
    feature space of the Frechet distance. Cached in cache/ch17_classifier.pt."""
    torch = _torch()
    import torch.nn as nn
    net = nn.Sequential(nn.Linear(784, 256), nn.ReLU(), nn.Linear(256, 10))
    path = os.path.join(CACHE_DIR, "ch17_classifier.pt")
    if os.path.exists(path):
        net.load_state_dict(torch.load(path))
        return net.eval()
    torch.manual_seed(2)
    X, y = mnist_tensors()
    opt = torch.optim.Adam(net.parameters(), 1e-3)
    for ep in range(2):
        perm = torch.randperm(len(X))
        for i in range(0, len(X), 128):
            idx = perm[i:i + 128]
            loss = nn.functional.cross_entropy(net(X[idx]), y[idx])
            opt.zero_grad()
            loss.backward()
            opt.step()
    os.makedirs(CACHE_DIR, exist_ok=True)
    torch.save(net.state_dict(), path)
    return net.eval()


def classifier_features(net, X):
    """(hidden features, class probabilities) of images X (torch tensor in [-1, 1])."""
    import torch
    with torch.no_grad():
        h = net[1](net[0](X))
        return h.numpy().astype(float), torch.softmax(net[2](h), 1).numpy().astype(float)


def frechet_distance(mu1, S1, mu2, S2):
    """||mu1 - mu2||^2 + Tr(S1 + S2 - 2 (S1^(1/2) S2 S1^(1/2))^(1/2)): the squared Wasserstein-2 distance
    between N(mu1, S1) and N(mu2, S2) (the FID formula, Result 17.13)."""
    r1 = linalg.sqrtm(S1)
    cross = linalg.sqrtm(r1 @ S2 @ r1)
    return float(np.sum((mu1 - mu2) ** 2) + np.trace(S1 + S2 - 2 * np.real(cross)))


def inception_score(P):
    """exp(E_x KL(p(y|x) || p(y))) = exp(I(x; y)) for class probabilities P (rows)."""
    py = P.mean(0)
    kl = np.sum(P * (np.log(P + 1e-12) - np.log(py + 1e-12)), 1)
    return float(np.exp(kl.mean()))


def mnist_eval(n=10000):
    """Frechet distance and inception score (classifier features) of the GAN checkpoints and of real
    test images; cached in cache/ch17_mnist_eval.json."""
    path = os.path.join(CACHE_DIR, "ch17_mnist_eval.json")
    if os.path.exists(path):
        with open(path) as f:
            return json.load(f)
    torch = _torch()
    net = mnist_classifier()
    Xt, yt = common.mnist(False)
    Xt = torch.tensor(Xt * 2 - 1, dtype=torch.float32)
    Xr, _ = mnist_tensors()
    hr, pr = classifier_features(net, Xr[:n])
    mu_r, S_r = hr.mean(0), np.cov(hr, rowvar=False)
    ht, pt = classifier_features(net, Xt[:n])
    acc = float((pt.argmax(1) == yt[:n]).mean())
    out = {"test_acc": acc, "real": {"FD": frechet_distance(ht.mean(0), np.cov(ht, rowvar=False), mu_r, S_r),
                                     "IS": inception_score(pt), "hist": np.bincount(pt.argmax(1), minlength=10).tolist()}}
    G, st = mnist_gan(False)
    out["ckpt"] = {}
    for t, sd in st["ckpt"].items():
        G.load_state_dict(sd)
        torch.manual_seed(100)
        with torch.no_grad():
            Xg = G(torch.randn(n, DZ))
        hg, pg = classifier_features(net, Xg)
        out["ckpt"][str(t)] = {"FD": frechet_distance(hg.mean(0), np.cov(hg, rowvar=False), mu_r, S_r),
                               "IS": inception_score(pg), "hist": np.bincount(pg.argmax(1), minlength=10).tolist()}
    Gc, _ = mnist_gan(True)
    torch.manual_seed(101)
    lab = torch.arange(10).repeat_interleave(n // 10)
    with torch.no_grad():
        Xc = Gc(torch.cat([torch.randn(len(lab), DZ), torch.nn.functional.one_hot(lab, 10).float()], 1))
    hc, pc = classifier_features(net, Xc)
    out["cond"] = {"acc": float((pc.argmax(1) == lab.numpy()).mean()),
                   "FD": frechet_distance(hc.mean(0), np.cov(hc, rowvar=False), mu_r, S_r), "IS": inception_score(pc)}
    with open(path, "w") as f:
        json.dump(out, f, indent=1)
    return out


# =========================================================================== CycleGAN in two dimensions
CYC_X = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, 1.0], [1.0, -1.0]])      # domain X: corners of a square
CYC_Y = np.array([[-3.0, 0.0], [-1.0, 0.0], [1.0, 0.0], [3.0, 0.0]])        # domain Y: four points on a line
CYC_S = 0.12


def _cyc_sample(torch, C, n, gen):
    k = torch.randint(0, len(C), (n,), generator=gen)
    return torch.tensor(C, dtype=torch.float32)[k] + CYC_S * torch.randn(n, 2, generator=gen), k


def cyclegan_2d(eta, steps=4000, seed=0):
    """Two generators gX: Y -> X and gY: X -> Y and two discriminators, non-saturating GAN losses (17.6)
    plus eta times the L1 cycle-consistency error (17.12), as in (17.13). Cached per eta."""
    torch = _torch()
    import torch.nn as nn
    path = os.path.join(CACHE_DIR, "ch17_cyclegan_eta%g.pt" % eta)

    def mlp(o):
        return nn.Sequential(nn.Linear(2, 64), nn.LeakyReLU(0.2), nn.Linear(64, 64), nn.LeakyReLU(0.2), nn.Linear(64, o))

    torch.manual_seed(seed)
    gX, gY, dX, dY = mlp(2), mlp(2), mlp(1), mlp(1)
    if os.path.exists(path):
        st = torch.load(path)
        gX.load_state_dict(st["gX"])
        gY.load_state_dict(st["gY"])
        return gX.eval(), gY.eval(), st["hist"]
    gen = torch.Generator().manual_seed(seed + 10)
    oG = torch.optim.Adam(list(gX.parameters()) + list(gY.parameters()), 1e-3, betas=(0.5, 0.999))
    oD = torch.optim.Adam(list(dX.parameters()) + list(dY.parameters()), 1e-3, betas=(0.5, 0.999))
    bce = nn.BCEWithLogitsLoss()
    B = 128
    one, zero = torch.ones(B, 1), torch.zeros(B, 1)
    hist = []
    for t in range(steps):
        x, _ = _cyc_sample(torch, CYC_X, B, gen)
        y, _ = _cyc_sample(torch, CYC_Y, B, gen)
        fy, fx = gY(x), gX(y)
        lD = bce(dY(y), one) + bce(dY(fy.detach()), zero) + bce(dX(x), one) + bce(dX(fx.detach()), zero)
        oD.zero_grad()
        lD.backward()
        oD.step()
        cyc = (gX(fy) - x).abs().sum(1).mean() + (gY(fx) - y).abs().sum(1).mean()
        lG = bce(dY(fy), one) + bce(dX(fx), one) + eta * cyc
        oG.zero_grad()
        lG.backward()
        oG.step()
        if t % 50 == 0:
            hist.append((t, cyc.item()))
    os.makedirs(CACHE_DIR, exist_ok=True)
    torch.save({"gX": gX.state_dict(), "gY": gY.state_dict(), "hist": hist}, path)
    return gX.eval(), gY.eval(), hist


# =========================================================================== figure helpers
def _ring_axes(ax, lim=2.8):
    ax.set_xlim(-lim, lim)
    ax.set_ylim(-lim, lim)
    ax.set_aspect("equal")
    ax.set_xticks([])
    ax.set_yticks([])


def _img_grid(ax, imgs, nrow, ncol, pad=1, vmin=-1, vmax=1):
    H = nrow * (28 + pad) + pad
    W = ncol * (28 + pad) + pad
    canvas = np.full((H, W), vmax, dtype=float)
    for k, im in enumerate(imgs):
        r, c = divmod(k, ncol)
        canvas[pad + r * (28 + pad):pad + r * (28 + pad) + 28, pad + c * (28 + pad):pad + c * (28 + pad) + 28] = im.reshape(28, 28)
    ax.imshow(-canvas, cmap="gray", vmin=-vmax, vmax=-vmin, interpolation="nearest")
    ax.set_xticks([])
    ax.set_yticks([])
    for s in ax.spines.values():
        s.set_visible(False)


# =========================================================================== Section 17.1.1 figures
FIG_S1_DATA = [["g", 0.5, -2.0, 0.5], ["g", 0.5, 2.0, 0.5]]


def fig_17_s1(out=None):
    """17.S1 The optimal discriminator, the JS objective and the reverse-KL objective of the NS loss."""
    torch = _torch()
    x, h = grid_x()
    lp = log_mix(x, FIG_S1_DATA)
    lq = log_mix(x, [["g", 1.0, 1.0, 1.0]])
    ds = d_star(lp, lq)
    # a discriminator network trained on samples approaches d*
    rng = np.random.default_rng(3)
    n = 20000
    xr = np.where(rng.random(n) < 0.5, -2.0, 2.0) + 0.5 * rng.standard_normal(n)
    xf = 1.0 + rng.standard_normal(n)
    net = torch.nn.Sequential(torch.nn.Linear(1, 32), torch.nn.Tanh(), torch.nn.Linear(32, 32), torch.nn.Tanh(), torch.nn.Linear(32, 1))
    opt = torch.optim.Adam(net.parameters(), 3e-3)
    Xt = torch.tensor(np.r_[xr, xf][:, None], dtype=torch.float32)
    Tt = torch.tensor(np.r_[np.ones(n), np.zeros(n)][:, None], dtype=torch.float32)
    for it in range(1500):
        idx = torch.randint(0, 2 * n, (512,))
        loss = torch.nn.functional.binary_cross_entropy_with_logits(net(Xt[idx]), Tt[idx])
        opt.zero_grad()
        loss.backward()
        opt.step()
    xs = np.linspace(-5, 5, 400)
    with torch.no_grad():
        dnet = torch.sigmoid(net(torch.tensor(xs[:, None], dtype=torch.float32))).numpy()[:, 0]
    mus = np.linspace(-4, 4, 81)
    sgs = np.linspace(0.2, 3.0, 57)
    JS = np.zeros((len(sgs), len(mus)))
    NS = np.zeros_like(JS)
    for i, s in enumerate(sgs):
        for j, m in enumerate(mus):
            d = grid_divergences(lp, log_mix(x, [["g", 1.0, m, s]]), h)
            JS[i, j] = d["C"]
            NS[i, j] = d["KLqp"] - 2 * d["JS"]
    fig, ax = plt.subplots(1, 3, figsize=(12.6, 3.4))
    sel = (x > -5) & (x < 5)
    ax[0].plot(x[sel], np.exp(lp[sel]), color=RED, label=r"$p_{\rm data}$")
    ax[0].plot(x[sel], np.exp(lq[sel]), color=BLUE, label=r"$p_G$")
    ax[0].plot(x[sel], ds[sel], color=GREEN, label=r"$d^\star$ (17.15)")
    ax[0].plot(xs, dnet, color="k", ls="--", lw=1.2, label="trained network")
    ax[0].axhline(0.5, color=GREY, lw=0.6, ls=":")
    ax[0].set_xlabel("$x$")
    ax[0].legend(loc="upper center", bbox_to_anchor=(0.6, 1.0), fontsize=7.5, frameon=False)
    ax[0].set_ylim(0, 1.18)
    ax[0].set_title("(a) optimal and trained discriminator")
    for a, Z, title, cmap in ((ax[1], JS, r"(b) $2\,\mathrm{JS}-\ln 4=-E(p_G,d^\star)$ (minimax)", "viridis"),
                              (ax[2], NS, r"(c) $\mathrm{KL}(p_G\|p_{\rm data})-2\,\mathrm{JS}$ (non-saturating)", "viridis")):
        Zc = np.clip(Z, None, np.percentile(Z, 85))
        cs = a.contourf(mus, sgs, Zc, 24, cmap=cmap)
        a.contour(mus, sgs, Zc, 12, colors="w", linewidths=0.4)
        i, j = np.unravel_index(np.argmin(Z), Z.shape)
        a.plot(mus[j], sgs[i], "*", color="w", ms=13, mec="k")
        if a is ax[2]:
            Zm = Z.copy()
            Zm[:, mus > 0] = np.inf
            i2, j2 = np.unravel_index(np.argmin(Zm), Z.shape)
            a.plot(mus[j2], sgs[i2], "*", color="w", ms=13, mec="k")
        a.set_xlabel(r"generator mean $\mu$")
        a.set_ylabel(r"generator s.d. $\sigma$")
        a.set_title(title, fontsize=9.5)
        fig.colorbar(cs, ax=a, fraction=0.046, pad=0.03)
    fig.tight_layout()
    savefig(fig, CH, "fig17-s1", out)


def fig_17_s2(out=None):
    """17.S2 Exercise 17.2: the bilinear game under four dynamics."""
    eta, steps = 0.2, 60
    P = bilinear_paths(eta, steps)
    fig, ax = plt.subplots(1, 3, figsize=(12.0, 3.6))
    sty = {"flow": ("k", "-", "continuous time (17.18)"), "sim": (RED, "-", "simultaneous GDA"),
           "alt": (BLUE, "-", "alternating GDA"), "extra": (GREEN, "-", "extragradient")}
    for k, (c, ls, lab) in sty.items():
        Q = P[k]
        if k == "flow":
            t = np.linspace(0, 2 * np.pi, 300)
            ax[0].plot(np.cos(t), -np.sin(t), color=c, lw=1.0, ls="--", label=lab)
        else:
            ax[0].plot(Q[:, 0], Q[:, 1], color=c, lw=1.1, marker="o", ms=2.2, label=lab)
        ax[1].semilogy(np.arange(steps + 1), np.sqrt((Q ** 2).sum(1)), color=c, ls="--" if k == "flow" else "-", label=lab)
        ax[2].plot(np.arange(steps + 1), Q[:, 0] * Q[:, 1], color=c, ls="--" if k == "flow" else "-", lw=1.2)
    k = np.arange(steps + 1)
    ax[1].semilogy(k, (1 + eta ** 2) ** (k / 2), color=RED, ls=":", lw=2, label=r"$(1+\eta^2)^{k/2}$")
    ax[1].semilogy(k, (1 - eta ** 2 + eta ** 4) ** (k / 2), color=GREEN, ls=":", lw=2, label=r"$(1-\eta^2+\eta^4)^{k/2}$")
    ax[0].plot(0, 0, "k+", ms=10)
    ax[0].set_aspect("equal")
    ax[0].set_xlabel("$a$ (generator, ascends)")
    ax[0].set_ylabel("$b$ (discriminator, descends)")
    ax[0].set_title(r"(a) paths from $(1,0)$, $\eta=0.2$")
    ax[0].legend(fontsize=7.5, loc="lower left", frameon=False)
    ax[0].set_xlim(-3.6, 3.6)
    ax[0].set_ylim(-3.6, 3.6)
    ax[1].set_xlabel("step $k$")
    ax[1].set_title(r"(b) distance $\|(a,b)\|$ from the saddle")
    ax[1].legend(fontsize=7.5, frameon=False)
    ax[2].set_xlabel("step $k$")
    ax[2].set_title(r"(c) the value $E=ab$ oscillates")
    ax[2].axhline(0, color=GREY, lw=0.6)
    fig.tight_layout()
    savefig(fig, CH, "fig17-s2", out)


def fig_17_s3(out=None):
    """17.S3 A conditional GAN on MNIST: one row per requested class (small-scale illustration)."""
    torch = _torch()
    G, _ = mnist_gan(True)
    torch.manual_seed(7)
    z = torch.randn(10, DZ).repeat(10, 1)               # the same ten latents in every row
    lab = torch.arange(10).repeat_interleave(10)
    with torch.no_grad():
        X = G(torch.cat([z, torch.nn.functional.one_hot(lab, 10).float()], 1)).numpy()
    fig, ax = plt.subplots(figsize=(4.6, 4.8))
    _img_grid(ax, X, 10, 10)
    ax.set_ylabel(r"class $c$ = 0, 1, ..., 9 (rows)", fontsize=9)
    ax.set_xlabel(r"ten latent vectors $z$, the same in every row", fontsize=9)
    fig.tight_layout()
    savefig(fig, CH, "fig17-s3", out)


# =========================================================================== Section 17.1.2 figures
RING_ROWS = [("ns", "non-saturating"), ("collapse", "NS, slow $d$\n(lr$_d$ = 1e-4)"),
             ("mm-far", "minimax,\nfar start"), ("ns-far", "non-saturating,\nfar start"),
             ("ls", "least squares"), ("wgan-gp", "WGAN-GP\n(5 critic steps)")]


def fig_17_s4(out=None):
    """17.S4 Ring of Gaussians: generated samples during training for six settings."""
    res = ring_runs()
    C = ring_centres()
    cols = RING_SNAPS
    fig, ax = plt.subplots(len(RING_ROWS), len(cols), figsize=(1.75 * len(cols), 1.8 * len(RING_ROWS)))
    for i, (key, lab) in enumerate(RING_ROWS):
        r = res[key]
        T = sorted(r["snaps"])
        for j in range(len(cols)):
            a = ax[i, j]
            a.scatter(C[:, 0], C[:, 1], s=55, facecolors="none", edgecolors=RED, lw=0.8)
            if j < len(T):
                t = T[j]
                F = r["snaps"][t]
                a.scatter(F[:, 0], F[:, 1], s=1.2, color=BLUE, alpha=0.6, lw=0)
                m, q = ring_quality(F)
                a.text(0.03, 0.03, "%d modes, %d%%" % (m, round(100 * q)), transform=a.transAxes, fontsize=6.5)
                a.text(0.97, 0.97, "it. %d" % t, transform=a.transAxes, fontsize=6.5, ha="right", va="top")
            else:
                a.axis("off")
            _ring_axes(a, 6.2 if key.endswith("far") else 2.8)
            if j == 0:
                a.set_ylabel(lab, fontsize=8)
    fig.tight_layout(h_pad=0.3, w_pad=0.2)
    savefig(fig, CH, "fig17-s4", out)


def fig_17_s5(out=None):
    """17.S5 Ring of Gaussians: modes covered, distance to the data and losses during training."""
    res = ring_runs()
    sty = {"ns": (BLUE, "-", "non-saturating"), "collapse": (ORANGE, "-", "NS, slow $d$"),
           "mm-far": (RED, "--", "minimax, far start"), "ns-far": (BLUE, "--", "NS, far start"),
           "ls": (GREEN, "-", "least squares"), "wgan-gp": (PURPLE, "-", "WGAN-GP")}
    fig, ax = plt.subplots(1, 3, figsize=(12.4, 3.3))
    for key, (c, ls, lab) in sty.items():
        hst = res[key]["hist"]
        it = np.array(hst["it"])
        ax[0].plot(it, hst["modes"], color=c, ls=ls, lw=1.0, label=lab, drawstyle="steps-post")
        ax[1].semilogy(it, hst["dist"], color=c, ls=ls, lw=1.1, label=lab)
        if key in ("ns", "collapse", "ls"):
            ax[2].plot(it, hst["lD"], color=c, ls=ls, lw=0.9, label=lab)
    ax[0].set_ylabel("modes with a sample within 3 s.d.")
    ax[0].set_ylim(-0.3, 8.5)
    ax[1].set_ylabel("mean distance to nearest mode")
    ax[1].axhline(RING_S * math.sqrt(math.pi / 2), color=GREY, ls=":", lw=1)
    ax[1].text(5000, RING_S * 1.4, "data", ha="right", fontsize=8, color=GREY)
    ax[2].set_ylabel("discriminator loss")
    ax[2].axhline(2 * LN2, color=GREY, ls=":", lw=1)
    ax[2].text(5000, 2 * LN2 + 0.03, r"$\ln 4$", ha="right", va="bottom", fontsize=8, color=GREY)
    for a, t in zip(ax, ("(a) mode coverage", "(b) distance of samples from the data", "(c) the loss is not a progress measure")):
        a.set_xlabel("iteration")
        a.set_title(t)
    ax[1].legend(fontsize=7, frameon=False, loc="upper right")
    fig.tight_layout()
    savefig(fig, CH, "fig17-s5", out)


FIG2 = dict(mu_d=0.0, mu_g=4.0, s=0.55, tau=1.4)


def fig_17_2(out=None):
    """17.2 The optimal discriminator for well-separated densities, and its smoothed version."""
    m0, m1, s, tau = FIG2["mu_d"], FIG2["mu_g"], FIG2["s"], FIG2["tau"]
    x = np.linspace(-2.5, 7.0, 800)
    lp = log_mix(x, [["g", 1.0, m0, s]])
    lq = log_mix(x, [["g", 1.0, m1, s]])
    lps = log_mix(x, [["g", 1.0, m0, math.hypot(s, tau)]])
    lqs = log_mix(x, [["g", 1.0, m1, math.hypot(s, tau)]])
    d, dt = d_star(lp, lq), d_star(lps, lqs)
    rng = np.random.default_rng(4)
    xr = m0 + s * rng.standard_normal(6)
    xg = m1 + s * rng.standard_normal(6)
    fig, ax = plt.subplots(1, 2, figsize=(11.0, 2.9), gridspec_kw={"width_ratios": [1.35, 1]})
    sc = 1.0 / np.exp(lp).max()
    ax[0].plot(x, np.exp(lp) * sc * 0.75, color=RED, label=r"$p_{\rm Data}(x)$")
    ax[0].plot(x, np.exp(lq) * sc * 0.75, color=BLUE, label=r"$p_G(x)$")
    ax[0].plot(x, d, color=GREEN, label=r"$d(x)$")
    ax[0].plot(x, dt, color=GREEN, ls="--", label=r"$\widetilde{d}(x)$")
    ax[0].plot(xr, np.zeros_like(xr) + 0.01, "o", color=RED, ms=4.5, clip_on=False)
    ax[0].plot(xg, np.zeros_like(xg) + 0.01, "o", color=BLUE, ms=4.5, clip_on=False)
    ax[0].set_ylim(0, 1.05)
    ax[0].set_xlim(x[0], x[-1])
    ax[0].set_xlabel("$x$")
    ax[0].set_yticks([0, 0.5, 1])
    ax[0].legend(loc="center left", bbox_to_anchor=(1.0, 0.5), frameon=True, fontsize=8.5)
    ax[0].set_title("(a) densities and discriminators")
    # slope of the minimax generator loss ln(1 - d(x)) with respect to x
    # d/dx ln(1 - d*) = d* (ln q)' - d* (ln p)' = d* (m1 - m0)/s^2 for two Gaussians of equal width s
    for dd, ss, ls, lab in ((d, s, "-", r"d"), (dt, math.hypot(s, tau), "--", r"\widetilde d")):
        ax[1].semilogy(x, dd * (m1 - m0) / ss ** 2, color=GREEN, ls=ls, label=r"$|\partial_x \ln(1-%s)|$" % lab)
    ax[1].axvspan(m1 - 2 * s, m1 + 2 * s, color=BLUE, alpha=0.1)
    ax[1].text(m1, 2e-14, "generated\nsamples", ha="center", fontsize=8, color=BLUE)
    ax[1].set_ylim(1e-15, 50)
    ax[1].set_xlim(x[0], x[-1])
    ax[1].set_xlabel("$x$")
    ax[1].legend(fontsize=8.5, frameon=False, loc="lower left")
    ax[1].set_title("(b) gradient reaching the generator")
    fig.tight_layout()
    savefig(fig, CH, "fig17-2", out)


def gan_1d(loss, mu0=6.0, iters=600, nd=5, lrD=0.2, lrG=0.1, B=64, seed=0):
    """A one-dimensional GAN: data N(0, 1), generator x = mu + z, logistic (or least-squares) linear
    discriminator d(x) = sigma(w x + b) (or w x + b) with nd SGD steps per generator step. Returns the
    path of mu and the final slope w."""
    r = np.random.default_rng(seed)
    w = b = 0.0
    mu = mu0
    path = [mu]
    for _ in range(iters):
        for _ in range(nd):
            xr = r.standard_normal(B)
            xf = mu + r.standard_normal(B)
            if loss == "ls":
                gr, gf = w * xr + b - 1, w * xf + b
            else:
                gr, gf = -(1 - sigm(w * xr + b)), sigm(w * xf + b)
            w, b = w - lrD * (np.mean(gr * xr) + np.mean(gf * xf)), b - lrD * (np.mean(gr) + np.mean(gf))
        a = w * (mu + r.standard_normal(B)) + b
        g = {"mm": np.mean(-sigm(a) * w), "ns": np.mean(-(1 - sigm(a)) * w), "ls": np.mean((a - 1) * w)}[loss]
        mu -= lrG * g
        path.append(mu)
    return np.array(path), w


def fig_17_s6(out=None):
    """17.S6 Logistic versus least-squares discriminators on separable data, and a 1-D GAN."""
    rng = np.random.default_rng(5)
    xr = 0.0 + 0.4 * rng.standard_normal(50)
    xf = 4.0 + 0.4 * rng.standard_normal(50)
    X = np.r_[xr, xf]
    t = np.r_[np.ones(50), np.zeros(50)]
    w, b = 0.0, 0.0
    ws, show = [], {10: None, 100: None, 1000: None, 10000: None}
    for it in range(1, 10001):
        g = sigm(w * X + b) - t
        w -= 0.5 * np.mean(g * X)
        b -= 0.5 * np.mean(g)
        ws.append(w)
        if it in show:
            show[it] = (w, b)
    wl, bl = np.linalg.lstsq(np.c_[X, np.ones_like(X)], t, rcond=None)[0]
    xs = np.linspace(-2, 6, 500)
    fig, ax = plt.subplots(1, 3, figsize=(13.2, 3.2))
    ax[0].plot(xr, np.zeros_like(xr), "o", color=RED, ms=3, label="real")
    ax[0].plot(xf, np.zeros_like(xf), "o", color=BLUE, ms=3, label="generated")
    cols = plt.cm.Greens(np.linspace(0.45, 1.0, len(show)))
    for c, (it, (w_, b_)) in zip(cols, show.items()):
        ax[0].plot(xs, sigm(w_ * xs + b_), color=c, lw=1.4, label="logistic, %d steps" % it)
    ax[0].plot(xs, wl * xs + bl, color=ORANGE, lw=2, ls="--", label="least squares")
    ax[0].set_ylim(-0.6, 1.75)
    ax[0].set_xlabel("$x$")
    ax[0].legend(fontsize=7, frameon=False, ncol=2, loc="upper right")
    ax[0].set_title("(a) linear discriminators, separable data")
    ax[1].semilogx(np.arange(1, 10001), np.abs(ws), color=GREEN, label=r"logistic: $|w|$ grows like $\ln k$")
    ax[1].axhline(abs(wl), color=ORANGE, ls="--", label="least squares: $|w|$ = %.3f" % abs(wl))
    ax[1].set_xlabel("gradient step $k$")
    ax[1].set_ylabel("slope $|w|$")
    ax[1].legend(fontsize=8, frameon=False)
    ax[1].set_title("(b) the logistic slope diverges")
    for loss, c, lab, lr in (("mm", BLUE, "minimax (17.9)", 0.2), ("ns", RED, "non-saturating (17.10)", 0.2),
                             ("ls", ORANGE, "least squares", 0.02)):
        P, _ = gan_1d(loss, lrD=lr)
        ax[2].plot(P, color=c, label=lab)
    ax[2].axhline(0, color=GREY, lw=0.6)
    ax[2].set_xlabel("generator step")
    ax[2].set_ylabel(r"generator mean $\mu$")
    ax[2].legend(fontsize=8, frameon=False)
    ax[2].set_title(r"(c) a 1-D GAN started at $\mu=6$")
    fig.tight_layout()
    savefig(fig, CH, "fig17-s6", out)


def fig_17_3(out=None):
    """17.3 The two generator losses -ln d and ln(1 - d), and their gradients with respect to the logit."""
    d = np.linspace(1e-3, 1 - 1e-3, 600)
    fig, ax = plt.subplots(1, 2, figsize=(9.6, 3.0))
    ax[0].plot(d, -np.log(d), color=RED)
    ax[0].plot(d, np.log(1 - d), color=BLUE)
    ax[0].axhline(0, color="k", lw=0.6)
    ax[0].text(0.55, 0.9, r"$-\ln(d)$", color=RED, fontsize=11)
    ax[0].text(0.35, -0.9, r"$\ln(1-d)$", color=BLUE, fontsize=11)
    ax[0].set_ylim(-2.6, 2.6)
    ax[0].set_xlim(0, 1)
    ax[0].set_xticks([0, 1])
    ax[0].set_yticks([-2, 0, 2])
    ax[0].set_xlabel("$d$")
    ax[0].set_title("(a) the two generator losses")
    ax[1].plot(d, d, color=BLUE, label=r"minimax: $|\partial_a\ln(1-\sigma(a))|=d$")
    ax[1].plot(d, 1 - d, color=RED, label=r"non-saturating: $|\partial_a\ln\sigma(a)|=1-d$")
    ax[1].set_xlabel(r"$d=\sigma(a)$")
    ax[1].set_ylabel("|gradient| w.r.t. logit $a$")
    ax[1].legend(fontsize=8, frameon=False, loc="upper center")
    ax[1].set_ylim(0, 1.25)
    ax[1].set_xlim(0, 1)
    ax[1].set_title("(b) gradients with respect to the logit")
    fig.tight_layout()
    savefig(fig, CH, "fig17-3", out)


def wgan_critic_1d(kind="gp", steps=4000, seed=0):
    """A 1 -> 64 -> 64 -> 1 critic trained to maximise E_P f - E_Q f for P = U[0,1], Q = U[3,4], with a
    gradient penalty (lambda = 10, interpolates) or weight clipping (0.05)."""
    torch = _torch()
    torch.manual_seed(seed)
    net = torch.nn.Sequential(torch.nn.Linear(1, 64), torch.nn.Tanh(), torch.nn.Linear(64, 64), torch.nn.Tanh(), torch.nn.Linear(64, 1))
    opt = torch.optim.Adam(net.parameters(), 1e-3, betas=(0.5, 0.999))
    for _ in range(steps):
        xp = torch.rand(256, 1)
        xq = 3 + torch.rand(256, 1)
        loss = net(xq).mean() - net(xp).mean()
        if kind == "gp":
            e = torch.rand(256, 1)
            xh = (e * xp + (1 - e) * xq).requires_grad_(True)
            g, = torch.autograd.grad(net(xh).sum(), xh, create_graph=True)
            loss = loss + 10.0 * ((g.abs() - 1) ** 2).mean()
        opt.zero_grad()
        loss.backward()
        opt.step()
        if kind == "clip":
            with torch.no_grad():
                for p in net.parameters():
                    p.clamp_(-0.05, 0.05)
    xs = torch.linspace(-1, 5, 601)[:, None]
    with torch.no_grad():
        f = net(xs)[:, 0].numpy().astype(float)
        est = float(net(torch.rand(20000, 1)).mean() - net(3 + torch.rand(20000, 1)).mean())
    return xs[:, 0].numpy().astype(float), f, est


def fig_17_s7(out=None):
    """17.S7 Wasserstein distance versus Jensen-Shannon divergence; optimal and learned critics."""
    th, js, w1 = shift_curves("uniform", 1.0, np.linspace(-6, 6, 241))
    th2, js2, w12 = shift_curves("gauss", 1.0, np.linspace(-6, 6, 241))
    fig, ax = plt.subplots(1, 3, figsize=(12.6, 3.3))
    ax[0].plot(th, w1, color=PURPLE, label=r"$W_1$, uniforms")
    ax[0].plot(th, js / LN2 * 1.0, color=GREEN, label=r"JS$/\ln 2$, uniforms")
    ax[0].plot(th2, js2 / LN2, color=GREEN, ls="--", label=r"JS$/\ln 2$, Gaussians")
    ax[0].plot(th2, w12, color=PURPLE, ls=":", lw=2.2, label=r"$W_1$, Gaussians")
    ax[0].set_xlabel(r"shift $\theta$ of $Q$")
    ax[0].set_ylim(0, 6.2)
    ax[0].legend(fontsize=7.5, frameon=False, loc="upper center")
    ax[0].set_title(r"(a) $P$ fixed, $Q$ moved by $\theta$")
    x, h = grid_x()
    lp = log_mix(x, shape_comps("uniform", 0.5))
    lq = log_mix(x, shape_comps("uniform", 3.5))
    sel = (x >= -1) & (x <= 5)
    fstar = optimal_critic(lp, lq, h)
    xs, fgp, egp = wgan_critic_1d("gp")
    _, fcl, ecl = wgan_critic_1d("clip")
    ref = lambda f, xx: f - np.interp(2.0, xx, f)  # noqa: E731
    ax[1].fill_between(x[sel], 0, np.exp(lp[sel]) * 0.8, color=RED, alpha=0.18, lw=0)
    ax[1].fill_between(x[sel], 0, np.exp(lq[sel]) * 0.8, color=BLUE, alpha=0.18, lw=0)
    ax[1].plot(x[sel], ref(fstar[sel], x[sel]), color="k", lw=2, label=r"optimal critic $f^\star$ (slope $-1$)")
    ax[1].plot(xs, ref(fgp, xs), color=PURPLE, ls="--", label="trained, gradient penalty")
    ax[1].plot(xs, ref(fcl, xs), color=ORANGE, ls="-.", label="trained, weight clipping")
    ax[1].plot(x[sel], d_star(lp, lq)[sel] * 3 - 1.5, color=GREEN, lw=1.2, label=r"$3d^\star-1.5$ (GAN)")
    ax[1].set_xlabel("$x$")
    ax[1].legend(fontsize=7, frameon=False, loc="lower left")
    ax[1].set_title(r"(b) $P=U[0,1]$ (red), $Q=U[3,4]$ (blue)")
    ax[1].set_ylim(-2.8, 3.0)
    pW = shift_descent("uniform", 4.0, "W1", lr=0.3, steps=30)
    pJ = shift_descent("uniform", 4.0, "JS", lr=0.3, steps=30)
    ax[2].plot(pW, color=PURPLE, marker="o", ms=3, label=r"descent on $W_1$")
    ax[2].plot(pJ, color=GREEN, marker="s", ms=3, label="descent on JS")
    ax[2].axhline(0, color=GREY, lw=0.6)
    ax[2].set_xlabel("step")
    ax[2].set_ylabel(r"$\theta$")
    ax[2].legend(fontsize=8, frameon=False)
    ax[2].set_title(r"(c) gradient descent on $\theta$ from 4")
    fig.tight_layout()
    savefig(fig, CH, "fig17-s7", out)
    return {"W1_gp_est": egp, "W1_clip_est": ecl}


# =========================================================================== Section 17.2 figures
def slerp(z0, z1, t):
    om = np.arccos(np.clip(np.dot(z0 / np.linalg.norm(z0), z1 / np.linalg.norm(z1)), -1, 1))
    return (np.sin((1 - t) * om) * z0 + np.sin(t * om) * z1) / np.sin(om)


def fig_17_9(out=None):
    """17.9 (small scale) Walks through the latent space of an MNIST GAN."""
    torch = _torch()
    G, _ = mnist_gan(False)
    rng = np.random.default_rng(9)
    rows, ncol = 6, 10
    Z = []
    for r in range(rows):
        z0, z1 = rng.standard_normal(DZ), rng.standard_normal(DZ)
        for t in np.linspace(0, 1, ncol):
            Z.append(slerp(z0, z1, t))
    with torch.no_grad():
        X = G(torch.tensor(np.array(Z), dtype=torch.float32)).numpy()
    fig, ax = plt.subplots(figsize=(6.6, 4.1))
    _img_grid(ax, X, rows, ncol)
    fig.tight_layout()
    savefig(fig, CH, "fig17-9", out)


def _arith_setup(n=20000, seed=11):
    torch = _torch()
    G, _ = mnist_gan(False)
    net = mnist_classifier()
    torch.manual_seed(seed)
    Z = torch.randn(n, DZ)
    with torch.no_grad():
        X = G(Z)
    _, P = classifier_features(net, X)
    ink = ((X.numpy() + 1) / 2).mean(1)
    return G, net, Z, X.numpy(), P.argmax(1), P.max(1), ink


def _arith_group(lab, conf, ink, rng, digit, thick, k=3):
    """k generated digits of a class (classifier confidence > 0.98) from the thickest (or thinnest) 10%."""
    idx = np.where((lab == digit) & (conf > 0.98))[0]
    q = np.quantile(ink[idx], [0.1, 0.9])
    pick = idx[ink[idx] >= q[1]] if thick else idx[ink[idx] <= q[0]]
    return rng.choice(pick, k, replace=False)


def latent_arithmetic(a=3, b=2, seed=11, pick=0):
    """'thick a - thin a + thin b' with three generated digits per group (as in Figure 17.10): returns
    the groups, the result and eight noisy neighbours (s.d. 0.25), and the same arithmetic on pixels."""
    torch = _torch()
    G, net, Z, X, lab, conf, ink = _arith_setup(seed=seed)
    rng = np.random.default_rng(pick)
    A, B_, C_ = (_arith_group(lab, conf, ink, rng, a, True), _arith_group(lab, conf, ink, rng, a, False),
                 _arith_group(lab, conf, ink, rng, b, False))
    zr = Z[A].mean(0) - Z[B_].mean(0) + Z[C_].mean(0)
    Zs = torch.cat([zr[None], zr[None] + 0.25 * torch.randn(8, DZ, generator=torch.Generator().manual_seed(3))])
    with torch.no_grad():
        Xr = G(Zs).numpy()
    _, Pr = classifier_features(net, torch.tensor(Xr))
    pix = X[A].mean(0) - X[B_].mean(0) + X[C_].mean(0)
    return {"X": X, "A": A, "B": B_, "C": C_, "Xr": Xr, "Pr": Pr, "ink": ink, "a": a, "b": b,
            "ink_r": ((Xr + 1) / 2).mean(1), "pix": np.clip(pix, -1, 1)}


def arithmetic_survey(reps=5, seed=0):
    """All 90 ordered digit pairs (a, b), five random triples each: fraction of results the classifier
    labels b, and the median ratio of their ink to that of the thin b inputs. Cached."""
    path = os.path.join(CACHE_DIR, "ch17_arith_survey.json")
    if os.path.exists(path):
        with open(path) as f:
            return json.load(f)
    torch = _torch()
    G, net, Z, X, lab, conf, ink = _arith_setup()
    rng = np.random.default_rng(seed)
    kept, ratio = [], []
    for a in range(10):
        for b in range(10):
            if a == b:
                continue
            for _ in range(reps):
                A, B_, C_ = (_arith_group(lab, conf, ink, rng, a, True), _arith_group(lab, conf, ink, rng, a, False),
                             _arith_group(lab, conf, ink, rng, b, False))
                zr = Z[A].mean(0) - Z[B_].mean(0) + Z[C_].mean(0)
                with torch.no_grad():
                    xr = G(zr[None])
                _, pr = classifier_features(net, xr)
                kept.append(int(pr.argmax() == b))
                ratio.append(float(((xr.numpy() + 1) / 2).mean() / ink[C_].mean()))
    out = {"kept": float(np.mean(kept)), "ink_ratio_median": float(np.median(ratio)), "trials": len(kept)}
    os.makedirs(CACHE_DIR, exist_ok=True)
    with open(path, "w") as f:
        json.dump(out, f)
    return out


def fig_17_10(out=None):
    """17.10 (small scale) Vector arithmetic in the latent space of an MNIST GAN."""
    r = latent_arithmetic()
    X, a, b = r["X"], r["a"], r["b"]
    fig = plt.figure(figsize=(9.0, 3.7))
    gs = fig.add_gridspec(2, 5, height_ratios=[3, 1], width_ratios=[1, 1, 1, 0.25, 3], hspace=0.3, wspace=0.15)
    titles = ["thick %d" % a, "$-$ thin %d" % a, "$+$ thin %d" % b]
    for j, (key, t) in enumerate(zip("ABC", titles)):
        ax = fig.add_subplot(gs[0, j])
        _img_grid(ax, X[r[key]], 3, 1)
        ax.set_title(t, fontsize=9.5)
        m = fig.add_subplot(gs[1, j])
        _img_grid(m, X[r[key]].mean(0)[None], 1, 1)
        if j == 0:
            m.set_ylabel("pixel\nmean", fontsize=8)
    ax = fig.add_subplot(gs[0, 4])
    _img_grid(ax, r["Xr"][[1, 2, 3, 4, 0, 5, 6, 7, 8]], 3, 3)
    ax.set_title(r"$=$ result (centre) and 8 noisy neighbours", fontsize=9.5)
    bx = fig.add_subplot(gs[1, 4])
    _img_grid(bx, r["pix"][None], 1, 1)
    bx.set_title("the same arithmetic on pixels", fontsize=8.5)
    savefig(fig, CH, "fig17-10", out)


def fig_17_6(out=None):
    """17.6 (small scale) CycleGAN between two two-dimensional domains, with and without cycle consistency."""
    torch = _torch()
    gen = torch.Generator().manual_seed(123)
    x, kx = _cyc_sample(torch, CYC_X, 800, gen)
    y, ky = _cyc_sample(torch, CYC_Y, 800, gen)
    cols = np.array([RED, BLUE, GREEN, ORANGE])
    fig, ax = plt.subplots(2, 4, figsize=(12.0, 5.2))
    stats = {}
    for i, eta in enumerate((0.0, 5.0)):
        gX, gY, hist = cyclegan_2d(eta)
        with torch.no_grad():
            fy = gY(x)
            rx = gX(fy)
            fx = gX(y)
            ry = gY(fx)
        err = float((rx - x).abs().sum(1).mean() + (ry - y).abs().sum(1).mean())
        tgt = np.argmin(((fy.numpy()[:, None, :] - CYC_Y[None]) ** 2).sum(-1), 1)
        purity = np.mean([np.bincount(tgt[kx.numpy() == k], minlength=4).max() / max(1, (kx.numpy() == k).sum()) for k in range(4)])
        stats[eta] = {"cyc": err, "purity": float(purity), "perm": [int(np.bincount(tgt[kx.numpy() == k], minlength=4).argmax()) for k in range(4)]}
        panels = [(x, kx, "samples $x$ from domain X", CYC_X), (fy, kx, r"$g_Y(x)$: translated to Y", CYC_Y),
                  (rx, kx, r"$g_X(g_Y(x))$: back to X", CYC_X), (fx, ky, r"$g_X(y)$, coloured by cluster of $y$", CYC_X)]
        for j, (P, k, t, Cc) in enumerate(panels):
            a = ax[i, j]
            P = P.numpy()
            a.scatter(Cc[:, 0], Cc[:, 1], s=200, facecolors="none", edgecolors=GREY, lw=0.8)
            a.scatter(P[:, 0], P[:, 1], s=3, c=cols[k.numpy()], lw=0, alpha=0.7)
            a.set_xlim(-4, 4)
            a.set_ylim(-2.2, 2.2)
            a.set_aspect("equal")
            a.set_xticks([])
            a.set_yticks([])
            a.set_title(t, fontsize=9)
            if j == 0:
                a.set_ylabel(r"$\eta=%g$" % eta + ("\n(GAN losses only)" if eta == 0 else "\n(with cycle loss)"), fontsize=9)
        ax[i, 2].text(0.03, 0.03, "cycle error %.2f" % err, transform=ax[i, 2].transAxes, fontsize=8)
    fig.tight_layout()
    savefig(fig, CH, "fig17-6", out)
    return stats


def fig_17_s8(out=None):
    """17.S8 Evaluating the MNIST GAN: class histogram, Frechet distance and inception score."""
    ev = mnist_eval()
    fig, ax = plt.subplots(1, 3, figsize=(12.0, 3.1))
    last = ev["ckpt"][str(MNIST_STEPS)]
    hg = np.array(last["hist"]) / sum(last["hist"])
    hr = np.array(ev["real"]["hist"]) / sum(ev["real"]["hist"])
    k = np.arange(10)
    ax[0].bar(k - 0.2, hr, 0.4, color=GREY, label="test images")
    ax[0].bar(k + 0.2, hg, 0.4, color=BLUE, label="GAN samples")
    ax[0].set_xticks(k)
    ax[0].set_xlabel("predicted class")
    ax[0].set_ylabel("fraction")
    ax[0].legend(fontsize=8, frameon=False)
    ax[0].set_title("(a) class frequencies (classifier)")
    T = sorted(int(t) for t in ev["ckpt"])
    ax[1].plot(T, [ev["ckpt"][str(t)]["FD"] for t in T], "o-", color=BLUE, label="GAN")
    ax[1].axhline(ev["real"]["FD"], color=GREY, ls="--", label="test vs training images")
    ax[1].axhline(ev["cond"]["FD"], color=GREEN, ls=":", label="conditional GAN (final)")
    ax[1].set_xscale("log")
    ax[1].set_yscale("log")
    ax[1].set_xlabel("training step")
    ax[1].set_title("(b) Frechet distance (lower is better)")
    ax[1].legend(fontsize=7.5, frameon=False)
    ax[2].plot(T, [ev["ckpt"][str(t)]["IS"] for t in T], "o-", color=BLUE, label="GAN")
    ax[2].axhline(ev["real"]["IS"], color=GREY, ls="--", label="test images")
    ax[2].axhline(ev["cond"]["IS"], color=GREEN, ls=":", label="conditional GAN (final)")
    ax[2].axhline(10, color="k", lw=0.6)
    ax[2].set_xscale("log")
    ax[2].set_ylim(1, 10.5)
    ax[2].set_xlabel("training step")
    ax[2].set_title("(c) inception score (at most 10)")
    ax[2].legend(fontsize=7.5, frameon=False, loc="lower right")
    fig.tight_layout()
    savefig(fig, CH, "fig17-s8", out)


FIGURES = {
    "17.S1": fig_17_s1, "17.S2": fig_17_s2, "17.S3": fig_17_s3, "17.S4": fig_17_s4, "17.S5": fig_17_s5,
    "17.2": fig_17_2, "17.S6": fig_17_s6, "17.3": fig_17_3, "17.S7": fig_17_s7,
    "17.9": fig_17_9, "17.10": fig_17_10, "17.6": fig_17_6, "17.S8": fig_17_s8,
}

if __name__ == "__main__":
    run(FIGURES, CH)
