"""Wasserman (2004), Chapter 11: Bayesian Inference.

Companion code for Chapter 11 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 175-192, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch11-bayesian-inference.html):

* Section 11.2, the Bayesian method: beta_post and beta_interval (Example 11.1, Bernoulli data and
  a Beta prior), normal_post (Example 11.2 and equation (11.7), a Normal mean with a Normal prior),
  qbeta and qgamma (quantiles by bisection on the regularised incomplete beta and gamma functions,
  the same algorithm as the browser demonstrations).
* Sections 11.3 and 11.4, functions of parameters and simulation: logodds_density (the posterior
  density of the log-odds in Example 11.3, as corrected in the author's errata) and
  logodds_density_misprint (the formula as printed), two_binomials and tau_density (Example 11.7:
  the posterior of p2 - p1 by simulation and by numerical convolution).
* Section 11.5, large-sample properties (Theorem 11.5, the Bernstein-von Mises theorem): bvm (the
  exact posterior against its Normal approximation, their total variation distance and the
  posterior probability of the Wald interval, in the natural or the log scale) for Bernoulli data
  with a Beta prior and for the nerve data (exponential model, Gamma prior).
* Section 11.6, flat, improper and Jeffreys priors: the priors that are flat in a chosen
  parametrisation (FLAT_BERN, FLAT_POIS), fisher_info_phi (the Fisher information in a new
  parametrisation, computed directly from the score), and jeffreys_check (Jeffreys' rule commutes
  with reparametrisation).
* Section 11.8, Bayesian testing: post_h0_normal (P(H0 | data) for H0: mu = 0 with a N(0, b^2)
  prior under H1, Exercise 8), the prior-free lower bound and the Wald p-value.
* Section 11.9, strengths and weaknesses: credible_coverage (exact frequentist coverage of the
  equal-tailed Beta posterior interval), prior_avg_coverage, robins_ritov (Example 11.9: the
  Horvitz-Thompson estimator against likelihood-based Bayes estimators) and normconst_postmean
  (Example 11.10).
* Exercises 11.2 to 11.8: ex2, ex3, ex4, ex5, ex6, ex8.
* JSRandom and js_uniforms: the seeded generator of the browser demonstrations (DK.rng in
  courses/demo-kit.js), so that the simulated data on the pages can be reproduced here.

Chapter 11 of the book has no figures, so nothing is reproduced and nothing is skipped. The module
draws seven supplementary figures of our own that illustrate the gaps filled in the notes:
  11.S1 the Bayesian method: Beta-Bernoulli updating, shrinkage of the posterior mean, Normal-Normal;
  11.S2 functions of parameters by simulation: the log-odds (Examples 11.3, 11.4) and p2 - p1 (11.7);
  11.S3 the Bernstein-von Mises theorem (Theorem 11.5) for Bernoulli data and the nerve data;
  11.S4 flat priors are not invariant; Jeffreys' prior (Section 11.6);
  11.S5 Bayesian testing and the Jeffreys-Lindley paradox (Section 11.8, Exercise 8);
  11.S6 frequentist coverage of Bayesian intervals, Examples 11.9 and 11.10 (Section 11.9);
  11.S7 Exercises 11.2 and 11.5.
Run `python3 ch11_bayesian_inference.py --list`, `python3 ch11_bayesian_inference.py` (all seven
figures, well under a minute on one CPU thread) or `python3 ch11_bayesian_inference.py --fig 11.S3`.
Figures go to ../img/ch11/ and the nerve data used by the Bernstein-von Mises demonstration to
../data/ch11-nerve.json (export_data()).
"""
import json
import math
import os

import numpy as np
from scipy import integrate, special, stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig

CH = 11
M32 = 0xFFFFFFFF
HERE = os.path.dirname(os.path.abspath(__file__))
JSON_DIR = os.path.normpath(os.path.join(HERE, "..", "data"))


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js): u() is
    mulberry32 and norm() the Box-Muller transform, exactly as in JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed), vectorised (mulberry32 adds a constant
    to its state at each call, so the j-th output is a hash of seed + j * 0x6D2B79F5 mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def bern_data(p, n, seed):
    """The Bernoulli sample of the demonstrations: x_k = 1 if the k-th uniform of DK.rng(seed) is < p."""
    return (js_uniforms(seed, n) < p).astype(int)


def zq(alpha):
    return float(stats.norm.ppf(1.0 - alpha / 2.0))


# =========================================================================== data
def nerve():
    """The 799 waiting times (seconds) between nerve pulses, nerve.dat of the book's website."""
    with open(common.aos_file("nerve.dat")) as f:
        return np.array([float(v) for v in f.read().split()])


def export_data(out_dir=JSON_DIR):
    """Write ../data/ch11-nerve.json for the Bernstein-von Mises demonstration."""
    x = nerve()
    os.makedirs(out_dir, exist_ok=True)
    path = os.path.join(out_dir, "ch11-nerve.json")
    with open(path, "w") as f:
        json.dump({"note": "Waiting times (seconds) between 799 successive pulses along a nerve fibre "
                           "(Cox and Lewis 1966), the data set nerve.dat of Wasserman (2004), from "
                           "https://www.stat.cmu.edu/~larry/all-of-statistics/", "x": [float(v) for v in x]}, f,
                  separators=(",", ":"))
    return path


# =========================================================================== quantiles by bisection
def qbeta(q, a, b, iters=100):
    """Quantile of Beta(a, b) by bisection on I_x(a, b), as qbeta() in the demonstrations."""
    lo, hi = 0.0, 1.0
    for _ in range(iters):
        mid = 0.5 * (lo + hi)
        if special.betainc(a, b, mid) < q:
            lo = mid
        else:
            hi = mid
    return 0.5 * (lo + hi)


def qgamma(q, a, rate, iters=200):
    """Quantile of Gamma(shape a, rate) by bisection on the regularised lower incomplete gamma."""
    lo, hi = 0.0, (a + 40.0 * math.sqrt(a) + 60.0) / rate
    for _ in range(iters):
        mid = 0.5 * (lo + hi)
        if special.gammainc(a, mid * rate) < q:
            lo = mid
        else:
            hi = mid
    return 0.5 * (lo + hi)


# =========================================================================== Section 11.2
def beta_post(s, n, a=1.0, b=1.0):
    """Posterior Beta(a + s, b + n - s) for s successes in n Bernoulli trials (Example 11.1)."""
    return a + s, b + n - s


def beta_interval(A, B, alpha=0.05):
    """Equal-tailed 1 - alpha posterior interval of Beta(A, B)."""
    return float(stats.beta.ppf(alpha / 2, A, B)), float(stats.beta.ppf(1 - alpha / 2, A, B))


def normal_post(xbar, n, sigma, a, b):
    """Posterior N(mean, tau^2) of a Normal mean with known sigma and prior N(a, b^2), equation (11.7).
    Returns (mean, tau, w) with w = (1/se^2) / (1/se^2 + 1/b^2), se = sigma / sqrt(n)."""
    se2 = sigma ** 2 / n
    w = (1 / se2) / (1 / se2 + 1 / b ** 2)
    tau2 = 1 / (1 / se2 + 1 / b ** 2)
    return w * xbar + (1 - w) * a, math.sqrt(tau2), w


# =========================================================================== Sections 11.3 and 11.4
def logodds_density(psi, s, n):
    """Posterior density of psi = log(p / (1 - p)) when p | x ~ Beta(s + 1, n - s + 1) (Example 11.3):
    h(psi) = e^{(s+1) psi} / (1 + e^psi)^{n+2} / B(s + 1, n - s + 1)."""
    psi = np.asarray(psi, dtype=float)
    lp = -np.logaddexp(0, -psi)          # log p
    lq = -np.logaddexp(0, psi)           # log(1 - p)
    return np.exp((s + 1) * lp + (n - s + 1) * lq - special.betaln(s + 1, n - s + 1))


def logodds_density_misprint(psi, s, n):
    """The density as printed on p. 180 (proportional to p^s (1 - p)^(n - s + 2)), normalised; see the
    author's errata. It is the log-odds density of Beta(s, n - s + 2), shifted to the left."""
    psi = np.asarray(psi, dtype=float)
    lp = -np.logaddexp(0, -psi)
    lq = -np.logaddexp(0, psi)
    return np.exp(s * lp + (n - s + 2) * lq - special.betaln(s, n - s + 2))


def two_binomials(x1, n1, x2, n2, B, rng):
    """Example 11.7 with the flat prior: independent posterior draws of p1, p2 and tau = p2 - p1."""
    p1 = rng.beta(x1 + 1, n1 - x1 + 1, B)
    p2 = rng.beta(x2 + 1, n2 - x2 + 1, B)
    return p1, p2, p2 - p1


def tau_density(t, x1, n1, x2, n2, m=4001):
    """Exact posterior density of tau = p2 - p1 (flat prior) by numerical convolution:
    f(t) = int f1(p) f2(p + t) dp over max(0, -t) < p < min(1, 1 - t)."""
    out = []
    for tt in np.atleast_1d(t):
        lo, hi = max(0.0, -tt), min(1.0, 1.0 - tt)
        if hi <= lo:
            out.append(0.0)
            continue
        p = np.linspace(lo, hi, m)
        g = stats.beta.pdf(p, x1 + 1, n1 - x1 + 1) * stats.beta.pdf(p + tt, x2 + 1, n2 - x2 + 1)
        out.append(integrate.simpson(g, x=p))
    return np.array(out)


# =========================================================================== Section 11.5: Bernstein-von Mises
NGRID_TV = 2001       # grid points for the total variation integral
HALF_TV = 12.0        # the grid spans the Normal approximation's mean +- 12 sd


def _lbeta_pdf(p, A, B):
    with np.errstate(divide="ignore", invalid="ignore"):
        out = (A - 1) * np.log(p) + (B - 1) * np.log1p(-p) - special.betaln(A, B)
    return np.where((p > 0) & (p < 1), out, -np.inf)


def _lgamma_pdf(lam, A, R):
    with np.errstate(divide="ignore", invalid="ignore"):
        out = A * math.log(R) - special.gammaln(A) + (A - 1) * np.log(lam) - R * lam
    return np.where(lam > 0, out, -np.inf)


def _trap(y, h):
    return h * (y.sum() - 0.5 * (y[0] + y[-1]))


def bvm(model, n, stat, a, b, scale="natural", alpha=0.05):
    """Exact posterior against the Normal approximation of Theorem 11.5 after n observations.

    model 'bern': stat = number of successes s, prior Beta(a, b), posterior Beta(a + s, b + n - s),
      approximation N(p_hat, p_hat (1 - p_hat) / n); scale 'log' means the log-odds psi, where the
      approximation is N(logit p_hat, 1 / (n p_hat (1 - p_hat))) (the Bayesian delta method).
    model 'exp': stat = sum of the observations, prior Gamma(a, rate b), posterior
      Gamma(a + n, b + sum), approximation N(lam_hat, lam_hat^2 / n) with lam_hat = n / sum; scale
      'log' means psi = log lam, approximation N(log lam_hat, 1 / n).
    Returns dict(center, se, tv, ppost): tv = 1 - int min(f, g) over the grid (exact even when the
    posterior has mass outside it), ppost = posterior probability of the Wald interval
    center +- z se. tv and ppost are None when the MLE is on the boundary (se = 0).
    """
    z = zq(alpha)
    if model == "bern":
        A, B = a + stat, b + n - stat
        ph = stat / n
        if ph <= 0 or ph >= 1:
            return dict(center=ph, se=0.0, tv=None, ppost=None)
        if scale == "natural":
            c, se = ph, math.sqrt(ph * (1 - ph) / n)
        else:
            c, se = math.log(ph / (1 - ph)), 1 / math.sqrt(n * ph * (1 - ph))
    else:
        A, R = a + n, b + stat
        lh = n / stat
        if scale == "natural":
            c, se = lh, lh / math.sqrt(n)
        else:
            c, se = math.log(lh), 1 / math.sqrt(n)
    t = np.linspace(c - HALF_TV * se, c + HALF_TV * se, NGRID_TV)
    h = t[1] - t[0]
    if model == "bern":
        if scale == "natural":
            lf = _lbeta_pdf(t, A, B)
        else:
            p = 1 / (1 + np.exp(-t))
            lf = _lbeta_pdf(p, A, B) + np.log(p) + np.log1p(-p)
    else:
        if scale == "natural":
            lf = _lgamma_pdf(t, A, R)
        else:
            lf = _lgamma_pdf(np.exp(t), A, R) + t
    lg = -0.5 * ((t - c) / se) ** 2 - math.log(se) - 0.5 * math.log(2 * math.pi)
    tv = 1 - _trap(np.exp(np.minimum(lf, lg)), h)
    lo, hi = c - z * se, c + z * se
    if model == "bern":
        if scale == "log":
            lo, hi = 1 / (1 + math.exp(-lo)), 1 / (1 + math.exp(-hi))
        lo, hi = max(lo, 0.0), min(hi, 1.0)
        pp = special.betainc(A, B, hi) - special.betainc(A, B, lo)
    else:
        if scale == "log":
            lo, hi = math.exp(lo), math.exp(hi)
        lo = max(lo, 0.0)
        pp = special.gammainc(A, R * hi) - special.gammainc(A, R * lo)
    return dict(center=c, se=se, tv=float(min(1.0, max(0.0, tv))), ppost=float(pp))


def bvm_ngrid(nmax):
    """The sample sizes of the demonstration's curves: about 60 values, evenly spaced on a log scale."""
    return np.unique(np.round(np.logspace(0, math.log10(nmax), 61)).astype(int))


def bvm_curve(model, data, a, b, scale="natural", alpha=0.05, ngrid=None):
    """tv and ppost along the grid of sample sizes, using the first n observations of data."""
    ns = bvm_ngrid(len(data)) if ngrid is None else ngrid
    cs = np.cumsum(data)
    out = []
    for n in ns:
        out.append(bvm(model, int(n), float(cs[n - 1]), a, b, scale, alpha))
    return ns, out


# =========================================================================== Section 11.6: flat and Jeffreys priors
# Bernoulli: the prior that is flat in phi = g(p) is |g'(p)| = p^c1 (1 - p)^c2 (up to a constant), so the
# posterior after s successes in n trials is Beta(s + 1 + c1, n - s + 1 + c2).
FLAT_BERN = {
    "p": dict(c=(0.0, 0.0), g=lambda p: p, ginv=lambda f: f, label="p"),
    "logit": dict(c=(-1.0, -1.0), g=lambda p: np.log(p / (1 - p)), ginv=lambda f: 1 / (1 + np.exp(-f)),
                  label="log(p/(1-p))"),
    "asin": dict(c=(-0.5, -0.5), g=lambda p: np.arcsin(np.sqrt(p)), ginv=lambda f: np.sin(f) ** 2,
                 label="arcsin(sqrt p)"),
    "log": dict(c=(-1.0, 0.0), g=lambda p: np.log(p), ginv=lambda f: np.exp(f), label="log p"),
    "sq": dict(c=(1.0, 0.0), g=lambda p: p ** 2, ginv=lambda f: np.sqrt(f), label="p^2"),
}
# Poisson: flat in phi = g(lambda) is lambda^c, posterior Gamma(s + 1 + c, rate n) with s = sum of counts.
FLAT_POIS = {
    "lam": dict(c=0.0, g=lambda l: l, ginv=lambda f: f, label="lambda"),
    "log": dict(c=-1.0, g=lambda l: np.log(l), ginv=lambda f: np.exp(f), label="log lambda"),
    "sqrt": dict(c=-0.5, g=lambda l: np.sqrt(l), ginv=lambda f: f ** 2, label="sqrt(lambda)"),
    "inv": dict(c=-2.0, g=lambda l: 1 / l, ginv=lambda f: 1 / f, label="1/lambda"),
}


def flat_posterior(model, key, s, n):
    """(shape1, shape2) of the posterior under the prior flat in parametrisation key: Beta(A, B) for
    'bern', Gamma(A, rate B) for 'pois'. None when the posterior is improper."""
    if model == "bern":
        c1, c2 = FLAT_BERN[key]["c"]
        A, B = s + 1 + c1, n - s + 1 + c2
        return (A, B) if A > 0 and B > 0 else None
    A = s + 1 + FLAT_POIS[key]["c"]
    return (A, float(n)) if A > 0 and n > 0 else None


def jeffreys_posterior(model, s, n):
    """Jeffreys' prior: Beta(1/2, 1/2) for Bernoulli, lambda^(-1/2) for Poisson."""
    return (s + 0.5, n - s + 0.5) if model == "bern" else (s + 0.5, float(n))


def fisher_info_phi(model, key, phi, eps=1e-5):
    """Fisher information of ONE observation in the parametrisation phi, computed from its definition
    I(phi) = E[(d/dphi log f(X; theta(phi)))^2] with a central difference for the score and the
    expectation as an exact sum over x (Bernoulli: x = 0, 1; Poisson: x = 0, 1, ..., until the mass
    is negligible). This does not use the chain rule I(phi) = I(theta) (dtheta/dphi)^2."""
    tab = FLAT_BERN if model == "bern" else FLAT_POIS
    ginv = tab[key]["ginv"]
    th0, thp, thm = ginv(phi), ginv(phi + eps), ginv(phi - eps)
    if model == "bern":
        out = 0.0
        for x in (0, 1):
            lf = lambda t: x * math.log(t) + (1 - x) * math.log(1 - t)
            sc = (lf(thp) - lf(thm)) / (2 * eps)
            out += (th0 if x == 1 else 1 - th0) * sc * sc
        return out
    lam = th0
    xs = np.arange(0, int(lam + 40 * math.sqrt(lam) + 60))
    sc = ((xs * math.log(thp) - thp) - (xs * math.log(thm) - thm)) / (2 * eps)
    pm = stats.poisson.pmf(xs, lam)
    return float(np.sum(pm * sc * sc))


def jeffreys_check(model, key, thetas):
    """Jeffreys' rule applied in phi and carried back to theta: sqrt(I_phi(g(theta))) |g'(theta)|,
    which equals sqrt(I(theta)); returned as a ratio to sqrt(I(theta)) at each theta (it should be 1)."""
    tab = FLAT_BERN if model == "bern" else FLAT_POIS
    g = tab[key]["g"]
    out = []
    for th in thetas:
        phi = g(th)
        dg = (g(th + 1e-6) - g(th - 1e-6)) / 2e-6
        lhs = math.sqrt(fisher_info_phi(model, key, phi)) * abs(dg)
        rhs = 1 / math.sqrt(th * (1 - th)) if model == "bern" else 1 / math.sqrt(th)
        out.append(lhs / rhs)
    return np.array(out)


# =========================================================================== Section 11.8: Bayesian testing
def post_h0_normal(xbar, b, n=1, prior_h0=0.5):
    """P(H0 | data) for H0: mu = 0 against H1: mu ~ N(0, b^2), with X_1..X_n ~ N(mu, 1) (Exercise 8).
    Under H0, xbar ~ N(0, 1/n); under H1 (marginally), xbar ~ N(0, b^2 + 1/n)."""
    n = np.asarray(n, dtype=float)
    b = np.asarray(b, dtype=float)
    f0 = stats.norm.pdf(xbar, 0, 1 / np.sqrt(n))
    f1 = stats.norm.pdf(xbar, 0, np.sqrt(b * b + 1 / n))
    return prior_h0 * f0 / (prior_h0 * f0 + (1 - prior_h0) * f1)


def post_h0_bound(z):
    """The prior-free lower bound L(theta0) / (L(theta0) + L(mle)) for a Normal mean: 1 / (1 + e^{z^2/2})."""
    return 1 / (1 + np.exp(0.5 * np.asarray(z, dtype=float) ** 2))


def wald_pvalue(z):
    return 2 * stats.norm.sf(np.abs(z))


# =========================================================================== Section 11.9
def credible_ends(n, a, b, alpha=0.05):
    """Equal-tailed 1 - alpha posterior intervals [L_y, U_y] for every outcome y = 0..n (Beta(a, b) prior),
    with the bisection quantiles of the demonstration."""
    lo = np.array([qbeta(alpha / 2, a + y, b + n - y) for y in range(n + 1)])
    hi = np.array([qbeta(1 - alpha / 2, a + y, b + n - y) for y in range(n + 1)])
    return lo, hi


def credible_coverage(n, p, a, b, alpha=0.05, ends=None):
    """Exact frequentist coverage P_p(L_Y < p < U_Y), Y ~ Binomial(n, p), of the Beta posterior interval."""
    lo, hi = credible_ends(n, a, b, alpha) if ends is None else ends
    y = np.arange(n + 1)
    pm = stats.binom.pmf(y, n, p)
    return float(np.sum(pm[(lo < p) & (p < hi)]))


def prior_avg_coverage(n, a, b, alpha=0.05, ends=None):
    """The coverage averaged over the prior, sum_y m(y) P(L_y < p < U_y | y) with m the beta-binomial
    marginal; it equals 1 - alpha (Filling the gap 11.19)."""
    lo, hi = credible_ends(n, a, b, alpha) if ends is None else ends
    y = np.arange(n + 1)
    m = stats.betabinom.pmf(y, n, a, b)
    post = special.betainc(a + y, b + n - y, hi) - special.betainc(a + y, b + n - y, lo)
    return float(np.sum(m * post))


def prior_avg_coverage_quad(n, a, b, alpha=0.05, ends=None):
    """The same average computed the other way round, int C(p) pi(p) dp, by adaptive quadrature over the
    pieces between interval endpoints (C is a step function times binomial polynomials)."""
    lo, hi = credible_ends(n, a, b, alpha) if ends is None else ends
    knots = np.unique(np.concatenate([[0.0, 1.0], lo, hi]))
    tot = 0.0
    for u, v in zip(knots[:-1], knots[1:]):
        if v - u < 1e-14:
            continue
        mid = 0.5 * (u + v)
        ys = np.nonzero((lo < mid) & (mid < hi))[0]
        if len(ys) == 0:
            continue
        f = lambda p: stats.beta.pdf(p, a, b) * np.sum(stats.binom.pmf(ys, n, p))
        tot += integrate.quad(f, u, v, limit=200, epsabs=1e-13)[0]
    return tot


def robins_ritov(n, R, B=10 ** 6, delta=0.05, seed=11):
    """Example 11.9 by simulation. Cells j = 1..B have u_j ~ Uniform(0, 1), theta_j = u_j^2 and
    xi_j = delta + (1 - 2 delta) u_j (well observed cells tend to have Y = 1). For R samples of size n
    returns the Horvitz-Thompson estimate (11.10), the posterior mean of psi under an exchangeable
    prior (theta_j iid given a mean mu with a uniform prior on mu; for B >> n this is the posterior
    mean of mu, (1 + sum R Y) / (2 + sum R)), and the posterior mean under independent uniform priors
    on the theta_j, (1/B) sum_j E(theta_j | data). Also returns psi = mean(theta)."""
    rng = np.random.default_rng(seed)
    u = rng.random(B)
    theta, xi = u ** 2, delta + (1 - 2 * delta) * u
    psi = float(theta.mean())
    ht, exch, indep = np.empty(R), np.empty(R), np.empty(R)
    for r in range(R):
        X = rng.integers(0, B, n)
        Rr = rng.random(n) < xi[X]
        Y = rng.random(n) < theta[X]
        RY = Rr & Y
        ht[r] = np.mean(RY / xi[X])
        exch[r] = (1 + RY.sum()) / (2 + Rr.sum())
        # independent Beta(1, 1) priors: E(theta_j | data) = (1 + n_j) / (2 + n_j + m_j)
        obs = X[Rr]
        cells, inv = np.unique(obs, return_inverse=True)
        nj = np.bincount(inv, weights=Y[Rr].astype(float))
        tj = np.bincount(inv).astype(float)
        indep[r] = ((B - len(cells)) * 0.5 + np.sum((1 + nj) / (2 + tj))) / B
    return dict(psi=psi, ht=ht, exch=exch, indep=indep, cc_limit=float(np.sum(theta * xi) / np.sum(xi)))


def normconst_postmean(n, prior):
    """Example 11.10: posterior mean of c, int c^{n+1} pi / int c^n pi, for three priors on c > 0."""
    n = np.asarray(n, dtype=float)
    if prior == "exp":            # pi(c) = e^{-c}: posterior Gamma(n + 1, 1), mean n + 1
        return n + 1
    if prior == "unif10":         # pi uniform on (0, 10): mean 10 (n + 1) / (n + 2)
        return 10 * (n + 1) / (n + 2)
    if prior == "halfnormal":     # pi(c) prop. to e^{-c^2/2}: ratio of Gamma functions
        return np.sqrt(2) * np.exp(special.gammaln((n + 2) / 2) - special.gammaln((n + 1) / 2))
    raise ValueError(prior)


# =========================================================================== exercises
def ex2(n=100, mu=5.0, B=1000, seed=2):
    """Exercise 2: N(mu, 1) data, flat prior. The posterior of mu is N(xbar, 1/n); theta = e^mu is
    log-Normal. Returns the data summary, posterior draws and intervals."""
    rng = np.random.default_rng(seed)
    x = rng.normal(mu, 1.0, n)
    xb = float(x.mean())
    s = 1 / math.sqrt(n)
    draws = rng.normal(xb, s, B)
    z = zq(0.05)
    return dict(x=x, xbar=xb, sd=s, draws=draws, theta_draws=np.exp(draws),
                mu_int=(xb - z * s, xb + z * s), theta_int=(math.exp(xb - z * s), math.exp(xb + z * s)),
                theta_delta=(math.exp(xb) * (1 - z * s), math.exp(xb) * (1 + z * s)),
                theta_sim=tuple(np.quantile(np.exp(draws), [0.025, 0.975])))


def ex3_post(theta, xmax, n):
    """Exercise 3: Uniform(0, theta) data with prior 1/theta: Pareto posterior n xmax^n / theta^(n+1)."""
    theta = np.asarray(theta, dtype=float)
    return np.where(theta >= xmax, n * xmax ** n / theta ** (n + 1), 0.0)


def ex4(B=100000, seed=4, alpha=0.10):
    """Exercise 4: 30/50 improve on placebo, 40/50 on treatment."""
    n1 = n2 = 50
    x1, x2 = 30, 40
    p1, p2 = x1 / n1, x2 / n2
    z = zq(alpha)
    tau = p2 - p1
    se_tau = math.sqrt(p1 * (1 - p1) / n1 + p2 * (1 - p2) / n2)
    psi = math.log(p1 / (1 - p1)) - math.log(p2 / (1 - p2))
    se_psi = math.sqrt(1 / (n1 * p1 * (1 - p1)) + 1 / (n2 * p2 * (1 - p2)))
    rng = np.random.default_rng(seed)
    # parametric bootstrap
    b1 = rng.binomial(n1, p1, B) / n1
    b2 = rng.binomial(n2, p2, B) / n2
    tb = b2 - b1
    se_boot = float(tb.std(ddof=1))
    # posterior with the flat prior
    q1 = rng.beta(x1 + 1, n1 - x1 + 1, B)
    q2 = rng.beta(x2 + 1, n2 - x2 + 1, B)
    tp = q2 - q1
    pp = np.log(q1 / (1 - q1)) - np.log(q2 / (1 - q2))
    return dict(tau=tau, se_tau=se_tau, ci_tau=(tau - z * se_tau, tau + z * se_tau),
                se_boot=se_boot, ci_boot=(tau - z * se_boot, tau + z * se_boot),
                ci_boot_pct=tuple(np.quantile(tb, [alpha / 2, 1 - alpha / 2])),
                post_tau_mean=float(tp.mean()), post_tau_int=tuple(np.quantile(tp, [alpha / 2, 1 - alpha / 2])),
                psi=psi, se_psi=se_psi, ci_psi=(psi - z * se_psi, psi + z * se_psi),
                post_psi_mean=float(pp.mean()), post_psi_int=tuple(np.quantile(pp, [alpha / 2, 1 - alpha / 2])),
                p_tau_pos=float(np.mean(tp > 0)))


EX5_DATA = np.array([0, 1, 0, 1, 0, 0, 0, 0, 0, 0])
EX5_PRIORS = ((0.5, 0.5), (1.0, 1.0), (10.0, 10.0), (100.0, 100.0))


def ex6_post(s, n, alpha_, beta_):
    """Exercise 6: Poisson data, Gamma(alpha, rate beta) prior: Gamma(alpha + s, rate beta + n)."""
    return alpha_ + s, beta_ + n


# =========================================================================== figures
def fig_11_s1(out_dir=None):
    """The Bayesian method: Beta-Bernoulli updating, shrinkage, Normal-Normal (Examples 11.1, 11.2)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    ax = axes[0, 0]
    n, s = 20, 6
    p = np.linspace(0.0005, 0.9995, 2000)
    A, B = beta_post(s, n)
    lo, hi = beta_interval(A, B)
    lik = stats.beta.pdf(p, s + 1, n - s + 1)          # the likelihood, scaled to integrate to 1
    ax.plot(p, np.ones_like(p), color=GREY, ls="--", lw=1.2, label="prior Beta(1, 1)")
    ax.plot(p, stats.beta.pdf(p, 10, 10), color=GREEN, ls=":", lw=1.3, label="prior Beta(10, 10)")
    ax.plot(p, stats.beta.pdf(p, s + 10, n - s + 10), color=GREEN, lw=1.3, label="posterior, Beta(10, 10) prior")
    ax.plot(p, lik, color=RED, lw=1.8, label="posterior, flat prior")
    m = (p > lo) & (p < hi)
    ax.fill_between(p[m], 0, lik[m], color=RED, alpha=0.15, lw=0)
    ax.axvline(s / n, color=BLUE, lw=0.8)
    ax.set_xlabel("p")
    ax.set_ylabel("density")
    ax.set_title("s = %d successes in n = %d" % (s, n))
    ax.set_ylim(0, 5.6)
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    common.panel_label(ax, "a", loc="upper left")

    ax = axes[0, 1]
    x = bern_data(0.3, 200, seed=11)
    k = np.arange(1, 201)
    cs = np.cumsum(x)
    ax.plot(k, cs / k, color=BLUE, lw=1.0, label="MLE s/n")
    for (a, b), col in (((1, 1), RED), ((10, 10), GREEN), ((50, 50), PURPLE)):
        ax.plot(k, (a + cs) / (a + b + k), color=col, lw=1.4, label="Beta(%d, %d)" % (a, b))
    ax.axhline(0.3, color=GREY, ls="--", lw=0.8)
    ax.set_xscale("log")
    ax.set_ylim(0, 1)
    ax.set_xlabel("n")
    ax.set_ylabel("posterior mean")
    ax.set_title("posterior means approach the MLE")
    ax.legend(loc="upper right", fontsize=7, frameon=False)
    common.panel_label(ax, "b", loc="lower left")

    ax = axes[0, 2]
    rng = np.random.default_rng(112)
    xs = rng.normal(2.0, 1.0, 5)
    xb = xs.mean()
    mpost, tau, w = normal_post(xb, 5, 1.0, 0.0, 1.0)
    th = np.linspace(-3, 5, 1000)
    se = 1 / math.sqrt(5)
    ax.plot(th, stats.norm.pdf(th, 0, 1), color=GREY, ls="--", lw=1.2, label="prior N(0, 1)")
    ax.plot(th, stats.norm.pdf(th, xb, se), color=BLUE, lw=1.2, label="likelihood N(xbar, se^2)")
    ax.plot(th, stats.norm.pdf(th, mpost, tau), color=RED, lw=1.8, label="posterior, w = %.3f" % w)
    z = zq(0.05)
    ax.plot([mpost - z * tau, mpost + z * tau], [0.05, 0.05], color=RED, lw=3, solid_capstyle="butt")
    ax.plot([xb - z * se, xb + z * se], [0.11, 0.11], color=BLUE, lw=3, solid_capstyle="butt")
    ax.set_xlabel(r"$\theta$")
    ax.set_ylabel("density")
    ax.set_title("Normal mean, n = 5 (11.7)")
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    ax.set_ylim(0, 1.25)
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S1", out_dir)


def fig_11_s2(out_dir=None):
    """Functions of parameters by simulation: the log-odds (Examples 11.3, 11.4) and p2 - p1 (11.7)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    rng = np.random.default_rng(113)
    ax = axes[0, 0]
    n, s = 20, 6
    P = rng.beta(s + 1, n - s + 1, 10000)
    psi = np.log(P / (1 - P))
    ax.hist(psi, bins=60, density=True, color=BLUE, alpha=0.35, label="10 000 draws")
    g = np.linspace(-4, 1.5, 600)
    ax.plot(g, logodds_density(g, s, n), color=RED, lw=1.8, label="h(psi), corrected")
    ax.plot(g, logodds_density_misprint(g, s, n), color=GREY, ls="--", lw=1.2, label="as printed")
    ax.set_xlabel(r"$\psi=\log\{p/(1-p)\}$")
    ax.set_ylabel("density")
    ax.set_title("log-odds, s = 6, n = 20")
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    ax.set_ylim(0, 1.05)
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    p1, p2, tau = two_binomials(30, 50, 40, 50, 4000, rng)
    ax.scatter(p1, p2, s=2, color=PURPLE, alpha=0.3, lw=0)
    ax.plot([0.3, 1], [0.3, 1], color=GREY, ls="--", lw=1)
    ax.set_xlim(0.35, 0.85)
    ax.set_ylim(0.55, 0.95)
    ax.set_xlabel(r"$p_1$ (placebo)")
    ax.set_ylabel(r"$p_2$ (treatment)")
    ax.set_title("joint posterior draws")
    ax.text(0.37, 0.92, "%.1f%% of draws above the line" % (100 * np.mean(p2 > p1)), fontsize=7.5)
    common.panel_label(ax, "b", loc="lower right")

    ax = axes[0, 2]
    _, _, tau = two_binomials(30, 50, 40, 50, 100000, rng)
    ax.hist(tau, bins=70, density=True, color=PURPLE, alpha=0.35, label="draws of p2 - p1")
    t = np.linspace(-0.25, 0.55, 300)
    ax.plot(t, tau_density(t, 30, 50, 40, 50, m=1201), color=RED, lw=1.6, label="exact (convolution)")
    q = np.quantile(tau, [0.05, 0.95])
    for v in q:
        ax.axvline(v, color=RED, ls=":", lw=1)
    ax.set_xlabel(r"$\tau=p_2-p_1$")
    ax.set_ylabel("density")
    ax.set_title("90%% interval (%.3f, %.3f)" % tuple(q))
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    ax.set_ylim(0, 5.8)
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S2", out_dir)


def fig_11_s3(out_dir=None):
    """Theorem 11.5: the posterior becomes Normal (Bernoulli data and the nerve data)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    xb = bern_data(0.1, 1000, seed=1)
    cs = np.cumsum(xb)
    ax = axes[0, 0]
    h = np.linspace(-4.5, 4.5, 900)
    for n, col in ((10, ORANGE), (40, GREEN), (640, BLUE)):
        s = int(cs[n - 1])
        ph = s / n
        se = math.sqrt(ph * (1 - ph) / n)
        pgrid = ph + se * h
        dens = np.where((pgrid > 0) & (pgrid < 1), np.exp(_lbeta_pdf(np.clip(pgrid, 1e-300, 1 - 1e-16), 1 + s, 1 + n - s)), 0) * se
        ax.plot(h, dens, color=col, lw=1.4, label="n = %d (s = %d)" % (n, s))
    ax.plot(h, stats.norm.pdf(h), color="k", ls="--", lw=1.1, label="N(0, 1)")
    ax.set_xlabel(r"$(p-\widehat p_n)/\widehat{\mathsf{se}}$")
    ax.set_ylabel("density")
    ax.set_title("Bernoulli(0.1), flat prior")
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    ax.set_ylim(0, 0.62)
    common.panel_label(ax, "a", loc="upper left")

    ax = axes[0, 1]
    nv = nerve()
    curves = [("bern", xb, 1, 1, "natural", BLUE, "-", "Bernoulli, p scale"),
              ("bern", xb, 1, 1, "log", BLUE, "--", "Bernoulli, log-odds"),
              ("exp", nv, 1, 0, "natural", RED, "-", "nerve, rate"),
              ("exp", nv, 1, 0, "log", RED, "--", "nerve, log rate")]
    for model, data, a, b, sc, col, ls, lab in curves:
        ns, out = bvm_curve(model, data, a, b, sc)
        keep = [i for i, o in enumerate(out) if o["tv"] is not None]
        ax.loglog(ns[keep], [out[i]["tv"] for i in keep], color=col, ls=ls, lw=1.3, label=lab)
    nn = np.array([5, 800])
    ax.loglog(nn, 0.6 / np.sqrt(nn), color=GREY, ls=":", lw=1, label=r"slope $-1/2$")
    ax.set_xlabel("n")
    ax.set_ylabel("total variation distance")
    ax.set_title("posterior vs Normal approximation")
    ax.legend(loc="lower left", fontsize=6.5, frameon=False)
    common.panel_label(ax, "b", loc="upper right")

    ax = axes[0, 2]
    for model, data, a, b, sc, col, ls, lab in curves[:1] + curves[2:3] + [("bern", xb, 20, 20, "natural", PURPLE, "-", "Bernoulli, Beta(20, 20) prior")]:
        ns, out = bvm_curve(model, data, a, b, sc)
        keep = [i for i, o in enumerate(out) if o["ppost"] is not None]
        ax.semilogx(ns[keep], [out[i]["ppost"] for i in keep], color=col, ls=ls, lw=1.3, label=lab)
    ax.axhline(0.95, color=GREY, ls="--", lw=0.9)
    ax.set_ylim(0, 1.02)
    ax.set_xlabel("n")
    ax.set_ylabel(r"$P(\theta\in C_n\mid X^n)$")
    ax.set_title("posterior mass of the Wald interval")
    ax.legend(loc="lower right", fontsize=6.5, frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S3", out_dir)


def fig_11_s4(out_dir=None):
    """Flat priors are not invariant; Jeffreys' prior (Section 11.6, Example 11.6)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    rng = np.random.default_rng(114)
    ax = axes[0, 0]
    U = rng.random(20000)
    ax.hist(np.log(U / (1 - U)), bins=80, range=(-8, 8), density=True, color=BLUE, alpha=0.35,
            label="logit of 20 000 uniforms")
    g = np.linspace(-8, 8, 500)
    ax.plot(g, np.exp(g) / (1 + np.exp(g)) ** 2, color=RED, lw=1.7, label=r"$e^\psi/(1+e^\psi)^2$")
    ax.set_xlabel(r"$\psi=\log\{p/(1-p)\}$")
    ax.set_ylabel("density")
    ax.set_title(r"flat on $p$ is not flat on $\psi$")
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    ax.set_ylim(0, 0.36)
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    p = np.linspace(0.001, 0.999, 1500)
    ax.plot(p, np.ones_like(p), color=GREY, ls="--", lw=1.2, label="flat on p, Beta(1, 1)")
    ax.plot(p, stats.beta.pdf(p, 0.5, 0.5), color=RED, lw=1.7, label="Jeffreys, Beta(1/2, 1/2)")
    phi = rng.random(20000) * math.pi / 2
    ax.hist(np.sin(phi) ** 2, bins=50, density=True, color=ORANGE, alpha=0.3,
            label=r"flat on $\arcsin\sqrt{p}$")
    ax.set_ylim(0, 3.2)
    ax.set_xlabel("p")
    ax.set_ylabel("density")
    ax.set_title("Jeffreys' prior for a proportion")
    ax.legend(loc="upper center", fontsize=6.8, frameon=False)
    common.panel_label(ax, "b", loc="upper left")

    ax = axes[0, 2]
    n, s = 10, 1
    for key, col, ls, lab in (("p", GREY, "--", "flat on p: Beta(2, 10)"),
                              ("logit", BLUE, "-", "flat on log-odds: Beta(1, 9)"),
                              ("asin", RED, "-", "Jeffreys: Beta(1.5, 9.5)"),
                              ("sq", GREEN, ":", r"flat on $p^2$: Beta(3, 10)")):
        A, B = flat_posterior("bern", key, s, n)
        ax.plot(p, stats.beta.pdf(p, A, B), color=col, ls=ls, lw=1.5, label=lab)
    ax.set_xlim(0, 0.6)
    ax.set_ylim(0, 9.5)
    ax.set_xlabel("p")
    ax.set_ylabel("posterior density")
    ax.set_title("s = 1 success in n = 10")
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S4", out_dir)


def fig_11_s5(out_dir=None):
    """Bayesian testing and the Jeffreys-Lindley paradox (Section 11.8, Exercise 8)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    ax = axes[0, 0]
    x = np.linspace(0, 4, 400)
    for b, col in ((0.5, GREEN), (1.0, BLUE), (3.0, PURPLE), (10.0, RED)):
        ax.plot(x, post_h0_normal(x, b), color=col, lw=1.4, label="b = %g" % b)
    ax.plot(x, wald_pvalue(x), color="k", ls="--", lw=1.1, label="Wald p-value")
    ax.plot(x, post_h0_bound(x), color=ORANGE, ls=":", lw=1.6, label="lower bound")
    ax.set_xlabel("x (n = 1)")
    ax.set_ylabel(r"$P(H_0\mid x)$")
    ax.set_title(r"$H_0:\mu=0$, prior $N(0,b^2)$ under $H_1$")
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    common.panel_label(ax, "a", loc="lower left")

    ax = axes[0, 1]
    ns = np.logspace(0, 6, 300)
    for z, col in ((1.96, RED), (2.576, BLUE), (3.29, GREEN)):
        ax.semilogx(ns, post_h0_normal(z / np.sqrt(ns), 1.0, ns), color=col, lw=1.4,
                    label="z = %.2f (p = %.3f)" % (z, wald_pvalue(z)))
    ax.set_ylim(0, 1)
    ax.set_xlabel("n")
    ax.set_ylabel(r"$P(H_0\mid \overline{x}_n)$")
    ax.set_title("fixed p-value, growing n (b = 1)")
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    common.panel_label(ax, "b", loc="lower right")

    ax = axes[0, 2]
    bs = np.logspace(-2, 3, 400)
    ax.semilogx(bs, post_h0_normal(1.96, bs), color=RED, lw=1.5, label="x = 1.96")
    ax.semilogx(bs, post_h0_normal(2.576, bs), color=BLUE, lw=1.5, label="x = 2.576")
    ax.axhline(post_h0_bound(1.96), color=RED, ls=":", lw=1)
    ax.axhline(post_h0_bound(2.576), color=BLUE, ls=":", lw=1)
    ax.set_ylim(0, 1)
    ax.set_xlabel("prior sd b under H1")
    ax.set_ylabel(r"$P(H_0\mid x)$")
    ax.set_title("dependence on the prior (n = 1)")
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    common.panel_label(ax, "c", loc="lower right")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S5", out_dir)


def fig_11_s6(out_dir=None):
    """Frequentist coverage of Bayesian intervals; Examples 11.9 and 11.10 (Section 11.9)."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    ax = axes[0, 0]
    n = 30
    ps = np.linspace(0.002, 0.998, 499)
    for (a, b), col, lab in (((1, 1), BLUE, "flat Beta(1, 1)"), ((0.5, 0.5), RED, "Jeffreys"),
                             ((10, 10), GREEN, "Beta(10, 10)")):
        e = credible_ends(n, a, b)
        ax.plot(ps, [credible_coverage(n, p, a, b, ends=e) for p in ps], color=col, lw=1.0, label=lab)
    ax.axhline(0.95, color=GREY, ls="--", lw=0.9)
    ax.set_ylim(0, 1.01)
    ax.set_xlabel("true p")
    ax.set_ylabel("frequentist coverage")
    ax.set_title("95% posterior intervals, n = 30")
    ax.legend(loc="lower center", fontsize=6.8, frameon=False)
    common.panel_label(ax, "a", loc="lower left")

    ax = axes[0, 1]
    rr = robins_ritov(1000, 1000)
    lo = min(rr["ht"].min(), rr["exch"].min(), rr["indep"].min())
    hi = max(rr["ht"].max(), rr["exch"].max())
    bins = np.linspace(lo, hi, 70)
    ax.hist(rr["ht"], bins=bins, density=True, color=BLUE, alpha=0.4, label="Horvitz-Thompson")
    ax.hist(rr["exch"], bins=bins, density=True, color=RED, alpha=0.4, label="Bayes, exchangeable prior")
    hmax = max(np.histogram(rr["ht"], bins=bins, density=True)[0].max(),
               np.histogram(rr["exch"], bins=bins, density=True)[0].max())
    ax.hist(rr["indep"], bins=bins, density=True, color=GREEN, alpha=0.6, label="Bayes, independent priors")
    ax.set_ylim(0, 1.9 * hmax)
    ax.axvline(rr["psi"], color="k", ls="--", lw=1)
    ax.text(rr["psi"], 1.3 * hmax, r" true $\psi$", fontsize=7.5, va="top")
    ax.set_xlabel(r"estimate of $\psi$")
    ax.set_ylabel("density")
    ax.set_title("Example 11.9, n = 1000, B = 10^6")
    ax.legend(loc="upper left", fontsize=6.3, frameon=False)
    common.panel_label(ax, "b", loc="upper right")

    ax = axes[0, 2]
    ns = np.arange(0, 201)
    for pr, col, lab in (("exp", RED, r"$\pi(c)=e^{-c}$"), ("halfnormal", BLUE, "half-Normal"),
                         ("unif10", GREEN, "Uniform(0, 10)")):
        ax.plot(ns, normconst_postmean(ns, pr), color=col, lw=1.4, label=lab)
    ax.axhline(1 / math.sqrt(2 * math.pi), color="k", ls="--", lw=1, label=r"true $c=1/\sqrt{2\pi}$")
    ax.set_yscale("log")
    ax.set_xlabel("n")
    ax.set_ylabel("posterior mean of c")
    ax.set_title("Example 11.10, g(x) = exp(-x^2/2)")
    ax.legend(loc="center right", bbox_to_anchor=(1.0, 0.3), fontsize=6.8, frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S6", out_dir)


def fig_11_s7(out_dir=None):
    """Exercises 11.2 and 11.5."""
    fig, axes = common.panel_grid(1, 3, size=(3.5, 2.9))
    r = ex2()
    ax = axes[0, 0]
    ax.hist(r["draws"], bins=35, density=True, color=BLUE, alpha=0.4, label="1000 draws")
    m = np.linspace(r["xbar"] - 4 * r["sd"], r["xbar"] + 4 * r["sd"], 400)
    ax.plot(m, stats.norm.pdf(m, r["xbar"], r["sd"]), color=RED, lw=1.7, label=r"$N(\overline{x},1/n)$")
    ax.set_xlabel(r"$\mu$")
    ax.set_ylabel("posterior density")
    ax.set_title(r"Ex. 2: $\overline{x}$ = %.3f, n = 100" % r["xbar"])
    ax.legend(loc="upper left", fontsize=6.8, frameon=False)
    common.panel_label(ax, "a", loc="upper right")

    ax = axes[0, 1]
    ax.hist(r["theta_draws"], bins=35, density=True, color=BLUE, alpha=0.4, label="exp of the draws")
    t = np.exp(m)
    ax.plot(t, stats.lognorm.pdf(t, s=r["sd"], scale=math.exp(r["xbar"])), color=RED, lw=1.7, label="log-Normal")
    ax.set_xlabel(r"$\theta=e^\mu$")
    ax.set_ylabel("posterior density")
    ax.set_title(r"Ex. 2(d): $\theta=e^\mu$")
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    common.panel_label(ax, "b", loc="upper left")

    ax = axes[0, 2]
    s, n = int(EX5_DATA.sum()), len(EX5_DATA)
    p = np.linspace(0.001, 0.999, 1500)
    for (a, b), col in zip(EX5_PRIORS, (RED, BLUE, GREEN, PURPLE)):
        ax.plot(p, stats.beta.pdf(p, a + s, b + n - s), color=col, lw=1.5,
                label="Beta(%g, %g) prior" % (a, b))
    ax.axvline(s / n, color=GREY, ls=":", lw=1)
    ax.set_xlabel("p")
    ax.set_ylabel("posterior density")
    ax.set_title("Ex. 5: 2 successes in 10")
    ax.legend(loc="upper right", fontsize=6.8, frameon=False)
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig11-S7", out_dir)


FIGURES = {"11.S1": fig_11_s1, "11.S2": fig_11_s2, "11.S3": fig_11_s3, "11.S4": fig_11_s4,
           "11.S5": fig_11_s5, "11.S6": fig_11_s6, "11.S7": fig_11_s7}

if __name__ == "__main__":
    export_data()
    common.run(FIGURES, CH)
