"""Wasserman (2004), Chapter 16: Causal Inference.

Companion code for Chapter 16 of L. Wasserman, All of Statistics: A Concise Course in Statistical
Inference (Springer, 2004), pp. 251-262, written from scratch with numpy, scipy and matplotlib. The
functions are short and meant to be read next to the lecture notes (../ch16-causal-inference.html):

* Section 16.1, the counterfactual model: the toy populations of Example 16.2 (causal_and_assoc),
  the four response types (survivors, responders, anti-responders, doomed) with type-dependent
  treatment probabilities (type_model: the causal effect theta, the association alpha, the selection
  bias that separates them, causal and associational risk ratios and odds ratios), a mixing path from
  random to self-selected assignment (mix_assignment) and a seeded simulation of the association
  estimator (types_sim), plus the large-sample standard error of the difference in means.
* Section 16.2, beyond binary treatments: the counterfactual curves of Figures 16.1 and 16.2 and a
  linear dose model with a confounder (dose_model, dose_sim), where the regression of Y on X has the
  wrong sign and the adjusted estimate (16.8) recovers the causal regression function.
* Section 16.3, observational studies and confounding: a stratified confounder model (conf_model,
  exact limits of the naive difference, of the regression estimator (16.8) and of the adjusted effect
  (16.7)) and a seeded simulation of the three estimators (conf_sim); inverse probability weighting
  (ipw) for comparison.
* Section 16.4, Simpson's paradox: the book's 2 x 2 x 2 table, the kidney stone data of Charig et al.
  (1986), summaries of any table (simpson_summary), tables built from allocation and recovery rates
  (simpson_table) and a seeded multinomial sample (simpson_sample).
* Exercises 16.1-16.5: a population with alpha > 0 > theta, the substitution argument behind
  Theorem 16.4, the no-assumption bounds of width one (manski_bounds), positive slopes with a decreasing
  regression, and the difference of medians under randomisation.
* JSRandom and js_uniforms: the seeded generator of the browser demonstrations (DK.rng in
  courses/demo-kit.js), so that the simulated numbers on the pages can be reproduced here number for
  number.

The chapter has two figures. Both are illustrations built on counterfactual curves rather than on data,
but each is computed (Figure 16.2 averages four curves and fits the observed points), so both are
reproduced with curves of my own: 16.1 (a counterfactual function and the observed point Y = C(X)) and
16.2 (four flat counterfactual curves: no causal effect, yet a decreasing regression). Nothing is skipped.
Five supplementary figures illustrate the gaps filled in the notes:
  16.S1 association against causation for the four response types (Theorems 16.1 and 16.3);
  16.S2 adjusting for a confounder as the confounding strength changes (Theorem 16.6, (16.8));
  16.S3 Simpson's paradox as two weighted averages (Section 16.4);
  16.S4 a continuous dose with a confounder: r(x), theta(x) and the estimate (16.8) (Sections 16.2-16.3);
  16.S5 Exercises 16.3 and 16.4 (bounds of width one; positive slopes, decreasing regression).
Run `python3 ch16_causal_inference.py --list`, `python3 ch16_causal_inference.py` (all seven figures,
a few seconds on one CPU thread) or `python3 ch16_causal_inference.py --fig 16.2`.
Files go to ../img/ch16/. The demonstrations compute everything in the browser and need no data files.
"""
import math

import numpy as np
from scipy import stats

import common
from common import BLUE, GREEN, GREY, MAGENTA, ORANGE, PURPLE, RED, savefig

CH = 16
M32 = 0xFFFFFFFF


# =========================================================================== the demos' generator
def _imul(a, b):
    return (a * b) & M32


class JSRandom:
    """The seeded random numbers of the browser demos (DK.rng in courses/demo-kit.js): u() is
    mulberry32 and norm() the Box-Muller transform that returns r cos(t) and keeps r sin(t) for the
    next call, exactly as in JavaScript."""

    def __init__(self, seed):
        self.a = int(seed) & M32
        self.spare = None

    def u(self):
        a = (self.a + 0x6D2B79F5) & M32
        self.a = a
        t = _imul(a ^ (a >> 15), 1 | a)
        t = ((t + _imul(t ^ (t >> 7), 61 | t)) & M32) ^ t
        return ((t ^ (t >> 14)) & M32) / 4294967296.0

    def norm(self):
        if self.spare is not None:
            s, self.spare = self.spare, None
            return s
        x = 0.0
        while x == 0.0:
            x = self.u()
        r = math.sqrt(-2.0 * math.log(x))
        th = 2.0 * math.pi * self.u()
        self.spare = r * math.sin(th)
        return r * math.cos(th)


def js_uniforms(seed, n, start=0):
    """Uniforms number start+1, ..., start+n of DK.rng(seed), vectorised (mulberry32 adds a constant
    to its state at each call, so the j-th output is a hash of seed + j * 0x6D2B79F5 mod 2^32)."""
    j = np.arange(start + 1, start + n + 1, dtype=np.uint64)
    m = np.uint64(M32)
    a = (np.uint64(int(seed) & M32) + j * np.uint64(0x6D2B79F5)) & m
    t = ((a ^ (a >> np.uint64(15))) * (np.uint64(1) | a)) & m
    t = ((t + (((t ^ (t >> np.uint64(7))) * (np.uint64(61) | t)) & m)) & m) ^ t
    return ((t ^ (t >> np.uint64(14))) & m).astype(np.float64) / 4294967296.0


def sigmoid(a):
    return 1.0 / (1.0 + math.exp(-a))


# =========================================================================== 16.1 the counterfactual model
# The toy data set of p. 252 (observed X and Y only) and the two populations of Example 16.2.
TOY_X = np.array([0, 0, 0, 0, 1, 1, 1, 1])
TOY_Y = np.array([4, 7, 2, 8, 3, 5, 8, 9])
EX162 = dict(X=np.array([0, 0, 0, 0, 1, 1, 1, 1]), C0=np.array([0, 0, 0, 0, 1, 1, 1, 1]),
             C1=np.array([0, 0, 0, 0, 1, 1, 1, 1]))
EX162_ADVICE = dict(X=np.array([0, 1, 1, 1, 1, 1, 1, 1]), C0=EX162["C0"].copy(), C1=EX162["C1"].copy())
# Exercise 16.1: three survivors who take the treatment, two anti-responders and three doomed who do not.
EX161 = dict(X=np.array([1, 1, 1, 0, 0, 0, 0, 0]), C0=np.array([1, 1, 1, 1, 1, 0, 0, 0]),
             C1=np.array([1, 1, 1, 0, 0, 0, 0, 0]))


def causal_and_assoc(X, C0, C1):
    """(theta, alpha) for a finite population listed subject by subject: theta = mean(C1) - mean(C0)
    is the average causal effect (16.2) and alpha = mean(Y | X=1) - mean(Y | X=0) the association
    (16.3), with Y = C_X the consistency relation (16.1)."""
    X, C0, C1 = (np.asarray(v, float) for v in (X, C0, C1))
    Y = np.where(X == 1, C1, C0)
    return C1.mean() - C0.mean(), Y[X == 1].mean() - Y[X == 0].mean()


TYPES = ("survivor", "responder", "anti-responder", "doomed")
TYPE_C0 = np.array([1, 0, 1, 0])
TYPE_C1 = np.array([1, 1, 0, 0])


def type_model(pi, e):
    """Exact population quantities for binary potential outcomes.

    pi: proportions of the four types (survivor, responder, anti-responder, doomed), normalised here;
    e: P(X = 1 | type). Returns a dict with theta = P(responder) - P(anti-responder), alpha,
    E(C_x | X = x') for all x, x', the selection-bias decomposition alpha - theta = p0 D1 + p1 D0 of
    Filling the gap 16.2, and causal / associational risk ratios and odds ratios."""
    pi = np.asarray(pi, float)
    pi = pi / pi.sum()
    e = np.asarray(e, float)
    p1 = float(np.sum(pi * e))
    p0 = 1.0 - p1
    w1 = pi * e / p1 if p1 > 0 else np.full(4, np.nan)       # type distribution among the treated
    w0 = pi * (1 - e) / p0 if p0 > 0 else np.full(4, np.nan)  # ... among the untreated
    EC1, EC0 = float(pi @ TYPE_C1), float(pi @ TYPE_C0)
    C1_t, C1_u = float(w1 @ TYPE_C1), float(w0 @ TYPE_C1)   # E(C1 | X=1), E(C1 | X=0)
    C0_t, C0_u = float(w1 @ TYPE_C0), float(w0 @ TYPE_C0)   # E(C0 | X=1), E(C0 | X=0)
    EY1, EY0 = C1_t, C0_u
    D1, D0 = C1_t - C1_u, C0_t - C0_u

    def odds(p):
        return p / (1 - p) if 0 < p < 1 else (math.inf if p >= 1 else 0.0)

    return dict(pi=pi, e=e, p1=p1, p0=p0, EC1=EC1, EC0=EC0, theta=EC1 - EC0,
                EY1=EY1, EY0=EY0, alpha=EY1 - EY0, C1_t=C1_t, C1_u=C1_u, C0_t=C0_t, C0_u=C0_u,
                D1=D1, D0=D0, bias=p0 * D1 + p1 * D0, w1=w1, w0=w0,
                rr_causal=EC1 / EC0 if EC0 > 0 else math.inf, or_causal=odds(EC1) / odds(EC0) if odds(EC0) > 0 else math.inf,
                rr_assoc=EY1 / EY0 if EY0 > 0 else math.inf, or_assoc=odds(EY1) / odds(EY0) if odds(EY0) > 0 else math.inf)


def mix_assignment(pi, e, lam):
    """Treatment probabilities on the path from random assignment (lam = 0: every type treated with
    the same probability P(X = 1)) to the given self-selection (lam = 1). P(X = 1) stays fixed."""
    pi = np.asarray(pi, float) / np.sum(pi)
    e = np.asarray(e, float)
    p1 = float(np.sum(pi * e))
    return (1 - lam) * p1 + lam * e


def types_sim(pi, e, n, R, seed):
    """R simulated studies of n subjects each, with one DK.rng(seed) stream used in order: for every
    subject one uniform picks the type (by the cumulative proportions) and one decides the treatment
    (X = 1 if u < e[type]). Returns the association estimates Ybar1 - Ybar0 (nan when an arm is empty)
    and the sample average causal effects mean(C1 - C0), as in the potential outcomes demo."""
    pi = np.asarray(pi, float)
    cum = np.cumsum(pi / pi.sum())
    u = js_uniforms(seed, 2 * n * R).reshape(R, n, 2)
    t = np.minimum(np.sum(u[:, :, :1] >= cum[None, None, :3], axis=2), 3)   # type index 0..3
    x = (u[:, :, 1] < np.asarray(e, float)[t]).astype(float)
    c0, c1 = TYPE_C0[t], TYPE_C1[t]
    y = np.where(x == 1, c1, c0)
    n1 = x.sum(1)
    n0 = n - n1
    with np.errstate(invalid="ignore", divide="ignore"):
        a = (y * x).sum(1) / n1 - (y * (1 - x)).sum(1) / n0
    a[(n1 == 0) | (n0 == 0)] = np.nan
    return dict(alpha_hat=a, sate=(c1 - c0).mean(1), n1=n1)


def types_sim_loop(pi, e, n, R, seed):
    """types_sim written subject by subject with JSRandom (a check of the vectorised version)."""
    pi = np.asarray(pi, float)
    cum = np.cumsum(pi / pi.sum())
    g = JSRandom(seed)
    out = []
    for _ in range(R):
        s1 = s0 = n1 = n0 = 0
        for _ in range(n):
            uu = g.u()
            t = 0
            while t < 3 and uu >= cum[t]:
                t += 1
            if g.u() < e[t]:
                n1 += 1
                s1 += TYPE_C1[t]
            else:
                n0 += 1
                s0 += TYPE_C0[t]
        out.append(s1 / n1 - s0 / n0 if n1 and n0 else math.nan)
    return np.array(out)


def diff_means_se(pi, e, n):
    """Large-sample standard deviation of Ybar1 - Ybar0 for binary outcomes:
    sqrt(v1 / (n p1) + v0 / (n p0)) with v_x = E(Y|X=x)(1 - E(Y|X=x)) (Filling the gap 16.3)."""
    M = type_model(pi, e)
    return math.sqrt(M["EY1"] * (1 - M["EY1"]) / (n * M["p1"]) + M["EY0"] * (1 - M["EY0"]) / (n * M["p0"]))


def frechet_responders(EC1, EC0):
    """Sharp bounds for P(responder) = P(C0 = 0, C1 = 1) given only the margins P(C1 = 1), P(C0 = 1)."""
    return max(0.0, EC1 - EC0), min(EC1, 1.0 - EC0)


# =========================================================================== 16.2 beyond binary treatments
def c_curve(x):
    """The counterfactual function drawn in Figure 16.1 (my own smooth dose-response curve)."""
    x = np.asarray(x, float)
    return 1.0 + 4.6 / (1.0 + np.exp(-(x - 4.5) / 1.1)) + 0.9 * np.exp(-0.5 * ((x - 8.5) / 1.0) ** 2)


FIG162_C = np.array([2.6, 2.0, 1.4, 0.8])   # flat counterfactual curves C_i(x) = c_i (Figure 16.2)
FIG162_X = np.array([0.8, 1.9, 3.1, 4.2])   # the doses the four subjects chose


def fig162_numbers():
    """theta(x) (constant, the mean of the four curves) and the least-squares line through the four
    observed points (X_i, C_i(X_i)): the association of Figure 16.2."""
    b, a = np.polyfit(FIG162_X, FIG162_C, 1)
    return dict(theta=FIG162_C.mean(), slope=b, intercept=a)


# Linear dose model with a confounder Z (Figure 16.S4):
#   Z ~ N(0, 1), X = mx + gx Z + sx E1, C(x) = a0 + az Z + sa E2 + b x.
DOSE = dict(mx=5.0, gx=-1.5, sx=0.8, a0=2.0, az=2.0, sa=0.5, b=0.5)


def dose_model(p=None):
    """Exact r(x) = E(Y | X = x) and theta(x) = E C(x) (both linear) for the dose model: returns the
    intercepts and slopes."""
    p = dict(DOSE, **(p or {}))
    vx = p["gx"] ** 2 + p["sx"] ** 2
    kz = p["gx"] / vx                       # E(Z | X = x) = kz (x - mx)
    r_slope = p["b"] + p["az"] * kz
    r_int = p["a0"] - p["az"] * kz * p["mx"]
    return dict(theta_int=p["a0"], theta_slope=p["b"], r_int=r_int, r_slope=r_slope)


def dose_sim(n, seed=16, p=None):
    """One observational data set from the dose model (numpy generator)."""
    p = dict(DOSE, **(p or {}))
    g = np.random.default_rng(seed)
    Z = g.standard_normal(n)
    X = p["mx"] + p["gx"] * Z + p["sx"] * g.standard_normal(n)
    A = p["a0"] + p["az"] * Z + p["sa"] * g.standard_normal(n)
    return Z, X, A + p["b"] * X, A


def ols(cols, y):
    """Least-squares coefficients for the design with columns cols (a list of arrays)."""
    A = np.column_stack(cols)
    return np.linalg.lstsq(A, y, rcond=None)[0]


def adjusted_linear(X, Z, Y, xs):
    """The estimate (16.8): theta_hat(x) = b0 + b1 x + b2 Zbar from the regression of Y on (1, X, Z)."""
    b = ols([np.ones_like(X), X, Z], Y)
    return b[0] + b[1] * np.asarray(xs) + b[2] * Z.mean(), b


# =========================================================================== 16.3 confounding
def strata_s(K):
    """Stratum scores s_z = -1 + 2 z / (K - 1), z = 0..K-1 (K >= 2): equally spaced in [-1, 1]."""
    return np.array([-1.0 + 2.0 * z / (K - 1) for z in range(K)])


def conf_model(K=3, gamma=0.0, delta=1.0, tau=1.0, eta=0.0, kappa=-1.0):
    """Exact quantities of the stratified confounder model used in the adjustment demo.

    Z uniform on {0, ..., K-1} with score s_z; P(X = 1 | Z = z) = e_z = sigmoid(kappa + gamma s_z)
    (kappa = -1: about a quarter of the middle stratum is treated);
    C0 = 1 + delta s_Z + noise and C1 = C0 + tau + eta s_Z (the effect may vary with Z).
    Returns the causal effect theta (= tau, since the s_z average to 0), the naive limit
    E(Y|X=1) - E(Y|X=0), the limit of the regression estimator (16.8) (the X coefficient of the
    least-squares fit of Y on (1, X, s_Z), which is theta_hat(1) - theta_hat(0)), the stratum
    effects and the propensities."""
    s = strata_s(K)
    pz = np.full(K, 1.0 / K)
    ez = np.array([sigmoid(kappa + gamma * v) for v in s])
    tz = tau + eta * s
    mu0 = 1.0 + delta * s
    p1 = float(pz @ ez)
    pz1 = pz * ez / p1
    pz0 = pz * (1 - ez) / (1 - p1)
    naive = float(pz1 @ (mu0 + tz) - pz0 @ mu0)
    # population normal equations for W = (1, X, s)
    EX, Es, EXs, Ess = p1, float(pz @ s), float(pz @ (ez * s)), float(pz @ (s * s))
    EY = float(pz @ (mu0 + ez * tz))
    EXY = float(pz @ (ez * (mu0 + tz)))
    EsY = float(pz @ (s * (mu0 + ez * tz)))
    G = np.array([[1, EX, Es], [EX, EX, EXs], [Es, EXs, Ess]])
    beta = np.linalg.solve(G, np.array([EY, EXY, EsY]))
    # with the interaction X s added, theta_hat(1) - theta_hat(0) = b1 + b3 * E(s) recovers tau
    return dict(K=K, s=s, pz=pz, ez=ez, tz=tz, mu0=mu0, p1=p1, pz1=pz1, pz0=pz0,
                theta=float(pz @ tz), naive=naive, reg=float(beta[1]), beta=beta)


def reg_slope_fw(x, s, y):
    """The X coefficient of the least-squares fit of y on (1, x, s), by the Frisch-Waugh formula with
    centred moments; written exactly as in the adjustment demo."""
    n = x.shape[-1]
    mx, ms, my = x.sum(axis=-1) / n, s.sum(axis=-1) / n, y.sum(axis=-1) / n
    cxx = (x * x).sum(axis=-1) / n - mx * mx
    css = (s * s).sum(axis=-1) / n - ms * ms
    cxs = (x * s).sum(axis=-1) / n - mx * ms
    cxy = (x * y).sum(axis=-1) / n - mx * my
    csy = (s * y).sum(axis=-1) / n - ms * my
    return (cxy * css - cxs * csy) / (cxx * css - cxs * cxs)


def conf_draws(n, R, seed, K, gamma, delta, tau, eta, sigma, kappa=-1.0):
    """The simulated data of the adjustment demo, vectorised. One DK.rng(seed) stream: for each subject
    in turn one uniform gives Z = floor(K u), one gives X = [u < e_Z], then rng.norm() gives the noise
    (Box-Muller: the first of each pair of subjects uses two fresh uniforms and the cosine, the second
    the cached sine). Y = 1 + delta s_Z + (tau + eta s_Z) X + sigma * noise. Arrays of shape (R, n)."""
    N = n * R
    npair = (N + 1) // 2
    u = js_uniforms(seed, 6 * npair).reshape(npair, 6)
    z = np.empty(2 * npair, dtype=int)
    ux = np.empty(2 * npair)
    z[0::2], ux[0::2] = np.floor(K * u[:, 0]), u[:, 1]
    z[1::2], ux[1::2] = np.floor(K * u[:, 4]), u[:, 5]
    if np.any(u[:, 2] == 0.0):
        raise RuntimeError("a zero uniform: use conf_draws_loop")
    r = np.sqrt(-2.0 * np.log(u[:, 2]))
    th = 2.0 * np.pi * u[:, 3]
    eps = np.empty(2 * npair)
    eps[0::2], eps[1::2] = r * np.cos(th), r * np.sin(th)
    z, ux, eps = z[:N], ux[:N], eps[:N]
    s = strata_s(K)
    ez = np.array([sigmoid(kappa + gamma * v) for v in s])
    x = (ux < ez[z]).astype(float)
    sz = s[z]
    y = 1.0 + delta * sz + (tau + eta * sz) * x + sigma * eps
    return z.reshape(R, n), x.reshape(R, n), y.reshape(R, n)


def conf_draws_loop(n, R, seed, K, gamma, delta, tau, eta, sigma, kappa=-1.0):
    """conf_draws subject by subject with JSRandom (a check of the vectorised version)."""
    g = JSRandom(seed)
    s = strata_s(K)
    ez = [sigmoid(kappa + gamma * v) for v in s]
    Z, X, Y = np.zeros((R, n), int), np.zeros((R, n)), np.zeros((R, n))
    for r_ in range(R):
        for i in range(n):
            z = int(math.floor(K * g.u()))
            x = 1.0 if g.u() < ez[z] else 0.0
            e = g.norm()
            Z[r_, i], X[r_, i] = z, x
            Y[r_, i] = 1.0 + delta * s[z] + (tau + eta * s[z]) * x + sigma * e
    return Z, X, Y


def conf_estimates(Z, X, Y, K):
    """The three estimators for each simulated study (rows): naive difference in means, the stratified
    (adjusted) estimator sum_z (n_z / n)(Ybar_1z - Ybar_0z), which is the plug-in estimate of (16.7) with
    the stratum means as r_hat, and the regression estimator (16.8). Strata with an empty arm are left
    out of the stratified estimator and the weights renormalised; 'dropped' counts them."""
    Z, X, Y = np.atleast_2d(Z), np.atleast_2d(X), np.atleast_2d(Y)
    R, n = X.shape
    n1 = X.sum(1)
    with np.errstate(invalid="ignore", divide="ignore"):
        naive = (X * Y).sum(1) / n1 - ((1 - X) * Y).sum(1) / (n - n1)
    num = np.zeros(R)
    wsum = np.zeros(R)
    dropped = np.zeros(R, int)
    for z in range(K):
        inz = (Z == z)
        a1 = (inz * X).sum(1)
        a0 = (inz * (1 - X)).sum(1)
        ok = (a1 > 0) & (a0 > 0)
        with np.errstate(invalid="ignore", divide="ignore"):
            d = (inz * X * Y).sum(1) / a1 - (inz * (1 - X) * Y).sum(1) / a0
        w = (a1 + a0) / n
        num += np.where(ok, w * np.nan_to_num(d), 0.0)
        wsum += np.where(ok, w, 0.0)
        dropped += (~ok) & (a1 + a0 > 0)
    with np.errstate(invalid="ignore", divide="ignore"):
        strat = num / wsum
    s = strata_s(K)[Z]
    reg = reg_slope_fw(X, s, Y)
    return dict(naive=naive, strat=strat, reg=reg, dropped=dropped)


def conf_sim(n, R, seed, K=3, gamma=0.0, delta=1.0, tau=1.0, eta=0.0, sigma=1.0, kappa=-1.0):
    """Simulated studies of the adjustment demo: the estimates of the three estimators."""
    Z, X, Y = conf_draws(n, R, seed, K, gamma, delta, tau, eta, sigma, kappa)
    return conf_estimates(Z, X, Y, K)


def ipw(Z, X, Y, K):
    """Inverse probability weighting with the stratum propensities estimated by the stratum treated
    fractions: mean(X Y / e_hat(Z)) - mean((1 - X) Y / (1 - e_hat(Z))) (one study, 1-D arrays)."""
    n = len(X)
    eh = np.array([X[Z == z].mean() if np.any(Z == z) else np.nan for z in range(K)])
    e = eh[Z]
    return float(np.sum(X * Y / e) / n - np.sum((1 - X) * Y / (1 - e)) / n)


# =========================================================================== 16.4 Simpson's paradox
# P[x, y, z]: the book's table (p. 260). Z = 1 men, Z = 0 women.
BOOK_TABLE = np.zeros((2, 2, 2))
BOOK_TABLE[1, 1, 1], BOOK_TABLE[1, 0, 1], BOOK_TABLE[1, 1, 0], BOOK_TABLE[1, 0, 0] = 0.15, 0.225, 0.10, 0.025
BOOK_TABLE[0, 1, 1], BOOK_TABLE[0, 0, 1], BOOK_TABLE[0, 1, 0], BOOK_TABLE[0, 0, 0] = 0.0375, 0.0875, 0.2625, 0.1125

# Kidney stones (Charig et al., 1986; Julious and Mullee, 1994): counts[x, y, z] with X = 1 open surgery,
# X = 0 percutaneous nephrolithotomy, Y = 1 success, Z = 1 large stones, Z = 0 small stones.
KIDNEY = np.zeros((2, 2, 2), int)
KIDNEY[1, 1, 0], KIDNEY[1, 0, 0], KIDNEY[1, 1, 1], KIDNEY[1, 0, 1] = 81, 6, 192, 71
KIDNEY[0, 1, 0], KIDNEY[0, 0, 0], KIDNEY[0, 1, 1], KIDNEY[0, 0, 1] = 234, 36, 55, 25

# the order in which the demo lays out the eight cells (for simpson_sample)
CELLS = [(1, 1, 1), (1, 0, 1), (1, 1, 0), (1, 0, 0), (0, 1, 1), (0, 0, 1), (0, 1, 0), (0, 0, 0)]


def simpson_table(q, a0, a1, p00, p01, p10, p11):
    """The joint table P[x, y, z] from P(Z = 1) = q, the allocations a_z = P(X = 1 | Z = z) and the
    recovery rates p_xz = P(Y = 1 | X = x, Z = z)."""
    P = np.zeros((2, 2, 2))
    pz = {0: 1 - q, 1: q}
    a = {0: a0, 1: a1}
    p = {(0, 0): p00, (0, 1): p01, (1, 0): p10, (1, 1): p11}
    for z in (0, 1):
        for x in (0, 1):
            px = a[z] if x == 1 else 1 - a[z]
            P[x, 1, z] = pz[z] * px * p[(x, z)]
            P[x, 0, z] = pz[z] * px * (1 - p[(x, z)])
    return P


def simpson_summary(P):
    """Everything Section 16.4 reads off a table P[x, y, z] (probabilities or counts)."""
    P = np.asarray(P, float)
    P = P / P.sum()
    pz = P.sum(axis=(0, 1))                   # P(Z = z)
    pxz = P.sum(axis=1)                       # P(X = x, Z = z)
    rate = P[:, 1, :] / pxz                   # P(Y = 1 | X = x, Z = z), indexed [x, z]
    px = pxz.sum(axis=1)
    pooled = P[:, 1, :].sum(axis=1) / px      # P(Y = 1 | X = x)
    zx = pxz[:, 1] / px                       # P(Z = 1 | X = x)
    alloc = pxz[1, :] / pz                    # P(X = 1 | Z = z)
    std = rate @ pz                           # standardised rates sum_z P(Y=1|x,z) P(z)
    return dict(pz=pz, rate=rate, pooled=pooled, zx=zx, alloc=alloc, std=std,
                diff_pooled=pooled[1] - pooled[0], diff_z=rate[1] - rate[0],
                adjusted=std[1] - std[0], marg=P.sum(axis=2))


def simpson_sample(P, n, seed):
    """Counts[x, y, z] of n subjects drawn from the table with one DK.rng(seed) uniform each, the cells
    taken in the order CELLS (the Simpson demo)."""
    P = np.asarray(P, float)
    if abs(P.sum() - 1.0) > 1e-12:    # counts or an unnormalised table (a table from simpson_table is used as is)
        P = P / P.sum()
    probs = np.array([P[c] for c in CELLS])
    cum = np.cumsum(probs)
    u = js_uniforms(seed, n)
    k = np.minimum(np.searchsorted(cum, u, side="right"), 7)
    cnt = np.bincount(k, minlength=8)
    C = np.zeros((2, 2, 2), int)
    for j, c in enumerate(CELLS):
        C[c] = cnt[j]
    return C


# =========================================================================== exercises
def manski_bounds(P_xy):
    """Exercise 16.3: bounds for theta from the joint law of (X, Y) alone (binary Y), with
    P_xy[x, y] = P(X = x, Y = y): [-(P(X=1,Y=0) + P(X=0,Y=1)), P(X=1,Y=1) + P(X=0,Y=0)], width 1."""
    P = np.asarray(P_xy, float) / np.sum(P_xy)
    return -(P[1, 0] + P[0, 1]), P[1, 1] + P[0, 0]


def type_model_xy(pi, e):
    """The joint law P[x, y] of (X, Y) under the type model."""
    pi = np.asarray(pi, float) / np.sum(pi)
    e = np.asarray(e, float)
    P = np.zeros((2, 2))
    for t in range(4):
        P[1, TYPE_C1[t]] += pi[t] * e[t]
        P[0, TYPE_C0[t]] += pi[t] * (1 - e[t])
    return P


def ex4_sim(n, seed=164):
    """Exercise 16.4: X ~ Uniform(1, 3) and, given X, a positive slope beta ~ Gamma(4, scale 1/(4 X^2)),
    so E(beta | X) = 1/X^2; Y = beta X has E(Y | X = x) = 1/x (decreasing) while every subject's curve
    C(x) = beta x increases. theta(x) = E(beta) x = x / 3."""
    g = np.random.default_rng(seed)
    X = g.uniform(1, 3, n)
    beta = g.gamma(4.0, 1.0 / (4.0 * X ** 2))
    return X, beta, beta * X


# =========================================================================== figures
def fig_16_1(out_dir=None):
    """Figure 16.1: a counterfactual function C(x) and the observed outcome Y = C(X)."""
    fig, ax = common.plt.subplots(figsize=(4.6, 3.0))
    xs = np.linspace(0, 11, 400)
    ax.plot(xs, c_curve(xs), color=BLUE, lw=2)
    X = 6.0
    Y = float(c_curve(X))
    ax.plot([X, X], [0, Y], color=GREY, ls=":", lw=1.2)
    ax.plot([0, X], [Y, Y], color=GREY, ls=":", lw=1.2)
    ax.plot(X, Y, "o", color=RED, ms=7, zorder=3)
    ax.annotate(r"$Y=C(X)$", (X, Y), xytext=(X + 0.6, Y - 1.5), fontsize=10, color=RED,
                arrowprops=dict(arrowstyle="->", color=RED, lw=0.8))
    ax.text(9.8, float(c_curve(9.8)) + 0.35, r"$C(x)$", color=BLUE, fontsize=11, ha="center")
    ax.set_xticks(range(12))
    ax.set_xticklabels([str(i) if i != 6 else "6\n$X$" for i in range(12)])
    ax.set_xlim(0, 11)
    ax.set_ylim(0, 7.2)
    ax.set_xlabel("dose $x$")
    ax.set_ylabel("$C(x)$")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-1", out_dir)


def fig_16_2(out_dir=None):
    """Figure 16.2: four flat counterfactual curves (no causal effect) and a decreasing regression."""
    fig, axes = common.panel_grid(2, 1, size=(4.6, 2.4))
    cols = (BLUE, GREEN, ORANGE, PURPLE)
    ax = axes[0, 0]
    for i in range(4):
        ax.plot([0, 5], [FIG162_C[i]] * 2, color=cols[i], lw=1.6)
        ax.plot(FIG162_X[i], FIG162_C[i], "o", color=cols[i], ms=6)
        ax.text(5.05, FIG162_C[i], "$C_%d(x)$" % (i + 1), va="center", fontsize=9, color=cols[i])
    ax.set_xlim(0, 5.6)
    ax.set_ylim(0, 3.2)
    ax.set_xticks(range(6))
    ax.set_ylabel("$C_i(x)$")
    common.panel_label(ax, "a")
    ax = axes[1, 0]
    nb = fig162_numbers()
    ax.plot([0, 5], [nb["theta"]] * 2, color=RED, lw=1.8)
    ax.text(5.05, nb["theta"], r"$\theta(x)$", va="center", fontsize=9, color=RED)
    xs = np.array([0.2, 4.8])
    ax.plot(xs, nb["intercept"] + nb["slope"] * xs, color="k", ls=":", lw=1.5)
    ax.text(3.7, nb["intercept"] + nb["slope"] * 3.7 - 0.5, r"$\widehat r(x)$", fontsize=9)
    for i in range(4):
        ax.plot(FIG162_X[i], FIG162_C[i], "o", color=cols[i], ms=6)
        ax.text(FIG162_X[i] + 0.12, FIG162_C[i] + 0.12, "$Y_%d$" % (i + 1), fontsize=9)
    ax.set_xlim(0, 5.6)
    ax.set_ylim(0, 3.2)
    ax.set_xticks(range(6))
    ax.set_xlabel("dose $x$")
    ax.set_ylabel("$Y$")
    common.panel_label(ax, "b")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-2", out_dir)


def fig_16_s1(out_dir=None):
    """Association against causation for the four response types (Theorems 16.1 and 16.3)."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.8))
    # (a) Example 16.2 and the advice population: alpha against the share of doomed who comply
    ax = axes[0, 0]
    pi = (0.5, 0, 0, 0.5)
    fr = np.linspace(0, 1, 101)
    al = [type_model(pi, (1, 0.5, 0.5, f))["alpha"] if f < 1 else np.nan for f in fr]
    ax.plot(fr, al, color=BLUE, label=r"association $\alpha$")
    ax.axhline(0, color=RED, lw=1.6, label=r"causal effect $\theta$")
    ax.plot([0, 0.75], [1, 4 / 7], "o", color="k", ms=5)
    ax.annotate("Example 16.2", (0, 1), xytext=(0.08, 0.9), fontsize=8)
    ax.annotate("after the advice", (0.75, 4 / 7), xytext=(0.32, 0.38), fontsize=8,
                arrowprops=dict(arrowstyle="->", lw=0.7))
    ax.set_xlabel("share of the unhealthy who take it")
    ax.set_ylabel("effect")
    ax.set_ylim(-0.15, 1.1)
    ax.legend(fontsize=7.5, frameon=False, loc="lower left")
    common.panel_label(ax, "a", loc="upper right")
    # (b) from random to self-selected assignment
    ax = axes[0, 1]
    pi = (0.3, 0.2, 0.1, 0.4)
    e = (0.9, 0.3, 0.6, 0.15)
    lam = np.linspace(0, 1, 101)
    ax.plot(lam, [type_model(pi, mix_assignment(pi, e, l))["alpha"] for l in lam], color=BLUE, label=r"$\alpha$")
    th = type_model(pi, e)["theta"]
    ax.axhline(th, color=RED, lw=1.6, label=r"$\theta$")
    ax.set_xlabel(r"self-selection $\lambda$ (0 = random)")
    ax.set_ylabel("effect")
    ax.set_ylim(0, 0.68)
    ax.legend(fontsize=7.5, frameon=False, loc="upper left")
    common.panel_label(ax, "b", loc="lower right")
    # (c) sampling distributions of the difference in means, n = 200
    ax = axes[0, 2]
    for lab, ee, col in (("random", mix_assignment(pi, e, 0.0), RED), ("self-selected", e, BLUE)):
        a = types_sim(pi, ee, 200, 2000, 7)["alpha_hat"]
        ax.hist(a[np.isfinite(a)], bins=50, range=(-0.2, 0.85), density=True, color=col, alpha=0.45, label=lab)
    ax.axvline(th, color=RED, lw=1.5)
    ax.axvline(type_model(pi, e)["alpha"], color=BLUE, lw=1.5, ls="--")
    ax.set_xlabel(r"$\overline{Y}_1-\overline{Y}_0$ ($n=200$)")
    ax.set_ylabel("density")
    ax.legend(fontsize=7.5, frameon=False, loc="upper right")
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-S1", out_dir)


def fig_16_s2(out_dir=None):
    """Adjusting for a confounder as the confounding strength changes (Theorem 16.6, (16.8))."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.8))
    gs = np.linspace(-3, 3, 81)
    for j, (eta, title) in enumerate(((0.0, "constant effect"), (1.5, "effect varies with $Z$"))):
        ax = axes[0, j]
        lim = [conf_model(3, g, 1.0, 1.0, eta) for g in gs]
        ax.plot(gs, [m["naive"] for m in lim], color=BLUE, label="naive difference")
        ax.plot(gs, [m["theta"] for m in lim], color=RED, lw=1.6, label=r"adjusted (16.7) $=\theta$")
        ax.plot(gs, [m["reg"] for m in lim], color=ORANGE, ls="--", label="regression (16.8)")
        ax.set_xlabel(r"confounding strength $\gamma$")
        ax.set_ylabel("limit of the estimator")
        ax.set_title(title, fontsize=9)
        ax.set_ylim(-1.5, 3.5)
        if j == 0:
            ax.legend(fontsize=7, frameon=False, loc="upper left")
        common.panel_label(ax, "ab"[j], loc="lower right")
    ax = axes[0, 2]
    est = conf_sim(400, 2000, 3, K=3, gamma=2.0, delta=1.0, tau=1.0, eta=1.5, sigma=1.0)
    for key, col, lab in (("naive", BLUE, "naive"), ("reg", ORANGE, "regression"), ("strat", RED, "stratified")):
        v = est[key][np.isfinite(est[key])]
        ax.hist(v, bins=60, range=(0.0, 3.6), density=True, color=col, alpha=0.45, label=lab)
    ax.axvline(1.0, color="k", lw=1.2)
    ax.set_xlabel(r"estimate ($n=400$, $\gamma=2$, $\eta=1.5$)")
    ax.set_ylabel("density")
    ax.set_ylim(0, 3.7)
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "c", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-S2", out_dir)


def _baker_kramer(ax, S, labels, title):
    cols = {1: BLUE, 0: ORANGE}
    for x in (1, 0):
        r0, r1 = S["rate"][x, 0], S["rate"][x, 1]
        ax.plot([0, 1], [r0, r1], color=cols[x], lw=1.6, label=labels[x])
        ax.plot(S["zx"][x], S["pooled"][x], "o", color=cols[x], ms=7, zorder=3)
        ax.plot(S["pz"][1], S["std"][x], "s", color=cols[x], ms=6, mfc="white", mew=1.6, zorder=3)
    ax.axvline(S["pz"][1], color=GREY, ls=":", lw=1)
    ax.set_xlim(-0.03, 1.03)
    ax.set_title(title, fontsize=9)
    ax.legend(fontsize=7, frameon=False, loc="best")


def fig_16_s3(out_dir=None):
    """Simpson's paradox: pooled rates are weighted averages with different weights (Section 16.4)."""
    fig, axes = common.panel_grid(1, 3, size=(3.3, 2.9))
    S = simpson_summary(BOOK_TABLE)
    ax = axes[0, 0]
    _baker_kramer(ax, S, {1: "treated ($X=1$)", 0: "untreated ($X=0$)"}, "the book's table")
    ax.set_xlabel("share of men, $P(Z=1\\mid X=x)$")
    ax.set_ylabel("recovery rate")
    ax.set_ylim(0.2, 0.9)
    common.panel_label(ax, "a", loc="lower left")
    K = simpson_summary(KIDNEY)
    ax = axes[0, 1]
    _baker_kramer(ax, K, {1: "open surgery", 0: "PCNL"}, "kidney stones")
    ax.set_xlabel("share of large stones")
    ax.set_ylabel("success rate")
    ax.set_ylim(0.6, 1.0)
    common.panel_label(ax, "b", loc="lower left")
    # (c) the pooled difference against the allocation of men, everything else fixed
    ax = axes[0, 2]
    a1s = np.linspace(0.01, 0.99, 99)
    rates = S["rate"]
    dp = [simpson_summary(simpson_table(0.5, S["alloc"][0], a1, rates[0, 0], rates[0, 1], rates[1, 0], rates[1, 1]))["diff_pooled"] for a1 in a1s]
    ax.plot(a1s, dp, color=BLUE, label="pooled difference")
    ax.axhline(S["adjusted"], color=RED, lw=1.6, label="adjusted difference")
    ax.axhline(0, color="k", lw=0.6)
    ax.plot(S["alloc"][1], S["diff_pooled"], "o", color="k", ms=5)
    ax.set_xlabel("$P(X=1\\mid Z=1)$ (women: 0.25)")
    ax.set_ylabel("$P(Y=1\\mid X=1)-P(Y=1\\mid X=0)$", fontsize=8)
    ax.legend(fontsize=7, frameon=False, loc="lower left")
    common.panel_label(ax, "c", loc="upper right")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-S3", out_dir)


def fig_16_s4(out_dir=None):
    """A continuous dose with a confounder: r(x), theta(x) and the adjusted estimate (16.8)."""
    fig, axes = common.panel_grid(1, 2, size=(3.9, 3.0))
    Z, X, Y, A = dose_sim(300, seed=16)
    M = dose_model()
    xs = np.linspace(X.min(), X.max(), 50)
    ax = axes[0, 0]
    sc = ax.scatter(X, Y, c=Z, cmap="coolwarm", s=9, alpha=0.8, lw=0)
    b = np.polyfit(X, Y, 1)
    ax.plot(xs, np.polyval(b, xs), color="k", ls=":", lw=1.6, label=r"least squares $\widehat r(x)$")
    ax.plot(xs, M["theta_int"] + M["theta_slope"] * xs, color=RED, lw=1.8, label=r"causal $\theta(x)$")
    th_hat, _ = adjusted_linear(X, Z, Y, xs)
    ax.plot(xs, th_hat, color=GREEN, ls="--", lw=1.6, label=r"adjusted (16.8)")
    ax.set_xlabel("dose $X$")
    ax.set_ylabel("$Y$")
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    cb = fig.colorbar(sc, ax=ax, fraction=0.05, pad=0.02)
    cb.set_label("$Z$", fontsize=8)
    common.panel_label(ax, "a", loc="lower left")
    # (b) a few counterfactual lines: each is increasing in x
    ax = axes[0, 1]
    idx = np.argsort(Z)[::30]
    cm = common.plt.get_cmap("coolwarm")
    zn = (Z - Z.min()) / (Z.max() - Z.min())
    for i in idx:
        ax.plot(xs, A[i] + M["theta_slope"] * xs, color=cm(zn[i]), lw=1)
        ax.plot(X[i], Y[i], "o", color=cm(zn[i]), ms=5, mec="k", mew=0.5)
    ax.set_xlabel("dose $x$")
    ax.set_ylabel("$C_i(x)$")
    common.panel_label(ax, "b", loc="lower left")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-S4", out_dir)


def fig_16_s5(out_dir=None):
    """Exercises 16.3 and 16.4."""
    fig, axes = common.panel_grid(1, 2, size=(3.9, 3.0))
    ax = axes[0, 0]
    pi = (0.3, 0.2, 0.1, 0.4)
    e = (0.9, 0.3, 0.6, 0.15)
    lam = np.linspace(0, 1, 101)
    lo, hi, al = [], [], []
    for l in lam:
        ee = mix_assignment(pi, e, l)
        b = manski_bounds(type_model_xy(pi, ee))
        lo.append(b[0])
        hi.append(b[1])
        al.append(type_model(pi, ee)["alpha"])
    ax.fill_between(lam, lo, hi, color=GREY, alpha=0.25, lw=0, label="bounds (width 1)")
    ax.axhline(type_model(pi, e)["theta"], color=RED, lw=1.6, label=r"$\theta$")
    ax.plot(lam, al, color=BLUE, label=r"$\alpha$")
    ax.axhline(0, color="k", lw=0.6)
    ax.set_xlabel(r"self-selection $\lambda$")
    ax.set_ylabel("effect")
    ax.set_ylim(-0.75, 0.95)
    ax.legend(fontsize=7, frameon=False, loc="lower right")
    common.panel_label(ax, "a", loc="upper left")
    ax = axes[0, 1]
    X, beta, Y = ex4_sim(400)
    ax.scatter(X, Y, s=7, color=BLUE, alpha=0.5, lw=0)
    xs = np.linspace(1, 3, 100)
    for i in range(0, 400, 50):
        ax.plot(xs, beta[i] * xs, color=GREY, lw=0.8)
    ax.plot(xs, 1 / xs, color="k", ls=":", lw=1.8, label=r"$E(Y\mid X=x)=1/x$")
    ax.plot(xs, xs / 3, color=RED, lw=1.8, label=r"$\theta(x)=x/3$")
    ax.set_xlabel("dose $x$")
    ax.set_ylabel("$Y$, $C_i(x)$")
    ax.set_ylim(0, 2.2)
    ax.legend(fontsize=7, frameon=False, loc="upper right")
    common.panel_label(ax, "b", loc="upper left")
    fig.tight_layout()
    return savefig(fig, CH, "fig16-S5", out_dir)


FIGURES = {"16.1": fig_16_1, "16.2": fig_16_2, "16.S1": fig_16_s1, "16.S2": fig_16_s2, "16.S3": fig_16_s3,
           "16.S4": fig_16_s4, "16.S5": fig_16_s5}

if __name__ == "__main__":
    common.run(FIGURES, CH)
