Add paired confidence intervals (E2), fixed-beta validation curve (E5), and mask-realization check

2026-07-27 14:04:02 +09:00
parent 0229c026dc
commit 5274ef8abe
4 changed files with 83 additions and 7 deletions
@@ -22,7 +22,8 @@ import os
 import numpy as np
 import torch

-from semantic_mac import (EmbeddingPool, affinity_matrix, matched_filter,
+from semantic_mac import (TAU, EmbeddingPool, affinity_matrix,
+                          matched_filter,
                          demux_lmmse, demux_dr, sample_latents_pool,
                          random_orthogonal, learn_structure_spectral,
                          subspace_error, metrics,
@@ -209,8 +210,14 @@ def main():
        f.write(f"spectral_err={err_spec}\nadapter_err={err_ad}\n")

    # --- performance ladder vs SNR (held-out pool sentences) ---------------
+    def inst_cos(e_hat, x_true):
+        """Per-instance cosine (batch, U) without consuming any RNG."""
+        e_n = e_hat / (np.linalg.norm(e_hat, axis=2, keepdims=True) + 1e-12)
+        return (e_n * x_true).sum(-1)
+
    snrs = list(range(0, 21, 4))
    rows = []
+    paired = []
    for s in snrs:
        rho = 10 ** (s / 10)
        z, x = gen_clean(pool, BATCH_EVAL, a, R, rng, idx_pool=IDX_EVAL)
@@ -218,22 +225,35 @@ def main():
        r = {}
        lm, _ = demux_lmmse(tilde, B, h, rho)
        r["LMMSE"] = metrics(lm, x)
-        r["DR-spec"] = metrics(demux_dr(tilde, B, h, rho, a, V_spec), x)
-        r["DR-adapt"] = metrics(
-            demux_dr(tilde, B, h, rho, a, W_ad.T[:, :DC]), x)
+        out_spec = demux_dr(tilde, B, h, rho, a, V_spec)
+        out_adapt = demux_dr(tilde, B, h, rho, a, W_ad.T[:, :DC])
+        r["DR-spec"] = metrics(out_spec, x)
+        r["DR-adapt"] = metrics(out_adapt, x)
        r["DR-oracle"] = metrics(demux_dr(tilde, B, h, rho, a, V_true), x)
        rng_c = np.random.default_rng(91000 + s)
        r["OMA"] = metrics(oma_observe(x, h, rho, rng_c), x)
        r["NOMA"] = metrics(demux_noma_genie(x, h, rho, rng_c), x)
        rows.append(r)
+        # paired spec-vs-adapter error difference on the SAME channel draws
+        err_s = (inst_cos(out_spec, x) < TAU).astype(float).ravel()
+        err_a = (inst_cos(out_adapt, x) < TAU).astype(float).ravel()
+        d_i = err_s - err_a
+        se = float(d_i.std(ddof=1) / math.sqrt(d_i.size))
+        paired.append((s, float(err_s.mean()), float(err_a.mean()),
+                       float(d_i.mean()), se))
        print(f"snr={s:2d} " + " ".join(
-            f"{k}:cos={v[0]:.3f},ser={v[2]:.3f}" for k, v in r.items()))
+            f"{k}:cos={v[0]:.3f},ser={v[2]:.3f}" for k, v in r.items())
+            + f" paired_diff={d_i.mean():+.5f} se={se:.5f}")
    keys = ["OMA", "NOMA", "LMMSE", "DR-spec", "DR-adapt", "DR-oracle"]
    with open(os.path.join(DATA, "e2_ladder.csv"), "w") as f:
        f.write("snr," + ",".join(f"{k}_cos,{k}_nmse,{k}_ser" for k in keys) + "\n")
        for s, r in zip(snrs, rows):
            f.write(f"{s}," + ",".join(
                f"{r[k][0]},{r[k][1]},{r[k][2]}" for k in keys) + "\n")
+    with open(os.path.join(DATA, "e2_paired.csv"), "w") as f:
+        f.write("snr,ser_spec,ser_adapt,mean_diff,se_diff\n")
+        for row in paired:
+            f.write(",".join(str(v) for v in row) + "\n")

    # figures are produced only by the canonical replot_all.py (uniform
    # geometry); experiment scripts write CSVs exclusively.
@@ -70,6 +70,14 @@ def main():
    print(f"learned receiver parameters: {n_par}")
    opt = torch.optim.Adam(net.parameters(), lr=1e-3)
    train_log = []
+    # fixed validation batch at beta=0.5 (dedicated RNG, so the training
+    # stream is untouched); evaluated every 100 steps as convergence
+    # evidence for the fixed training budget
+    rng_v = np.random.default_rng(777)
+    z_v, tilde_v, h_v, a_v, B_v = gen_batch(rng_v, 2000, 0.5)
+    zv_t = torch.tensor(z_v, dtype=torch.float32)
+    tv_t = torch.tensor(tilde_v, dtype=torch.float32)
+    val_log = []
    for it in range(STEPS):
        beta = float(rng.uniform(0.05, 0.9))
        z, tilde, h, a, B = gen_batch(rng, BATCH, beta)
@@ -82,13 +90,23 @@ def main():
        opt.step()
        if (it + 1) % 50 == 0:
            train_log.append((it + 1, float(loss.item())))
+        if (it + 1) % 100 == 0:
+            with torch.no_grad():
+                ov = net(tv_t)
+                vloss = float((1.0 - (ov * zv_t).sum(-1)).mean())
+            val_log.append((it + 1, vloss))
        if (it + 1) % 500 == 0:
-            print(f"  step {it+1}: loss={loss.item():.4f}")
-    # convergence evidence for the fixed training budget
+            print(f"  step {it+1}: loss={loss.item():.4f} "
+                  f"val={val_log[-1][1]:.4f}")
    with open(os.path.join(DATA, "e5_train_log.csv"), "w") as f:
        f.write("step,loss\n")
        for st, lo in train_log:
            f.write(f"{st},{lo}\n")
+    # fixed-beta validation curve (convergence evidence)
+    with open(os.path.join(DATA, "e5_valcurve.csv"), "w") as f:
+        f.write("step,val_loss\n")
+        for st, lo in val_log:
+            f.write(f"{st},{lo}\n")

    # evaluation across the affinity sweep
    betas = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
@@ -0,0 +1,7 @@
+snr,ser_spec,ser_adapt,mean_diff,se_diff
+0,1.0,1.0,0.0,0.0
+4,0.999875,0.9999375,-6.25e-05,6.250000000000001e-05
+8,0.99425,0.99675,-0.0025,0.0004415133728197241
+12,0.8074375,0.823875,-0.0164375,0.001822954877359744
+16,0.2864375,0.278125,0.0083125,0.001970442288335313
+20,0.033125,0.0274375,0.0056875,0.0008671398487004276
@@ -0,0 +1,31 @@
+step,val_loss
+100,0.47339609265327454
+200,0.4509868323802948
+300,0.43674764037132263
+400,0.4313800036907196
+500,0.4262669086456299
+600,0.4236183166503906
+700,0.4218555688858032
+800,0.4200989902019501
+900,0.41954243183135986
+1000,0.4185897707939148
+1100,0.41777464747428894
+1200,0.41801124811172485
+1300,0.4173048734664917
+1400,0.41738399863243103
+1500,0.4171726703643799
+1600,0.417477011680603
+1700,0.41742151975631714
+1800,0.4169759750366211
+1900,0.41719603538513184
+2000,0.4168195426464081
+2100,0.4161660969257355
+2200,0.41649919748306274
+2300,0.41610589623451233
+2400,0.416185587644577
+2500,0.4159914553165436
+2600,0.416018009185791
+2700,0.4158671498298645
+2800,0.4158063232898712
+2900,0.4162437915802002
+3000,0.41600432991981506