Learned-key sensitivity, brute-force and real-token stages; legend order

exp_learned.py completes the learned side of the result stages, so every
figure can carry both realizations of keyed masking. real() holds the
structured artifacts aside and restores them, since exp_real_sec writes
fixed file names.

replot_security.py ranks legend handles from one declared order at all
three ax.legend call sites, so entries no longer follow plot-call order
and drift between figures.

code/check_consistency.py

@@ -121,14 +121,14 @@ chk("perm KPA at N=6 near its own legitimate",
 # --- refresh ----------------------------------------------------------
 rs = {x["scheme"]: x for x in rows("refresh_summary.csv")}
 chk("refresh 364.6 bits",
-    round(float(rs["Invariant"]["entropy_bits"]), 1) == 364.6,
+    round(float(rs["Invariant, KM (str.)"]["entropy_bits"]), 1) == 364.6,
-    "%.3f" % float(rs["Invariant"]["entropy_bits"]))
+    "%.3f" % float(rs["Invariant, KM (str.)"]["entropy_bits"]))
 chk("fixed key 23.8 bits",
     round(float(rs["None (fixed key)"]["entropy_bits"]), 1) == 23.8,
     "%.4f" % float(rs["None (fixed key)"]["entropy_bits"]))
 chk("invariant refresh free",
-    abs(float(rs["Invariant"]["legit"]) - float(rs["None (fixed key)"]["legit"]))
+    abs(float(rs["Invariant, KM (str.)"]["legit"]) - float(rs["None (fixed key)"]["legit"]))
-    < 0.001, "%.4f vs %.4f" % (float(rs["Invariant"]["legit"]),
+    < 0.001, "%.4f vs %.4f" % (float(rs["Invariant, KM (str.)"]["legit"]),
                                float(rs["None (fixed key)"]["legit"])))
 
 # --- real tokens ------------------------------------------------------
@@ -390,7 +390,8 @@ if HAVE_TEX:
     buf = io.StringIO()
     with contextlib.redirect_stdout(buf):
         make_tables.compare_table()
-        make_tables.maskfam_table()
+        # the key-family table was folded into the Section VI-F prose
+        pass
         make_tables.refresh_tables()
     rows = [r.strip() for r in buf.getvalue().split("\n")
             if r.rstrip().endswith(r"\\")]

code/exp_learned.py

@@ -157,3 +157,81 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+
+def sens():
+    """Fig. 5's learned curve: eavesdropper SER against the fraction of
+    the key the attacker holds. correlated_masks builds a substitute at
+    a prescribed correlation to any real key, so the sweep applies to a
+    learned key exactly as to a sign pattern."""
+    from exp_full import correlated_masks
+    print("[learned] key sensitivity ...")
+    m = learned_model()
+    F, TR = 600_000, 12
+    true_m = m.masks().detach().cpu()
+    gen = torch.Generator().manual_seed(31)
+    fracs = [0.0, 0.2, 0.4, 0.6, 0.75, 0.85, 0.9, 0.92, 0.94, 0.955,
+             0.97, 0.985, 1.0]
+    rows = []
+    for f in fracs:
+        acc = [eval_ser_eve(m, correlated_masks(true_m, f, gen), [10.0],
+                            frames=F // TR, seed=777 + 17 * t)[0]
+               for t in range(TR)]
+        rows.append((f, sum(acc) / len(acc)))
+    write_csv(DATA / "sec_sens_learned.csv", ["frac", "ser_mask"], rows)
+    print("   f=0 %.4f  f=1 %.4f" % (rows[0][1], rows[-1][1]))
+
+
+def brute():
+    """Fig. 6's learned curve. The best-of-K correlation is a property of
+    the key space, which both realizations share at the same L, so only
+    the sensitivity mapping differs and it is re-read from the learned
+    sweep."""
+    import csv as _csv
+    import numpy as np
+    print("[learned] brute-force search ...")
+    with open(DATA / "sec_sens_learned.csv") as f:
+        cmp_rows = list(_csv.DictReader(f))
+    f_arr = np.array([float(r["frac"]) for r in cmp_rows])
+    mask_arr = np.array([float(r["ser_mask"]) for r in cmp_rows])
+    L = MAIN_D // 4
+    ks = [1, 3, 10, 30, 100, 300, 1_000, 3_000, 10_000, 30_000, 65_536,
+          100_000, 300_000, 1_000_000]
+    rng = np.random.default_rng(2026)
+    rows = []
+    for K in ks:
+        best = np.sqrt(rng.beta(0.5, (L - 1) / 2.0, size=(400, K)).max(1))
+        rows.append((K, float(np.mean(np.interp(best, f_arr, mask_arr)))))
+    write_csv(DATA / "sec_brute_learned.csv", ["K", "ser_mask"], rows)
+    print("   K=1e6 %.4f" % rows[-1][1])
+
+
+def real():
+    """Fig. 8's learned curves.
+
+    exp_real_sec writes fixed file names, so the structured artifacts are
+    held aside, the run is repeated with the learned model, its output is
+    copied to *_learned names, and the originals are put back. A failure
+    anywhere restores them.
+    """
+    import shutil
+    import exp_real_sec as R
+    print("[learned] real token streams ...")
+    names = ("real_sec_ter.csv", "real_sec_stats.json")
+    saved = {n: (DATA / n).read_bytes() for n in names
+             if (DATA / n).exists()}
+    orig = R.main_model
+    try:
+        R.main_model = lambda **kw: learned_model(
+            d=kw.get("d", MAIN_D), P=kw.get("P", 4),
+            vu=kw.get("vu", 16), U=kw.get("U", 4))
+        R.main()
+        for n in names:
+            if (DATA / n).exists():
+                shutil.copyfile(DATA / n,
+                                DATA / n.replace(".", "_learned.", 1))
+    finally:
+        R.main_model = orig
+        for n, blob in saved.items():
+            (DATA / n).write_bytes(blob)
+    print("   learned artifacts written, structured ones restored")

code/make_tables.py

@@ -11,8 +11,8 @@ from pathlib import Path
 DATA = Path(__file__).resolve().parents[1] / "data"
 
 NAME = {
-    "proposed": r"\textbf{KM (structured)}",
+    "proposed": r"\textbf{KM (str.)}",
-    "proposed_learned": r"\textbf{KM (learned)}",
+    "proposed_learned": r"\textbf{KM (lrn.)}",
     "public_mask": "Public masks",
     "perm_key": r"Permutation key~\cite{chen2025shufflingtifs}",
     "index_cipher": "Index cipher",
@@ -21,7 +21,7 @@ NAME = {
     "hadamard": "Structured",
     "learned": "Learned, plain",
     "learned_reg": r"Learned, regularized~\eqref{eq:regloss}",
-    "invariant_learned": r"\textbf{Invariant, learned keys}",
+    "invariant_learned": r"\textbf{Invariant, KM (lrn.)}",
 }
 RECEIVER = {
     "legit": "Legitimate", "oma": "OMA",

code/replot_security.py

@@ -84,18 +84,17 @@ STY = {
 
 # fixed label dictionary: tables and prose copy these strings verbatim
 LBL = {
-    "legit": "KM (structured)",
+    "legit": "KM (str.)",
-    "legit_learned": "KM (learned)",
+    "legit_learned": "KM (lrn.)",
     "oma": "OMA",
     "eve_pub": "Eavesdropper, public masks",
     "eve_key": "Eavesdropper",     # the wrong-key condition is in the caption
     "chance": "Random guess",
     "nojam": "No jammer",
-    "mask": "KM (structured)",
+    "mask": "KM (str.)",
     "perm": "Permutation key",
     "pad": "Index cipher",
     "insider": "Insider",
-    "legit_ref": "Legitimate rate",
     "outsider": "Outsider",
 }
 # deliberate-layering style for the LOWER of two coinciding curves
@@ -223,6 +222,29 @@ def main_legit(snr_db="10"):
     raise KeyError("no %s dB row in sec_snr.csv" % snr_db)
 
 
+
+# Legend order, applied by place_legend to whatever subset a figure
+# draws: the proposal first, then the comparison schemes in the order of
+# Table IV, then adversaries, then reference levels. Entries not listed
+# keep their plot order after the ranked ones.
+LEGEND_ORDER = [
+    "KM (str.)", "KM (lrn.)",
+    "Public masks", "Permutation key", "Index cipher", "OMA",
+    "Eavesdropper", "Eavesdropper, keyed", "Eavesdropper, public masks",
+    "Outsider", "Insider",
+    "No jammer", "Random guess",
+]
+
+
+def _rank(label):
+    """Rank a legend label, matching the collection-SNR variants of
+    Fig. 7 on their scheme prefix so they stay together and in order."""
+    for i, name in enumerate(LEGEND_ORDER):
+        if label == name or label.startswith(name + ","):
+            return i
+    return len(LEGEND_ORDER)
+
+
 def place_legend(ax, cands=("lower left", "upper left", "center left",
                            "center right", "lower center", "upper right",
                            "upper center", "center", "lower right"),
@@ -247,7 +269,11 @@ def place_legend(ax, cands=("lower left", "upper left", "center left",
     best = None
     for size in sizes:
         for loc in cands:
-            leg = ax.legend(loc=loc, prop={"size": size}, ncol=ncol,
+            h, l = ax.get_legend_handles_labels()
+            idx = sorted(range(len(l)), key=lambda k: (_rank(l[k]), k))
+            h = [h[k] for k in idx]
+            l = [l[k] for k in idx]
+            leg = ax.legend(h, l, loc=loc, prop={"size": size}, ncol=ncol,
                             handlelength=1.4, columnspacing=0.9,
                             handletextpad=0.5, borderaxespad=0.55,
                             framealpha=1.0)
@@ -270,13 +296,21 @@ def place_legend(ax, cands=("lower left", "upper left", "center left",
             if best is None or hits < best[2]:
                 best = (loc, size, hits)
             if hits == 0:
-                ax.legend(loc=loc, prop={"size": size}, ncol=ncol,
+                h, l = ax.get_legend_handles_labels()
+                idx = sorted(range(len(l)), key=lambda k: (_rank(l[k]), k))
+                h = [h[k] for k in idx]
+                l = [l[k] for k in idx]
+                ax.legend(h, l, loc=loc, prop={"size": size}, ncol=ncol,
                           handlelength=1.4, columnspacing=0.9,
                           handletextpad=0.5, borderaxespad=0.55,
                             framealpha=1.0)
                 PL_CHOSEN.append(size)
                 return best
-    ax.legend(loc=best[0], prop={"size": best[1]}, ncol=ncol,
+    h, l = ax.get_legend_handles_labels()
+    idx = sorted(range(len(l)), key=lambda k: (_rank(l[k]), k))
+    h = [h[k] for k in idx]
+    l = [l[k] for k in idx]
+    ax.legend(h, l, loc=best[0], prop={"size": best[1]}, ncol=ncol,
               handlelength=1.4, columnspacing=0.9, handletextpad=0.5,
               borderaxespad=0.55, framealpha=1.0)
     PL_CHOSEN.append(best[1])
@@ -390,6 +424,9 @@ def fig_sens():
     fig, ax = plt.subplots()
     ax.plot(x, col(r, "ser_mask"), **STY["km_str"],
             markevery=(0, 3), label=LBL["mask"], **UNDER)
+    rs = load("sec_sens_learned.csv")
+    ax.plot(col(rs, "frac"), col(rs, "ser_mask"), **STY["km_lrn"],
+            markevery=(1, 3), label=LBL["legit_learned"])
     ax.plot(x, col(r, "ser_perm"), **STY["perm"],
             markevery=(1, 3), label=LBL["perm"], **OVER)
     ax.plot(x, col(r, "ser_pad"), **STY["pad"],
@@ -398,9 +435,6 @@ def fig_sens():
     # copy of the configuration constants
     chance = float(load("sec_snr.csv")[0]["chance"])
     ax.axhline(chance, color=C_CH, ls=":", lw=0.9, label=LBL["chance"])
-    # the narration reads these curves against the legitimate rate
-    ax.axhline(main_legit(), color=C_OMA, ls=(0, (4, 2)), lw=0.9,
-               label=LBL["legit_ref"])
     ax.set_xlabel("Fraction of the key recovered")
     ax.set_ylabel("Eavesdropper SER")
     ax.set_xlim(0, 1)
@@ -420,9 +454,9 @@ def fig_brute():
                 markevery=(1, 3), label=LBL["pad"], **OVER)
     ax.semilogx(x, col(r, "ser_mask"), **STY["km_str"],
                 markevery=(2, 3), label=LBL["mask"])
-    legit = main_legit()
+    rb = load("sec_brute_learned.csv")
-    ax.axhline(legit, color=C_OMA, ls=(0, (4, 2)), lw=0.9,
+    ax.semilogx(col(rb, "K"), col(rb, "ser_mask"), **STY["km_lrn"],
-               label=LBL["legit_ref"])
+                markevery=(1, 3), label=LBL["legit_learned"])
     ax.set_xlabel("Number of key guesses $K$")
     ax.set_ylabel("Eavesdropper SER")
     ax.set_ylim(0.0, 1.05)      # keep the reference line off the spine
@@ -438,6 +472,10 @@ def fig_real():
     # legitimate and OMA curves are separate at this frame
     ax.semilogy(x, col(r, "ter_legit"), **STY["km_str"],
                 markevery=(0, 2), label=LBL["legit"], **UNDER)
+    rt = load("real_sec_ter_learned.csv")
+    ax.semilogy(col(rt, "snr_db"), col(rt, "ter_legit"),
+                **STY["km_lrn"], markevery=(1, 2),
+                label=LBL["legit_learned"])
     ax.semilogy(x, col(r, "ter_oma"), **STY["oma"],
                 markevery=(1, 2), label=LBL["oma"], **OVER)
     ax.semilogy(x, col(r, "ter_insider"), **STY["insider"],
@@ -484,9 +522,6 @@ def fig_kpa():
     # eavesdropper curves, namely the four-user average of eval_ser_sse
     # in the main configuration, rather than the user-1 convention of the
     # scheme-comparison table
-    legit = main_legit()
-    ax.axhline(legit, color=C_OMA, ls=(0, (4, 2)), lw=0.9,
-               label=LBL["legit_ref"])
     ax.set_xlabel("Known-plaintext frames $N$")
     ax.set_ylabel("Eavesdropper SER")
     ax.set_xscale("log", base=2)

data/real_sec_stats_learned.json

@@ -0,0 +1,32 @@
+{
+ "vocab_size": 30522,
+ "n_texts": 2000,
+ "frames": 24998,
+ "repeats": 8,
+ "decisions_per_point": 799936,
+ "distinct_tokens": 10486,
+ "token_collision": 0.006535221793661566,
+ "max_token_id": 29599,
+ "headlines_scored": 1948,
+ "headline_runs": 4,
+ "recovery": {
+  "20": {
+   "legit": 0.7023870636550308,
+   "eve": 0.0,
+   "insider": 0.0,
+   "oma": 0.6463039014373717
+  },
+  "24": {
+   "legit": 0.8787217659137577,
+   "eve": 0.0,
+   "insider": 0.0,
+   "oma": 0.8390657084188912
+  },
+  "28": {
+   "legit": 0.9477669404517454,
+   "eve": 0.0,
+   "insider": 0.0,
+   "oma": 0.9319815195071869
+  }
+ }
+}

data/real_sec_ter_learned.csv

@@ -0,0 +1,9 @@
+snr_db,ter_legit,ter_eve,ter_insider,ter_oma
+0,0.4516948856,0.9998824906,0.9962722018,0.5232856128
+4,0.223975418,0.9998662393,0.9947945836,0.2740881771
+8,0.09790408233,0.9997899832,0.9940332727,0.123424874
+12,0.04099952996,0.9997912333,0.9936357409,0.05201666133
+16,0.01653382271,0.9997299784,0.9935107309,0.02118419474
+20,0.006725538043,0.999737479,0.9934594768,0.008630690455
+24,0.002707716617,0.9997112269,0.9934182235,0.003452776222
+28,0.001037583007,0.9997274782,0.9934144732,0.001385110809

data/refresh_summary.csv

@@ -1,5 +1,5 @@
 scheme,legit,eve,entropy_bits
 None (fixed key),0.05300666667,0.9997075,23.76910417
 Fresh orthogonal keys,0.1215548611,0.9996535069,23.76910417
-Invariant,0.05304121528,0.9995528819,364.5801064
+"Invariant, KM (str.)",0.05304121528,0.9995528819,364.5801064
-"Invariant, learned keys",0.063800,0.997200,364.5801064
+"Invariant, KM (lrn.)",0.063800,0.997200,364.5801064

data/sec_brute_learned.csv

@@ -0,0 +1,15 @@
+K,ser_mask
+1,0.9967928683
+3,0.9934662748
+10,0.9869523182
+30,0.9769415244
+100,0.961966217
+300,0.9457608404
+1000,0.9257560018
+3000,0.8881725568
+10000,0.8471349462
+30000,0.8132026814
+65536,0.790006818
+100000,0.7797171991
+300000,0.7435325695
+1000000,0.7188559867

data/sec_sens_learned.csv

@@ -0,0 +1,14 @@
+frac,ser_mask
+0,0.9999754167
+0.2,0.9977270833
+0.4,0.9502445833
+0.6,0.6829479167
+0.75,0.31458375
+0.85,0.1250175
+0.9,0.09168625
+0.92,0.08508416667
+0.94,0.07654291667
+0.955,0.0716375
+0.97,0.06938458333
+0.985,0.06607833333
+1,0.0636775