Reproducible V8, stored V5 row, token collision probability

2026-08-26 15:02:21 +09:00
parent d061a9be88
commit 8d29dfa5ad
5 changed files with 51 additions and 7 deletions
@@ -189,6 +189,10 @@ def main():
        "repeats": REPEATS,
        "decisions_per_point": N * U * REPEATS,
        "distinct_tokens": int(torch.unique(ids_all).numel()),
+        # the chance that two independently drawn tokens coincide, which
+        # is the floor the insider TER is measured against
+        "token_collision": float(((torch.bincount(ids_all.reshape(-1)).double()
+                                  / ids_all.numel()) ** 2).sum()),
        "max_token_id": int(ids_all.max()),
        "headlines_scored": sum(len(b) for b in bounds),
        "headline_runs": REC_RUNS,