changed to new test method for singificance

sim/rl/behavior_loader/loader.py

@@ -2,6 +2,7 @@ import os
 import json
 from pydantic import BaseModel as Base
 
+
 class PayloadModel(Base):
     sessionId: str
     experimentId: str | None
@@ -13,6 +14,7 @@ class PayloadModel(Base):
     userAgent: str
     ts: str
 
+
 class ValueModel(Base):
     payload: PayloadModel
     encoding: str
@@ -20,6 +22,7 @@ class ValueModel(Base):
     schemaId: int
     size: int
 
+
 class InteractionModel(Base):
     partitionID: int
     offset: int
@@ -30,14 +33,17 @@ class InteractionModel(Base):
     key: dict
     value: ValueModel
 
+
 def _is_admin(page: str | None) -> bool:
     return page is not None and page.startswith("/admin/")
 
+
 class Loader:
     def __init__(self, src_dir: str):
         self.src_dir = src_dir
         self.entries = os.listdir(src_dir)
-        if not self.entries: raise ValueError("empty directory")
+        if not self.entries:
+            raise ValueError("empty directory")
         self.data = self._load_sessions()
 
     def _load_sessions(self) -> dict:
@@ -55,16 +61,21 @@ class Loader:
     def get_entries(self) -> tuple[list[str], int]:
         return self.entries, len(self.entries)
 
+
 class AgentLoader(Loader):
     def _load_sessions(self) -> dict:
         sessions = {}
         for entry in self.entries:
-            with open(f"{self.src_dir}/{entry}/int.json") as f:
+            path = f"{self.src_dir}/{entry}/int.json"
+            if not os.path.isfile(path):
+                continue
+            with open(path) as f:
                 raw = json.load(f)
             ints = [PayloadModel(**i) for i in raw]
             sessions[entry] = [i for i in ints if not _is_admin(i.page)]
         return sessions
 
+
 class JointLoader:
     def __init__(self, human_dir: str, agent_dir: str):
         self.human_loader = Loader(human_dir)
@@ -74,10 +85,14 @@ class JointLoader:
 
     def _merge(self) -> dict:
         return {
-            **{f"human_{sid}": [e.value.payload for e in evts]
-               for sid, evts in self.human_loader.get_data().items()},
-            **{f"agent_{sid}": evts
-               for sid, evts in self.agent_loader.get_data().items()}
+            **{
+                f"human_{sid}": [e.value.payload for e in evts]
+                for sid, evts in self.human_loader.get_data().items()
+            },
+            **{
+                f"agent_{sid}": evts
+                for sid, evts in self.agent_loader.get_data().items()
+            },
         }
 
     def get_data(self) -> dict:
@@ -86,12 +101,17 @@ class JointLoader:
     def get_entries(self) -> tuple[list[str], int]:
         return self.entries, len(self.entries)
 
+
 if __name__ == "__main__":
     agent_dir = "/home/velocitatem/Documents/Projects/PHANTOM/experiments/agents/collected_data/"
-    human_dir = "/home/velocitatem/Documents/Projects/PHANTOM/experiments/collected_data/"
+    human_dir = (
+        "/home/velocitatem/Documents/Projects/PHANTOM/experiments/collected_data/"
+    )
 
-    for name, cls, path in [("agent", AgentLoader, agent_dir),
-                             ("human", Loader, human_dir),
-                             ("joint", lambda d: JointLoader(human_dir, d), agent_dir)]:
+    for name, cls, path in [
+        ("agent", AgentLoader, agent_dir),
+        ("human", Loader, human_dir),
+        ("joint", lambda d: JointLoader(human_dir, d), agent_dir),
+    ]:
         ldr = cls(path) if name != "joint" else cls(agent_dir)
         print(f"Loaded {len(ldr.get_entries()[0])} {name} sessions")

sim/rl/behavior_loader/models.py

@@ -260,6 +260,29 @@ def _avg_event_kl(
     return float(np.mean([kl_divergence(src_evt[e], dst_evt[e]) for e in common]))
 
 
+def per_session_divergence(
+    model: BehaviorModel,
+    reference_evt: Dict[str, Dict[str, float]],
+) -> List[float]:
+    """KL from each session's event-level transition dist to a reference kernel. Returns one scalar per session."""
+    scores = []
+    for sid, evts in model.data.items():
+        if len(evts) < 2:
+            continue
+        subset_mdp = _build_subset_mdp(model, [sid])
+        sess_evt = aggregate_event_transitions(subset_mdp)
+        common = set(sess_evt.keys()) & set(reference_evt.keys())
+        if not common:
+            scores.append(0.0)
+            continue
+        scores.append(
+            float(
+                np.mean([kl_divergence(sess_evt[e], reference_evt[e]) for e in common])
+            )
+        )
+    return scores
+
+
 def bootstrap_intra_class_divergence(
     model: BehaviorModel,
     n_bootstrap: int = 100,
@@ -412,3 +435,36 @@ if __name__ == "__main__":
         f"  Lift vs pooled intra mean: {inter_class_avg / max(float(np.mean(pooled_null)), 1e-10):.2f}x"
     )
     print(f"  Empirical p-value (inter > intra): {p_empirical:.4f}")
+
+    # per-session divergence scores: delta_H - delta_A per session (positive means closer to agent behavior)
+    from scipy.stats import mannwhitneyu
+
+    human_dH = per_session_divergence(
+        human_model, human_evt
+    )  # human session vs human centroid
+    human_dA = per_session_divergence(
+        human_model, agent_evt
+    )  # human session vs agent centroid
+    agent_dH = per_session_divergence(
+        agent_model, human_evt
+    )  # agent session vs human centroid
+    agent_dA = per_session_divergence(
+        agent_model, agent_evt
+    )  # agent session vs agent centroid
+    # score = delta_H - delta_A: high means far from humans, close to agents
+    n_h = min(len(human_dH), len(human_dA))
+    n_a = min(len(agent_dH), len(agent_dA))
+    human_diff = [human_dH[i] - human_dA[i] for i in range(n_h)]
+    agent_diff = [agent_dH[i] - agent_dA[i] for i in range(n_a)]
+    print(f"\nPer-session divergence gap (delta_H - delta_A):")
+    print(
+        f"  Human sessions (n={n_h}): mean={np.mean(human_diff):.4f}, std={np.std(human_diff):.4f}"
+    )
+    print(
+        f"  Agent sessions (n={n_a}): mean={np.mean(agent_diff):.4f}, std={np.std(agent_diff):.4f}"
+    )
+    if n_h >= 2 and n_a >= 2:
+        U, mw_p = mannwhitneyu(human_diff, agent_diff, alternative="two-sided")
+        print(f"  Mann-Whitney U={U:.1f}, p={mw_p:.4f}")
+    else:
+        print("  Insufficient sessions for Mann-Whitney test")