fixing cookie import

* baseline setup of agent abstract

* feat: new implementation of simple AI agent that can follow a goal and return

* refactored import structure and created full tests

* pytest setup a github workflow to run tests + more ignores

* singularity for pushing

* fixing builds of PDFs

* inital structure of docs

* init styles and docs

* basic style implementation

* 13 create outline for research paper draft (#18)

* updated outline for paper from issue

* extra paper sections and some formalization of series data

* algorithms and acknowledgements

* updated outline for paper from issue

* Refactor docker-compose services to use individual Dockerfiles (#20)

* Initial plan

* Refactor services into individual Dockerfiles

Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

* Add EXPOSE directives to all Dockerfiles with port documentation

Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

* 2 nextjs scaffold with store mode shop and admin session experiment wiring event emission v1 (#17)

* chore: cleaning gitignore

* formating and env documentation

* feat: context switching of hotel/airline depndent on env var via middleware

* fixed alignment and building

* wrong file

* prods

* fixed applying style

* better session cookie management

* tentative session storage with maybe using airtable

* migrated api of ingestion

* events and products apge

* fixing build

* 13 create outline for research paper draft (#18)

* updated outline for paper from issue

* extra paper sections and some formalization of series data

* algorithms and acknowledgements

* updated outline for paper from issue

* upadted text formating

* event unification

* refactor tracking to ues callbacks instead of refs

* implement a pricing display api with session passing

* moved middleware to proxy according to new changes in Nextjs

* refactoed kafka ingestion to go via backend not web-db

* Refactor docker-compose services to use individual Dockerfiles (#20)

* Initial plan

* Refactor services into individual Dockerfiles

Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

* Add EXPOSE directives to all Dockerfiles with port documentation

Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: velocitatem <60182044+velocitatem@users.noreply.github.com>

* fixing small bugs and adding exepriments to tracking

* added some doc

* fixing prod

* prod kafka server logging

* topic auto create

* pytest setup a github workflow to run tests + more ignores

* getting data from agents properly

* proper pipeline to handle data and build matrices

* fixing backend dumping

* fixing agents and ignore

* fixing import for tests

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>

.github/workflows/pytest.yml

@@ -0,0 +1,30 @@
+name: Run Tests
+on:
+  push:
+    paths:
+      - 'experiments/**'
+      - 'backend/**'
+      - 'requirements.txt'
+      - '.github/workflows/pytest.yml'
+  pull_request:
+    paths:
+      - 'experiments/**'
+      - 'backend/**'
+      - 'requirements.txt'
+      - '.github/workflows/pytest.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - name: Install dependencies
+        run: |
+          python -m venv .venv
+          .venv/bin/pip install --upgrade pip
+          .venv/bin/pip install -r requirements.txt
+      - name: Run tests
+        run: .venv/bin/pytest -v

.gitignore

@@ -1,6 +1,8 @@
 **/.env
 **/.venv
-PHANTOM.wiki/
+**/__pycache__
+**/.ipynb_checkpoints/
 **/.virtual_documents/
-**/__pycache__/
+**/session_*.svg
-**/.ipynb_checkpoints/
+**/*graph.svg
+paper/src/bib/auto

Makefile

@@ -4,6 +4,10 @@ BUILDDIR  := build
 TEX       := main.tex
 JOBNAME   := main
 PDF       := paper/$(BUILDDIR)/$(JOBNAME).pdf
+VENV      := .venv
+PYTHON    := $(VENV)/bin/python
+PIP       := $(VENV)/bin/pip
+PYTEST    := $(VENV)/bin/pytest
 
 .DEFAULT_GOAL := help
 
@@ -35,5 +39,14 @@ clean:
 	$(LATEXMK) -C -jobname=$(JOBNAME) -outdir=../$(BUILDDIR) || true
 	rm -rf paper/$(BUILDDIR)/*
 
+$(VENV):
+	python3 -m venv $(VENV)
+	$(PIP) install --upgrade pip
 
-.PHONY: all pdf clean watch run.webapp
+install: $(VENV)
+	$(PIP) install -r requirements.txt
+
+test: $(VENV)
+	$(PYTEST) -v
+
+.PHONY: all pdf clean watch run.webapp install test

backend/server/app.py

@@ -7,7 +7,9 @@ import uvicorn
 import os
 import json
 from datetime import datetime
-from kafka import KafkaProducer
+from kafka import KafkaProducer, KafkaAdminClient, KafkaConsumer
+from kafka.admin import NewTopic
+from kafka.errors import TopicAlreadyExistsError
 from dotenv import load_dotenv
 load_dotenv()
 
@@ -22,15 +24,24 @@ def get_producer() -> KafkaProducer:
         host = os.getenv('KAFKA_HOST', 'localhost')
         port = os.getenv('KAFKA_PORT', '9092')
         broker = f'{host}:{port}' if port else host
+        print(f"[KAFKA_INIT] Connecting to broker: {broker}")
         _producer = KafkaProducer(
             bootstrap_servers=[broker],
             value_serializer=lambda v: json.dumps(v).encode('utf-8'),
             key_serializer=lambda k: k.encode('utf-8') if k else None,
+            acks=1,
+            retries=3,
+            max_in_flight_requests_per_connection=5,
+            request_timeout_ms=30000,
+            api_version_auto_timeout_ms=10000,
+            max_block_ms=5000,  # don't block send() for more than 5s
         )
+        print(f"[KAFKA_INIT] Producer created successfully")
     return _producer
 
 class EventPayload(BaseModel):
     sessionId: str
+    experimentId: Optional[str] = None
     eventName: str
     page: str
     productId: Optional[str] = None
@@ -47,6 +58,33 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+@app.on_event("startup")
+async def startup_event():
+    """create kafka topics on startup"""
+    host = os.getenv('KAFKA_HOST', 'localhost')
+    port = os.getenv('KAFKA_PORT', '9092')
+    broker = f'{host}:{port}'
+
+    try:
+        print(f"[STARTUP] Creating Kafka topics on {broker}")
+        admin = KafkaAdminClient(
+            bootstrap_servers=[broker],
+            request_timeout_ms=10000,
+        )
+
+        topics = [
+            NewTopic(name='user-interactions', num_partitions=3, replication_factor=1)
+        ]
+
+        admin.create_topics(new_topics=topics, validate_only=False)
+        print(f"[STARTUP] Topics created successfully")
+        admin.close()
+    except TopicAlreadyExistsError:
+        print(f"[STARTUP] Topics already exist, skipping creation")
+    except Exception as e:
+        print(f"[STARTUP] Failed to create topics: {e}")
+        print(f"[STARTUP] Will rely on auto-creation on first message")
+
 @app.get("/health")
 async def health():
     kafka_status = "unknown"
@@ -72,12 +110,13 @@ async def ingest_logs(event: EventPayload):
             event.ts = datetime.utcnow().isoformat() + 'Z'
 
         producer = get_producer()
-        producer.send(
+        future = producer.send(
             'user-interactions',
             key=event.sessionId,
             value=event.model_dump()
         )
-        producer.flush(timeout=5)
+        # add callback for error logging but don't block
+        future.add_errback(lambda e: print(f"[KAFKA_SEND_ERROR] {e}"))
 
         return {"success": True}
     except Exception as e:
@@ -87,10 +126,62 @@ async def ingest_logs(event: EventPayload):
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/api/kafka/dump")
-def dump_logs():
+def dump_logs(
-    # TODO: implement a dump of logs of time period t_start to t_end (params of get)
+    last_n: Optional[int] = None,
-    # OR: allow for params of last_n logs as a param - creating two modes of the dumping
+    t_start: Optional[str] = None,
-    pass
+    t_end: Optional[str] = None
+):
+    """dump all messages from user-interactions topic
+
+    params:
+        last_n: return only last n messages (default: all)
+        t_start: filter by start timestamp iso format (future use)
+        t_end: filter by end timestamp iso format (future use)
+    """
+    host = os.getenv('KAFKA_HOST', 'localhost')
+    port = os.getenv('KAFKA_PORT', '9092')
+    broker = f'{host}:{port}'
+
+    try:
+        consumer = KafkaConsumer(
+            'user-interactions',
+            bootstrap_servers=[broker],
+            auto_offset_reset='earliest',
+            enable_auto_commit=False,
+            value_deserializer=lambda x: json.loads(x.decode('utf-8')),
+            consumer_timeout_ms=5000
+        )
+
+        events = []
+        for msg in consumer:
+            events.append(msg.value)
+
+        consumer.close()
+
+        # apply filters
+        if t_start or t_end:
+            # filter by timestamp range if provided
+            filtered = []
+            for e in events:
+                ts = e.get('ts')
+                if ts:
+                    if t_start and ts < t_start:
+                        continue
+                    if t_end and ts > t_end:
+                        continue
+                filtered.append(e)
+            events = filtered
+
+        if last_n and last_n > 0:
+            events = events[-last_n:]
+
+        return {"success": True, "count": len(events), "data": events}
+
+    except Exception as e:
+        import traceback
+        print(f"[DUMP_ERROR] {e}")
+        print(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=str(e))
 
 
 

experiments/agents/__init__.py

@@ -0,0 +1 @@
+"""Agentic behavior runner for PHANTOM research platform."""

experiments/agents/agent.py

@@ -0,0 +1,44 @@
+from .base import Agent as BaseAgent
+from browser_use import Browser, Agent, ChatOpenAI
+from enum import Enum
+
+class AgentTypes(str, Enum):
+    GENERIC_BROWSER_USE_AGENT = "generic_browser_use_agent"
+
+def _build_prompt(goal : str, environment_url : str) -> str:
+    #TODO: Improve prompt engineering here and experiment with
+    return f"""You are an autonomous agent tasked with achieving the following goal: {goal}
+You have access to a web browser to interact with the environment at {environment_url}.
+Use the browser to navigate, gather information, and perform actions necessary to accomplish your goal.
+Be thorough and ensure you complete the task fully."""
+
+class GenericBrowserUseAgent(BaseAgent):
+    def __init__(self,
+                 goal: str,
+                 url: str = "http://localhost:3000",
+                 timeout: int = 300,
+                 llm_model: str = "gpt-5-mini",
+                 headless: bool = True):
+        super().__init__(goal, url, timeout)
+        self.llm_model = ChatOpenAI(model=llm_model)
+        self.browser = Browser(headless=headless)
+        self.agent = Agent(task=_build_prompt(goal, url),
+                           llm=self.llm_model,
+                           browser=self.browser)
+    async def act(self) -> str:
+        self.result = await self.agent.run()
+        # https://github.com/browser-use/browser-use/blob/main/browser_use/agent/views.py#L301
+        return self.result.final_result()
+
+def get_agent(agent_type: AgentTypes,  **kwargs) -> Agent:
+    if agent_type == AgentTypes.GENERIC_BROWSER_USE_AGENT:
+        return GenericBrowserUseAgent(**kwargs)
+    else:
+        raise ValueError(f"Unknown agent type: {agent_type}")
+
+if __name__ == "__main__":
+    import asyncio
+    JTBD= "Name all the products on this site and try to find out more about each product by clicking into them (they might not open)"
+    agent = get_agent(AgentTypes.GENERIC_BROWSER_USE_AGENT, goal=JTBD, url="http://localhost:3000/products", timeout=300)
+    R=asyncio.run(agent.act())
+    print(R)

experiments/agents/base.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+class Agent(ABC):
+    """Base interface for browser automation agents"""
+
+    def __init__(self, goal: str, url: str = "http://localhost:3000", timeout: int = 300):
+        self.goal = goal
+        self.url = url
+        self.timeout = timeout
+        self.result: Optional[str] = None
+
+    @abstractmethod
+    async def act(self) -> str:
+        """Execute goal and return result text"""
+        pass
+
+    def final_result(self) -> Optional[str]:
+        return self.result

experiments/agents/test.py

@@ -0,0 +1,30 @@
+import pytest
+import asyncio
+from experiments.agents.agent import get_agent, AgentTypes
+import os
+
+
+def test_agent_init():
+    agent = get_agent(AgentTypes.GENERIC_BROWSER_USE_AGENT, goal="test", url="http://example.com", timeout=100)
+    assert agent.goal == "test"
+    assert agent.url == "http://example.com"
+    assert agent.timeout == 100
+
+
+def test_invalid_agent():
+    with pytest.raises(ValueError):
+        get_agent("invalid", goal="test")
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif("OPENAI_API_KEY" not in os.environ, reason="OPENAI_API_KEY not set")
+async def test_agent_execution():
+    agent = get_agent(AgentTypes.GENERIC_BROWSER_USE_AGENT, goal="get page title", url="https://example.com", timeout=60)
+
+    result = await agent.act()
+    assert result
+    assert agent.final_result()
+    assert agent.final_result().history[-1].result[-1].is_done == True
+    assert isinstance(result, str)
+    assert "example" in result.lower()
+    assert len(result) > 0

experiments/procesing/extract.py

@@ -0,0 +1,120 @@
+import pandas as pd
+import json
+import numpy as np
+import os
+import requests
+from dotenv import load_dotenv
+from sklearn.base import BaseEstimator, TransformerMixin
+from supabase import create_client, Client
+load_dotenv()
+
+BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:5000")
+SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
+SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
+N_PRICE_BUCKETS = 5
+
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+def get_data_from_kafka() -> pd.DataFrame:
+    """fetch all events from backend dump endpoint"""
+    resp = requests.get(f"{BACKEND_URL}/api/kafka/dump")
+    resp.raise_for_status()
+    data = resp.json()
+
+    if not data.get('success') or not data.get('data'):
+        return pd.DataFrame()
+
+    df = pd.DataFrame(data['data'])
+    # explode metadata col json
+    if 'metadata' in df.columns:
+        df = df.join(pd.json_normalize(df.pop("metadata"), sep=".").add_prefix("metadata_"))
+    df = df.dropna(subset=['eventName'])
+    return df
+
+
+def join_with_experiments(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty or 'experimentId' not in df.columns:
+        return df
+
+    unique_exp_ids = df['experimentId'].dropna().unique()
+    if len(unique_exp_ids) == 0:
+        return df
+
+    resp = supabase.table('experiments').select(
+        'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, task:tasks(task_name, task_description, task_def_of_done)'
+    ).in_('id', unique_exp_ids.tolist()).execute()
+
+    if not resp.data:
+        return df
+
+    exp_df = pd.DataFrame(resp.data)
+
+    # flatten task nested object if present
+    if 'task' in exp_df.columns and exp_df['task'].notnull().any():
+        task_normalized = pd.json_normalize(exp_df['task'].dropna())
+        task_normalized.index = exp_df[exp_df['task'].notnull()].index
+        exp_df = exp_df.drop(columns=['task']).join(task_normalized, rsuffix='_task')
+
+    # rename experiment columns for clarity
+    exp_df = exp_df.rename(columns={
+        'id': 'experimentId',
+        'subject_name': 'exp_subject',
+        'xp_human_only': 'exp_human_only',
+        'xp_market_mode': 'exp_market_mode',
+        'xp_task_id': 'exp_task_id'
+    })
+
+    df = df.merge(exp_df, on='experimentId', how='left')
+    return df
+
+
+def augment_event_titles(df: pd.DataFrame) -> pd.DataFrame:
+    # from taking standard view_item_page in eventName to view_item_page_{metadata_schema}
+    # we want metadata schema to create product specific event names
+
+    # only create price buckets if we have enough unique prices
+    if df["metadata_price"].notnull().sum() > 0:
+        try:
+            price_buckets = pd.qcut(
+                df["metadata_price"],
+                q=N_PRICE_BUCKETS,
+                labels=[f"PB_{i+1}" for i in range(N_PRICE_BUCKETS)],
+                duplicates='drop'  # handle duplicate bin edges
+            )
+        except ValueError:
+            # fallback: if still not enough unique values, use cut with fixed ranges or just use raw price
+            price_buckets = df["metadata_price"].apply(lambda x: f"P_{int(x)}" if pd.notnull(x) else "")
+    else:
+        price_buckets = pd.Series([""] * len(df), index=df.index)
+
+    # metadata_schema: _product_id@price_bucket_{i} only if we have product metadata otherswise keep original event name
+    # TODO: make this adaptive, if we have hover_over_title we append the title, if its view_page we say which page
+    df["metadata_schema"] = np.where(
+        df["productId"].notnull() & df["metadata_price"].notnull(),
+        "_" + df["productId"].astype(str) + "@" + price_buckets.astype(str),
+        ""
+    )
+    df["eventName"] = df["eventName"] + df["metadata_schema"].astype(str)
+    return df
+
+
+def extract() -> pd.DataFrame:
+    df = get_data_from_kafka()
+    df = join_with_experiments(df)
+    df = augment_event_titles(df)
+    return df
+
+
+class DataExtractor(BaseEstimator, TransformerMixin):
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, X=None):
+        return extract()
+
+
+if __name__ == "__main__":
+    df = extract()
+    print(df.head())
+    print(df.tail())
+    print(df.info())

experiments/procesing/mapping.py

@@ -0,0 +1,158 @@
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+def build_transition_prob_matrix(df: pd.DataFrame):
+    df = df.dropna(subset=['eventName'])
+    events = df['eventName'].tolist()
+    labels = pd.Index(events).unique().tolist()
+    idx = {e:i for i,e in enumerate(labels)}
+    M = np.zeros((len(labels), len(labels)), dtype=float)
+    for a, b in zip(events, events[1:]):
+        M[idx[a], idx[b]] += 1
+    row_sums = M.sum(axis=1, keepdims=True)
+    with np.errstate(divide='ignore', invalid='ignore'):
+        P = np.divide(M, row_sums, where=row_sums>0)  # row-normalized
+    return P, labels
+
+# https://medium.com/data-science/time-series-data-markov-transition-matrices-7060771e362b
+from graphviz import Digraph
+import numpy as np
+import pandas as pd
+
+def _as_prob_df(matrix, labels=None):
+    """Return a square DataFrame with index=columns=labels."""
+    if isinstance(matrix, pd.DataFrame):
+        # Ensure square and aligned
+        assert (matrix.index == matrix.columns).all(), "Index/columns must match."
+        return matrix
+    matrix = np.asarray(matrix, dtype=float)
+    assert matrix.shape[0] == matrix.shape[1], "Matrix must be square."
+    if labels is None:
+        raise ValueError("labels are required when matrix is not a DataFrame")
+    assert len(labels) == matrix.shape[0], "labels length must match matrix size."
+    return pd.DataFrame(matrix, index=list(labels), columns=list(labels))
+
+def _df_to_edgelist(P: pd.DataFrame, threshold=0.0, round_digits=2):
+    """Build weighted edges > threshold."""
+    edges = []
+    for src in P.index:
+        for dst in P.columns:
+            w = float(P.loc[src, dst])
+            if w > threshold:
+                edges.append((str(src), str(dst), f"{w:.{round_digits}f}"))
+    return edges
+
+def render_graph(fname, matrix, ls_index=None, threshold=0.0, fmt="svg", view=False):
+    """
+    fname: output file stem (no extension)
+    matrix: NumPy array or pandas DataFrame of transition PROBABILITIES
+    ls_index: ordered labels (required if matrix is not a DataFrame)
+    threshold: hide edges with weight <= threshold
+    fmt: 'svg'|'png'|'pdf' etc.
+    view: open after rendering
+    """
+    P = _as_prob_df(matrix, labels=ls_index)
+    edges = _df_to_edgelist(P, threshold=threshold)
+
+    g = Digraph(format=fmt)
+    g.attr(rankdir="LR", size="30")
+    g.attr("node", shape="circle")
+
+    # ensure isolated nodes appear
+    for node in P.index:
+        g.node(str(node), width="1", height="1")
+
+    for src, dst, label in edges:
+        g.edge(src, dst, label=label)
+
+    g.render(fname, view=view, cleanup=True)
+    return g
+
+
+class TransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0):
+        self.threshold = threshold
+        self.P_ = None
+        self.labels_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        P, labels = build_transition_prob_matrix(X)
+        self.P_ = P
+        self.labels_ = labels
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        return self.P_, self.labels_
+
+    def render(self, fname: str, fmt="svg", view=False):
+        if self.P_ is None or self.labels_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return render_graph(
+            fname,
+            self.P_,
+            ls_index=self.labels_,
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+
+
+class SessionTransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0, session_col='sessionId'):
+        self.threshold = threshold
+        self.session_col = session_col
+        self.session_matrices_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        if self.session_col not in X.columns:
+            raise ValueError(f"Column '{self.session_col}' not found in DataFrame")
+
+        session_matrices = {}
+        for session_id, grp in X.groupby(self.session_col):
+            if len(grp) > 1:  # need at least 2 events for transitions
+                P, labels = build_transition_prob_matrix(grp)
+                session_matrices[session_id] = {'matrix': P, 'labels': labels}
+
+        self.session_matrices_ = session_matrices
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return pd.Series(self.session_matrices_)
+
+    def render_session(self, session_id: str, fname: str, fmt="svg", view=False):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        if session_id not in self.session_matrices_:
+            raise ValueError(f"Session '{session_id}' not found in fitted data.")
+
+        sess_data = self.session_matrices_[session_id]
+        return render_graph(
+            fname,
+            sess_data['matrix'],
+            ls_index=sess_data['labels'],
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+if __name__ == "__main__":
+    # Example usage
+    data = {
+        'eventName': [
+            'A', 'B', 'A', 'C', 'B', 'A', 'A', 'C', 'B', 'C',
+            'A', 'B', 'C', 'A', 'B', 'C', 'A', 'B', 'C', 'A'
+        ]
+    }
+    df = pd.DataFrame(data)
+
+    transformer = TransitionProbMatrixTransformer(threshold=0.1)
+    transformer.fit(df)
+    P, labels = transformer.transform(None)
+
+    print("Transition Probability Matrix:")
+    print(pd.DataFrame(P, index=labels, columns=labels))
+
+    # Render the graph
+    transformer.render("transition_graph", fmt="svg", view=False)

experiments/procesing/pipeline.py

@@ -0,0 +1,15 @@
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from extract import DataExtractor
+from mapping import SessionTransitionProbMatrixTransformer, render_graph
+
+
+if __name__ == "__main__":
+    steps = [
+        ('data_extraction', DataExtractor()),
+        #('transition_matrix', SessionTransitionProbMatrixTransformer(threshold=0.05)),
+    ]
+    pipeline = Pipeline(steps)
+    result = pipeline.fit_transform(None)
+    print(result)
+    print(result.info())

paper/concat_code.sh

@@ -16,10 +16,11 @@ mkdir -p "$(dirname "$OUTPUT_FILE")"
 add_file() {
     local filepath="$1"
     local relpath="${filepath#$PROJECT_ROOT/}"
+    local escaped_path="${relpath//_/\\_}"
 
     # Add section header and code listing (no language-specific highlighting)
-    echo "\\subsection{${relpath}}" >> "$OUTPUT_FILE"
+    echo "\\subsection{${escaped_path}}" >> "$OUTPUT_FILE"
-    echo "\\begin{lstlisting}[caption={${relpath}}]" >> "$OUTPUT_FILE"
+    echo "\\begin{lstlisting}[caption={${escaped_path}}]" >> "$OUTPUT_FILE"
     cat "$filepath" >> "$OUTPUT_FILE"
     echo "" >> "$OUTPUT_FILE"
     echo "\\end{lstlisting}" >> "$OUTPUT_FILE"

pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+testpaths = experiments
+python_files = test*.py
+python_classes = Test*
+python_functions = test_*
+asyncio_mode = auto
+asyncio_default_fixture_loop_scope = function

requirements.txt

@@ -5,3 +5,9 @@ jupyter
 ipykernel
 matplotlib
 graphviz
+browser-use
+pytest
+pytest-asyncio
+uv
+scikit-learn
+supabase

web/package-lock.json

@@ -8,6 +8,8 @@
       "name": "web",
       "version": "0.1.0",
       "dependencies": {
+        "@supabase/ssr": "^0.7.0",
+        "@supabase/supabase-js": "^2.81.1",
         "next": "16.0.0",
         "react": "19.2.0",
         "react-dom": "19.2.0",
@@ -657,6 +659,97 @@
         "node": ">= 10"
       }
     },
+    "node_modules/@supabase/auth-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.81.1.tgz",
+      "integrity": "sha512-K20GgiSm9XeRLypxYHa5UCnybWc2K0ok0HLbqCej/wRxDpJxToXNOwKt0l7nO8xI1CyQ+GrNfU6bcRzvdbeopQ==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "2.8.1"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
+    "node_modules/@supabase/functions-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.81.1.tgz",
+      "integrity": "sha512-sYgSO3mlgL0NvBFS3oRfCK4OgKGQwuOWJLzfPyWg0k8MSxSFSDeN/JtrDJD5GQrxskP6c58+vUzruBJQY78AqQ==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "2.8.1"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
+    "node_modules/@supabase/postgrest-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.81.1.tgz",
+      "integrity": "sha512-DePpUTAPXJyBurQ4IH2e42DWoA+/Qmr5mbgY4B6ZcxVc/ZUKfTVK31BYIFBATMApWraFc8Q/Sg+yxtfJ3E0wSg==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "2.8.1"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
+    "node_modules/@supabase/realtime-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.81.1.tgz",
+      "integrity": "sha512-ViQ+Kxm8BuUP/TcYmH9tViqYKGSD1LBjdqx2p5J+47RES6c+0QHedM0PPAjthMdAHWyb2LGATE9PD2++2rO/tw==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/phoenix": "^1.6.6",
+        "@types/ws": "^8.18.1",
+        "tslib": "2.8.1",
+        "ws": "^8.18.2"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
+    "node_modules/@supabase/ssr": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/@supabase/ssr/-/ssr-0.7.0.tgz",
+      "integrity": "sha512-G65t5EhLSJ5c8hTCcXifSL9Q/ZRXvqgXeNo+d3P56f4U1IxwTqjB64UfmfixvmMcjuxnq2yGqEWVJqUcO+AzAg==",
+      "license": "MIT",
+      "dependencies": {
+        "cookie": "^1.0.2"
+      },
+      "peerDependencies": {
+        "@supabase/supabase-js": "^2.43.4"
+      }
+    },
+    "node_modules/@supabase/storage-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.81.1.tgz",
+      "integrity": "sha512-UNmYtjnZnhouqnbEMC1D5YJot7y0rIaZx7FG2Fv8S3hhNjcGVvO+h9We/tggi273BFkiahQPS/uRsapo1cSapw==",
+      "license": "MIT",
+      "dependencies": {
+        "tslib": "2.8.1"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
+    "node_modules/@supabase/supabase-js": {
+      "version": "2.81.1",
+      "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.81.1.tgz",
+      "integrity": "sha512-KSdY7xb2L0DlLmlYzIOghdw/na4gsMcqJ8u4sD6tOQJr+x3hLujU9s4R8N3ob84/1bkvpvlU5PYKa1ae+OICnw==",
+      "license": "MIT",
+      "dependencies": {
+        "@supabase/auth-js": "2.81.1",
+        "@supabase/functions-js": "2.81.1",
+        "@supabase/postgrest-js": "2.81.1",
+        "@supabase/realtime-js": "2.81.1",
+        "@supabase/storage-js": "2.81.1"
+      },
+      "engines": {
+        "node": ">=20.0.0"
+      }
+    },
     "node_modules/@swc/helpers": {
       "version": "0.5.15",
       "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz",
@@ -941,12 +1034,17 @@
       "version": "20.19.23",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.23.tgz",
       "integrity": "sha512-yIdlVVVHXpmqRhtyovZAcSy0MiPcYWGkoO4CGe/+jpP0hmNuihm4XhHbADpK++MsiLHP5MVlv+bcgdF99kSiFQ==",
-      "dev": true,
       "license": "MIT",
       "dependencies": {
         "undici-types": "~6.21.0"
       }
     },
+    "node_modules/@types/phoenix": {
+      "version": "1.6.6",
+      "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.6.tgz",
+      "integrity": "sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==",
+      "license": "MIT"
+    },
     "node_modules/@types/react": {
       "version": "19.2.2",
       "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.2.tgz",
@@ -967,6 +1065,15 @@
         "@types/react": "^19.2.0"
       }
     },
+    "node_modules/@types/ws": {
+      "version": "8.18.1",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
+      "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/caniuse-lite": {
       "version": "1.0.30001751",
       "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001751.tgz",
@@ -993,6 +1100,15 @@
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==",
       "license": "MIT"
     },
+    "node_modules/cookie": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz",
+      "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/csstype": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
@@ -1605,9 +1721,29 @@
       "version": "6.21.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
       "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
-      "dev": true,
       "license": "MIT"
     },
+    "node_modules/ws": {
+      "version": "8.18.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+      "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/zod": {
       "version": "4.1.12",
       "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz",

web/package.json

@@ -8,6 +8,8 @@
     "start": "next start"
   },
   "dependencies": {
+    "@supabase/ssr": "^0.7.0",
+    "@supabase/supabase-js": "^2.81.1",
     "next": "16.0.0",
     "react": "19.2.0",
     "react-dom": "19.2.0",

web/src/app/admin/experiments/page.tsx

@@ -1,20 +1,26 @@
 'use client';
 
 import { useEffect, useState } from 'react';
-import { useSession } from '@/hooks/useSession';
+import { TaskManager } from '@/components/admin/TaskManager';
+import { ExperimentForm } from '@/components/admin/ExperimentForm';
 
 type Experiment = {
   id: string;
-  status: 'active' | 'stopped';
+  subject_name: string;
-  sessionIds: string[];
+  xp_human_only: boolean;
-  createdAt: number;
+  xp_market_mode: string;
+  created_at: string;
+  task?: {
+    id: string;
+    task_name: string;
+  };
 };
 
 export default function ExperimentsAdmin() {
-  const { sessionId, isLoading: sessionLoading } = useSession();
   const [exps, setExps] = useState<Experiment[]>([]);
-  const [loading, setLoading] = useState(false);
+  const [selectedTaskId, setSelectedTaskId] = useState<string | undefined>();
   const [error, setError] = useState<string | null>(null);
+  const [showForm, setShowForm] = useState(false);
 
   const fetchExps = async () => {
     try {
@@ -31,86 +37,22 @@ export default function ExperimentsAdmin() {
     fetchExps();
   }, []);
 
-  const handleStart = async () => {
+  const handleExperimentCreated = async () => {
-    if (!sessionId) {
+    setShowForm(false);
-      setError('no session available');
+    setSelectedTaskId(undefined);
-      return;
+    await fetchExps();
-    }
-
-    setLoading(true);
-    setError(null);
-
-    try {
-      const res = await fetch('/api/admin/experiments/start', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ sessionId }),
-      });
-
-      if (!res.ok) {
-        const data = await res.json();
-        throw new Error(data.error || 'start failed');
-      }
-
-      await fetchExps(); // refresh list
-    } catch (err: any) {
-      setError(err.message);
-    } finally {
-      setLoading(false);
-    }
   };
 
-  const handleStop = async (expId: string) => {
-    setLoading(true);
-    setError(null);
-
-    try {
-      const res = await fetch('/api/admin/experiments/stop', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ experimentId: expId }),
-      });
-
-      if (!res.ok) {
-        const data = await res.json();
-        throw new Error(data.error || 'stop failed');
-      }
-
-      await fetchExps(); // refresh list
-    } catch (err: any) {
-      setError(err.message);
-    } finally {
-      setLoading(false);
-    }
-  };
-
-  if (sessionLoading) {
-    return (
-      <div className="flex min-h-screen items-center justify-center bg-zinc-50 dark:bg-black">
-        <p className="text-zinc-600 dark:text-zinc-400">loading session...</p>
-      </div>
-    );
-  }
-
   return (
     <div className="min-h-screen bg-zinc-50 px-6 py-12 dark:bg-black">
-      <div className="mx-auto max-w-5xl">
+      <div className="mx-auto max-w-7xl">
-        <div className="mb-8 flex items-center justify-between">
+        <div className="mb-8">
-          <div>
+          <h1 className="text-3xl font-semibold tracking-tight text-black dark:text-zinc-50">
-            <h1 className="text-3xl font-semibold tracking-tight text-black dark:text-zinc-50">
+            Experiment Management
-              Experiments
+          </h1>
-            </h1>
+          <p className="mt-2 text-sm text-zinc-600 dark:text-zinc-400">
-            <p className="mt-2 text-sm text-zinc-600 dark:text-zinc-400">
+            configure tasks and run experiments
-              current session: {sessionId || 'none'}
+          </p>
-            </p>
-          </div>
-          <button
-            onClick={handleStart}
-            disabled={loading || !sessionId}
-            className="rounded-lg bg-black px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-zinc-800 disabled:opacity-50 dark:bg-zinc-50 dark:text-black dark:hover:bg-zinc-200"
-          >
-            {loading ? 'starting...' : 'start experiment'}
-          </button>
         </div>
 
         {error && (
@@ -119,79 +61,123 @@ export default function ExperimentsAdmin() {
           </div>
         )}
 
-        <div className="overflow-hidden rounded-lg border border-zinc-200 bg-white dark:border-zinc-800 dark:bg-zinc-950">
+        <div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
-          <table className="w-full text-left text-sm">
+          {/* left column: task manager */}
-            <thead className="border-b border-zinc-200 bg-zinc-50 dark:border-zinc-800 dark:bg-zinc-900">
+          <div className="lg:col-span-1">
-              <tr>
+            <TaskManager
-                <th className="px-6 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+              onTaskSelect={setSelectedTaskId}
-                  experiment id
+              selectedTaskId={selectedTaskId}
-                </th>
+            />
-                <th className="px-6 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+          </div>
-                  status
+
-                </th>
+          {/* right column: experiment form + list */}
-                <th className="px-6 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+          <div className="space-y-6 lg:col-span-2">
-                  session count
+            <div className="flex items-center justify-between">
-                </th>
+              <h2 className="text-lg font-semibold text-zinc-900 dark:text-zinc-100">
-                <th className="px-6 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+                Experiments
-                  created
+              </h2>
-                </th>
+              <button
-                <th className="px-6 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+                onClick={() => setShowForm(!showForm)}
-                  action
+                className="rounded-lg bg-black px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-zinc-800 dark:bg-zinc-50 dark:text-black dark:hover:bg-zinc-200"
-                </th>
+              >
-              </tr>
+                {showForm ? 'hide form' : 'new experiment'}
-            </thead>
+              </button>
-            <tbody className="divide-y divide-zinc-200 dark:divide-zinc-800">
+            </div>
-              {exps.length === 0 ? (
+
-                <tr>
+            {showForm && (
-                  <td
+              <ExperimentForm
-                    colSpan={5}
+                selectedTaskId={selectedTaskId}
-                    className="px-6 py-8 text-center text-zinc-500 dark:text-zinc-400"
+                onSuccess={handleExperimentCreated}
-                  >
+              />
-                    no experiments yet
+            )}
-                  </td>
+
-                </tr>
+            <div className="overflow-hidden rounded-lg border border-zinc-200 bg-white dark:border-zinc-800 dark:bg-zinc-950">
-              ) : (
+              <table className="w-full text-left text-sm">
-                exps.map((exp) => (
+                <thead className="border-b border-zinc-200 bg-zinc-50 dark:border-zinc-800 dark:bg-zinc-900">
-                  <tr
+                  <tr>
-                    key={exp.id}
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                    className="hover:bg-zinc-50 dark:hover:bg-zinc-900"
+                      subject
-                  >
+                    </th>
-                    <td className="px-6 py-4 font-mono text-xs text-zinc-700 dark:text-zinc-300">
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                      {exp.id.slice(0, 8)}...
+                      mode
-                    </td>
+                    </th>
-                    <td className="px-6 py-4">
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                      <span
+                      human
-                        className={`inline-block rounded-full px-2 py-1 text-xs font-medium ${
+                    </th>
-                          exp.status === 'active'
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                            ? 'bg-green-100 text-green-800 dark:bg-green-950 dark:text-green-200'
+                      task
-                            : 'bg-zinc-100 text-zinc-800 dark:bg-zinc-800 dark:text-zinc-200'
+                    </th>
-                        }`}
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                      >
+                      created
-                        {exp.status}
+                    </th>
-                      </span>
+                    <th className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
-                    </td>
+                      link
-                    <td className="px-6 py-4 text-zinc-700 dark:text-zinc-300">
+                    </th>
-                      {exp.sessionIds.length}
-                    </td>
-                    <td className="px-6 py-4 text-zinc-700 dark:text-zinc-300">
-                      {new Date(exp.createdAt).toLocaleString()}
-                    </td>
-                    <td className="px-6 py-4">
-                      {exp.status === 'active' && (
-                        <button
-                          onClick={() => handleStop(exp.id)}
-                          disabled={loading}
-                          className="text-sm font-medium text-red-600 hover:text-red-700 disabled:opacity-50 dark:text-red-400 dark:hover:text-red-300"
-                        >
-                          stop
-                        </button>
-                      )}
-                    </td>
                   </tr>
-                ))
+                </thead>
-              )}
+                <tbody className="divide-y divide-zinc-200 dark:divide-zinc-800">
-            </tbody>
+                  {exps.length === 0 ? (
-          </table>
+                    <tr>
+                      <td
+                        colSpan={6}
+                        className="px-4 py-8 text-center text-zinc-500 dark:text-zinc-400"
+                      >
+                        no experiments yet
+                      </td>
+                    </tr>
+                  ) : (
+                    exps.map((exp) => {
+                      const baseUrl = exp.xp_market_mode === 'airline'
+                        ? 'https://phantom-airline.vercel.app'
+                        : 'https://phantom-hotel.vercel.app';
+                      const link = `${baseUrl}/start-task?uuid=${exp.id}`;
+
+                      return (
+                        <tr
+                          key={exp.id}
+                          className="hover:bg-zinc-50 dark:hover:bg-zinc-900"
+                        >
+                          <td className="px-4 py-3 font-medium text-zinc-900 dark:text-zinc-100">
+                            {exp.subject_name}
+                          </td>
+                          <td className="px-4 py-3">
+                            <span className="inline-block rounded-full bg-zinc-100 px-2 py-1 text-xs font-medium text-zinc-800 dark:bg-zinc-800 dark:text-zinc-200">
+                              {exp.xp_market_mode || 'none'}
+                            </span>
+                          </td>
+                          <td className="px-4 py-3">
+                            {exp.xp_human_only ? (
+                              <span className="text-xs text-green-600 dark:text-green-400">
+                                yes
+                              </span>
+                            ) : (
+                              <span className="text-xs text-zinc-500">no</span>
+                            )}
+                          </td>
+                          <td className="px-4 py-3 text-xs text-zinc-600 dark:text-zinc-400">
+                            {exp.task ? exp.task.task_name : '—'}
+                          </td>
+                          <td className="px-4 py-3 text-xs text-zinc-600 dark:text-zinc-400">
+                            {new Date(exp.created_at).toLocaleDateString()}
+                          </td>
+                          <td className="px-4 py-3">
+                            <button
+                              onClick={() => {
+                                navigator.clipboard.writeText(link);
+                              }}
+                              className="text-xs font-medium text-zinc-900 hover:text-zinc-600 dark:text-zinc-100 dark:hover:text-zinc-400"
+                            >
+                              copy link
+                            </button>
+                          </td>
+                        </tr>
+                      );
+                    })
+                  )}
+                </tbody>
+              </table>
+            </div>
+          </div>
         </div>
       </div>
     </div>

web/src/app/api/admin/experiments/route.ts

@@ -1,10 +1,40 @@
-import { NextResponse } from 'next/server';
+import { NextRequest, NextResponse } from 'next/server';
-import { getAllExperiments } from '@/lib/sessionStore';
+import { createClient } from '@/utils/supabase/server';
+import { cookies } from 'next/headers';
 
-export async function GET() {
+export async function GET(req: NextRequest) {
   try {
-    const exps = getAllExperiments();
+    const cookieStore = await cookies();
-    return NextResponse.json({ experiments: exps });
+    const supabase = createClient(cookieStore);
+
+    const { searchParams } = new URL(req.url);
+    const id = searchParams.get('id');
+
+    if (id) {
+      const { data, error } = await supabase
+        .from('experiments')
+        .select(`
+          *,
+          task:tasks(*)
+        `)
+        .eq('id', id)
+        .single();
+
+      if (error) throw error;
+      return NextResponse.json({ experiment: data });
+    }
+
+    const { data, error } = await supabase
+      .from('experiments')
+      .select(`
+        *,
+        task:tasks(*)
+      `)
+      .order('created_at', { ascending: false });
+
+    if (error) throw error;
+
+    return NextResponse.json({ experiments: data || [] });
   } catch (err: any) {
     console.error('experiments list error:', err);
     return NextResponse.json(
@@ -13,3 +43,44 @@ export async function GET() {
     );
   }
 }
+
+export async function POST(req: NextRequest) {
+  try {
+    const cookieStore = await cookies();
+    const supabase = createClient(cookieStore);
+    const body = await req.json();
+
+    const { subject_name, xp_human_only, xp_market_mode, xp_task_id } = body;
+
+    if (!subject_name) {
+      return NextResponse.json(
+        { error: 'subject_name is required' },
+        { status: 400 }
+      );
+    }
+
+    const { data, error } = await supabase
+      .from('experiments')
+      .insert([{
+        subject_name,
+        xp_human_only: xp_human_only ?? false,
+        xp_market_mode: xp_market_mode || null,
+        xp_task_id: xp_task_id || null,
+      }])
+      .select(`
+        *,
+        task:tasks(*)
+      `)
+      .single();
+
+    if (error) throw error;
+
+    return NextResponse.json({ experiment: data });
+  } catch (err: any) {
+    console.error('experiment creation error:', err);
+    return NextResponse.json(
+      { error: err.message || 'unknown error' },
+      { status: 500 }
+    );
+  }
+}

web/src/app/api/admin/tasks/route.ts

@@ -0,0 +1,58 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createClient } from '@/utils/supabase/server';
+import { cookies } from 'next/headers';
+
+export async function GET() {
+  try {
+    const cookieStore = await cookies();
+    const supabase = createClient(cookieStore);
+
+    const { data, error } = await supabase
+      .from('tasks')
+      .select('*')
+      .order('created_at', { ascending: false });
+
+    if (error) throw error;
+
+    return NextResponse.json({ tasks: data || [] });
+  } catch (err: any) {
+    console.error('tasks fetch error:', err);
+    return NextResponse.json(
+      { error: err.message || 'unknown error' },
+      { status: 500 }
+    );
+  }
+}
+
+export async function POST(req: NextRequest) {
+  try {
+    const cookieStore = await cookies();
+    const supabase = createClient(cookieStore);
+    const body = await req.json();
+
+    const { task_name, task_description, task_def_of_done } = body;
+
+    if (!task_name) {
+      return NextResponse.json(
+        { error: 'task_name is required' },
+        { status: 400 }
+      );
+    }
+
+    const { data, error } = await supabase
+      .from('tasks')
+      .insert([{ task_name, task_description, task_def_of_done }])
+      .select()
+      .single();
+
+    if (error) throw error;
+
+    return NextResponse.json({ task: data });
+  } catch (err: any) {
+    console.error('task creation error:', err);
+    return NextResponse.json(
+      { error: err.message || 'unknown error' },
+      { status: 500 }
+    );
+  }
+}

web/src/app/api/session/route.ts

@@ -1,13 +1,12 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { randomUUID } from 'crypto';
-import { getSession, createSession } from '@/lib/sessionStore';
+import { getSession, createSession, setExperiment } from '@/lib/sessionStore';
 
 const COOKIE_NAME = 'phantom_session_id';
 const isProd = process.env.NODE_ENV === 'production';
 
 export async function GET(req: NextRequest) {
     try {
-        // check for existing session cookie
         const existingSession = req.cookies.get(COOKIE_NAME)?.value;
 
         if (existingSession) {
@@ -18,13 +17,11 @@ export async function GET(req: NextRequest) {
             });
         }
 
-        // mint new session id
         const sessionId = randomUUID();
         createSession(sessionId);
 
         const res = NextResponse.json({ sessionId, experimentId: undefined });
 
-        // set httpOnly cookie with security flags
         res.cookies.set({
             name: COOKIE_NAME,
             value: sessionId,
@@ -32,7 +29,7 @@ export async function GET(req: NextRequest) {
             sameSite: 'lax',
             secure: isProd,
             path: '/',
-            maxAge: 60 * 60 * 24 * 30, // 30 days
+            maxAge: 60 * 60 * 24 * 30,
         });
 
         return res;
@@ -44,3 +41,52 @@ export async function GET(req: NextRequest) {
         );
     }
 }
+
+export async function POST(req: NextRequest) {
+    try {
+        const body = await req.json();
+        const { experimentId } = body;
+
+        if (!experimentId) {
+            return NextResponse.json(
+                { error: 'experimentId is required' },
+                { status: 400 }
+            );
+        }
+
+        let sessionId = req.cookies.get(COOKIE_NAME)?.value;
+
+        if (!sessionId) {
+            sessionId = randomUUID();
+            createSession(sessionId);
+        }
+
+        setExperiment(sessionId, experimentId);
+
+        const res = NextResponse.json({
+            sessionId,
+            experimentId,
+            success: true
+        });
+
+        if (!req.cookies.get(COOKIE_NAME)) {
+            res.cookies.set({
+                name: COOKIE_NAME,
+                value: sessionId,
+                httpOnly: true,
+                sameSite: 'lax',
+                secure: isProd,
+                path: '/',
+                maxAge: 60 * 60 * 24 * 30,
+            });
+        }
+
+        return res;
+    } catch (err: any) {
+        console.error('session update error:', err);
+        return NextResponse.json(
+            { error: err.message || 'unknown error' },
+            { status: 500 }
+        );
+    }
+}

web/src/app/start-task/page.tsx

@@ -0,0 +1,93 @@
+'use client';
+
+import { useEffect, useState, Suspense } from 'react';
+import { useSearchParams, useRouter } from 'next/navigation';
+
+const StartTaskContent = () => {
+  const searchParams = useSearchParams();
+  const router = useRouter();
+  const [status, setStatus] = useState<'loading' | 'error' | 'redirecting'>('loading');
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    const uuid = searchParams.get('uuid');
+
+    if (!uuid) {
+      setError('no experiment UUID provided');
+      setStatus('error');
+      return;
+    }
+
+    const validateAndStore = async () => {
+      try {
+        const res = await fetch(`/api/admin/experiments?id=${uuid}`);
+        if (!res.ok) throw new Error('experiment not found');
+
+        const data = await res.json();
+        const exp = data.experiment;
+
+        if (!exp) throw new Error('invalid experiment UUID');
+
+        localStorage.setItem('phantom_experiment_id', uuid);
+
+        await fetch('/api/session', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ experimentId: uuid }),
+        });
+
+        setStatus('redirecting');
+
+        setTimeout(() => {
+          router.push("/");
+        }, 800);
+
+      } catch (err: any) {
+        setError(err.message || 'failed to start task');
+        setStatus('error');
+      }
+    };
+
+    validateAndStore();
+  }, [searchParams, router]);
+
+  return (
+    <div className="flex min-h-screen items-center justify-center bg-zinc-50 dark:bg-black">
+      <div className="text-center">
+        {status === 'loading' && (
+          <div>
+            <div className="mb-4 h-8 w-8 animate-spin rounded-full border-4 border-zinc-200 border-t-zinc-900 dark:border-zinc-800 dark:border-t-zinc-100 mx-auto" />
+            <p className="text-zinc-600 dark:text-zinc-400">validating browser...</p>
+          </div>
+        )}
+
+        {status === 'redirecting' && (
+          <div>
+            <div className="mb-4 text-4xl">✓</div>
+            <p className="text-zinc-900 dark:text-zinc-100 font-medium">website loaded</p>
+            <p className="mt-2 text-sm text-zinc-600 dark:text-zinc-400">redirecting to page...</p>
+          </div>
+        )}
+
+        {status === 'error' && (
+          <div className="rounded-lg bg-red-50 p-6 dark:bg-red-950">
+            <p className="text-red-900 dark:text-red-100 font-medium">error</p>
+            <p className="mt-2 text-sm text-red-700 dark:text-red-300">{error}</p>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+};
+
+export default function StartTaskPage() {
+  return (
+    <Suspense fallback={
+      <div className="flex min-h-screen items-center justify-center bg-zinc-50 dark:bg-black">
+        <p className="text-zinc-600 dark:text-zinc-400">loading...</p>
+      </div>
+    }>
+      <StartTaskContent />
+    </Suspense>
+  );
+}

web/src/components/admin/ExperimentForm.tsx

@@ -0,0 +1,118 @@
+'use client';
+
+import { useState } from 'react';
+
+type ExperimentFormProps = {
+  selectedTaskId?: string;
+  onSuccess?: () => void;
+};
+
+export const ExperimentForm = ({ selectedTaskId, onSuccess }: ExperimentFormProps) => {
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [form, setForm] = useState({
+    subject_name: '',
+    xp_human_only: false,
+    xp_market_mode: 'hotel' as 'hotel' | 'airline',
+  });
+
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    setLoading(true);
+    setError(null);
+
+    try {
+      const res = await fetch('/api/admin/experiments', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          ...form,
+          xp_task_id: selectedTaskId || null,
+        }),
+      });
+
+      if (!res.ok) {
+        const data = await res.json();
+        throw new Error(data.error || 'creation failed');
+      }
+
+      setForm({ subject_name: '', xp_human_only: false, xp_market_mode: 'hotel' });
+      onSuccess?.();
+    } catch (err: any) {
+      setError(err.message);
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  return (
+    <form onSubmit={handleSubmit} className="space-y-4 rounded-lg border border-zinc-200 bg-white p-6 dark:border-zinc-800 dark:bg-zinc-950">
+      <h2 className="text-lg font-semibold text-zinc-900 dark:text-zinc-100">
+        Create Experiment
+      </h2>
+
+      {error && (
+        <div className="rounded-lg bg-red-50 p-3 text-sm text-red-800 dark:bg-red-950 dark:text-red-200">
+          {error}
+        </div>
+      )}
+
+      <div>
+        <label className="block text-sm font-medium text-zinc-700 dark:text-zinc-300">
+          subject name
+        </label>
+        <input
+          type="text"
+          value={form.subject_name}
+          onChange={(e) => setForm({ ...form, subject_name: e.target.value })}
+          className="mt-1 w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 focus:border-zinc-900 focus:outline-none dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-100"
+          placeholder="e.g., baseline_dynamic_pricing_v1"
+          required
+        />
+      </div>
+
+      <div>
+        <label className="block text-sm font-medium text-zinc-700 dark:text-zinc-300">
+          market mode
+        </label>
+        <select
+          value={form.xp_market_mode}
+          onChange={(e) => setForm({ ...form, xp_market_mode: e.target.value as 'hotel' | 'airline' })}
+          className="mt-1 w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 focus:border-zinc-900 focus:outline-none dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-100"
+        >
+          <option value="hotel">hotel</option>
+          <option value="airline">airline</option>
+        </select>
+      </div>
+
+      <div className="flex items-center gap-2">
+        <input
+          type="checkbox"
+          id="human-only"
+          checked={form.xp_human_only}
+          onChange={(e) => setForm({ ...form, xp_human_only: e.target.checked })}
+          className="h-4 w-4 rounded border-zinc-300 text-zinc-900 focus:ring-zinc-900 dark:border-zinc-700 dark:bg-zinc-900"
+        />
+        <label htmlFor="human-only" className="text-sm text-zinc-700 dark:text-zinc-300">
+          human participants only
+        </label>
+      </div>
+
+      {selectedTaskId && (
+        <div className="rounded-lg bg-zinc-50 p-3 dark:bg-zinc-900">
+          <p className="text-sm text-zinc-600 dark:text-zinc-400">
+            task selected: <span className="font-mono text-xs">{selectedTaskId.slice(0, 8)}...</span>
+          </p>
+        </div>
+      )}
+
+      <button
+        type="submit"
+        disabled={loading}
+        className="w-full rounded-lg bg-black px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-zinc-800 disabled:opacity-50 dark:bg-zinc-50 dark:text-black dark:hover:bg-zinc-200"
+      >
+        {loading ? 'creating experiment...' : 'create experiment'}
+      </button>
+    </form>
+  );
+};

web/src/components/admin/TaskManager.tsx

@@ -0,0 +1,178 @@
+'use client';
+
+import { useState, useEffect } from 'react';
+
+type Task = {
+  id: string;
+  task_name: string;
+  task_description: string;
+  task_def_of_done: string;
+  created_at: string;
+};
+
+type TaskManagerProps = {
+  onTaskSelect?: (taskId: string) => void;
+  selectedTaskId?: string;
+};
+
+export const TaskManager = ({ onTaskSelect, selectedTaskId }: TaskManagerProps) => {
+  const [tasks, setTasks] = useState<Task[]>([]);
+  const [loading, setLoading] = useState(false);
+  const [showForm, setShowForm] = useState(false);
+  const [form, setForm] = useState({
+    task_name: '',
+    task_description: '',
+    task_def_of_done: '',
+  });
+  const [error, setError] = useState<string | null>(null);
+
+  const fetchTasks = async () => {
+    try {
+      const res = await fetch('/api/admin/tasks');
+      if (!res.ok) throw new Error(`fetch failed: ${res.status}`);
+      const data = await res.json();
+      setTasks(data.tasks || []);
+    } catch (err: any) {
+      setError(err.message);
+    }
+  };
+
+  useEffect(() => {
+    fetchTasks();
+  }, []);
+
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    setLoading(true);
+    setError(null);
+
+    try {
+      const res = await fetch('/api/admin/tasks', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(form),
+      });
+
+      if (!res.ok) {
+        const data = await res.json();
+        throw new Error(data.error || 'creation failed');
+      }
+
+      setForm({ task_name: '', task_description: '', task_def_of_done: '' });
+      setShowForm(false);
+      await fetchTasks();
+    } catch (err: any) {
+      setError(err.message);
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="space-y-4">
+      <div className="flex items-center justify-between">
+        <h2 className="text-lg font-semibold text-zinc-900 dark:text-zinc-100">
+          Tasks
+        </h2>
+        <button
+          onClick={() => setShowForm(!showForm)}
+          className="rounded-lg bg-zinc-900 px-3 py-1.5 text-sm font-medium text-white transition-colors hover:bg-zinc-700 dark:bg-zinc-100 dark:text-black dark:hover:bg-zinc-300"
+        >
+          {showForm ? 'cancel' : 'new task'}
+        </button>
+      </div>
+
+      {error && (
+        <div className="rounded-lg bg-red-50 p-3 text-sm text-red-800 dark:bg-red-950 dark:text-red-200">
+          {error}
+        </div>
+      )}
+
+      {showForm && (
+        <form onSubmit={handleSubmit} className="space-y-3 rounded-lg border border-zinc-200 bg-white p-4 dark:border-zinc-800 dark:bg-zinc-950">
+          <div>
+            <label className="block text-sm font-medium text-zinc-700 dark:text-zinc-300">
+              task name
+            </label>
+            <input
+              type="text"
+              value={form.task_name}
+              onChange={(e) => setForm({ ...form, task_name: e.target.value })}
+              className="mt-1 w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 focus:border-zinc-900 focus:outline-none dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-100"
+              placeholder="e.g., Book cheapest flight to Paris"
+              required
+            />
+          </div>
+
+          <div>
+            <label className="block text-sm font-medium text-zinc-700 dark:text-zinc-300">
+              description
+            </label>
+            <textarea
+              value={form.task_description}
+              onChange={(e) => setForm({ ...form, task_description: e.target.value })}
+              className="mt-1 w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 focus:border-zinc-900 focus:outline-none dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-100"
+              placeholder="User should find and book the cheapest available flight..."
+              rows={3}
+            />
+          </div>
+
+          <div>
+            <label className="block text-sm font-medium text-zinc-700 dark:text-zinc-300">
+              definition of done
+            </label>
+            <textarea
+              value={form.task_def_of_done}
+              onChange={(e) => setForm({ ...form, task_def_of_done: e.target.value })}
+              className="mt-1 w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 focus:border-zinc-900 focus:outline-none dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-100"
+              placeholder="Booking is completed and confirmation page is shown"
+              rows={2}
+            />
+          </div>
+
+          <button
+            type="submit"
+            disabled={loading}
+            className="w-full rounded-lg bg-black px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-zinc-800 disabled:opacity-50 dark:bg-zinc-50 dark:text-black dark:hover:bg-zinc-200"
+          >
+            {loading ? 'creating...' : 'create task'}
+          </button>
+        </form>
+      )}
+
+      <div className="space-y-2">
+        {tasks.length === 0 ? (
+          <p className="py-8 text-center text-sm text-zinc-500 dark:text-zinc-400">
+            no tasks yet
+          </p>
+        ) : (
+          tasks.map((task) => (
+            <div
+              key={task.id}
+              onClick={() => onTaskSelect?.(task.id)}
+              className={`cursor-pointer rounded-lg border p-3 transition-colors ${
+                selectedTaskId === task.id
+                  ? 'border-zinc-900 bg-zinc-50 dark:border-zinc-100 dark:bg-zinc-900'
+                  : 'border-zinc-200 bg-white hover:border-zinc-300 dark:border-zinc-800 dark:bg-zinc-950 dark:hover:border-zinc-700'
+              }`}
+            >
+              <h3 className="font-medium text-zinc-900 dark:text-zinc-100">
+                {task.task_name}
+              </h3>
+              {task.task_description && (
+                <p className="mt-1 text-sm text-zinc-600 dark:text-zinc-400">
+                  {task.task_description}
+                </p>
+              )}
+              {task.task_def_of_done && (
+                <p className="mt-1 text-xs text-zinc-500 dark:text-zinc-500">
+                  done: {task.task_def_of_done}
+                </p>
+              )}
+            </div>
+          ))
+        )}
+      </div>
+    </div>
+  );
+};

web/src/hooks/useInteractionTracking.ts

@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from 'react';
-import '@/lib/experiments' // ensure experiments lib is loaded
+import '@/lib/experiments'
 import type { EventName } from '@/lib/events';
 
 const fetchSessionId = async (): Promise<string> => {
@@ -21,10 +21,14 @@ const track = async (ev: {
   metadata?: Record<string, unknown>;
 }) => {
   try {
+    const experimentId = localStorage.getItem('phantom_experiment_id');
     await fetch('/api/ingest', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(ev),
+      body: JSON.stringify({
+        ...ev,
+        experimentId: experimentId || undefined,
+      }),
     });
   } catch (err) {
     console.error('track failed:', err);

web/src/proxy.ts

@@ -10,6 +10,7 @@ export function proxy(req: NextRequest) {
         pathname.startsWith('/admin') ||
         pathname.startsWith('/_next') ||
         pathname.startsWith('/static') ||
+        pathname.startsWith('/start-task') ||
         pathname.includes('.')
         // TODO: add robots.txt and sitemap.xml if needed here
     ) {

web/src/utils/supabase/client.ts

@@ -0,0 +1,10 @@
+import { createBrowserClient } from "@supabase/ssr";
+
+const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
+const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
+
+export const createClient = () =>
+    createBrowserClient(
+        supabaseUrl!,
+        supabaseKey!,
+    );

web/src/utils/supabase/middleware.ts

@@ -0,0 +1,37 @@
+import { createServerClient, type CookieOptions } from "@supabase/ssr";
+import { type NextRequest, NextResponse } from "next/server";
+
+const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
+const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
+
+export const createClient = (request: NextRequest) => {
+  // Create an unmodified response
+  let supabaseResponse = NextResponse.next({
+    request: {
+      headers: request.headers,
+    },
+  });
+
+  const supabase = createServerClient(
+    supabaseUrl!,
+    supabaseKey!,
+    {
+      cookies: {
+        getAll() {
+          return request.cookies.getAll()
+        },
+        setAll(cookiesToSet) {
+          cookiesToSet.forEach(({ name, value, options }) => request.cookies.set(name, value))
+          supabaseResponse = NextResponse.next({
+            request,
+          })
+          cookiesToSet.forEach(({ name, value, options }) =>
+            supabaseResponse.cookies.set(name, value, options)
+          )
+        },
+      },
+    },
+  );
+
+  return supabaseResponse
+};

web/src/utils/supabase/server.ts

@@ -0,0 +1,27 @@
+import { createServerClient, type CookieOptions } from "@supabase/ssr";
+import { cookies } from "next/headers";
+import { ReadonlyRequestCookies } from "next/dist/server/web/spec-extension/adapters/request-cookies";
+
+const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
+const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
+
+export const createClient = (cookieStore: ReadonlyRequestCookies) => {
+    return createServerClient(
+        supabaseUrl!,
+        supabaseKey!,
+        {
+            cookies: {
+                getAll() {
+                    return cookieStore.getAll()
+                },
+                setAll(cookiesToSet) {
+                    try {
+                        cookiesToSet.forEach(({ name, value, options }) => cookieStore.set(name, value, options))
+                    } catch {
+                        // `setAll` called from Server Component - ignored if middleware handles session refresh
+                    }
+                },
+            },
+        },
+    );
+};