chore: cleaning shitty code

* feat: training pipeline + tensorboard

* tesnorboard forgot

* chore: ml basic boilerplate

* feat: naive architecture as start

* eval setup

* chore: parquet exporting of data

* chore: updating requirements necesary

* feat: separating modules and adding training logs paths

* Update experiments/ml/train.py

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* fix: new path for runs

* fix: undoing ai slop code

* chore: modules and reqs

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

.gitignore

@@ -6,3 +6,11 @@
 **/session_*.svg
 **/*graph.svg
 paper/src/bib/auto
+
+# Airflow logs - exclude DAG run logs
+experiments/airflow/logs/*
+experiments/airflow/logs/scheduler/
+experiments/airflow/logs/dag_processor_manager/
+tests/e2e/node_modules/**
+**/auto/*.el
+*.old

Makefile

@@ -11,42 +11,72 @@ PYTEST    := $(VENV)/bin/pytest
 
 .DEFAULT_GOAL := help
 
-all: pdf
-
-run.webapp:
-	@cd web && npm install && npm run dev
+.PHONY: help
+help:
+	@echo "pdf.build pdf.watch pdf.clean | test.backend test.e2e test.all | web.dev | install | stats.lines"
 
 $(BUILDDIR):
 	mkdir -p paper/$(BUILDDIR)
 
-pdf: $(BUILDDIR)
-	@echo "Concatenating source code..."
+.PHONY: pdf.build
+pdf.build: $(BUILDDIR)
 	@bash paper/concat_code.sh
 	@cd $(SRCDIR) && \
 	$(LATEXMK) -pdf -jobname=$(JOBNAME) \
 		-interaction=nonstopmode -file-line-error \
 		-outdir=../$(BUILDDIR) $(TEX)
 
-watch: $(BUILDDIR)
+.PHONY: pdf.watch
+pdf.watch: $(BUILDDIR)
 	@cd $(SRCDIR) && \
 	$(LATEXMK) -pvc -pdf -jobname=$(JOBNAME) \
 		-interaction=nonstopmode -file-line-error \
 		-r ../.latexmkrc \
 		-outdir=../$(BUILDDIR) $(TEX)
 
-clean:
+.PHONY: pdf.clean
+pdf.clean:
 	@cd $(SRCDIR) && \
 	$(LATEXMK) -C -jobname=$(JOBNAME) -outdir=../$(BUILDDIR) || true
 	rm -rf paper/$(BUILDDIR)/*
 
+.PHONY: test.backend
+test.backend: $(VENV)
+	$(PYTEST) -v
+
+.PHONY: test.e2e
+test.e2e:
+	@cd tests/e2e && npm install
+	@cd tests/e2e && npx playwright install chromium
+	@timeout 30 bash -c 'until curl -sf http://localhost:5000/health > /dev/null 2>&1; do sleep 1; done' || (echo "Backend not ready" && exit 1)
+	@timeout 30 bash -c 'until curl -sf http://localhost:3000 > /dev/null 2>&1; do sleep 1; done' || (echo "Web app not ready" && exit 1)
+	@cd tests/e2e && npm test
+
+.PHONY: test.all
+test.all: test.backend test.e2e
+
+.PHONY: web.dev
+web.dev:
+	@cd web && npm install && npm run dev
+
 $(VENV):
 	python3 -m venv $(VENV)
 	$(PIP) install --upgrade pip
 
+.PHONY: install
 install: $(VENV)
 	$(PIP) install -r requirements.txt
 
-test: $(VENV)
-	$(PYTEST) -v
+.PHONY: stats.lines
+stats.lines:
+	@find . \( -path '*/node_modules' -o -path '*/.venv' -o -path '*/venv' \) -prune -o \
+	\( -name "*.ts" -o -name "*.py" \) -type f -print0 | xargs -0 cat | wc -l
 
-.PHONY: all pdf clean watch run.webapp install test
+.PHONY: pdf clean watch run.webapp test count-lines all
+pdf: pdf.build
+clean: pdf.clean
+watch: pdf.watch
+run.webapp: web.dev
+test: test.backend
+count-lines: stats.lines
+all: pdf.build

README.md

@@ -1,5 +1,12 @@
+<img width="200" align="left" src="https://github.com/user-attachments/assets/d148b00d-e9f9-4280-89cc-0cc866e17251" />
+
+### PHANTOM
+
 [![Build PDF](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml/badge.svg)](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml)
+[![TPU Research Cloud](https://img.shields.io/badge/TPU%20Research%20Cloud-TRC%20supported-4285F4?logo=googlecloud&logoColor=white)](https://sites.research.google/trc/faq/)
+[![Vercel Deploy](https://deploy-badge.vercel.app/?url=https://phantom-hotel.vercel.app&name=Hotel)](https://phantom-hotel.vercel.app)
+[![Vercel Deploy](https://deploy-badge.vercel.app/?url=https://phantom-airline.vercel.app&name=Airline)](https://phantom-airline.vercel.app)
+
+
 
-- https://phantom-hotel.vercel.app/
-- https://phantom-airline.vercel.app/
 

backend/provider/app.py

@@ -0,0 +1,113 @@
+from fastapi import FastAPI, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import Literal, Optional
+import uvicorn, os, sys
+from supabase import create_client, Client
+from dotenv import load_dotenv
+import numpy as np
+import pandas as pd
+load_dotenv()
+
+# Local imports of registry and pricing function
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../experiments/")
+from procesing.providers import SupabaseProvider, BackendAPIProvider
+from procesing.pricers import (
+    StaticPricer,
+    RandomPricer,
+    ElasticityBasedPricer
+)
+from procesing.steps import (
+    PredictPricesStep
+)
+from procesing import PipelineContext
+sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
+print(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
+from lib.model_registry import ModelRegistry
+
+# Config
+app = FastAPI(title="PHANTOM Pricing Provider")
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
+
+supabase: Client = create_client(os.getenv("NEXT_PUBLIC_SUPABASE_URL"), os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY"))
+registry = ModelRegistry()
+
+class PriceResponse(BaseModel):
+    productId: str
+    price: float
+    base_price: float
+    markup: float
+    elasticity: Optional[float] = None
+    model_version: str = 'latest'
+
+@app.get("/health")
+def health() -> dict:
+    return {"status": "healthy", "redis": registry.health_check()}
+
+@app.get("/api/{mode}/price/{productId}", response_model=PriceResponse)
+def get_price(mode: Literal['hotel', 'airline'], productId: str, sessionId: Optional[str] = Query(None), experimentId: Optional[str] = Query(None)):
+    product = supabase.table(f'{mode}_products').select("metadata").eq('id', productId).execute().data[0]
+    if not product: raise HTTPException(404, f"Product {productId} not found")
+
+    metadata = product['metadata']
+    base_price = metadata.get('base_price', 100.0)
+
+    # fetch pre-computed prices from registry
+    prices_df = registry.get_prices('latest')
+    elasticity_df = registry.get_elasticity('latest')
+
+    if prices_df is None:
+        # fallback: no pre-computed prices available
+        return PriceResponse(
+            productId=productId,
+            price=base_price,
+            base_price=base_price,
+            markup=1.0,
+            elasticity=None
+        )
+
+    # lookup pre-computed price for this product
+    product_price_row = prices_df[prices_df['productId'] == productId]
+    if product_price_row.empty:
+        # product not in pre-computed prices, fallback to base
+        return PriceResponse(
+            productId=productId,
+            price=base_price,
+            base_price=base_price,
+            markup=1.0,
+            elasticity=None
+        )
+
+    optimal_price = float(product_price_row['optimal_price'].iloc[0]) # TODO: use optimal_price everywhere as  aresult
+
+    # get elasticity if available
+    product_elasticity = None
+    if elasticity_df is not None:
+        product_elasticity_row = elasticity_df[elasticity_df['productId'] == productId]
+        if not product_elasticity_row.empty:
+            product_elasticity = float(product_elasticity_row['elasticity'].iloc[0])
+
+    return PriceResponse(
+        productId=productId,
+        price=optimal_price,
+        base_price=base_price,
+        markup=optimal_price/base_price,
+        elasticity=product_elasticity
+    )
+
+@app.get("/models")
+def list_models(): return registry.list_models()
+
+@app.post("/models/reload")
+def reload_models():
+    elasticity, pricing_model = registry.get_elasticity('latest'), registry.get_pricing_model('latest')
+    return {
+        "elasticity_loaded": bool(elasticity),
+        "n_products": len(elasticity) if elasticity is not None else 0,
+        "pricing_model_loaded": bool(pricing_model),
+        "model_class": pricing_model.__class__.__name__ if pricing_model else None
+    }
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PROVIDER_PORT", "5001")))

backend/provider/requirements.txt

@@ -0,0 +1,16 @@
+fastapi
+uvicorn[standard]
+pydantic
+numpy
+pandas
+scikit-learn
+redis
+supabase
+confluent-kafka>=2.3.0
+kafka-python
+graphviz
+python-dotenv>=1.0.0
+requests>=2.31.0
+typing-extensions>=4.8.0
+pypickle
+pymc

backend/server/app.py

@@ -290,6 +290,7 @@ async def get_products(
         query = supabase.table(table).select('*')
 
         # filter by exact date_index if provided
+        # dateIndex from frontend is days from today, convert to days since epoch
         if dateIndex is not None:
             query = query.eq('date_index', dateIndex)
 

docker-compose.yml

@@ -1,4 +1,15 @@
 services:
+
+  tensorboard:
+    image: tensorflow/tensorflow:latest
+    container_name: "PHANTOM-tensorboard"
+    ports:
+      - "6006:6006"
+    volumes:
+      - ./experiments/ml/runs:/logs
+    command: tensorboard --logdir=/logs --host=0.0.0.0 --port=6006
+    restart: unless-stopped
+
   backend:
     container_name: "PHANTOM-backend"
     build:
@@ -71,6 +82,133 @@ services:
       - "${REDPANDA_CONSOLE_PORT:-8080}:8080"
     restart: unless-stopped
 
+  postgres:
+    container_name: "PHANTOM-postgres"
+    image: postgres:13
+    environment:
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+    ports:
+      - "5433:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    restart: unless-stopped
+
+  airflow-init:
+    container_name: "PHANTOM-airflow-init"
+    build:
+      context: .
+      dockerfile: docker/Airflow.dockerfile
+    depends_on:
+      - postgres
+    environment:
+      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
+      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
+      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW__CORE__LOAD_EXAMPLES=false
+      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
+      - _AIRFLOW_DB_MIGRATE=true
+      - _AIRFLOW_WWW_USER_CREATE=true
+      - _AIRFLOW_WWW_USER_USERNAME=admin
+      - _AIRFLOW_WWW_USER_PASSWORD=admin
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+    command: version
+    restart: "no"
+
+  airflow-webserver:
+    container_name: "PHANTOM-airflow-webserver"
+    build:
+      context: .
+      dockerfile: docker/Airflow.dockerfile
+    depends_on:
+      - postgres
+      - airflow-init
+      - redis
+    environment:
+      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
+      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
+      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
+      - AIRFLOW__CORE__LOAD_EXAMPLES=false
+      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
+      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
+      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
+      - KAFKA_HOST=kafka
+      - KAFKA_PORT=29092
+      - BACKEND_URL=http://backend:5000
+      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
+      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+    ports:
+      - "${AIRFLOW_WEBSERVER_PORT:-8085}:8080"
+    command: webserver
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  airflow-scheduler:
+    container_name: "PHANTOM-airflow-scheduler"
+    build:
+      context: .
+      dockerfile: docker/Airflow.dockerfile
+    depends_on:
+      airflow-webserver:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    environment:
+      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
+      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
+      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
+      - AIRFLOW__CORE__LOAD_EXAMPLES=false
+      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
+      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
+      - KAFKA_HOST=kafka
+      - KAFKA_PORT=29092
+      - BACKEND_URL=http://backend:5000
+      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
+      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+    command: scheduler
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+
+  pricing-provider:
+    container_name: "PHANTOM-pricing-provider"
+    build:
+      context: .
+      dockerfile: docker/Provider.dockerfile
+    depends_on:
+      - redis
+      - kafka
+    environment:
+      - PROVIDER_PORT=5001
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - KAFKA_HOST=kafka
+      - KAFKA_PORT=29092
+      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
+      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+      - BACKEND_URL=http://localhost:5000
+    ports:
+      - "${PROVIDER_PORT:-5001}:5001"
+    restart: unless-stopped
+
 volumes:
   phantom_kafka_data:
   phantom_redis_data:
+  postgres_data:

docker/Airflow.dockerfile

@@ -0,0 +1,30 @@
+FROM apache/airflow:2.7.3-python3.11
+
+USER root
+
+# install system deps if needed
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+USER airflow
+
+# copy requirements for pipeline dependencies
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+# install postgres driver and providers
+RUN pip install --no-cache-dir \
+    psycopg2-binary \
+    apache-airflow-providers-postgres
+
+# set airflow home
+ENV AIRFLOW_HOME=/opt/airflow
+
+COPY --chown=airflow:root experiments/airflow/dags ${AIRFLOW_HOME}/dags
+COPY --chown=airflow:root experiments/procesing ${AIRFLOW_HOME}/procesing
+COPY --chown=airflow:root lib ${AIRFLOW_HOME}/lib
+
+# create logs and plugins dirs (airflow expects them)
+RUN mkdir -p ${AIRFLOW_HOME}/logs ${AIRFLOW_HOME}/plugins

docker/Airflow.railway.dockerfile

@@ -0,0 +1,41 @@
+FROM apache/airflow:2.7.3-python3.11
+
+USER root
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    supervisor \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+USER airflow
+
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+RUN pip install --no-cache-dir \
+    psycopg2-binary \
+    apache-airflow-providers-postgres
+
+ENV AIRFLOW_HOME=/opt/airflow
+ENV AIRFLOW__CORE__EXECUTOR=SequentialExecutor
+ENV AIRFLOW__CORE__LOAD_EXAMPLES=false
+ENV AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
+ENV AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
+
+# copy all code into image (standalone - no volume mounts needed)
+COPY --chown=airflow:root experiments/airflow/dags ${AIRFLOW_HOME}/dags
+COPY --chown=airflow:root experiments/procesing ${AIRFLOW_HOME}/procesing
+COPY --chown=airflow:root lib ${AIRFLOW_HOME}/lib
+
+RUN mkdir -p ${AIRFLOW_HOME}/logs ${AIRFLOW_HOME}/plugins
+
+# copy entrypoint script
+COPY --chown=airflow:root docker/airflow-railway-entrypoint.sh /entrypoint.sh
+USER root
+RUN chmod +x /entrypoint.sh
+USER airflow
+
+EXPOSE 8080
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Provider.dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies including graphviz
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    graphviz \
+    libgraphviz-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy and install Python dependencies
+COPY backend/provider/requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code into image
+COPY lib/ /app/lib/
+COPY experiments/procesing/ /app/procesing/
+COPY backend/provider/ /app/provider/
+
+ENV PYTHONPATH=/app:/app/lib:/app/procesing
+
+WORKDIR /app/provider
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5001"]

docker/airflow-railway-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# init db and create admin user on first run
+airflow db migrate
+
+# create admin user if not exists
+airflow users create \
+    --username "${AIRFLOW_ADMIN_USER:-admin}" \
+    --password "${AIRFLOW_ADMIN_PASSWORD:-admin}" \
+    --firstname Admin \
+    --lastname User \
+    --role Admin \
+    --email admin@example.com || true
+
+# start scheduler in background
+airflow scheduler &
+
+# start webserver in foreground (Railway needs one foreground process)
+exec airflow webserver --port ${PORT:-8080}

experiments/README.md

@@ -0,0 +1,8 @@
+
+# Products
+# Agents
+# Pipeline
+
+Our pipeline technically should follow principles in a style like this:
+- Each step should be defined as an inheriting child of an scikit pipeline step, the granularity of the steps is dictated by the following: a step should be a transformation, augmentation or computation independently, no single stage should run multiple in-itself. This way we can modularize properly all the components and track properly in airflow. A stage can be defined as an sklearn step but then must be transalted to a function that takes the context in our DAG of airflow. All parametrization must be done via contexts.
+

experiments/airflow/dags/ml_training_pipeline.py

@@ -0,0 +1,115 @@
+from airflow import DAG, Dataset
+from airflow.decorators import task
+from airflow.utils.dates import days_ago
+from datetime import timedelta
+import pandas as pd
+import logging
+import sys
+import pickle
+
+sys.path.insert(0, '/opt/airflow')
+
+from procesing.context import PipelineContext
+from procesing.providers import SupabaseProvider, BackendAPIProvider
+from procesing.steps import (
+    FetchInteractionsStep,
+    ValidateDataStep,
+    ExtractSessionFeaturesStep,
+    JoinLabelsStep,
+)
+
+TRAINING_DATASET = Dataset('phantom://ml/training-data')
+
+DEFAULT_ARGS = {
+    'owner': 'phantom-research',
+    'depends_on_past': False,
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 2,
+    'retry_delay': timedelta(minutes=5),
+}
+
+
+class CompositeProvider(SupabaseProvider, BackendAPIProvider):
+    def __init__(self):
+        SupabaseProvider.__init__(self)
+        BackendAPIProvider.__init__(self)
+
+
+def _get_context(store_mode: str = 'hotel') -> PipelineContext:
+    return PipelineContext(provider=CompositeProvider(), store_mode=store_mode)
+
+
+with DAG(
+    'ml_training_pipeline',
+    default_args=DEFAULT_ARGS,
+    description='ML training data pipeline: fetch -> validate -> extract features -> label -> publish',
+    schedule=None,
+    start_date=days_ago(1),
+    catchup=False,
+    max_active_runs=1,
+    tags=['ml', 'training', 'features', 'research'],
+) as dag:
+
+    @task
+    def fetch_interactions(**kwargs) -> bytes:
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
+        df = FetchInteractionsStep(ctx).transform(None)
+        logging.info(f"Fetched {len(df)} interactions, {df['sessionId'].nunique()} sessions")
+        return pickle.dumps(df)
+
+    @task
+    def validate_data(raw_data: bytes, **kwargs) -> bytes:
+        df = pickle.loads(raw_data)
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
+        validated = ValidateDataStep(ctx).transform(df)
+        report = ctx.get_cached('validation_report') or {}
+        logging.info(f"Validation: {report.get('status')}, {report.get('sessions', 0)} sessions")
+        return pickle.dumps(validated)
+
+    @task
+    def extract_session_features(validated_data: bytes, **kwargs) -> bytes:
+        df = pickle.loads(validated_data)
+        if df.empty:
+            logging.warning("Empty input, skipping feature extraction")
+            return pickle.dumps(pd.DataFrame())
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
+        features = ExtractSessionFeaturesStep(ctx).transform(df)
+        logging.info(f"Extracted {len(features.columns)} features for {len(features)} sessions")
+        return pickle.dumps(features)
+
+    @task
+    def join_labels(features_data: bytes, **kwargs) -> bytes:
+        features_df = pickle.loads(features_data)
+        if features_df.empty:
+            logging.warning("Empty features, skipping label join")
+            return pickle.dumps(pd.DataFrame())
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
+        labeled = JoinLabelsStep(ctx).transform(features_df)
+        n_agents = labeled['is_agent'].sum() if 'is_agent' in labeled.columns else 0
+        logging.info(f"Labeled {len(labeled)} sessions: {n_agents} agents")
+        return pickle.dumps(labeled)
+
+    @task(outlets=[TRAINING_DATASET])
+    def publish_training_data(labeled_data: bytes, **kwargs) -> dict:
+        labeled_df = pickle.loads(labeled_data)
+        if labeled_df.empty:
+            return {'status': 'skipped', 'reason': 'empty_data'}
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+        return {
+            'status': 'success',
+            'n_sessions': len(labeled_df),
+            'n_features': len([c for c in labeled_df.columns if c not in ['sessionId', 'experimentId', 'is_agent']]),
+            'store_mode': dag_conf.get('store_mode', 'hotel'),
+            'timestamp': pd.Timestamp.now().isoformat(),
+        }
+
+    raw = fetch_interactions()
+    validated = validate_data(raw)
+    features = extract_session_features(validated)
+    labeled = join_labels(features)
+    publish_training_data(labeled)

experiments/airflow/dags/surge_pricing_factory.py

@@ -0,0 +1,210 @@
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+from airflow.utils.dates import days_ago
+from datetime import timedelta
+import pandas as pd
+import logging
+import sys
+import pickle
+
+sys.path.insert(0, '/opt/airflow')
+
+from procesing.context import PipelineContext
+from procesing.providers import SupabaseProvider, BackendAPIProvider
+from procesing.steps import (
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    ComputeDemandStep,
+    AggregatePriceLogsStep,
+    JoinProductFeaturesStep,
+)
+from procesing.pricers.simple import SimpleSurgePricer
+
+DEFAULT_ARGS = {
+    'owner': 'phantom-research',
+    'depends_on_past': False,
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 2,
+    'retry_delay': timedelta(minutes=5),
+}
+
+class CompositeProvider(SupabaseProvider, BackendAPIProvider):
+    def __init__(self):
+        SupabaseProvider.__init__(self)
+        BackendAPIProvider.__init__(self)
+
+def _get_provider():
+    return CompositeProvider()
+
+def _make_task_callables(store_mode: str):
+    """Generate task callables bound to a specific store_mode."""
+
+    def get_context(**kwargs):
+        return PipelineContext(provider=_get_provider(), store_mode=store_mode)
+
+    def fetch_interactions(**kwargs):
+        ctx = get_context(**kwargs)
+        df = FetchInteractionsStep(ctx).transform(None)
+        kwargs['ti'].xcom_push(key='interactions_raw', value=pickle.dumps(df))
+        logging.info(f"[{store_mode}] Fetched {len(df)} interaction records")
+        return len(df)
+
+    def fetch_price_logs(**kwargs):
+        ctx = get_context(**kwargs)
+        df = FetchPriceLogsStep(ctx).transform(None)
+        kwargs['ti'].xcom_push(key='price_logs_raw', value=pickle.dumps(df))
+        logging.info(f"[{store_mode}] Fetched {len(df)} price records")
+        return len(df)
+
+    def compute_demand(**kwargs):
+        ti = kwargs['ti']
+        df = pickle.loads(ti.xcom_pull(key='interactions_raw'))
+        ctx = get_context(**kwargs)
+        demand_df = ComputeDemandStep(ctx).transform(df)
+        ti.xcom_push(key='demand_data', value=pickle.dumps(demand_df))
+        logging.info(f"[{store_mode}] Computed demand for {len(demand_df)} products")
+        return len(demand_df)
+
+    def aggregate_price_logs(**kwargs):
+        ti = kwargs['ti']
+        df = pickle.loads(ti.xcom_pull(key='price_logs_raw'))
+        ctx = get_context(**kwargs)
+        price_df = AggregatePriceLogsStep(ctx).transform(df)
+        ti.xcom_push(key='price_data', value=pickle.dumps(price_df))
+        logging.info(f"[{store_mode}] Aggregated price logs for {len(price_df)} products")
+        return len(price_df)
+
+    def join_product_features(**kwargs):
+        ti = kwargs['ti']
+        demand_df = pickle.loads(ti.xcom_pull(key='demand_data'))
+        price_df = pickle.loads(ti.xcom_pull(key='price_data'))
+        ctx = get_context(**kwargs)
+        joined_df = JoinProductFeaturesStep(ctx).transform((demand_df, price_df))
+        ti.xcom_push(key='product_features', value=pickle.dumps(joined_df))
+        logging.info(f"[{store_mode}] Joined features for {len(joined_df)} products")
+        return len(joined_df)
+
+    def apply_surge_pricing(**kwargs):
+        ti = kwargs['ti']
+        product_features = pickle.loads(ti.xcom_pull(key='product_features'))
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+
+        data = product_features.rename(columns={'demand_score': 'demand'})
+        surge_pricer = SimpleSurgePricer(
+            high_threshold=dag_conf.get('high_threshold', 10),
+            low_threshold=dag_conf.get('low_threshold', 2),
+            surge_multiplier=dag_conf.get('surge_multiplier', 1.2),
+            discount_multiplier=dag_conf.get('discount_multiplier', 0.9)
+        )
+        surge_pricer.fit(data)
+        data['optimal_price'] = surge_pricer.predict()
+
+        prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
+            'price': 'current_price', 'demand': 'demand_score'
+        })
+        ti.xcom_push(key='predicted_prices', value=pickle.dumps(prices_df))
+        logging.info(f"[{store_mode}] Applied surge pricing for {len(prices_df)} products")
+        return len(prices_df)
+
+    def publish_results(**kwargs):
+        ti = kwargs['ti']
+        prices_df = pickle.loads(ti.xcom_pull(key='predicted_prices'))
+        from lib.model_registry import ModelRegistry
+
+        registry = ModelRegistry()
+        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+
+        metadata = {
+            'timestamp': pd.Timestamp.now().isoformat(),
+            'store_mode': store_mode,
+            'dag_run_id': kwargs['dag_run'].run_id if kwargs.get('dag_run') else 'manual',
+            'pricing_method': 'surge',
+            'high_threshold': dag_conf.get('high_threshold', 10),
+            'low_threshold': dag_conf.get('low_threshold', 2),
+            'surge_multiplier': dag_conf.get('surge_multiplier', 1.2),
+            'discount_multiplier': dag_conf.get('discount_multiplier', 0.9)
+        }
+        registry.publish_prices(prices_df, model_name=f'{store_mode}_latest', metadata=metadata)
+        logging.info(f"[{store_mode}] Published surge pricing for {len(prices_df)} products")
+
+        return {
+            'n_products': len(prices_df),
+            'registry_status': 'success',
+            'store_mode': store_mode,
+            'mean_demand': float(prices_df['demand_score'].mean()) if 'demand_score' in prices_df.columns else None
+        }
+
+    return {
+        'fetch_interactions': fetch_interactions,
+        'fetch_price_logs': fetch_price_logs,
+        'compute_demand': compute_demand,
+        'aggregate_price_logs': aggregate_price_logs,
+        'join_product_features': join_product_features,
+        'apply_surge_pricing': apply_surge_pricing,
+        'publish_results': publish_results,
+    }
+
+
+def create_surge_pricing_dag(store_mode: str) -> DAG:
+    """Factory: generates a surge pricing DAG for a given store_mode."""
+    callables = _make_task_callables(store_mode)
+
+    dag = DAG(
+        f'surge_pricing_{store_mode}',
+        default_args=DEFAULT_ARGS,
+        description=f'Surge pricing pipeline for {store_mode} store mode',
+        schedule_interval='*/15 * * * *',
+        start_date=days_ago(1),
+        catchup=False,
+        max_active_runs=1,
+        tags=['pricing', 'surge', 'research', store_mode],
+    )
+
+    with dag:
+        t_fetch_interactions = PythonOperator(
+            task_id='fetch_interactions',
+            python_callable=callables['fetch_interactions'],
+            provide_context=True,
+        )
+        t_fetch_price_logs = PythonOperator(
+            task_id='fetch_price_logs',
+            python_callable=callables['fetch_price_logs'],
+            provide_context=True,
+        )
+        t_compute_demand = PythonOperator(
+            task_id='compute_demand',
+            python_callable=callables['compute_demand'],
+            provide_context=True,
+        )
+        t_aggregate_prices = PythonOperator(
+            task_id='aggregate_price_logs',
+            python_callable=callables['aggregate_price_logs'],
+            provide_context=True,
+        )
+        t_join_features = PythonOperator(
+            task_id='join_product_features',
+            python_callable=callables['join_product_features'],
+            provide_context=True,
+        )
+        t_surge_pricing = PythonOperator(
+            task_id='apply_surge_pricing',
+            python_callable=callables['apply_surge_pricing'],
+            provide_context=True,
+        )
+        t_publish = PythonOperator(
+            task_id='publish_results',
+            python_callable=callables['publish_results'],
+            provide_context=True,
+        )
+
+        t_fetch_interactions >> t_compute_demand
+        t_fetch_price_logs >> t_aggregate_prices
+        [t_compute_demand, t_aggregate_prices] >> t_join_features >> t_surge_pricing >> t_publish
+
+    return dag
+
+
+# instantiate DAGs for Airflow to discover
+dag_airline = create_surge_pricing_dag('airline')
+dag_hotel = create_surge_pricing_dag('hotel')

experiments/airflow/dags/surge_pricing_pipeline.py

@@ -0,0 +1,237 @@
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+from airflow.utils.dates import days_ago
+from datetime import timedelta
+import pandas as pd
+import logging
+import sys
+import pickle
+import io
+
+# add parent dir to path so procesing package can be imported
+sys.path.insert(0, '/opt/airflow')
+
+from procesing.context import PipelineContext
+from procesing.providers import SupabaseProvider, BackendAPIProvider
+from procesing.steps import (
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    ComputeDemandStep,
+    AggregatePriceLogsStep,
+    JoinProductFeaturesStep,
+)
+from procesing.pricers.simple import SimpleSurgePricer
+
+default_args = {
+    'owner': 'phantom-research',
+    'depends_on_past': False,
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 2,
+    'retry_delay': timedelta(minutes=5),
+}
+
+def get_provider():
+    """Factory to create composite provider"""
+    class CompositeProvider(SupabaseProvider, BackendAPIProvider): # TODO: Fix this into one global provider singelton instead of multiple inheritance declarations acoss the codebase
+        def __init__(self):
+            SupabaseProvider.__init__(self)
+            BackendAPIProvider.__init__(self)
+    return CompositeProvider()
+
+def get_context(**kwargs):
+    """Build pipeline context from Airflow config"""
+    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+    return PipelineContext(
+        provider=get_provider(),
+        store_mode=dag_conf.get('store_mode', 'hotel'),
+    )
+
+# atomic task functions (each wraps one sklearn step)
+def fetch_interactions(**kwargs):
+    """Task: Fetch interaction data from Kafka"""
+    context = get_context(**kwargs)
+    step = FetchInteractionsStep(context)
+    df = step.transform(None)
+
+    kwargs['ti'].xcom_push(key='interactions_raw', value=pickle.dumps(df))
+    logging.info(f"Fetched {len(df)} interaction records")
+    return len(df)
+
+def fetch_price_logs(**kwargs):
+    """Task: Fetch price logs from Kafka"""
+    context = get_context(**kwargs)
+    step = FetchPriceLogsStep(context)
+    df = step.transform(None)
+
+    kwargs['ti'].xcom_push(key='price_logs_raw', value=pickle.dumps(df))
+    logging.info(f"Fetched {len(df)} price records")
+    return len(df)
+
+def compute_demand(**kwargs):
+    """Task: Compute demand scores from interactions"""
+    ti = kwargs['ti']
+    df = pickle.loads(ti.xcom_pull(key='interactions_raw'))
+
+    context = get_context(**kwargs)
+    step = ComputeDemandStep(context)
+    demand_df = step.transform(df)
+    # TODO: clear the xcom
+
+
+    ti.xcom_push(key='demand_data', value=pickle.dumps(demand_df))
+    logging.info(f"Computed demand for {len(demand_df)} products")
+    return len(demand_df)
+
+def aggregate_price_logs(**kwargs):
+    """Task: Aggregate price logs"""
+    ti = kwargs['ti']
+    df = pickle.loads(ti.xcom_pull(key='price_logs_raw'))
+
+    context = get_context(**kwargs)
+    step = AggregatePriceLogsStep(context)
+    price_df = step.transform(df)
+
+    ti.xcom_push(key='price_data', value=pickle.dumps(price_df))
+    logging.info(f"Aggregated price logs for {len(price_df)} products")
+    return len(price_df)
+
+def join_product_features(**kwargs):
+    """Task: Join demand and price data"""
+    ti = kwargs['ti']
+    demand_df = pickle.loads(ti.xcom_pull(key='demand_data'))
+    price_df = pickle.loads(ti.xcom_pull(key='price_data'))
+
+    context = get_context(**kwargs)
+    step = JoinProductFeaturesStep(context)
+    joined_df = step.transform((demand_df, price_df))
+
+    ti.xcom_push(key='product_features', value=pickle.dumps(joined_df))
+    logging.info(f"Joined features for {len(joined_df)} products")
+    return len(joined_df)
+
+def apply_surge_pricing(**kwargs):
+    """Task: Apply surge pricing rules to generate optimal prices"""
+    ti = kwargs['ti']
+    product_features = pickle.loads(ti.xcom_pull(key='product_features'))
+
+    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+
+    # rename demand_score to demand for pricer compatibility
+    data = product_features.rename(columns={'demand_score': 'demand'})
+
+    surge_pricer = SimpleSurgePricer(
+        high_threshold=dag_conf.get('high_threshold', 10),
+        low_threshold=dag_conf.get('low_threshold', 2),
+        surge_multiplier=dag_conf.get('surge_multiplier', 1.2),
+        discount_multiplier=dag_conf.get('discount_multiplier', 0.9)
+    )
+    surge_pricer.fit(data)
+    data['optimal_price'] = surge_pricer.predict()
+
+    prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
+        'price': 'current_price',
+        'demand': 'demand_score'
+    })
+
+    ti.xcom_push(key='predicted_prices', value=pickle.dumps(prices_df))
+    logging.info(f"Applied surge pricing for {len(prices_df)} products")
+    return len(prices_df)
+
+def publish_results(**kwargs):
+    """Task: Publish surge pricing results to registry"""
+    ti = kwargs['ti']
+    prices_df = pickle.loads(ti.xcom_pull(key='predicted_prices'))
+
+    sys.path.insert(0, '/opt/airflow')
+    from lib.model_registry import ModelRegistry
+
+    registry = ModelRegistry()
+    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
+
+    metadata = {
+        'timestamp': pd.Timestamp.now().isoformat(),
+        'store_mode': dag_conf.get('store_mode', 'hotel'),
+        'dag_run_id': kwargs['dag_run'].run_id if kwargs.get('dag_run') else 'manual',
+        'pricing_method': 'surge',
+        'high_threshold': dag_conf.get('high_threshold', 10),
+        'low_threshold': dag_conf.get('low_threshold', 2),
+        'surge_multiplier': dag_conf.get('surge_multiplier', 1.2),
+        'discount_multiplier': dag_conf.get('discount_multiplier', 0.9)
+    }
+
+    registry.publish_prices(prices_df, model_name='latest', metadata=metadata)
+
+    logging.info(f"Published surge pricing for {len(prices_df)} products")
+
+    return {
+        'n_products': len(prices_df),
+        'registry_status': 'success',
+        'mean_demand': float(prices_df['demand_score'].mean()) if 'demand_score' in prices_df.columns else None
+    }
+
+
+# DAG definition
+with DAG(
+    'surge_pricing_pipeline',
+    default_args=default_args,
+    description='Simple surge pricing pipeline: demand aggregation + rule-based pricing',
+    schedule_interval='*/15 * * * *',
+    start_date=days_ago(1),
+    catchup=False,
+    max_active_runs=1,
+    tags=['pricing', 'surge', 'research', 'simplified'],
+) as dag:
+
+    # parallel data fetching
+    t_fetch_interactions = PythonOperator(
+        task_id='fetch_interactions',
+        python_callable=fetch_interactions,
+        provide_context=True,
+    )
+
+    t_fetch_price_logs = PythonOperator(
+        task_id='fetch_price_logs',
+        python_callable=fetch_price_logs,
+        provide_context=True,
+    )
+
+    # compute demand from interactions
+    t_compute_demand = PythonOperator(
+        task_id='compute_demand',
+        python_callable=compute_demand,
+        provide_context=True,
+    )
+
+    # aggregate price logs
+    t_aggregate_prices = PythonOperator(
+        task_id='aggregate_price_logs',
+        python_callable=aggregate_price_logs,
+        provide_context=True,
+    )
+
+    # join demand and prices
+    t_join_features = PythonOperator(
+        task_id='join_product_features',
+        python_callable=join_product_features,
+        provide_context=True,
+    )
+
+    # apply surge pricing
+    t_surge_pricing = PythonOperator(
+        task_id='apply_surge_pricing',
+        python_callable=apply_surge_pricing,
+        provide_context=True,
+    )
+
+    # publish to registry
+    t_publish = PythonOperator(
+        task_id='publish_results',
+        python_callable=publish_results,
+        provide_context=True,
+    )
+
+    # dependency graph: parallel fetch -> process -> join -> surge -> publish
+    t_fetch_interactions >> t_compute_demand
+    t_fetch_price_logs >> t_aggregate_prices
+    [t_compute_demand, t_aggregate_prices] >> t_join_features >> t_surge_pricing >> t_publish

experiments/ml/__init__.py

@@ -0,0 +1,11 @@
+from .evals import evaluate
+from .arch import (
+    XGBoostAgentClassifier,
+    LightGBMAgentClassifier
+)
+
+__all__ =[
+    'evaluate',
+    'XGBoostAgentClassifier',
+    'LightGBMAgentClassifier'
+]

experiments/ml/arch.py

@@ -0,0 +1,122 @@
+# sklearn compatible models for agent detection
+from sklearn.base import BaseEstimator, ClassifierMixin
+from procesing.context import PipelineContext
+from typing import Any, Optional, Tuple
+from abc import ABC, abstractmethod
+import xgboost as xgb
+import lightgbm as lgb
+import numpy as np
+import pandas as pd
+
+TASK = 'classification'
+LABELS = ['human', 'agent']
+
+
+class BaseAgentClassifier(BaseEstimator, ClassifierMixin, ABC):
+    """Base class for tree-based agent detection classifiers with common logic"""
+
+    def __init__(self, context: Optional[PipelineContext] = None, n_estimators: int = 200,
+                 max_depth: int = 6, learning_rate: float = 0.05,
+                 early_stopping_rounds: int = 20):
+        self.context = context
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.learning_rate = learning_rate
+        self.early_stopping_rounds = early_stopping_rounds
+        self.model_ = None
+        self.feature_names_ = None
+
+    def _to_array(self, X):
+        """Convert pandas structures to numpy arrays"""
+        return X.values if isinstance(X, (pd.DataFrame, pd.Series)) else X
+
+    def _compute_pos_weight(self, y_arr):
+        """Calculate scale_pos_weight for class imbalance handling"""
+        n_neg, n_pos = (y_arr == 0).sum(), (y_arr == 1).sum()
+        return n_neg / n_pos if n_pos > 0 else 1.0
+
+    def _prepare_eval_set(self, eval_set):
+        """Convert eval_set to numpy arrays if needed"""
+        if not eval_set:
+            return None
+        X_val, y_val = eval_set[0]
+        return [(self._to_array(X_val), self._to_array(y_val))]
+
+    @abstractmethod
+    def _build_model(self, scale_pos: float):
+        """Build the underlying model instance (must be implemented by subclasses)"""
+        pass
+
+    @abstractmethod
+    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
+        """Fit model with evaluation set (must be implemented by subclasses)"""
+        pass
+
+    def fit(self, X, y, eval_set=None):
+        X_arr, y_arr = self._to_array(X), self._to_array(y)
+
+        if isinstance(X, pd.DataFrame):
+            self.feature_names_ = X.columns.tolist()
+
+        scale_pos = self._compute_pos_weight(y_arr)
+        self.model_ = self._build_model(scale_pos)
+
+        eval_arr = self._prepare_eval_set(eval_set)
+        if eval_arr:
+            self._fit_with_eval(X_arr, y_arr, eval_arr)
+        else:
+            self.model_.fit(X_arr, y_arr)
+
+        return self
+
+    def predict(self, X):
+        return self.model_.predict(self._to_array(X))
+
+    def predict_proba(self, X):
+        return self.model_.predict_proba(self._to_array(X))
+
+    @property
+    def feature_importances_(self):
+        return self.model_.feature_importances_ if self.model_ else None
+
+
+class XGBoostAgentClassifier(BaseAgentClassifier):
+    """XGBoost binary classifier for agent detection with class imbalance handling"""
+
+    def _build_model(self, scale_pos: float):
+        return xgb.XGBClassifier(
+            n_estimators=self.n_estimators,
+            max_depth=self.max_depth,
+            learning_rate=self.learning_rate,
+            scale_pos_weight=scale_pos,
+            eval_metric='auc',
+            early_stopping_rounds=self.early_stopping_rounds,
+            random_state=42,
+            tree_method='hist',
+            enable_categorical=False
+        )
+
+    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
+        self.model_.fit(X_arr, y_arr, eval_set=eval_arr, verbose=False)
+
+
+class LightGBMAgentClassifier(BaseAgentClassifier):
+    """LightGBM binary classifier for agent detection with class imbalance handling"""
+
+    def _build_model(self, scale_pos: float):
+        return lgb.LGBMClassifier(
+            n_estimators=self.n_estimators,
+            max_depth=self.max_depth,
+            learning_rate=self.learning_rate,
+            scale_pos_weight=scale_pos,
+            metric='auc',
+            random_state=42,
+            verbosity=-1
+        )
+
+    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
+        self.model_.fit(
+            X_arr, y_arr,
+            eval_set=eval_arr,
+            callbacks=[lgb.early_stopping(self.early_stopping_rounds, verbose=False)]
+        )

experiments/ml/evals.py

@@ -0,0 +1,103 @@
+from sklearn.metrics import (accuracy_score, precision_score, recall_score,
+                             f1_score, roc_auc_score, confusion_matrix, roc_curve)
+from torch.utils.tensorboard import SummaryWriter
+from logging import getLogger
+import numpy as np
+import matplotlib.pyplot as plt
+import io
+from PIL import Image
+
+logger = getLogger(__name__)
+
+
+def log_feature_importance(writer, model, feature_names, epoch):
+    """Visualize and log feature importance to TensorBoard"""
+    if not hasattr(model, 'feature_importances_') or model.feature_importances_ is None:
+        return
+
+    importance = model.feature_importances_
+    indices = np.argsort(importance)[::-1][:20]  # top 20
+    top_features = [feature_names[i] for i in indices]
+    top_importance = importance[indices]
+
+    for i, (feat, imp) in enumerate(zip(top_features, top_importance)):
+        writer.add_scalar(f'FeatureImportance/{feat}', imp, epoch)
+
+    fig, ax = plt.subplots(figsize=(10, 8))
+    ax.barh(range(len(top_features)), top_importance, align='center')
+    ax.set_yticks(range(len(top_features)))
+    ax.set_yticklabels(top_features)
+    ax.invert_yaxis()
+    ax.set_xlabel('Importance')
+    ax.set_title(f'Top 20 Feature Importance (Epoch {epoch})')
+    ax.grid(axis='x', alpha=0.3)
+
+    buf = io.BytesIO()
+    plt.tight_layout()
+    plt.savefig(buf, format='png', dpi=100)
+    buf.seek(0)
+    img = Image.open(buf)
+    img_arr = np.array(img)
+    writer.add_image('FeatureImportance/Chart', img_arr, epoch, dataformats='HWC')
+    plt.close()
+
+def evaluate(perdicted_class, predicted_proba, true_class, writer: SummaryWriter, epoch: int):
+    accuracy = accuracy_score(true_class, perdicted_class)
+    precision = precision_score(true_class, perdicted_class, zero_division=0)
+    recall = recall_score(true_class, perdicted_class, zero_division=0)
+    f1 = f1_score(true_class, perdicted_class, zero_division=0)
+    roc_auc = roc_auc_score(true_class, predicted_proba)
+
+    writer.add_scalar('Eval/Accuracy', accuracy, epoch)
+    writer.add_scalar('Eval/Precision', precision, epoch)
+    writer.add_scalar('Eval/Recall', recall, epoch)
+    writer.add_scalar('Eval/F1_Score', f1, epoch)
+    writer.add_scalar('Eval/ROC_AUC', roc_auc, epoch)
+
+    # confusion matrix
+    cm = confusion_matrix(true_class, perdicted_class)
+    tn, fp, fn, tp = cm.ravel()
+    writer.add_scalar('Eval/TrueNeg', tn, epoch)
+    writer.add_scalar('Eval/FalsePos', fp, epoch)
+    writer.add_scalar('Eval/FalseNeg', fn, epoch)
+    writer.add_scalar('Eval/TruePos', tp, epoch)
+
+    # specificity and sensitivity
+    specificity = tn / (tn + fp) if (tn + fp) > 0 else 0
+    sensitivity = recall  # same as recall/TPR
+    writer.add_scalar('Eval/Specificity', specificity, epoch)
+    writer.add_scalar('Eval/Sensitivity', sensitivity, epoch)
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
+    ax1.matshow(cm, cmap='Blues', alpha=0.7)
+    for i in range(2):
+        for j in range(2):
+            ax1.text(j, i, str(cm[i, j]), ha='center', va='center', fontsize=14)
+    ax1.set_xlabel('Predicted')
+    ax1.set_ylabel('True')
+    ax1.set_title(f'Confusion Matrix (Epoch {epoch})')
+    ax1.set_xticks([0, 1])
+    ax1.set_yticks([0, 1])
+    ax1.set_xticklabels(['Human', 'Agent'])
+    ax1.set_yticklabels(['Human', 'Agent'])
+
+    # ROC curve
+    fpr, tpr, _ = roc_curve(true_class, predicted_proba)
+    ax2.plot(fpr, tpr, label=f'AUC={roc_auc:.3f}', linewidth=2)
+    ax2.plot([0, 1], [0, 1], 'k--', label='Random')
+    ax2.set_xlabel('False Positive Rate')
+    ax2.set_ylabel('True Positive Rate')
+    ax2.set_title('ROC Curve')
+    ax2.legend()
+    ax2.grid(alpha=0.3)
+
+    buf = io.BytesIO()
+    plt.tight_layout()
+    plt.savefig(buf, format='png', dpi=100)
+    buf.seek(0)
+    img = Image.open(buf)
+    img_arr = np.array(img)
+    writer.add_image('Eval/Metrics', img_arr, epoch, dataformats='HWC')
+    plt.close()
+
+    logger.info(f"Eval {epoch}: Acc={accuracy:.4f} Prec={precision:.4f} Rec={recall:.4f} F1={f1:.4f} AUC={roc_auc:.4f}")

experiments/ml/requirements.txt

@@ -0,0 +1,6 @@
+torch
+tensorboard
+fastparquet
+pyarrow
+xgboost
+lightgbm

experiments/ml/train.py

@@ -0,0 +1,137 @@
+from torch.utils.tensorboard import SummaryWriter
+from sklearn.model_selection import train_test_split
+from logging import getLogger
+from pathlib import Path
+import pandas as pd
+import numpy as np
+import joblib
+from datetime import datetime
+from ml.evals import evaluate, log_feature_importance
+from ml.arch import XGBoostAgentClassifier, LightGBMAgentClassifier, LABELS
+
+logger = getLogger(__name__)
+
+FEATURE_COLS_EXCLUDE = ['sessionId', 'experimentId', 'is_agent', 'xp_human_only', 'xp_market_mode', 'browser_family']
+RUNS_DIR = Path('ml/runs')
+CHECKPOINTS_DIR = Path('ml/checkpoints')
+
+
+def prepare_data(df):
+    """
+    Prepare feature matrix and labels from raw dataframe
+    Handles missing labels, feature selection, and categorical encoding
+    Returns: (X, y, feature_cols)
+    """
+    # drop rows with missing labels
+    n_before = len(df)
+    df = df[df['is_agent'].notna()].copy()
+    n_dropped = n_before - len(df)
+    if n_dropped > 0:
+        logger.warning(f"Dropped {n_dropped} sessions with missing labels")
+
+    if len(df) == 0:
+        logger.error("No labeled data available")
+        return None, None, None
+
+    feature_cols = [c for c in df.columns if c not in FEATURE_COLS_EXCLUDE]
+
+    # handle categorical browser_family via one-hot encoding
+    if 'browser_family' in df.columns:
+        browser_dummies = pd.get_dummies(df['browser_family'], prefix='browser', drop_first=True)
+        df = pd.concat([df, browser_dummies], axis=1)
+        feature_cols.extend(browser_dummies.columns.tolist())
+
+    X = df[feature_cols].fillna(0)
+    y = df['is_agent'].astype(int)
+
+    return X, y, feature_cols
+
+
+def train(data_path=None, model_type='xgboost', test_size=0.2, random_state=42,
+          n_estimators=200, max_depth=6, learning_rate=0.05):
+    """
+    Train agent detection classifier
+    Args:
+        data_path: path to labeled feature matrix CSV or parquet
+        model_type: 'xgboost' or 'lightgbm'
+        test_size: fraction for test split
+        random_state: seed for reproducibility
+    """
+    RUNS_DIR.mkdir(exist_ok=True)
+    CHECKPOINTS_DIR.mkdir(exist_ok=True)
+
+    run_name = f"{model_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+    writer = SummaryWriter(log_dir=RUNS_DIR / run_name)
+    logger.info(f"Starting training run: {run_name}")
+
+    # load data
+    if data_path is None:
+        logger.error("data_path required")
+        return
+    df = pd.read_parquet(data_path)
+    logger.info(f"Loaded {len(df)} sessions from {data_path}")
+
+    # prepare features and labels
+    if 'is_agent' not in df.columns:
+        logger.error("Missing is_agent column")
+        return
+
+    X, y, feature_cols = prepare_data(df)
+    if X is None:
+        return
+
+    # class distribution
+    n_agents = y.sum()
+    n_humans = (y == 0).sum()
+    logger.info(f"Class distribution: {n_humans} humans, {n_agents} agents" + (f" (ratio {n_humans / n_agents:.2f})" if n_agents > 0 else ""))
+
+    # train/test split with stratification
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=random_state, stratify=y
+    )
+    logger.info(f"Train: {len(X_train)}, Test: {len(X_test)}")
+
+    # init model
+    if model_type == 'xgboost':
+        model = XGBoostAgentClassifier(
+            n_estimators=n_estimators,
+            max_depth=max_depth,
+            learning_rate=learning_rate
+        )
+    elif model_type == 'lightgbm':
+        model = LightGBMAgentClassifier(
+            n_estimators=n_estimators,
+            max_depth=max_depth,
+            learning_rate=learning_rate
+        )
+    else:
+        logger.error(f"Unknown model type: {model_type}")
+        return
+
+    # train with eval set for early stopping
+    model.fit(X_train, y_train, eval_set=[(X_test, y_test)])
+    logger.info("Training complete")
+
+    # evaluate on test set
+    y_pred = model.predict(X_test)
+    y_prob = model.predict_proba(X_test)[:, 1]
+
+    evaluate(y_pred, y_prob, y_test, writer, epoch=0)
+
+    # log feature importance
+    log_feature_importance(writer, model, X.columns.tolist(), epoch=0)
+
+    # save model
+    model_path = CHECKPOINTS_DIR / f"{run_name}.pkl"
+    joblib.dump({'model': model, 'feature_cols': X.columns.tolist(), 'run_name': run_name}, model_path)
+    logger.info(f"Model saved to {model_path}")
+
+    writer.close()
+    return model, X.columns.tolist()
+
+
+if __name__ == "__main__":
+    import sys
+    data_path = sys.argv[1]
+    model_type = sys.argv[2] if len(sys.argv) > 2 else 'xgboost'
+    train(data_path, model_type=model_type)

experiments/procesing/__init__.py

@@ -1,19 +1,51 @@
-from .extract import (
-    KafkaDataFetcher,
-    ExperimentJoiner,
-    EventTitleAugmenter,
+from procesing.context import PipelineContext
+from procesing.providers import DataProvider, SupabaseProvider, BackendAPIProvider
+from procesing.steps import (
+    BaseContextStep,
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    FetchExperimentsStep,
+    JoinExperimentsStep,
+    CreatePriceBucketsStep,
+    AugmentEventNamesStep,
+    ChunkByTimeWindowStep,
+    ComputeDemandStep,
+    ComputeDemandForChunksStep,
+    AggregatePriceLogsStep,
+    # StateSpace,
+    # BuildStateSpaceStep,
+    FitPricingFunctionStep,
+    PredictPricesStep,
+)
+from procesing.pipelines import (
+    interaction_extraction_pipeline,
+    price_extraction_pipeline,
+    pricing_pipeline,
+    full_pipeline,
 )
-from .demand import DemandEstimator
-from .mapping import SessionTransitionProbMatrixTransformer, render_graph
-from .pipeline import etl_pipeline, pricing_pipeline
 
 __all__ = [
-    'KafkaDataFetcher',
-    'ExperimentJoiner',
-    'EventTitleAugmenter',
-    'DemandEstimator',
-    'SessionTransitionProbMatrixTransformer',
-    'render_graph',
-    'etl_pipeline',
+    'PipelineContext',
+    'DataProvider',
+    'SupabaseProvider',
+    'BackendAPIProvider',
+    'BaseContextStep',
+    'FetchInteractionsStep',
+    'FetchPriceLogsStep',
+    'FetchExperimentsStep',
+    'JoinExperimentsStep',
+    'CreatePriceBucketsStep',
+    'AugmentEventNamesStep',
+    'ChunkByTimeWindowStep',
+    'ComputeDemandStep',
+    'ComputeDemandForChunksStep',
+    'AggregatePriceLogsStep',
+    # 'StateSpace',
+    # 'BuildStateSpaceStep',
+    'FitPricingFunctionStep',
+    'PredictPricesStep',
+    'interaction_extraction_pipeline',
+    'price_extraction_pipeline',
     'pricing_pipeline',
+    'full_pipeline',
 ]

experiments/procesing/context.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict
+import pandas as pd
+from procesing.providers.base import DataProvider
+
+class PipelineContext:
+    """
+    Context for pipeline execution holding config, provider, and cached data.
+    Enables dependency injection and eliminates global state.
+    """
+
+    def __init__(self,
+                 provider: DataProvider,
+                 store_mode: str,
+                 window_size: str = '30s',
+                 **config):
+        self.provider = provider
+        self.store_mode = store_mode
+        self.window_size = window_size
+        self.config = config
+        self._cache: Dict[str, Any] = {}
+
+    def get_cached(self, key: str, default=None):
+        return self._cache.get(key, default)
+
+    def cache(self, key: str, value):
+        self._cache[key] = value
+        return value
+
+    @property
+    def products(self) -> pd.DataFrame:
+        """Lazy-load and cache product catalog, single fetch per pipeline run"""
+        if 'products' not in self._cache:
+            self._cache['products'] = self.provider.fetch_products(self.store_mode)
+        return self._cache['products']

experiments/procesing/demand.py

@@ -1,119 +0,0 @@
-from sklearn.base import BaseEstimator, TransformerMixin
-import numpy as np
-import pandas as pd
-from supabase import create_client, Client
-from typing import Optional, Literal
-import os
-import logging
-log = logging.getLogger(__name__)
-
-SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
-SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
-
-supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-
-class ChunkInteractionsIntoSteps(BaseEstimator, TransformerMixin):
-    """
-    Split interaction data into time windows for temporal analysis.
-    Returns a list of dataframes, one per time window.
-    """
-    def __init__(self,
-                 window_size:str='1h',
-                 ts_col:str='ts',
-                 return_metadata:bool=True):
-        """
-        Args:
-            window_size: pandas freq string ('1h', '30T', '1D', etc)
-            ts_col: timestamp column name
-            return_metadata: if True, return dict with metadata per chunk
-        """
-        self.window_size = window_size
-        self.ts_col = ts_col
-        self.return_metadata = return_metadata
-
-    def fit(self, X):
-        return self
-
-    def transform(self, interactions: pd.DataFrame):
-        """
-        Returns:
-            if return_metadata=False: list of dataframes, one per window
-            if return_metadata=True: list of dicts with keys:
-                - 'data': dataframe for this window
-                - 'window_start': start timestamp
-                - 'window_end': end timestamp
-                - 'window_idx': integer index
-        """
-        if interactions.empty:
-            return []
-
-        df = interactions.copy()
-
-        # ensure timestamp is datetime
-        if not pd.api.types.is_datetime64_any_dtype(df[self.ts_col]):
-            df[self.ts_col] = pd.to_datetime(df[self.ts_col])
-
-        # sort by time
-        df = df.sort_values(self.ts_col)
-
-        # assign window
-        df['_window'] = df[self.ts_col].dt.floor(self.window_size)
-
-        # group by window
-        chunks = []
-        for idx, (window_start, group) in enumerate(df.groupby('_window')):
-            chunk_data = group.drop(columns=['_window'])
-
-            if self.return_metadata:
-                chunks.append({
-                    'data': chunk_data,
-                    'window_start': window_start,
-                    'window_end': window_start + pd.Timedelta(self.window_size),
-                    'window_idx': idx
-                })
-            else:
-                chunks.append(chunk_data)
-
-        return chunks
-
-
-class DemandEstimator(BaseEstimator, TransformerMixin):
-    def __init__(self,
-                 store_mode:str='hotel',
-                 session_filter:str="",
-                 experiment_filter:str=""):
-        self.store=store_mode
-        self.session_filter=session_filter if len(session_filter)>0 else None
-        self.experiment_filter=experiment_filter if len(experiment_filter)>0 else None
-    def fit(self, X):
-        return self
-
-    def transform(self, interactions : pd.DataFrame):
-        if interactions.empty:
-            return pd.DataFrame(columns=["productId", "demand_score"])
-        if self.session_filter:
-            interactions = interactions[interactions['sessionId'] == self.session_filter]
-        if self.experiment_filter:
-            interactions = interactions[interactions['experimentId'] == self.experiment_filter]
-
-        products=supabase.table(f'{self.store}_products').select("id, room_type, date_index, metadata, availability").execute()
-        products = pd.DataFrame(products.data)
-        unique_products = products['id'].unique()
-        log.info(f"Demand estimator found {len(unique_products)} in data")
-
-        # filter out rows without productId
-        interactions_with_products = interactions.dropna(subset=['productId'])
-
-        if interactions_with_products.empty:
-            # no interactions with products, return all zeros
-            return pd.DataFrame({
-                'productId': unique_products,
-                'demand_score': 0
-            })
-
-        # TODO: improve demand score calculation rather than just counting interactions (use weights..)
-        # while maintaining simplicity of a simple cross tab approach
-        product_demand = pd.crosstab(interactions_with_products['productId'], "no_of_interactions")
-        product_demand = product_demand.reindex(unique_products, fill_value=0).reset_index()
-        product_demand.columns = ['productId', 'demand_score']
-        return product_demand

experiments/procesing/elasticity.py

@@ -130,25 +130,24 @@ class TemporalElasticityEstimator(BaseEstimator, TransformerMixin):
 
     def _build_product_timeseries(self, aligned_chunks):
         """Build time series [price, quantity] per product."""
-        series_by_product = {}
-
+        # vectorize chunk merging instead of iterating rows
+        all_merged = []
         for chunk in aligned_chunks:
-            demand_df = chunk['demand']
-            price_df = chunk['prices']
+            merged = chunk['demand'].merge(chunk['prices'], on='productId', how='inner')
+            merged['timestamp'] = chunk['window_start']
+            all_merged.append(merged[['productId', 'timestamp', 'price', 'demand_score']])
 
-            # merge on productId
-            merged = demand_df.merge(price_df, on='productId', how='inner')
+        if not all_merged:
+            return {}
 
-            for _, row in merged.iterrows():
-                pid = row['productId']
-                if pid not in series_by_product:
-                    series_by_product[pid] = []
-
-                series_by_product[pid].append({
-                    'timestamp': chunk['window_start'],
-                    'price': row['price'],
-                    'quantity': row['demand_score']
-                })
+        # concat all chunks and group by productId in one pass
+        combined = pd.concat(all_merged, ignore_index=True)
+        series_by_product = {
+            pid: group[['timestamp', 'price', 'demand_score']].rename(
+                columns={'demand_score': 'quantity'}
+            ).to_dict('records')
+            for pid, group in combined.groupby('productId')
+        }
 
         return series_by_product
 

experiments/procesing/extract.py

@@ -1,207 +0,0 @@
-import pandas as pd
-import json
-import numpy as np
-import os
-import requests
-from dotenv import load_dotenv
-from sklearn.base import BaseEstimator, TransformerMixin
-from supabase import create_client, Client
-from typing import Tuple, List, Dict
-load_dotenv()
-
-BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:5000")
-SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
-SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
-N_PRICE_BUCKETS = 5
-
-supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-
-
-class KafkaDataFetcher(BaseEstimator, TransformerMixin):
-    def __init__(self, topic: str = "user-interactions"):
-        self.topic = topic # also can be price-logs
-    def fit(self, X=None, y=None):
-        return self
-
-    def transform(self, X=None):
-        resp = requests.get(f"{BACKEND_URL}/api/kafka/dump?topic={self.topic}")
-        resp.raise_for_status()
-        data = resp.json()
-
-        if not data.get('success') or not data.get('data'):
-            return pd.DataFrame()
-
-        df = pd.DataFrame(data['data'])
-        if self.topic == 'user-interactions':
-            if 'metadata' in df.columns: # explode metadata col json
-                df = df.join(pd.json_normalize(df.pop("metadata"), sep=".").add_prefix("metadata_"))
-            df = df.dropna(subset=['eventName'])
-            # remape dateIndex
-            df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
-        return df
-
-
-class ExperimentJoiner(BaseEstimator, TransformerMixin):
-    def fit(self, X=None, y=None):
-        return self
-
-    def transform(self, df):
-        if df.empty or 'experimentId' not in df.columns:
-            return df
-
-        unique_exp_ids = df['experimentId'].dropna().unique()
-        if len(unique_exp_ids) == 0:
-            return df
-
-        resp = supabase.table('experiments').select(
-            'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, task:tasks(task_name, task_description, task_def_of_done)'
-        ).in_('id', unique_exp_ids.tolist()).execute()
-
-        if not resp.data:
-            return df
-
-        exp_df = pd.DataFrame(resp.data)
-
-        # flatten task nested object if present
-        if 'task' in exp_df.columns and exp_df['task'].notnull().any():
-            task_normalized = pd.json_normalize(exp_df['task'].dropna())
-            task_normalized.index = exp_df[exp_df['task'].notnull()].index
-            exp_df = exp_df.drop(columns=['task']).join(task_normalized, rsuffix='_task')
-
-        # rename experiment columns for clarity
-        exp_df = exp_df.rename(columns={
-            'id': 'experimentId',
-            'subject_name': 'exp_subject',
-            'xp_human_only': 'exp_human_only',
-            'xp_market_mode': 'exp_market_mode',
-            'xp_task_id': 'exp_task_id'
-        })
-
-        df = df.merge(exp_df, on='experimentId', how='left')
-        return df
-
-
-class EventTitleAugmenter(BaseEstimator, TransformerMixin):
-    def fit(self, X=None, y=None):
-        return self
-
-    def transform(self, df):
-        # from taking standard view_item_page in eventName to view_item_page_{metadata_schema}
-        # we want metadata schema to create product specific event names
-
-        # only create price buckets if we have enough unique prices
-        if df["metadata_price"].notnull().sum() > 0:
-            try:
-                price_buckets = pd.qcut(
-                    df["metadata_price"],
-                    q=N_PRICE_BUCKETS,
-                    labels=[f"PB_{i+1}" for i in range(N_PRICE_BUCKETS)],
-                    duplicates='drop'  # handle duplicate bin edges
-                )
-            except ValueError:
-                # fallback: if still not enough unique values, use cut with fixed ranges or just use raw price
-                price_buckets = df["metadata_price"].apply(lambda x: f"P_{int(x)}" if pd.notnull(x) else "")
-        else:
-            price_buckets = pd.Series([""] * len(df), index=df.index)
-
-        # metadata_schema: _product_id@price_bucket_{i} only if we have product metadata otherswise keep original event name
-        # TODO: make this adaptive, if we have hover_over_title we append the title, if its view_page we say which page
-        df["metadata_schema"] = np.where(
-            df["productId"].notnull() & df["metadata_price"].notnull(),
-            "_" + df["productId"].astype(str) + "@" + price_buckets.astype(str),
-            ""
-        )
-        df["eventName"] = df["eventName"] + df["metadata_schema"].astype(str)
-        return df
-
-
-def chunk_shared_data(interactions_df: pd.DataFrame,
-                      price_logs_df: pd.DataFrame,
-                      window_size: str = '30s',
-                      ts_col: str = 'ts') -> Tuple[List[Dict], List[Dict]]:
-    """
-    Chunk interaction and price data into aligned time windows.
-
-    Args:
-        interactions_df: interaction data with timestamp column
-        price_logs_df: price log data with timestamp column
-        window_size: pandas freq string ('30s', '1min', '1h', etc)
-        ts_col: name of timestamp column
-
-    Returns:
-        tuple of (interaction_chunks, price_chunks) where each is list of dicts:
-        {
-            'window_start': timestamp,
-            'window_end': timestamp,
-            'data': dataframe for this window
-        }
-    """
-    if interactions_df.empty and price_logs_df.empty:
-        return [], []
-
-    # convert timestamps to datetime
-    interactions_df = interactions_df.copy()
-    price_logs_df = price_logs_df.copy()
-
-    if not interactions_df.empty:
-        if not pd.api.types.is_datetime64_any_dtype(interactions_df[ts_col]):
-            interactions_df[ts_col] = pd.to_datetime(interactions_df[ts_col])
-
-    if not price_logs_df.empty:
-        if not pd.api.types.is_datetime64_any_dtype(price_logs_df[ts_col]):
-            price_logs_df[ts_col] = pd.to_datetime(price_logs_df[ts_col])
-
-    # find global time bounds
-    times = []
-    if not interactions_df.empty:
-        times.extend([interactions_df[ts_col].min(), interactions_df[ts_col].max()])
-    if not price_logs_df.empty:
-        times.extend([price_logs_df[ts_col].min(), price_logs_df[ts_col].max()])
-
-    if not times:
-        return [], []
-
-    earliest = min(times)
-    latest = max(times)
-
-    # create shared time windows
-    windows = pd.date_range(start=earliest, end=latest, freq=window_size)
-
-    if len(windows) < 2:
-        return [], []
-
-    # chunk both datasets
-    interaction_chunks = []
-    price_chunks = []
-
-    for i in range(len(windows) - 1):
-        window_start = windows[i]
-        window_end = windows[i + 1]
-
-        # filter interactions in this window
-        if not interactions_df.empty:
-            mask = (interactions_df[ts_col] >= window_start) & (interactions_df[ts_col] < window_end)
-            interaction_chunk = interactions_df[mask]
-        else:
-            interaction_chunk = pd.DataFrame()
-
-        interaction_chunks.append({
-            'window_start': window_start,
-            'window_end': window_end,
-            'data': interaction_chunk
-        })
-
-        # filter price logs in this window
-        if not price_logs_df.empty:
-            mask = (price_logs_df[ts_col] >= window_start) & (price_logs_df[ts_col] < window_end)
-            price_chunk = price_logs_df[mask]
-        else:
-            price_chunk = pd.DataFrame()
-
-        price_chunks.append({
-            'window_start': window_start,
-            'window_end': window_end,
-            'data': price_chunk
-        })
-
-    return interaction_chunks, price_chunks

experiments/procesing/mapping.py

@@ -1,158 +0,0 @@
-import numpy as np
-import pandas as pd
-from sklearn.base import BaseEstimator, TransformerMixin
-
-def build_transition_prob_matrix(df: pd.DataFrame):
-    df = df.dropna(subset=['eventName'])
-    events = df['eventName'].tolist()
-    labels = pd.Index(events).unique().tolist()
-    idx = {e:i for i,e in enumerate(labels)}
-    M = np.zeros((len(labels), len(labels)), dtype=float)
-    for a, b in zip(events, events[1:]):
-        M[idx[a], idx[b]] += 1
-    row_sums = M.sum(axis=1, keepdims=True)
-    with np.errstate(divide='ignore', invalid='ignore'):
-        P = np.divide(M, row_sums, where=row_sums>0)  # row-normalized
-    return P, labels
-
-# https://medium.com/data-science/time-series-data-markov-transition-matrices-7060771e362b
-from graphviz import Digraph
-import numpy as np
-import pandas as pd
-
-def _as_prob_df(matrix, labels=None):
-    """Return a square DataFrame with index=columns=labels."""
-    if isinstance(matrix, pd.DataFrame):
-        # Ensure square and aligned
-        assert (matrix.index == matrix.columns).all(), "Index/columns must match."
-        return matrix
-    matrix = np.asarray(matrix, dtype=float)
-    assert matrix.shape[0] == matrix.shape[1], "Matrix must be square."
-    if labels is None:
-        raise ValueError("labels are required when matrix is not a DataFrame")
-    assert len(labels) == matrix.shape[0], "labels length must match matrix size."
-    return pd.DataFrame(matrix, index=list(labels), columns=list(labels))
-
-def _df_to_edgelist(P: pd.DataFrame, threshold=0.0, round_digits=2):
-    """Build weighted edges > threshold."""
-    edges = []
-    for src in P.index:
-        for dst in P.columns:
-            w = float(P.loc[src, dst])
-            if w > threshold:
-                edges.append((str(src), str(dst), f"{w:.{round_digits}f}"))
-    return edges
-
-def render_graph(fname, matrix, ls_index=None, threshold=0.0, fmt="svg", view=False):
-    """
-    fname: output file stem (no extension)
-    matrix: NumPy array or pandas DataFrame of transition PROBABILITIES
-    ls_index: ordered labels (required if matrix is not a DataFrame)
-    threshold: hide edges with weight <= threshold
-    fmt: 'svg'|'png'|'pdf' etc.
-    view: open after rendering
-    """
-    P = _as_prob_df(matrix, labels=ls_index)
-    edges = _df_to_edgelist(P, threshold=threshold)
-
-    g = Digraph(format=fmt)
-    g.attr(rankdir="LR", size="30")
-    g.attr("node", shape="circle")
-
-    # ensure isolated nodes appear
-    for node in P.index:
-        g.node(str(node), width="1", height="1")
-
-    for src, dst, label in edges:
-        g.edge(src, dst, label=label)
-
-    g.render(fname, view=view, cleanup=True)
-    return g
-
-
-class TransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
-    def __init__(self, threshold=0.0):
-        self.threshold = threshold
-        self.P_ = None
-        self.labels_ = None
-
-    def fit(self, X: pd.DataFrame, y=None):
-        P, labels = build_transition_prob_matrix(X)
-        self.P_ = P
-        self.labels_ = labels
-        return self
-
-    def transform(self, X: pd.DataFrame = None):
-        return self.P_, self.labels_
-
-    def render(self, fname: str, fmt="svg", view=False):
-        if self.P_ is None or self.labels_ is None:
-            raise ValueError("Transformer has not been fitted yet.")
-        return render_graph(
-            fname,
-            self.P_,
-            ls_index=self.labels_,
-            threshold=self.threshold,
-            fmt=fmt,
-            view=view
-        )
-
-
-class SessionTransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
-    def __init__(self, threshold=0.0, session_col='sessionId'):
-        self.threshold = threshold
-        self.session_col = session_col
-        self.session_matrices_ = None
-
-    def fit(self, X: pd.DataFrame, y=None):
-        if self.session_col not in X.columns:
-            raise ValueError(f"Column '{self.session_col}' not found in DataFrame")
-
-        session_matrices = {}
-        for session_id, grp in X.groupby(self.session_col):
-            if len(grp) > 1:  # need at least 2 events for transitions
-                P, labels = build_transition_prob_matrix(grp)
-                session_matrices[session_id] = {'matrix': P, 'labels': labels}
-
-        self.session_matrices_ = session_matrices
-        return self
-
-    def transform(self, X: pd.DataFrame = None):
-        if self.session_matrices_ is None:
-            raise ValueError("Transformer has not been fitted yet.")
-        return pd.Series(self.session_matrices_)
-
-    def render_session(self, session_id: str, fname: str, fmt="svg", view=False):
-        if self.session_matrices_ is None:
-            raise ValueError("Transformer has not been fitted yet.")
-        if session_id not in self.session_matrices_:
-            raise ValueError(f"Session '{session_id}' not found in fitted data.")
-
-        sess_data = self.session_matrices_[session_id]
-        return render_graph(
-            fname,
-            sess_data['matrix'],
-            ls_index=sess_data['labels'],
-            threshold=self.threshold,
-            fmt=fmt,
-            view=view
-        )
-if __name__ == "__main__":
-    # Example usage
-    data = {
-        'eventName': [
-            'A', 'B', 'A', 'C', 'B', 'A', 'A', 'C', 'B', 'C',
-            'A', 'B', 'C', 'A', 'B', 'C', 'A', 'B', 'C', 'A'
-        ]
-    }
-    df = pd.DataFrame(data)
-
-    transformer = TransitionProbMatrixTransformer(threshold=0.1)
-    transformer.fit(df)
-    P, labels = transformer.transform(None)
-
-    print("Transition Probability Matrix:")
-    print(pd.DataFrame(P, index=labels, columns=labels))
-
-    # Render the graph
-    transformer.render("transition_graph", fmt="svg", view=False)

experiments/procesing/metrics.py

@@ -0,0 +1,245 @@
+"""
+Revenue and KPI benchmark framework for pricing strategies.
+
+Computes session-level and aggregate metrics to compare pricing functions:
+    - Revenue: R_T = Σ P_t^T · Q_t
+    - Conversion rate
+    - Average order value (AOV)
+    - Agent exploitation loss: L_agent = R_oracle - R_observed
+"""
+from typing import Dict, List, Any, Optional
+from dataclasses import dataclass, field, asdict
+import pandas as pd
+import numpy as np
+
+
+@dataclass
+class SessionMetrics:
+    """KPIs for single session."""
+    session_id: str
+    experiment_id: Optional[str] = None
+
+    # interaction metrics
+    total_interactions: int = 0
+    page_views: int = 0
+    item_views: int = 0
+    searches: int = 0
+    cart_adds: int = 0
+
+    # revenue metrics
+    items_purchased: int = 0
+    total_revenue: float = 0.0
+    avg_item_price: float = 0.0
+    conversion_rate: float = 0.0
+
+    # pricing signals
+    total_price_shown: float = 0.0  # sum of all prices displayed
+    avg_markup: float = 0.0  # avg (price / base_price)
+
+    # behavioral features (for agent detection)
+    interaction_velocity: float = 0.0  # interactions per minute
+    session_duration_sec: float = 0.0
+    unique_products_viewed: int = 0
+
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class AggregateMetrics:
+    """Aggregate KPIs across sessions/experiments."""
+    experiment_id: Optional[str] = None
+    n_sessions: int = 0
+
+    # revenue aggregates
+    total_revenue: float = 0.0
+    avg_revenue_per_session: float = 0.0
+    median_revenue_per_session: float = 0.0
+
+    # conversion aggregates
+    total_conversions: int = 0
+    conversion_rate: float = 0.0  # purchases / sessions
+
+    # pricing aggregates
+    avg_markup: float = 0.0
+    median_markup: float = 0.0
+
+    # agent exploitation metrics
+    estimated_agent_sessions: int = 0  # sessions flagged as agent-driven
+    agent_revenue: float = 0.0
+    human_revenue: float = 0.0
+    agent_loss: float = 0.0  # L_agent = R_oracle - R_observed (if available)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class MetricsComputer:
+    """Compute session and aggregate metrics from interaction/price logs."""
+
+    @staticmethod
+    def compute_session_metrics(
+        session_id: str,
+        interactions: pd.DataFrame,
+        price_logs: pd.DataFrame,
+        purchases: Optional[pd.DataFrame] = None,
+        experiment_id: Optional[str] = None
+    ) -> SessionMetrics:
+        """
+        Compute metrics for single session.
+
+        Args:
+            session_id: session identifier
+            interactions: user-interactions events for this session
+            price_logs: price-logs for this session
+            purchases: purchase events (if available)
+            experiment_id: experiment identifier
+        """
+        metrics = SessionMetrics(session_id=session_id, experiment_id=experiment_id)
+
+        if interactions.empty:
+            return metrics
+
+        # interaction counts
+        event_counts = interactions['eventName'].value_counts().to_dict()
+        metrics.total_interactions = len(interactions)
+        metrics.page_views = event_counts.get('page_view', 0) + event_counts.get('view_item_page', 0)
+        metrics.item_views = event_counts.get('view_item_page', 0)
+        metrics.searches = event_counts.get('search', 0)
+        metrics.cart_adds = event_counts.get('add_item_to_cart', 0)
+
+        # unique products viewed
+        metrics.unique_products_viewed = interactions['productId'].dropna().nunique()
+
+        # session duration
+        if 'ts' in interactions.columns:
+            timestamps = pd.to_datetime(interactions['ts'])
+            metrics.session_duration_sec = (timestamps.max() - timestamps.min()).total_seconds()
+            if metrics.session_duration_sec > 0:
+                metrics.interaction_velocity = (metrics.total_interactions / metrics.session_duration_sec) * 60
+
+        # revenue from purchases
+        if purchases is not None and not purchases.empty:
+            metrics.items_purchased = len(purchases)
+            metrics.total_revenue = purchases['price'].sum() if 'price' in purchases.columns else 0.0
+            metrics.avg_item_price = metrics.total_revenue / metrics.items_purchased if metrics.items_purchased > 0 else 0.0
+            metrics.conversion_rate = 1.0 if metrics.items_purchased > 0 else 0.0
+
+        # pricing metrics
+        if not price_logs.empty:
+            metrics.total_price_shown = price_logs['price'].sum()
+            # compute markup if base_price available in price logs or join with product catalog
+            if 'base_price' in price_logs.columns:
+                valid_markup = price_logs[price_logs['base_price'] > 0]
+                if not valid_markup.empty:
+                    metrics.avg_markup = (valid_markup['price'] / valid_markup['base_price']).mean()
+
+        return metrics
+
+    @staticmethod
+    def compute_aggregate_metrics(
+        session_metrics_list: List[SessionMetrics],
+        experiment_id: Optional[str] = None,
+        agent_detector_fn: Optional[callable] = None
+    ) -> AggregateMetrics:
+        """
+        Aggregate metrics across sessions.
+
+        Args:
+            session_metrics_list: list of SessionMetrics
+            experiment_id: experiment identifier
+            agent_detector_fn: optional function to classify session as agent (returns bool)
+        """
+        agg = AggregateMetrics(experiment_id=experiment_id)
+        agg.n_sessions = len(session_metrics_list)
+
+        if agg.n_sessions == 0:
+            return agg
+
+        df = pd.DataFrame([m.to_dict() for m in session_metrics_list])
+
+        # revenue aggregates
+        agg.total_revenue = df['total_revenue'].sum()
+        agg.avg_revenue_per_session = df['total_revenue'].mean()
+        agg.median_revenue_per_session = df['total_revenue'].median()
+
+        # conversion aggregates
+        agg.total_conversions = (df['items_purchased'] > 0).sum()
+        agg.conversion_rate = agg.total_conversions / agg.n_sessions
+
+        # pricing aggregates
+        valid_markups = df[df['avg_markup'] > 0]
+        if not valid_markups.empty:
+            agg.avg_markup = valid_markups['avg_markup'].mean()
+            agg.median_markup = valid_markups['avg_markup'].median()
+
+        # agent detection (if detector provided)
+        if agent_detector_fn is not None:
+            agent_flags = [agent_detector_fn(m) for m in session_metrics_list]
+            agg.estimated_agent_sessions = sum(agent_flags)
+
+            agent_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if is_agent)
+            human_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if not is_agent)
+
+            agg.agent_revenue = agent_revenue
+            agg.human_revenue = human_revenue
+
+        return agg
+
+    @staticmethod
+    def compare_pricing_strategies(
+        experiments: Dict[str, List[SessionMetrics]],
+        baseline_experiment_id: Optional[str] = None
+    ) -> pd.DataFrame:
+        """
+        Compare multiple pricing strategies/experiments.
+
+        Args:
+            experiments: dict mapping experiment_id -> list of SessionMetrics
+            baseline_experiment_id: experiment to use as baseline for comparison
+
+        Returns:
+            DataFrame with comparative metrics
+        """
+        results = []
+        baseline_agg = None
+
+        for exp_id, session_metrics in experiments.items():
+            agg = MetricsComputer.compute_aggregate_metrics(session_metrics, experiment_id=exp_id)
+            result = agg.to_dict()
+
+            if exp_id == baseline_experiment_id:
+                baseline_agg = agg
+
+            results.append(result)
+
+        df = pd.DataFrame(results)
+
+        # add relative metrics if baseline exists
+        if baseline_agg is not None:
+            df['revenue_lift_pct'] = ((df['total_revenue'] - baseline_agg.total_revenue) / baseline_agg.total_revenue * 100)
+            df['conversion_lift_pct'] = ((df['conversion_rate'] - baseline_agg.conversion_rate) / baseline_agg.conversion_rate * 100)
+
+        return df
+
+
+def simple_agent_detector(session_metrics: SessionMetrics, velocity_threshold: float = 5.0) -> bool:
+    """
+    Simple heuristic agent detector based on interaction velocity.
+
+    Args:
+        session_metrics: SessionMetrics instance
+        velocity_threshold: interactions per minute threshold (default: 5.0)
+
+    Returns:
+        True if session likely agent-driven
+    """
+    # agents tend to have higher interaction velocity and lower session duration
+    if session_metrics.interaction_velocity > velocity_threshold:
+        return True
+    # agents often view many products quickly without converting
+    if session_metrics.unique_products_viewed > 10 and session_metrics.conversion_rate == 0:
+        return True
+    return False

experiments/procesing/pipeline.py

@@ -1,90 +0,0 @@
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
-import pandas as pd
-import logging
-log = logging.getLogger(__name__)
-
-from extract import KafkaDataFetcher, ExperimentJoiner, EventTitleAugmenter, chunk_shared_data
-from mapping import SessionTransitionProbMatrixTransformer, render_graph
-from demand import DemandEstimator, ChunkInteractionsIntoSteps
-from elasticity import TemporalElasticityEstimator, aggregate_price_logs
-
-
-
-# elasticity pipeline components (not sklearn compatible, manual orchestration)
-def elasticity_pipeline(interactions_df, price_logs_df, window_size='30s', store_mode='hotel'):
-    """
-    Compute price elasticity from interaction and price data.
-
-    Args:
-        interactions_df: raw interaction data from demand_data_pipeline
-        price_logs_df: price log data from price_data_pipeline
-        window_size: time window for chunking
-        store_mode: 'hotel' or 'airline'
-
-    Returns:
-        df with [productId, elasticity, std_error, n_obs]
-    """
-    # step 1: chunk interactions into time windows
-    chunker = ChunkInteractionsIntoSteps(window_size=window_size, return_metadata=True)
-    interaction_chunks = chunker.transform(interactions_df)
-    log.info(f"Chunked interactions into {len(interaction_chunks)} windows of size {window_size}")
-
-    if not interaction_chunks:
-        return None
-
-    # step 2: compute demand per window
-    demand_estimator = DemandEstimator(store_mode=store_mode)
-    demand_chunks = []
-    for chunk in interaction_chunks:
-        demand_vector = demand_estimator.transform(chunk['data'])
-        demand_chunks.append({
-            'window_start': chunk['window_start'],
-            'window_end': chunk['window_end'],
-            'demand_vector': demand_vector # each has a full list of all products, even if demand is 0
-        })
-    # [q_chunk1, q_chunk2, ...]
-
-    # step 3: aggregate price logs into windows
-    price_chunks = aggregate_price_logs(price_logs_df, window_size=window_size)
-
-    # step 4: compute elasticity
-    elasticity_estimator = TemporalElasticityEstimator(method='point', min_observations=2)
-    elasticity_df = elasticity_estimator.transform(demand_chunks, price_chunks, store_mode=store_mode)
-
-    return elasticity_df
-
-
-# exposable pipelines
-interaction_pipeline = Pipeline([
-    ('kafka_fetch', KafkaDataFetcher(topic='user-interactions')),
-    ('experiment_join', ExperimentJoiner()),
-    ('event_augment', EventTitleAugmenter()),
-])
-
-price_data_pipeline = Pipeline([
-    ('kafka_fetch', KafkaDataFetcher(topic='price-logs')),
-])
-
-# interaction_data + price_data -> elasticity (demand)
-# elasticity -> pricing
-
-pricing_pipeline = Pipeline([
-    ('demand_estimation', DemandEstimator()),
-])
-if __name__ == "__main__":
-    # fetch both datasets
-    interaction_data = interaction_pipeline.fit_transform(None)
-    pricing_data = price_data_pipeline.fit_transform(None)
-    if interaction_data.empty or pricing_data.empty:
-        print("Insufficient data for elasticity computation"); exit(0)
-    # compute elasticity via unified pipeline
-    window_size = "30s"
-    elasticity_results = elasticity_pipeline(interaction_data, pricing_data, window_size=window_size)
-    elasticity_value_array = elasticity_results['elasticity'].values if elasticity_results is not None else np.array([])
-    print(elasticity_value_array)
-
-    if elasticity_results is not None and not elasticity_results.empty:
-        print(elasticity_results.to_string(index=False))
-    else:
-        print("\nInsufficient data for elasticity computation")

experiments/procesing/pipelines.py

@@ -0,0 +1,174 @@
+from sklearn.pipeline import Pipeline
+import pandas as pd
+from procesing.context import PipelineContext
+from procesing.providers import SupabaseProvider, BackendAPIProvider
+import os
+from procesing.steps import (
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    FetchExperimentsStep,
+    JoinExperimentsStep,
+    CreatePriceBucketsStep,
+    AugmentEventNamesStep,
+    ChunkByTimeWindowStep,
+    ComputeDemandForChunksStep,
+    AggregatePriceLogsStep,
+    FitPricingFunctionStep,
+    PredictPricesStep,
+    ComputeDemandStep,
+    JoinProductFeaturesStep,
+    ExtractSessionFeaturesStep,
+    JoinLabelsStep,
+    ValidateDataStep,
+)
+from procesing.pricers import SimpleSurgePricer
+
+def interaction_extraction_pipeline(context: PipelineContext):
+    """Pipeline for extracting and augmenting interaction data"""
+    return Pipeline([
+        ('fetch', FetchInteractionsStep(context)),
+        ('create_buckets', CreatePriceBucketsStep(context)),
+        ('augment_events', AugmentEventNamesStep(context)),
+    ])
+
+
+def price_extraction_pipeline(context: PipelineContext):
+    """Pipeline for extracting price logs"""
+    return Pipeline([
+        ('fetch', FetchPriceLogsStep(context)),
+    ])
+
+
+def product_features_pipeline(context: PipelineContext,
+                                    interactions_df: pd.DataFrame,
+                                    price_logs_df: pd.DataFrame):
+    demand_step = ComputeDemandStep(context)
+    price_step = AggregatePriceLogsStep(context)
+    join_step = JoinProductFeaturesStep(context)
+
+
+    demand_data = demand_step.transform(interactions_df)
+    price_data= price_step.transform(price_logs_df)
+    joined_data = join_step.transform((demand_data, price_data))
+
+    return joined_data
+
+
+
+def pricing_pipeline(context: "PipelineContext",
+                     data: pd.DataFrame,
+                     high_threshold: int = 10,
+                     low_threshold: int = 2,
+                     surge_multiplier: float = 1.2,
+                     discount_multiplier: float = 0.9) -> pd.DataFrame:
+
+    if data.empty or 'productId' not in data.columns:
+        return pd.DataFrame()
+
+    surge_pricer = SimpleSurgePricer()
+    surge_pricer.fit(data)
+    data['optimal_price'] = surge_pricer.predict()
+    return data
+
+
+def full_pipeline(context: PipelineContext,
+                  high_threshold: int = 10,
+                  low_threshold: int = 2,
+                  surge_multiplier: float = 1.2,
+                  discount_multiplier: float = 0.9):
+    """
+    Complete end-to-end pipeline: data extraction -> demand/price aggregation -> surge pricing
+
+    Args:
+        context: Pipeline context
+        high_threshold: Demand threshold for surge pricing
+        low_threshold: Demand threshold for discounts
+        surge_multiplier: Price multiplier for high demand
+        discount_multiplier: Price multiplier for low demand
+
+    Returns:
+        tuple: (product_features_df, optimal_prices_df)
+            - product_features_df: [productId, demand_score, price]
+            - optimal_prices_df: [productId, current_price, optimal_price, demand_score]
+    """
+    interaction_pipe = interaction_extraction_pipeline(context)
+    price_pipe = price_extraction_pipeline(context)
+
+    interactions_df = interaction_pipe.fit_transform(None)
+    price_logs_df = price_pipe.fit_transform(None)
+    product_features_df = product_features_pipeline(context, interactions_df, price_logs_df)
+    print(product_features_df.to_string())
+
+    # generate optimal prices using surge rules
+    optimal_prices_df = pricing_pipeline(context, product_features_df,
+                                          high_threshold=high_threshold,
+                                          low_threshold=low_threshold,
+                                          surge_multiplier=surge_multiplier,
+                                          discount_multiplier=discount_multiplier)
+
+    return product_features_df, optimal_prices_df
+
+
+def ml_training_pipeline(context: PipelineContext) -> pd.DataFrame:
+    """
+    Build labeled session-level feature matrix for ML model training.
+    Pipeline: fetch -> validate -> extract features -> join labels
+
+    Returns:
+        DataFrame with ~25 features per session + is_agent label
+        Columns: sessionId, experimentId, temporal/behavioral/product/ua features, is_agent
+    """
+    # fetch raw interactions
+    interactions_df = FetchInteractionsStep(context).transform(None)
+
+    # validate data quality (report cached in context)
+    interactions_df = ValidateDataStep(context).transform(interactions_df)
+    if interactions_df.empty:
+        return pd.DataFrame()
+
+    # extract vectorized session features
+    features_df = ExtractSessionFeaturesStep(context).transform(interactions_df)
+    if features_df.empty:
+        return pd.DataFrame()
+
+    # join experiment labels (is_agent = ~xp_human_only)
+    labeled_df = JoinLabelsStep(context).transform(features_df)
+
+    return labeled_df
+
+
+
+
+if __name__ == '__main__':
+
+    class ExperimentsProvider(SupabaseProvider, BackendAPIProvider):
+        def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
+            base_path = "/home/velocitatem/Documents/Projects/PHANTOM/experiments/collected_data/" # os.path.join(os.path.dirname(__file__), "collected_data")
+            if not os.path.isdir(base_path):
+                return pd.DataFrame()
+
+            files = {"user-interactions": "int.json", "price-logs": "price.json"}
+            file_to_read = files.get(topic, files["user-interactions"])
+            frames = []
+
+            for d in os.listdir(base_path):
+                full_path = os.path.join(base_path, d, file_to_read)
+                if not os.path.isfile(full_path):
+                    continue
+                try:
+                    data = pd.read_json(full_path)
+                    payloads = pd.DataFrame([r['payload'] for r in data['value'].to_list()])
+                    frames.append(payloads)
+                except Exception as e:
+                    print(f"Warning: Could not process {full_path}: {e}")
+
+            return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
+
+    # demo: run ML training pipeline
+    context = PipelineContext(provider=ExperimentsProvider(), store_mode='hotel')
+    features = ml_training_pipeline(context)
+    print(f"Feature matrix: {features.shape}")
+    print(features.head())
+    print(features.info())
+
+    features.to_parquet("features.parquet")

experiments/procesing/pricers/__init__.py

@@ -0,0 +1,14 @@
+from procesing.pricers.base import PricingFunction
+from procesing.pricers.elasticity import ElasticityBasedPricer
+from procesing.pricers.simple import StaticPricer, RandomPricer, SimpleSurgePricer
+from procesing.pricers.session_aware import SessionAwarePricer, ProductSpecificSessionPricer
+
+__all__ = [
+    'PricingFunction',
+    'ElasticityBasedPricer',
+    'StaticPricer',
+    'RandomPricer',
+    'SimpleSurgePricer',
+    'SessionAwarePricer',
+    'ProductSpecificSessionPricer'
+]

experiments/procesing/pricers/base.py

@@ -0,0 +1,70 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, List
+import numpy as np
+import pandas as pd
+
+
+class PricingFunction(ABC):
+    """
+    Abstract base for pricing functions.
+
+    Defines mapping: f(Q_t, P_t, S_t, H_t) -> P_{t+1}
+
+    Where:
+        Q_t ∈ R^n: demand vector at time t
+        P_t ∈ R^n: price vector at time t
+        S_t: session features (behavioral signals, interactions)
+        H_t = {Q_{t-k}, P_{t-k}, S_{t-k}}: historical state trajectory
+
+    Objective:
+        maximize E[R_T] = E[Σ P_t^T · Q_t]
+        subject to:
+            Q_t = g(P_t, S_t)  (demand response via elasticity)
+            P_t ≥ C  (cost floor)
+            minimize L_agent = R_oracle - R_observed
+    """
+
+    @abstractmethod
+    def fit(self, *kwargs):
+        """
+        Offline training on historical data.
+
+        Args:
+            historical_data: DataFrame with elasticity, prices, demand signals
+            **kwargs: additional training parameters
+        """
+        pass
+
+    @abstractmethod
+    def predict(self, *kwargs) -> np.ndarray:
+        """
+        Generate optimal prices given current state.
+
+        Args:
+            state_space: StateSpace object containing Q_t, P_t, S_t, H_t
+
+        Returns:
+            P_{t+1}: price vector in R^n
+        """
+        pass
+
+    def update(self, observation: Dict[str, Any]):
+        """
+        Online learning update (optional).
+
+        Args:
+            observation: dict with {state, action, reward, next_state}
+                - state: StateSpace before pricing decision
+                - action: prices shown (P_t)
+                - reward: revenue/conversion signal
+                - next_state: StateSpace after user interaction
+        """
+        pass  # default: no online learning
+
+    def get_params(self) -> Dict[str, Any]:
+        """Return pricing function parameters for serialization."""
+        return {}
+
+    def set_params(self, params: Dict[str, Any]):
+        """Load pricing function parameters from dict."""
+        pass

experiments/procesing/pricers/elasticity.py

@@ -0,0 +1,59 @@
+import numpy as np
+import pandas as pd
+from procesing.pricers.base import PricingFunction
+
+
+class ElasticityBasedPricer(PricingFunction):
+    """
+    Pricing based on demand elasticity estimates.
+    f(Q, S) = base_price * (1 + alpha * elasticity * demand_deviation)
+    """
+
+    def __init__(self, alpha: float = 0.1, price_floor: float = 0.0, price_ceil: float = np.inf):
+        self.alpha = alpha
+        self.price_floor = price_floor
+        self.price_ceil = price_ceil
+        self.elasticity = None
+        self.base_prices = None
+        self.mean_demand = None
+
+    def fit(self, historical_data: pd.DataFrame):
+        """
+        Calibrate from historical elasticity estimates.
+        Expects: [productId, elasticity, base_price, mean_demand]
+        """
+        if 'elasticity' not in historical_data.columns:
+            raise ValueError("historical_data must contain 'elasticity' column")
+
+        self.elasticity = historical_data['elasticity'].values
+        self.base_prices = (historical_data['base_price'].values
+                           if 'base_price' in historical_data.columns
+                           else np.ones(len(historical_data)) * 100)
+        self.mean_demand = (historical_data['mean_demand'].values
+                           if 'mean_demand' in historical_data.columns
+                           else np.ones(len(historical_data)) * 10)
+        return self
+
+    def predict(self, state_space) -> np.ndarray:
+        """
+        Adjust prices based on demand deviation and elasticity.
+        Higher demand -> increase price (but less for elastic goods)
+        """
+        if self.elasticity is None:
+            raise ValueError("Must call fit() before predict()")
+
+        demand = np.asarray(state_space.demand)
+        if len(demand) != len(self.elasticity):
+            raise ValueError(f"Demand vector size {len(demand)} != elasticity size {len(self.elasticity)}")
+
+        # compute demand deviation from mean
+        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
+
+        # adjust price: if demand high and elastic, don't increase much
+        # if demand high and inelastic, increase more
+        price_multiplier = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
+        prices = self.base_prices * price_multiplier
+
+        # enforce bounds
+        prices = np.clip(prices, self.price_floor, self.price_ceil)
+        return prices

experiments/procesing/pricers/session_aware.py

@@ -0,0 +1,172 @@
+"""
+Session-aware pricing functions that leverage behavioral features S_t.
+These pricers aim to minimize L_agent = R_oracle - R_observed.
+"""
+import numpy as np
+import pandas as pd
+from procesing.pricers.base import PricingFunction
+from procesing.pricers.elasticity import ElasticityBasedPricer
+
+
+class SessionAwarePricer(PricingFunction):
+    """
+    Extends elasticity-based pricing with session behavioral signals.
+
+    f(Q, P, S) = base_price * elasticity_factor * session_factor
+
+    Where session_factor adjusts for:
+        - interaction_velocity (agent detection proxy)
+        - product_view_depth (interest signal)
+        - cart_to_view_ratio (conversion intent)
+
+    Strategy: charge higher prices to suspected agents (high velocity)
+    to recover oracle revenue from reconnaissance sessions.
+    """
+
+    def __init__(self,
+                 alpha: float = 0.1,
+                 beta_velocity: float = 0.05,
+                 beta_attention: float = 0.03,
+                 agent_velocity_threshold: float = 5.0,
+                 agent_markup: float = 1.2,
+                 price_floor: float = 0.0,
+                 price_ceil: float = np.inf):
+        """
+        Args:
+            alpha: elasticity sensitivity
+            beta_velocity: interaction velocity weight
+            beta_attention: product attention weight
+            agent_velocity_threshold: velocity above which to apply agent markup
+            agent_markup: price multiplier for suspected agent sessions
+            price_floor, price_ceil: price bounds
+        """
+        self.alpha = alpha
+        self.beta_velocity = beta_velocity
+        self.beta_attention = beta_attention
+        self.agent_velocity_threshold = agent_velocity_threshold
+        self.agent_markup = agent_markup
+        self.price_floor = price_floor
+        self.price_ceil = price_ceil
+
+        # fitted parameters
+        self.elasticity = None
+        self.base_prices = None
+        self.mean_demand = None
+
+    def fit(self, historical_data: pd.DataFrame, **kwargs):
+        """Calibrate from historical elasticity data."""
+        if 'elasticity' not in historical_data.columns:
+            raise ValueError("historical_data must contain 'elasticity'")
+
+        self.elasticity = historical_data['elasticity'].values
+        self.base_prices = (historical_data['base_price'].values
+                           if 'base_price' in historical_data.columns
+                           else np.ones(len(historical_data)) * 100)
+        self.mean_demand = (historical_data['mean_demand'].values
+                           if 'mean_demand' in historical_data.columns
+                           else np.ones(len(historical_data)) * 10)
+        return self
+
+    def predict(self, state_space) -> np.ndarray:
+        """Generate prices with session awareness."""
+        if self.elasticity is None:
+            raise ValueError("Must call fit() before predict()")
+
+        demand = np.asarray(state_space.demand)
+        n_products = len(demand)
+
+        # base elasticity-driven pricing
+        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
+        elasticity_factor = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
+
+        # session-aware adjustments
+        session_factor = np.ones(n_products)
+
+        if not state_space.session_features.empty:
+            sf = state_space.session_features.iloc[0]  # single session features
+
+            # agent detection via velocity
+            velocity = sf.get('interaction_velocity', 0.0)
+            if velocity > self.agent_velocity_threshold:
+                # suspected agent: apply markup to recover oracle revenue
+                session_factor *= self.agent_markup
+
+            # attention signal: higher view depth -> user interested -> can charge more
+            view_depth = sf.get('product_view_depth', 0)
+            if view_depth > 0:
+                attention_boost = 1 + self.beta_attention * np.log1p(view_depth)
+                session_factor *= attention_boost
+
+            # cart presence: if user has items in cart, slightly increase prices
+            cart_to_view = sf.get('cart_to_view_ratio', 0.0)
+            if cart_to_view > 0.1:
+                session_factor *= (1 + 0.02)  # small boost for conversion intent
+
+        prices = self.base_prices * elasticity_factor * session_factor
+        prices = np.clip(prices, self.price_floor, self.price_ceil)
+
+        return prices
+
+
+class ProductSpecificSessionPricer(PricingFunction):
+    """
+    Session-aware pricer with product-specific demand signals.
+
+    Uses S_t to extract per-product interaction counts and adjusts pricing
+    for products the user has already viewed/hovered.
+
+    Strategy: products viewed multiple times = high interest -> price up
+    """
+
+    def __init__(self,
+                 alpha: float = 0.1,
+                 view_boost: float = 0.02,
+                 max_view_boost: float = 0.15,
+                 price_floor: float = 0.0,
+                 price_ceil: float = np.inf):
+        self.alpha = alpha
+        self.view_boost = view_boost
+        self.max_view_boost = max_view_boost
+        self.price_floor = price_floor
+        self.price_ceil = price_ceil
+
+        self.elasticity = None
+        self.base_prices = None
+        self.mean_demand = None
+        self.product_ids = None
+
+    def fit(self, historical_data: pd.DataFrame, **kwargs):
+        if 'elasticity' not in historical_data.columns or 'productId' not in historical_data.columns:
+            raise ValueError("historical_data must contain 'elasticity' and 'productId'")
+
+        self.elasticity = historical_data['elasticity'].values
+        self.base_prices = (historical_data['base_price'].values
+                           if 'base_price' in historical_data.columns
+                           else np.ones(len(historical_data)) * 100)
+        self.mean_demand = (historical_data['mean_demand'].values
+                           if 'mean_demand' in historical_data.columns
+                           else np.ones(len(historical_data)) * 10)
+        self.product_ids = historical_data['productId'].values
+        return self
+
+    def predict(self, state_space) -> np.ndarray:
+        if self.elasticity is None:
+            raise ValueError("Must call fit() before predict()")
+
+        demand = np.asarray(state_space.demand)
+        n_products = len(demand)
+
+        # base pricing
+        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
+        base_prices = self.base_prices * (1 + self.alpha * np.abs(self.elasticity) * demand_dev)
+
+        # product-specific session adjustments
+        if not state_space.session_features.empty and state_space.product_ids is not None:
+            # extract product interaction counts from session metadata
+            # (this would require session features to include per-product signals)
+            # for now, use uniform boost as placeholder
+            # TODO: extend session feature extraction to include product-specific counts
+            pass
+
+        prices = np.clip(base_prices, self.price_floor, self.price_ceil)
+        return prices

experiments/procesing/pricers/simple.py

@@ -0,0 +1,91 @@
+import numpy as np
+import pandas as pd
+from procesing.pricers.base import PricingFunction
+
+
+class StaticPricer(PricingFunction):
+    """Static pricing: always return fixed base prices"""
+
+    def __init__(self, base_prices: np.ndarray = None):
+        self.base_prices = base_prices
+
+    def fit(self, historical_data: pd.DataFrame):
+        """Extract base prices from historical data"""
+        if 'base_price' in historical_data.columns:
+            self.base_prices = historical_data['base_price'].values
+        elif 'price' in historical_data.columns:
+            self.base_prices = historical_data['price'].values
+        else:
+            raise ValueError("historical_data must contain 'base_price' or 'price' column")
+        return self
+
+    def predict(self, state_space) -> np.ndarray:
+        """Return static base prices regardless of state"""
+        if self.base_prices is None:
+            raise ValueError("Must call fit() or provide base_prices in constructor")
+        return self.base_prices.copy()
+
+
+class RandomPricer(PricingFunction):
+    """Random pricing within bounds (for baseline comparison)"""
+
+    def __init__(self, price_min: float = 50.0, price_max: float = 500.0, seed: int = None):
+        self.price_min = price_min
+        self.price_max = price_max
+        self.seed = seed
+        self.n_products = None
+        self.rng = np.random.default_rng(seed)
+
+    def fit(self, historical_data: pd.DataFrame):
+        """Learn number of products"""
+        self.n_products = len(historical_data)
+        return self
+
+    def predict(self, state_space) -> np.ndarray:
+        """Generate random prices"""
+        if self.n_products is None:
+            self.n_products = len(state_space.demand)
+        return self.rng.uniform(self.price_min, self.price_max, size=self.n_products)
+
+
+class SimpleSurgePricer(PricingFunction):
+    """
+    Rule-based surge pricer adjusting prices via demand thresholds.
+    Logic: if demand > high_threshold -> surge, if demand < low_threshold -> discount.
+    Simpler and more controllable than curve fitting approaches.
+    """
+
+    def __init__(self,
+                 base_prices: np.ndarray = None,
+                 high_threshold: int = 10,
+                 low_threshold: int = 2,
+                 surge_multiplier: float = 1.2,
+                 discount_multiplier: float = 0.9):
+        self.base_prices = base_prices
+        self.high_threshold = high_threshold
+        self.low_threshold = low_threshold
+        self.surge_multiplier = surge_multiplier
+        self.discount_multiplier = discount_multiplier
+
+    def fit(self, market_data : pd.DataFrame):
+        """Extract base prices from product catalog or historical averages"""
+        self.base_prices = market_data['base_price'].to_numpy() if 'base_price' in market_data.columns else market_data['price'].values
+        self.demand_history = market_data['demand'].to_numpy() if 'demand' in market_data.columns else np.zeros_like(self.base_prices)
+
+    def predict(self) -> np.ndarray:
+        """
+        Adjust prices based on current demand using surge rules.
+        state_space.demand: demand counts per product
+        state_space.prices: current prices (fallback if base_prices not set)
+        """
+        current_prices = self.base_prices if self.base_prices is not None else np.ones_like(demand_vector) * 99.99
+        demand = self.demand_history if self.demand_history is not None else np.zeros_like(current_prices)
+        new_prices = current_prices.copy()
+
+        high_mask = demand >= self.high_threshold
+        new_prices[high_mask] *= self.surge_multiplier
+
+        low_mask = demand <= self.low_threshold
+        new_prices[low_mask] *= self.discount_multiplier
+
+        return new_prices

experiments/procesing/pricing.py

@@ -35,8 +35,9 @@ from sklearn.base import BaseEstimator, TransformerMixin
 import numpy as np
 import pandas as pd
 import os
+from dotenv import load_dotenv
+load_dotenv()
 from supabase import create_client, Client
-from pipeline import interaction_pipeline, price_data_pipeline, elasticity_pipeline
 
 SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
 SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
@@ -79,18 +80,136 @@ class PricingFunction(BaseEstimator, TransformerMixin, ABC):
 class SimpleLinearPricingFunction(PricingFunction):
     def __init__(self, price_sensitivity: float = -0.1):
         super().__init__()
-        self.price_sensitivity = price_sensitivity  # simple coefficient
+        self.price_sensitivity = price_sensitivity
 
     def fit(self, historical_data):
         return self
 
     def transform(self, state_space: StateSpace) -> np.ndarray:
-        # Simple linear adjustment: P_{t+1} = P_t + sensitivity * Q_t
-        new_prices = state_space.prices + self.price_sensitivity * state_space.demand # this is not great
+        new_prices = state_space.prices + self.price_sensitivity * state_space.demand
         return np.maximum(new_prices, 0)
 
+
+class ElasticityBasedPricingFunction(PricingFunction):
+    """
+    Revenue-maximizing pricing using elasticity estimates.
+
+    For each product, optimal price P* maximizes R = P * Q(P)
+    where Q(P) follows power law: Q(P) = Q_0 * (P/P_0)^ε
+
+    Taking derivative dR/dP = 0 gives optimal markup:
+    P* = P_0 * (1 + 1/ε) if ε < -1 (elastic)
+
+    For inelastic demand (|ε| < 1), we apply bounded markup.
+    """
+
+    def __init__(self,
+                 cost_floor: float = 0.5,
+                 max_markup: float = 2.0,
+                 min_markup: float = 1.0,
+                 inelastic_markup: float = 1.3):
+        super().__init__()
+        self.cost_floor = cost_floor  # prices as fraction of base
+        self.max_markup = max_markup  # max price = base * max_markup
+        self.min_markup = min_markup  # min price = base * min_markup
+        self.inelastic_markup = inelastic_markup  # default for |ε| < 1
+        self.elasticity_map = {}  # productId -> elasticity
+
+    def fit(self, elasticity_df: pd.DataFrame):
+        """
+        Args:
+            elasticity_df: df with [productId, elasticity, std_error, n_obs]
+        """
+        if elasticity_df is not None and not elasticity_df.empty:
+            self.elasticity_map = dict(zip(
+                elasticity_df['productId'],
+                elasticity_df['elasticity']
+            ))
+        return self
+
+    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
+        """
+        Args:
+            state_space: current state (prices = base prices)
+            product_ids: array of productIds aligned with state_space.prices
+
+        Returns:
+            optimized prices P_{t+1}
+        """
+        base_prices = state_space.prices
+
+        if product_ids is None:
+            # fallback: use positional index as productId (not ideal)
+            product_ids = np.arange(len(base_prices))
+
+        new_prices = np.zeros_like(base_prices)
+
+        for i, (base_p, pid) in enumerate(zip(base_prices, product_ids)):
+            elasticity = self.elasticity_map.get(pid, 0.0)
+
+            if elasticity < -1:  # elastic demand
+                # optimal markup: (1 + 1/ε)
+                markup = 1 + (1 / elasticity)
+                optimal_p = base_p * markup
+            elif elasticity > -1 and elasticity < 0:  # inelastic
+                # conservative markup
+                optimal_p = base_p * self.inelastic_markup
+            else:  # ε ≥ 0 (demand increases with price, or no data)
+                # no elasticity data or anomalous, keep base price
+                optimal_p = base_p
+
+            # apply bounds
+            optimal_p = np.clip(
+                optimal_p,
+                base_p * self.min_markup,
+                base_p * self.max_markup
+            )
+            optimal_p = max(optimal_p, self.cost_floor)
+
+            new_prices[i] = optimal_p
+
+        return new_prices
+
+
+class ContextualElasticityPricing(PricingFunction):
+    """
+    Revenue optimization with contextual adjustments based on session features.
+
+    Combines elasticity-based pricing with surge/demand-based multipliers.
+    """
+
+    def __init__(self,
+                 base_pricer: ElasticityBasedPricingFunction = None,
+                 demand_sensitivity: float = 0.1,
+                 surge_threshold: float = 0.7):
+        super().__init__()
+        self.base_pricer = base_pricer or ElasticityBasedPricingFunction()
+        self.demand_sensitivity = demand_sensitivity
+        self.surge_threshold = surge_threshold
+
+    def fit(self, elasticity_df: pd.DataFrame):
+        self.base_pricer.fit(elasticity_df)
+        return self
+
+    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
+        # get base optimal prices from elasticity
+        base_optimal = self.base_pricer.transform(state_space, product_ids)
+
+        # compute surge multiplier from demand
+        if len(state_space.demand) > 0:
+            demand_normalized = state_space.demand / (state_space.demand.max() + 1e-8)
+            surge_multiplier = 1 + self.demand_sensitivity * np.maximum(
+                demand_normalized - self.surge_threshold, 0
+            )
+        else:
+            surge_multiplier = np.ones_like(base_optimal)
+
+        return base_optimal * surge_multiplier
+
 # Example usage:
 if __name__ == "__main__":
+    from pipeline import interaction_pipeline, price_data_pipeline, elasticity_pipeline
+
     store_mode = 'hotel'
     interaction_data = interaction_pipeline.fit_transform(None)
     price_data = price_data_pipeline.fit_transform(None)

experiments/procesing/providers/__init__.py

@@ -0,0 +1,5 @@
+from procesing.providers.base import DataProvider
+from procesing.providers.supabase import SupabaseProvider
+from procesing.providers.backend import BackendAPIProvider
+
+__all__ = ['DataProvider', 'SupabaseProvider', 'BackendAPIProvider']

experiments/procesing/providers/backend.py

@@ -0,0 +1,19 @@
+import os
+import pandas as pd
+import requests
+from typing import List
+from procesing.providers.base import DataProvider
+
+class BackendAPIProvider(DataProvider):
+    """Concrete backend API implementation"""
+    def __init__(self, backend_url: str = None):
+        self.backend_url = backend_url or os.getenv("BACKEND_URL", "http://localhost:5000")
+    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
+        resp = requests.get(f"{self.backend_url}/api/kafka/dump?topic={topic}")
+        resp.raise_for_status()
+        data = resp.json()
+
+        if not data.get('success') or not data.get('data'):
+            return pd.DataFrame()
+
+        return pd.DataFrame(data['data'])

experiments/procesing/providers/base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+from typing import List
+import pandas as pd
+
+class DataProvider(ABC):
+    """Abstract interface for data access, enables DI and testing"""
+
+    @abstractmethod
+    def fetch_products(self, store_mode: str) -> pd.DataFrame:
+        """Fetch product catalog for given store mode"""
+        pass
+
+    @abstractmethod
+    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
+        """Fetch experiment metadata for given IDs"""
+        pass
+
+    @abstractmethod
+    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
+        """Fetch data from Kafka topic via backend API"""
+        pass

experiments/procesing/providers/supabase.py

@@ -0,0 +1,42 @@
+import os
+import pandas as pd
+import requests
+from typing import List
+from supabase import create_client, Client
+from procesing.providers.base import DataProvider
+from dotenv import load_dotenv
+
+class SupabaseProvider(DataProvider):
+    """Concrete Supabase + backend API implementation"""
+
+    def __init__(self,
+                 supabase_url: str = None,
+                 supabase_key: str = None,):
+        load_dotenv()
+        self.supabase_url = supabase_url or os.getenv("NEXT_PUBLIC_SUPABASE_URL")
+        self.supabase_key = supabase_key or os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
+        self.supabase: Client = create_client(self.supabase_url, self.supabase_key)
+
+    def fetch_products(self, store_mode: str) -> pd.DataFrame:
+        # hotel uses room_type, airline uses flight_type; select all and normalize
+        resp = self.supabase.table(f'{store_mode}_products').select("*").execute()
+        if not resp.data:
+            return pd.DataFrame()
+        df = pd.DataFrame(resp.data)
+        # normalize type column: hotel has room_type, airline has flight_type
+        if 'room_type' in df.columns:
+            df['product_type'] = df['room_type']
+        elif 'flight_type' in df.columns:
+            df['product_type'] = df['flight_type']
+        return df
+
+    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
+        if not experiment_ids:
+            return pd.DataFrame()
+
+        resp = self.supabase.table('experiments').select(
+            'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, '
+            'task:tasks(task_name, task_description, task_def_of_done)'
+        ).in_('id', experiment_ids).execute()
+
+        return pd.DataFrame(resp.data) if resp.data else pd.DataFrame()

experiments/procesing/steps/__init__.py

@@ -0,0 +1,39 @@
+from procesing.steps.base import BaseContextStep
+from procesing.steps.fetch import FetchInteractionsStep, FetchPriceLogsStep, FetchExperimentsStep
+from procesing.steps.join import JoinExperimentsStep, JoinProductFeaturesStep
+from procesing.steps.augment import CreatePriceBucketsStep, AugmentEventNamesStep, AugmentInteractionsStep
+from procesing.steps.chunk import ChunkByTimeWindowStep
+from procesing.steps.demand import ComputeDemandStep, ComputeDemandForChunksStep
+from procesing.steps.elasticity import AggregatePriceLogsStep
+from procesing.steps.pricing import FitPricingFunctionStep, PredictPricesStep
+from procesing.steps.session import (
+    ExtractSessionFeaturesStep, JoinLabelsStep, ValidateDataStep,
+    TemporalFeatureStep, BehavioralFeatureStep, ProductFeatureStep, UserAgentFeatureStep,
+    _extract_features_for_session
+)
+
+__all__ = [
+    'BaseContextStep',
+    'FetchInteractionsStep',
+    'FetchPriceLogsStep',
+    'FetchExperimentsStep',
+    'JoinExperimentsStep',
+    'JoinProductFeaturesStep',
+    'CreatePriceBucketsStep',
+    'AugmentEventNamesStep',
+    'AugmentInteractionsStep',
+    'ChunkByTimeWindowStep',
+    'ComputeDemandStep',
+    'ComputeDemandForChunksStep',
+    'AggregatePriceLogsStep',
+    'FitPricingFunctionStep',
+    'PredictPricesStep',
+    'ExtractSessionFeaturesStep',
+    'JoinLabelsStep',
+    'ValidateDataStep',
+    'TemporalFeatureStep',
+    'BehavioralFeatureStep',
+    'ProductFeatureStep',
+    'UserAgentFeatureStep',
+    '_extract_features_for_session',
+]

experiments/procesing/steps/augment.py

@@ -0,0 +1,140 @@
+import numpy as np
+import pandas as pd
+from procesing.steps.base import BaseContextStep
+
+
+class AugmentInteractionsStep(BaseContextStep):
+    """
+    Consolidated step: create price buckets, augment event names, join experiments.
+    Input: (interactions_df, price_logs_df)
+    Output: enriched interactions_df
+    """
+
+    def transform(self, data: tuple):
+        interactions_df, price_logs_df = data
+
+        if interactions_df.empty:
+            return interactions_df
+
+        # Step 1: Create price buckets
+        interactions_df = self._create_price_buckets(interactions_df)
+
+        # Step 2: Augment event names
+        interactions_df = self._augment_event_names(interactions_df)
+
+        # Step 3: Join experiments (optional)
+        if 'experimentId' in interactions_df.columns:
+            interactions_df = self._join_experiments(interactions_df)
+
+        return interactions_df
+
+    def _create_price_buckets(self, df: pd.DataFrame):
+        """Create price bucket labels from price data"""
+        if 'metadata_price' not in df.columns:
+            df['price_bucket'] = ""
+            return df
+
+        n_buckets = self.context.config.get('n_price_buckets', 5)
+
+        if df['metadata_price'].notnull().sum() > 0:
+            try:
+                price_buckets = pd.qcut(
+                    df['metadata_price'],
+                    q=n_buckets,
+                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
+                    duplicates='drop'
+                )
+            except ValueError:
+                # fallback for insufficient unique values
+                price_buckets = df['metadata_price'].apply(
+                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
+                )
+        else:
+            price_buckets = pd.Series([""] * len(df), index=df.index)
+
+        df['price_bucket'] = price_buckets
+        return df
+
+    def _augment_event_names(self, df: pd.DataFrame):
+        """Augment event names with product and price bucket schema"""
+        # Create schema: _productId@price_bucket
+        has_product = df.get('productId', pd.Series()).notnull()
+        has_bucket = df.get('price_bucket', pd.Series()).notnull()
+
+        df['metadata_schema'] = np.where(
+            has_product & has_bucket,
+            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
+            ""
+        )
+
+        df['eventName'] = df['eventName'] + df['metadata_schema']
+        return df
+
+    def _join_experiments(self, df: pd.DataFrame):
+        """Join experiment metadata if experimentId present"""
+        exp_ids = df['experimentId'].dropna().unique().tolist()
+        if not exp_ids:
+            return df
+
+        experiments_df = self.context.provider.fetch_experiments(exp_ids)
+        if experiments_df.empty:
+            return df
+
+        return df.merge(
+            experiments_df,
+            left_on='experimentId',
+            right_on='id',
+            how='left',
+            suffixes=('', '_exp')
+        )
+
+
+class CreatePriceBucketsStep(BaseContextStep):
+    """Create price bucket labels from price data"""
+
+    def transform(self, df: pd.DataFrame):
+        if df.empty or 'metadata_price' not in df.columns:
+            df['price_bucket'] = ""
+            return df
+
+        n_buckets = self.context.config.get('n_price_buckets', 5)
+
+        if df['metadata_price'].notnull().sum() > 0:
+            try:
+                price_buckets = pd.qcut(
+                    df['metadata_price'],
+                    q=n_buckets,
+                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
+                    duplicates='drop'
+                )
+            except ValueError:
+                # fallback for insufficient unique values
+                price_buckets = df['metadata_price'].apply(
+                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
+                )
+        else:
+            price_buckets = pd.Series([""] * len(df), index=df.index)
+
+        df['price_bucket'] = price_buckets
+        return df
+
+
+class AugmentEventNamesStep(BaseContextStep):
+    """Augment event names with product and price bucket schema"""
+
+    def transform(self, df: pd.DataFrame):
+        if df.empty:
+            return df
+
+        # Create schema: _productId@price_bucket
+        has_product = df.get('productId', pd.Series()).notnull()
+        has_bucket = df.get('price_bucket', pd.Series()).notnull()
+
+        df['metadata_schema'] = np.where(
+            has_product & has_bucket,
+            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
+            ""
+        )
+
+        df['eventName'] = df['eventName'] + df['metadata_schema']
+        return df

experiments/procesing/steps/base.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+from sklearn.base import BaseEstimator, TransformerMixin
+from procesing.context import PipelineContext
+from typing import Any
+
+class BaseContextStep(BaseEstimator, TransformerMixin, ABC):
+    """
+    Base for all pipeline steps.
+    Each step is stateless, context-driven, and performs ONE transformation.
+    """
+
+    def __init__(self, context: PipelineContext):
+        self.context = context
+
+    def fit(self, X=None, y=None):
+        """Most steps don't need training"""
+        return self
+
+    @abstractmethod
+    def transform(self, X) -> Any:
+        """Transform input using context. Must be implemented by subclass."""
+        pass
+
+    def get_params(self, deep=True):
+        """sklearn compatibility"""
+        return {'context': self.context}
+
+    def set_params(self, **params):
+        """sklearn compatibility"""
+        if 'context' in params:
+            self.context = params['context']
+        return self

experiments/procesing/steps/chunk.py

@@ -0,0 +1,34 @@
+import pandas as pd
+from procesing.steps.base import BaseContextStep
+
+class ChunkByTimeWindowStep(BaseContextStep):
+    """
+    Chunk dataframe into time windows.
+    Returns list of dicts with window metadata.
+    """
+
+    def transform(self, df: pd.DataFrame):
+        if df.empty:
+            return []
+
+        df = df.copy()
+        ts_col = self.context.config.get('ts_col', 'ts')
+        window_size = self.context.window_size
+
+        # ensure datetime
+        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
+            df[ts_col] = pd.to_datetime(df[ts_col])
+
+        df = df.sort_values(ts_col)
+        df['_window'] = df[ts_col].dt.floor(window_size)
+
+        chunks = []
+        for idx, (window_start, group) in enumerate(df.groupby('_window')):
+            chunks.append({
+                'window_start': window_start,
+                'window_end': window_start + pd.Timedelta(window_size),
+                'window_idx': idx,
+                'data': group.drop(columns=['_window'])
+            })
+
+        return chunks

experiments/procesing/steps/demand.py

@@ -0,0 +1,61 @@
+import pandas as pd
+from procesing.steps.base import BaseContextStep
+
+class ComputeDemandStep(BaseContextStep):
+    """
+    Compute demand vector for a single time window or dataframe.
+    Input: single chunk dict OR raw dataframe
+    Output: demand dataframe with [productId, demand_score]
+    """
+
+    def transform(self, chunk):
+        # handle both chunk dict and raw dataframe
+        if isinstance(chunk, dict):
+            interactions = chunk['data']
+            window_meta = {k: v for k, v in chunk.items() if k != 'data'}
+        else:
+            interactions = chunk
+            window_meta = {}
+
+        products = self.context.products
+        unique_products = products['id'].unique()
+
+        # apply filters if configured
+        session_filter = self.context.config.get('session_filter')
+        experiment_filter = self.context.config.get('experiment_filter')
+
+        if session_filter and 'sessionId' in interactions.columns:
+            interactions = interactions[interactions['sessionId'] == session_filter]
+        if experiment_filter and 'experimentId' in interactions.columns:
+            interactions = interactions[interactions['experimentId'] == experiment_filter]
+
+        interactions_with_products = interactions.dropna(subset=['productId'])
+
+        if interactions_with_products.empty:
+            demand_df = pd.DataFrame({
+                'productId': unique_products,
+                'demand_score': 0
+            })
+        else:
+            # crosstab for simple demand count
+            demand_df = pd.crosstab(
+                interactions_with_products['productId'],
+                'count'
+            ).reindex(unique_products, fill_value=0).reset_index()
+            demand_df.columns = ['productId', 'demand_score']
+
+        # attach window metadata if present
+        if window_meta:
+            return {**window_meta, 'demand_vector': demand_df}
+        return demand_df
+
+
+class ComputeDemandForChunksStep(BaseContextStep):
+    """Apply ComputeDemandStep to list of chunks"""
+
+    def transform(self, chunks: list):
+        if not chunks:
+            return []
+
+        demand_step = ComputeDemandStep(self.context)
+        return [demand_step.transform(chunk) for chunk in chunks]

experiments/procesing/steps/elasticity.py

@@ -0,0 +1,42 @@
+import numpy as np
+import pandas as pd
+from typing import Dict, List
+from procesing.steps.base import BaseContextStep
+
+class AggregatePriceLogsStep(BaseContextStep):
+    """
+    Aggregate price logs into time windows using VECTORIZED operations.
+    Input: price_logs_df
+    Output: DataFrame with columns [productId, price]
+    """
+
+    def transform(self, price_logs_df: pd.DataFrame):
+        if price_logs_df.empty:
+            return pd.DataFrame(columns=['productId', 'price'])
+
+        df = price_logs_df.copy()
+        ts_col = self.context.config.get('ts_col', 'ts')
+        #window_size = self.context.window_size WE ARE NOT USING CHUNKS ANYMORE
+
+        # ensure datetime
+        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
+            df[ts_col] = pd.to_datetime(df[ts_col])
+
+        df = df.sort_values([ts_col, 'productId'])
+        products = self.context.products
+        # get base price from metadata if available 1) read the metadata col as json and get the base_price
+        products['base_price'] = products.apply(
+            lambda row: row['metadata'].get('base_price', 0) if isinstance(row['metadata'], dict) else 0,
+            axis=1
+        )
+
+        unique_products = products['id'].unique()
+
+        df_indexed = df.set_index(ts_col)
+        # we return a df of average price per product over the entire period
+        # TODO: maybe consider different opration to handle price aggregation over time
+        avg_prices = df_indexed.groupby('productId')['price'].mean().reindex(unique_products, fill_value=0).reset_index()
+        avg_prices.columns = ['productId', 'price']
+        # fill 0s with base_price from products
+        base_price_map = products.set_index('id')['base_price'].to_dict()
+        return avg_prices

experiments/procesing/steps/fetch.py

@@ -0,0 +1,81 @@
+import pandas as pd
+from procesing.steps.base import BaseContextStep
+
+class FetchInteractionsStep(BaseContextStep):
+    """Fetch raw interaction data from Kafka topic with optional time and store_mode filtering"""
+
+    def __init__(self, context, lookback: str = None):
+        super().__init__(context)
+        self.lookback = lookback
+
+    def transform(self, X=None):
+        df = self.context.provider.fetch_kafka_topic('user-interactions')
+
+        if df.empty:
+            return df
+
+        # Explode metadata JSON column
+        if 'metadata' in df.columns:
+            df = df.join(
+                pd.json_normalize(df.pop('metadata'), sep='.').add_prefix('metadata_')
+            )
+
+        df = df.dropna(subset=['eventName'])
+        # drop all where page has /admin/
+        df = df[~df['page'].str.contains('/admin/', na=False)]
+
+        # filter by store_mode from context
+        if 'storeMode' in df.columns:
+            df = df[df['storeMode'] == self.context.store_mode]
+
+        # Remap dateIndex if present
+        if 'metadata_dateIndex' in df.columns:
+            df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
+
+        # Apply time filtering if lookback specified
+        if self.lookback and 'ts' in df.columns:
+            df['ts'] = pd.to_datetime(df['ts'])
+            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
+            df = df[df['ts'] >= cutoff]
+
+        return df
+
+
+class FetchPriceLogsStep(BaseContextStep):
+    """Fetch price log data from Kafka topic with optional time and store_mode filtering"""
+
+    def __init__(self, context, lookback: str = None):
+        super().__init__(context)
+        self.lookback = lookback
+
+    def transform(self, X=None):
+        df = self.context.provider.fetch_kafka_topic('price-logs')
+
+        if df.empty:
+            return df
+
+        # filter by store_mode from context
+        if 'storeMode' in df.columns:
+            df = df[df['storeMode'] == self.context.store_mode]
+
+        # Apply time filtering if lookback specified
+        if self.lookback and 'ts' in df.columns:
+            df['ts'] = pd.to_datetime(df['ts'])
+            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
+            df = df[df['ts'] >= cutoff]
+
+        return df
+
+
+class FetchExperimentsStep(BaseContextStep):
+    """Fetch experiment metadata for given interaction data"""
+
+    def transform(self, interactions_df: pd.DataFrame):
+        if interactions_df.empty or 'experimentId' not in interactions_df.columns:
+            return pd.DataFrame()
+
+        exp_ids = interactions_df['experimentId'].dropna().unique().tolist()
+        if not exp_ids:
+            return pd.DataFrame()
+
+        return self.context.provider.fetch_experiments(exp_ids)

experiments/procesing/steps/join.py

@@ -0,0 +1,58 @@
+import pandas as pd
+from procesing.steps.base import BaseContextStep
+
+class JoinExperimentsStep(BaseContextStep):
+    """Join experiment metadata to interactions"""
+
+    def transform(self, data: tuple):
+        """
+        Args:
+            data: (interactions_df, experiments_df)
+        Returns:
+            merged interactions dataframe
+        """
+        interactions_df, experiments_df = data
+
+        if experiments_df.empty:
+            return interactions_df
+
+        # Flatten nested task field if present
+        if 'task' in experiments_df.columns and experiments_df['task'].notnull().any():
+            task_norm = pd.json_normalize(experiments_df['task'].dropna())
+            task_norm.index = experiments_df[experiments_df['task'].notnull()].index
+            experiments_df = experiments_df.drop('task', axis=1).join(task_norm, rsuffix='_task')
+
+        # Rename for clarity
+        experiments_df = experiments_df.rename(columns={
+            'id': 'experimentId',
+            'subject_name': 'exp_subject',
+            'xp_human_only': 'exp_human_only',
+            'xp_market_mode': 'exp_market_mode',
+            'xp_task_id': 'exp_task_id'
+        })
+
+        return interactions_df.merge(experiments_df, on='experimentId', how='left')
+
+class JoinProductFeaturesStep(BaseContextStep):
+    """Join product features to interactions"""
+
+    def transform(self, data: tuple):
+        """
+        Args:
+            data: (interactions_df, products_df)
+        Returns:
+            merged interactions dataframe
+        """
+        demand_df, price_df = data
+
+        # get base prices from products if available
+        products = self.context.products
+        products['base_price'] = products.apply(
+            lambda row: float(row['metadata'].get('base_price', 0.0)) if isinstance(row['metadata'], dict) else 0,
+            axis=1
+        )
+        products = products[['id', 'base_price']].rename(columns={'id': 'productId'})
+
+        if price_df.empty:
+            return demand_df
+        return demand_df.merge(price_df, on='productId', how='left').merge(products, on='productId', how='left')

experiments/procesing/steps/pricing.py

@@ -0,0 +1,55 @@
+import numpy as np
+import pandas as pd
+from typing import Optional, List, Dict, Any
+from dataclasses import dataclass, field
+from procesing.pricers.simple import StaticPricer
+from procesing.steps.base import BaseContextStep
+from procesing.pricers import ElasticityBasedPricer
+
+class State:
+    def __init__(self,
+                 last_action : str,
+                 last_productId : str,
+                 last_price : float,
+                 session_features : np.ndarray
+                 ):
+        pass
+
+
+
+class FitPricingFunctionStep(BaseContextStep):
+    """
+    Fit pricing function using data.
+    Input: pricing_data
+    Output: fitted pricing function instance
+    """
+
+    def transform(self, pricing_data: pd.DataFrame):
+        pricing_class = self.context.config.get('pricing_function_class', StaticPricer)
+        pricing_params = self.context.config.get('pricing_function_params', {})
+
+        pricer = pricing_class(**pricing_params)
+        pricer.fit(pricing_data)
+
+        return pricer
+
+
+class PredictPricesStep(BaseContextStep):
+    """
+    Predict optimal prices using fitted pricing function.
+    Input: (pricer, state_space)
+    Output: prices_df [productId, predicted_price]
+    """
+
+    def transform(self, data: tuple):
+        pricer, state_space = data
+
+        products = self.context.products
+        product_ids = products['id'].values
+
+        predicted_prices = pricer.predict(state_space)
+
+        return pd.DataFrame({
+            'productId': product_ids,
+            'predicted_price': predicted_prices
+        })

experiments/procesing/steps/session.py

@@ -0,0 +1,261 @@
+"""
+Session feature extraction for ML training pipeline.
+"""
+import pandas as pd
+import numpy as np
+import re
+from typing import Dict, Any
+from procesing.steps.base import BaseContextStep
+
+EVENT_CATS = {
+    'page_view': ['page_view'],
+    'item_view': ['view_item_page', 'learn_more_about_item'],
+    'cart_add': ['add_item_to_cart'],
+    'purchase': ['purchase', 'checkout_complete'],
+    'hover': ['hover_over_title', 'hover_over_paragraph', 'hover_over_link', 'hover_over_button'],
+    # 'filter': ['filter', 'search', 'apply_filter'],
+}
+HEADLESS_RE = re.compile(r'HeadlessChrome|Headless|PhantomJS', re.I)
+AUTOMATION_RE = re.compile(r'Selenium|Playwright|Puppeteer|WebDriver|chromedriver|geckodriver', re.I)
+BROWSER_PATTERNS = [('Chrome', r'Chrome/[\d.]+'), ('Firefox', r'Firefox/[\d.]+'),
+                    ('Safari', r'Safari/[\d.]+'), ('Edge', r'Edg/[\d.]+')]
+
+
+def _get_browser(s: str) -> str:
+    if pd.isna(s): return 'Unknown'
+    for name, pat in BROWSER_PATTERNS:
+        if re.search(pat, s): return name
+    return 'Other'
+
+
+class TemporalFeatureStep(BaseContextStep):
+    """Vectorized time-based features: durations, velocities, gaps."""
+
+    def __init__(self, context, timeout_sec: float = 900, velocity_window: str = '5min'):
+        super().__init__(context)
+        self.timeout_sec = timeout_sec
+        self.velocity_window = velocity_window
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        df = X.copy()
+        if df.empty or 'ts' not in df.columns:
+            return pd.DataFrame(columns=pd.Series(['sessionId']))
+
+        df['ts_dt'] = pd.to_datetime(df['ts'])
+        df = df.sort_values(['sessionId', 'ts_dt'])
+        df['time_diff'] = df.groupby('sessionId')['ts_dt'].diff().dt.total_seconds()
+        df['active_diff'] = df['time_diff'].where(df['time_diff'] <= self.timeout_sec, 0)
+
+        agg = df.groupby('sessionId').agg(
+            session_duration_sec=('active_diff', 'sum'),
+            total_interactions=('sessionId', 'count'),
+            avg_time_between_events=('time_diff', 'mean'),
+            std_time_between_events=('time_diff', 'std'),
+            min_time_between_events=('time_diff', 'min'),
+            session_start_hour=('ts_dt', lambda x: x.min().hour),
+        ).reset_index()
+        agg['std_time_between_events'] = agg['std_time_between_events'].fillna(0)
+        agg['interaction_velocity'] = np.where(
+            agg['session_duration_sec'] > 0,
+            (agg['total_interactions'] / agg['session_duration_sec']) * 60, 0)
+
+        vel = df.set_index('ts_dt').groupby('sessionId').resample(self.velocity_window, include_groups=False).size()
+        max_velocity = vel.groupby('sessionId').max().rename('max_velocity_5min')
+        agg = agg.merge(max_velocity, on='sessionId', how='left')
+        agg['max_velocity_5min'] = agg['max_velocity_5min'].fillna(0)
+        return agg
+
+
+class BehavioralFeatureStep(BaseContextStep):
+    """Vectorized event counts and ratios per session."""
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        df = X.copy()
+        if df.empty or 'eventName' not in df.columns:
+            return pd.DataFrame(columns=pd.Series(['sessionId']))
+
+        for cat, events in EVENT_CATS.items():
+            df[f'is_{cat}'] = df['eventName'].isin(events)
+        df['is_hover'] = df['is_hover'] | df['eventName'].str.startswith('hover_over_')
+
+        agg = df.groupby('sessionId').agg(
+            total_events=('eventName', 'count'), unique_pages=('page', 'nunique'),
+            page_views=('is_page_view', 'sum'), item_views=('is_item_view', 'sum'),
+            cart_adds=('is_cart_add', 'sum'), purchases=('is_purchase', 'sum'),
+            hover_events=('is_hover', 'sum'),
+            # filter_events=('is_filter', 'sum'),
+        ).reset_index()
+        agg['cart_to_view_ratio'] = np.where(agg['item_views'] > 0, agg['cart_adds'] / agg['item_views'], 0)
+        agg['conversion_rate'] = np.where(agg['item_views'] > 0, agg['purchases'] / agg['item_views'], 0)
+        agg['hover_intensity'] = np.where(agg['total_events'] > 0, agg['hover_events'] / agg['total_events'], 0)
+        return agg
+
+
+class ProductFeatureStep(BaseContextStep):
+    """Vectorized product interaction features: diversity, depth, price sensitivity."""
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        df = X.copy()
+        if df.empty:
+            return pd.DataFrame(columns=pd.Series(['sessionId']))
+        price_col = next((c for c in ['metadata_base_price', 'metadata_price', 'base_price'] if c in df.columns), None)
+        df['price_seen'] = pd.to_numeric(df[price_col], errors='coerce') if price_col else np.nan
+
+        prod_df = df[df['productId'].notna()]
+        if prod_df.empty:
+            return pd.DataFrame(columns=pd.Series(['sessionId', 'unique_products_viewed', 'product_view_depth', 'avg_price_seen', 'min_price_seen', 'max_price_seen', 'price_range']))
+
+        agg = prod_df.groupby('sessionId').agg(
+            unique_products_viewed=('productId', 'nunique'),
+            product_view_depth=('productId', lambda x: x.value_counts().iloc[0] if len(x) > 0 else 0),
+            avg_price_seen=('price_seen', 'mean'), min_price_seen=('price_seen', 'min'),
+            max_price_seen=('price_seen', 'max'),
+        ).reset_index()
+        agg['price_range'] = (agg['max_price_seen'] - agg['min_price_seen']).fillna(0)
+        return agg
+
+
+class UserAgentFeatureStep(BaseContextStep):
+    """Parse userAgent into bot-detection signals."""
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame|pd.Series:
+        df = X.copy()
+        if df.empty or 'userAgent' not in df.columns:
+            return pd.DataFrame(columns=pd.Series(['sessionId']))
+
+        ua = df.groupby('sessionId')['userAgent'].first().reset_index()
+        ua['is_headless'] = ua['userAgent'].str.contains(HEADLESS_RE, na=False)
+        ua['is_automation'] = ua['userAgent'].str.contains(AUTOMATION_RE, na=False)
+        ua['browser_family'] = ua['userAgent'].apply(_get_browser)
+        return ua[['sessionId', 'is_headless', 'is_automation', 'browser_family']]
+
+
+class ExtractSessionFeaturesStep(BaseContextStep):
+    """
+    Vectorized session feature extraction - replaces O(n^2) per-row loop.
+    Input: interactions_df
+    Output: session-level feature matrix
+    """
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if X.empty:
+            return pd.DataFrame()
+        df = X.copy()
+
+        # run all feature steps and merge on sessionId
+        temporal = TemporalFeatureStep(self.context).transform(df)
+        behavioral = BehavioralFeatureStep(self.context).transform(df)
+        product = ProductFeatureStep(self.context).transform(df)
+        ua = UserAgentFeatureStep(self.context).transform(df)
+
+        result = temporal
+        for other in [behavioral, product, ua]:
+            if not other.empty and 'sessionId' in other.columns:
+                result = result.merge(other, on='sessionId', how='left')
+
+        # carry forward experimentId for label joining
+        if 'experimentId' in df.columns:
+            exp_map = df.groupby('sessionId')['experimentId'].first()
+            result = result.merge(exp_map, on='sessionId', how='left')
+
+        return result
+
+
+class JoinLabelsStep(BaseContextStep):
+    """
+    Join experiment labels to session features.
+    Input: (features_df, experiments_df) or features_df (fetches experiments)
+    Output: labeled feature matrix with is_agent column
+    """
+
+    def transform(self, X : tuple) -> pd.DataFrame:
+        data = X;
+        if isinstance(data, tuple):
+            features_df, experiments_df = data
+        else:
+            features_df = data
+            if 'experimentId' not in features_df.columns:
+                return features_df
+            exp_ids = features_df['experimentId'].dropna().unique().tolist()
+            experiments_df = self.context.provider.fetch_experiments(exp_ids) if exp_ids else pd.DataFrame()
+
+        if features_df.empty:
+            return features_df
+        if experiments_df.empty:
+            features_df['is_agent'] = np.nan
+            return features_df
+
+        exp = experiments_df.copy()
+        if 'id' in exp.columns:
+            exp = exp.rename(columns={'id': 'experimentId'})
+        if 'xp_human_only' in exp.columns:
+            exp['is_agent'] = ~exp['xp_human_only']
+
+        cols = ['experimentId'] + [c for c in ['is_agent', 'xp_human_only', 'xp_market_mode'] if c in exp.columns]
+        return features_df.merge(exp[cols].drop_duplicates(), on='experimentId', how='left')
+
+
+class ValidateDataStep(BaseContextStep):
+    """
+    Data quality checks before training.
+    Input: df
+    Output: df (unchanged, but logs validation report to context)
+    """
+    REQUIRED = ['sessionId', 'eventName', 'ts']
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        df = X.copy()
+        report = {'status': 'valid', 'rows': len(df), 'sessions': 0}
+        if df.empty:
+            report['status'] = 'empty'
+            self.context.cache('validation_report', report)
+            return df
+
+        missing = [c for c in self.REQUIRED if c not in df.columns]
+        if missing:
+            report['status'] = 'invalid'
+            report['missing_cols'] = missing
+
+        report['sessions'] = df['sessionId'].nunique() if 'sessionId' in df.columns else 0
+        report['null_sessions'] = int(df['sessionId'].isna().sum()) if 'sessionId' in df.columns else 0
+        if 'experimentId' in df.columns:
+            report['null_experiments'] = int(df['experimentId'].isna().sum())
+
+        self.context.cache('validation_report', report)
+        return df
+
+
+# legacy compat - kept for backwards compatibility with existing code
+def _extract_features_for_session(session_df: pd.DataFrame, session_timeout_sec: float = 900) -> Dict[str, Any]:
+    """Single-session feature extraction (legacy interface)."""
+    defaults = {k: 0 for k in ['total_interactions', 'page_views', 'item_views', 'searches',
+                               'cart_adds', 'hovers', 'unique_products_viewed', 'product_view_depth',
+                               'session_duration_sec', 'interaction_velocity',
+                               'avg_time_between_events', 'std_time_between_events', 'cart_to_view_ratio']}
+    if session_df.empty:
+        return defaults
+
+    session_df = session_df.copy()
+    if 'sessionId' not in session_df.columns:
+        session_df['sessionId'] = 'tmp'
+
+    # use a dummy context for the steps
+    class DummyCtx: config = {} # should maybe inherit but whatever
+    ctx = DummyCtx()
+
+    t = TemporalFeatureStep(ctx, timeout_sec=session_timeout_sec).transform(session_df)
+    b = BehavioralFeatureStep(ctx).transform(session_df)
+    p = ProductFeatureStep(ctx).transform(session_df)
+
+    result = {}
+    for df in [t, b, p]:
+        if not df.empty:
+            for col in df.columns:
+                if col != 'sessionId':
+                    result[col] = df[col].iloc[0] if len(df) > 0 else 0
+
+    remap = {'hover_events': 'hovers', 'filter_events': 'searches', 'unique_pages': 'unique_pages_visited'}
+    for old, new in remap.items():
+        if old in result:
+            result[new] = result.pop(old)
+    return result

experiments/procesing/tests/conftest.py

@@ -0,0 +1,281 @@
+import pytest
+import pandas as pd
+from typing import List
+from procesing.providers.base import DataProvider
+from procesing.context import PipelineContext
+
+
+class MockProvider(DataProvider):
+    """Mock provider for testing, holds in-memory fixtures"""
+
+    def __init__(self, products_df=None, experiments_df=None, kafka_data=None):
+        self._products = products_df if products_df is not None else pd.DataFrame()
+        self._experiments = experiments_df if experiments_df is not None else pd.DataFrame()
+        self._kafka_data = kafka_data if kafka_data is not None else {}
+
+    def fetch_products(self, store_mode: str) -> pd.DataFrame:
+        return self._products.copy()
+
+    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
+        if self._experiments.empty:
+            return pd.DataFrame()
+        return self._experiments[
+            self._experiments['id'].isin(experiment_ids)
+        ].copy()
+
+    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
+        return self._kafka_data.get(topic, pd.DataFrame()).copy()
+
+
+@pytest.fixture
+def mock_products():
+    """Standard product catalog fixture with realistic IDs from test data"""
+    return pd.DataFrame({
+        'id': [
+            'd018efc1-25e9-4284-b276-80386e048b25',
+            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
+            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
+        ],
+        'name': ['Junior Suite', 'Superior Room', 'Deluxe Room'],
+        'base_price': [200.0, 150.0, 180.0]
+    })
+
+
+@pytest.fixture
+def mock_interactions_raw_kafka():
+    """Raw Kafka message structure for interactions, matches production format"""
+    return [
+        {
+            'partitionID': 0, 'offset': 203, 'timestamp': 1764102082676,
+            'value': {
+                'payload': {
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'eventName': 'learn_more_about_item',
+                    'page': '/hotel/products/d018efc1-25e9-4284-b276-80386e048b25',
+                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
+                    'metadata': {'type': 'hotel', 'dateIndex': 1, 'roomType': 'Junior Suite'},
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T20:21:22.674Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 204, 'timestamp': 1764102086982,
+            'value': {
+                'payload': {
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'eventName': 'page_view',
+                    'page': '/hotel/products',
+                    'productId': None,
+                    'metadata': {'referrer': ''},
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T20:21:26.947Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 205, 'timestamp': 1764102091825,
+            'value': {
+                'payload': {
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'eventName': 'hover_over_title',
+                    'page': '/hotel/products',
+                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
+                    'metadata': {'elementText': 'Superior Room', 'dateIndex': 1, 'dwellTime': 1200},
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T20:21:31.823Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 206, 'timestamp': 1764102094193,
+            'value': {
+                'payload': {
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
+                    'eventName': 'hover_over_paragraph',
+                    'page': '/hotel/products',
+                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
+                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1307},
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T20:21:34.191Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 207, 'timestamp': 1764102101970,
+            'value': {
+                'payload': {
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
+                    'eventName': 'hover_over_paragraph',
+                    'page': '/hotel/products',
+                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
+                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1201},
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T20:21:41.967Z'
+                }
+            }
+        }
+    ]
+
+
+@pytest.fixture
+def mock_interactions(mock_interactions_raw_kafka):
+    """Processed interaction DataFrame (what provider.fetch_kafka_topic returns)"""
+    records = [msg['value']['payload'] for msg in mock_interactions_raw_kafka]
+    df = pd.DataFrame(records)
+    df['timestamp'] = pd.to_datetime(df['ts'])
+    return df
+
+
+@pytest.fixture
+def mock_price_logs_raw_kafka():
+    """Raw Kafka message structure for price logs, matches production format"""
+    return [
+        {
+            'partitionID': 0, 'offset': 32, 'timestamp': 1764104757969,
+            'value': {
+                'payload': {
+                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
+                    'price': 162.47,
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T21:05:57.967Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 33, 'timestamp': 1764104757995,
+            'value': {
+                'payload': {
+                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
+                    'price': 743.49,
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T21:05:57.993Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 34, 'timestamp': 1764104758011,
+            'value': {
+                'payload': {
+                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
+                    'price': 163.87,
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T21:05:58.009Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 35, 'timestamp': 1764104758050,
+            'value': {
+                'payload': {
+                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
+                    'price': 397.46,
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T21:05:58.049Z'
+                }
+            }
+        },
+        {
+            'partitionID': 0, 'offset': 36, 'timestamp': 1764104768865,
+            'value': {
+                'payload': {
+                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
+                    'price': 401.66,
+                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
+                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
+                    'storeMode': 'hotel',
+                    'ts': '2025-11-25T21:06:08.864Z'
+                }
+            }
+        }
+    ]
+
+
+@pytest.fixture
+def mock_price_logs(mock_price_logs_raw_kafka):
+    """Processed price logs DataFrame (what provider.fetch_kafka_topic returns)"""
+    # extract payloads and flatten
+    records = [msg['value']['payload'] for msg in mock_price_logs_raw_kafka]
+    df = pd.DataFrame(records)
+    df['timestamp'] = pd.to_datetime(df['ts'])
+    return df
+
+
+@pytest.fixture
+def mock_experiments():
+    """Standard experiment metadata fixture matching Supabase schema"""
+    return pd.DataFrame({
+        'id': ['53aefd07-f66a-4d7f-ba8b-7ea1fc562d35', 'bbbbcccc-dddd-eeee-ffff-000011112222'],
+        'created_at': pd.to_datetime(['2025-11-25T20:00:00Z', '2025-11-26T10:00:00Z']),
+        'subject_name': ['Session A', 'Session B'],
+        'xp_human_only': [True, False],
+        'xp_market_mode': ['hotel', 'airline'],
+        'xp_task_id': [None, None]
+    })
+
+
+@pytest.fixture
+def mock_provider(mock_products, mock_experiments, mock_interactions, mock_price_logs):
+    """Fully configured mock provider"""
+    return MockProvider(
+        products_df=mock_products,
+        experiments_df=mock_experiments,
+        kafka_data={
+            'user-interactions': mock_interactions,
+            'price-logs': mock_price_logs
+        }
+    )
+
+
+@pytest.fixture
+def pipeline_context(mock_provider):
+    """Standard pipeline context for testing"""
+    return PipelineContext(
+        provider=mock_provider,
+        store_mode='hotel',
+        window_size='30s',
+        n_price_buckets=3
+    )
+
+
+@pytest.fixture
+def empty_provider():
+    """Provider with no data, for edge case testing"""
+    return MockProvider(
+        products_df=pd.DataFrame(columns=['id', 'name', 'base_price']),
+        experiments_df=pd.DataFrame(columns=['id', 'created_at', 'subject_name', 'xp_human_only', 'xp_market_mode', 'xp_task_id']),
+        kafka_data={'user-interactions': pd.DataFrame(), 'price-logs': pd.DataFrame()}
+    )
+
+
+@pytest.fixture
+def empty_context(empty_provider):
+    """Context with empty provider"""
+    return PipelineContext(
+        provider=empty_provider,
+        store_mode='hotel',
+        window_size='30s'
+    )
+
+
+@pytest.fixture
+def session_interactions(mock_interactions):
+    """Enriched interaction data for session feature extraction tests"""
+    df = mock_interactions.copy()
+    df['userAgent'] = ['Mozilla/5.0 Chrome/120', 'Mozilla/5.0 Chrome/120',
+                       'HeadlessChrome/120', 'HeadlessChrome/120', 'HeadlessChrome/120']
+    df['metadata_base_price'] = [None, None, 150.0, 150.0, 200.0]
+    return df

experiments/procesing/tests/test_augement.py

@@ -0,0 +1,45 @@
+import pytest
+import random
+import pandas as pd
+from procesing.steps import (
+    CreatePriceBucketsStep,
+    AugmentEventNamesStep
+)
+
+def test_bucketing(pipeline_context):
+    step = CreatePriceBucketsStep(context=pipeline_context)
+
+    # Test with normal price data
+    df = pd.DataFrame({
+        'metadata_price': random.sample(range(10, 1000), 100)
+    })
+    result = step.transform(df)
+    assert 'price_bucket' in result.columns
+    # test if is categorical
+    assert isinstance(result['price_bucket'].dtype, pd.CategoricalDtype)
+    assert result['price_bucket'].nunique() == 3 # as per context config
+    # distribution check
+    counts = result['price_bucket'].value_counts()
+    assert all(counts > 0)
+    assert counts.max() - counts.min() <= 10  # roughly equal distribution for 100 samples
+    # Test with empty DataFrame
+    df = pd.DataFrame()
+    result = step.transform(df)
+    assert 'price_bucket' in result.columns
+    assert result.empty
+
+
+def test_augment_names(pipeline_context):
+    df = pd.DataFrame({
+        'eventName': ['click', 'view', 'purchase'],
+        'productId': ['prod_1', 'prod_2', None],
+        'price_bucket': ['PB_1', None, 'PB_3']
+    })
+    step = AugmentEventNamesStep(context=pipeline_context)
+    result = step.transform(df)
+    expected_event_names = [
+        'click_prod_1@PB_1',
+        'view',
+        'purchase'
+    ]
+    assert result['eventName'].tolist() == expected_event_names

experiments/procesing/tests/test_demand.py

@@ -0,0 +1,49 @@
+import pytest
+import random
+import pandas as pd
+from procesing.steps import (
+    ComputeDemandStep
+)
+
+def test_compute_demand(pipeline_context):
+    step = ComputeDemandStep(context=pipeline_context)
+
+    # Test with normal interaction data
+    df = pd.DataFrame({
+        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
+        'productId': random.choices([
+            'd018efc1-25e9-4284-b276-80386e048b25',
+            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
+            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
+        ], k=100),
+        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
+    })
+    result = step.transform(df)
+    assert type(result) == pd.DataFrame
+    assert not result.empty
+    assert set(result['productId']) == set(pipeline_context.products['id'])
+    assert all(result['demand_score'] > 100/3 -10)
+
+
+def test_compute_demand_skewed(pipeline_context):
+    step = ComputeDemandStep(context=pipeline_context)
+
+    # Test with normal interaction data
+    df = pd.DataFrame({
+        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
+        'productId': random.choices([
+            'd018efc1-25e9-4284-b276-80386e048b25',
+            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
+            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
+        ], weights=[0.7, 0.2, 0.1], k=100),
+        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
+    })
+    result = step.transform(df)
+    assert type(result) == pd.DataFrame
+    assert not result.empty
+    assert set(result['productId']) == set(pipeline_context.products['id'])
+    # test for skewness
+    scores = result.set_index('productId')['demand_score'].to_dict()
+    assert scores['d018efc1-25e9-4284-b276-80386e048b25'] > \
+           scores['51266ddb-5b07-47b7-89ee-5b5cae94bb11'] > \
+           scores['2cd7f756-fc65-4ba0-ab01-74521c1fff43']

experiments/procesing/tests/test_fetch.py

@@ -0,0 +1,51 @@
+import pytest
+import pandas as pd
+from procesing.steps import (
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    FetchExperimentsStep,
+)
+
+
+def test_fetch_interactions_data(pipeline_context):
+    step = FetchInteractionsStep(pipeline_context)
+    data = step.transform(None)
+    assert data is not None
+    assert isinstance(data, pd.DataFrame)
+    expected_cols = [
+        "eventName",
+        "dateIndex",
+        "experimentId",
+        "storeMode",
+        "metadata_elementText"
+    ]
+    for expected in expected_cols:
+        assert expected in data.columns
+
+def test_fetch_price_logs(pipeline_context):
+    step = FetchPriceLogsStep(pipeline_context)
+    data = step.transform(None)
+    assert data is not None
+    assert isinstance(data, pd.DataFrame)
+    expected_cols = [
+        "price",
+        "productId"
+    ]
+    for expected in expected_cols:
+        assert expected in data.columns
+    prices = data['price'].to_list()
+    assert min(prices) >= 0
+    assert max(prices) <= 9999
+
+
+def test_experiments_fetching(pipeline_context):
+    interactions = FetchInteractionsStep(pipeline_context).transform(None)
+    assert interactions is not None
+    experiments = FetchExperimentsStep(pipeline_context)
+    experiment_data = experiments.transform(interactions)
+    assert experiment_data is not None
+    assert isinstance(experiment_data, pd.DataFrame)
+    assert not experiment_data.empty
+    assert 'id' in experiment_data.columns
+    assert len(experiment_data) == 2
+    assert '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35' in experiment_data['id'].values

experiments/procesing/tests/test_pricing.py

@@ -0,0 +1,87 @@
+import pytest
+import pandas as pd
+
+from procesing.pricers import (
+    StaticPricer,
+    RandomPricer,
+    ElasticityBasedPricer
+)
+
+
+def test_static_pricer_fit_and_predict():
+    # Sample historical data
+    historical_data = pd.DataFrame({
+        'product_id': [1, 2, 3],
+        'base_price': [100.0, 150.0, 200.0]
+    })
+
+    # Initialize and fit StaticPricer
+    pricer = StaticPricer()
+    pricer.fit(historical_data)
+
+    # Predict prices
+    predicted_prices = pricer.predict(None)
+
+    # Assert that predicted prices match base prices
+    expected_prices = historical_data['base_price'].values
+    assert all(predicted_prices == expected_prices), "Predicted prices do not match base prices"
+
+
+def test_random_pricer_fit_and_predict():
+    # Sample historical data
+    historical_data = pd.DataFrame({
+        'product_id': [1, 2, 3],
+        'base_price': [100.0, 150.0, 200.0]
+    })
+
+    # Initialize and fit RandomPricer
+    pricer = RandomPricer(price_min=50.0, price_max=250.0, seed=42)
+    pricer.fit(historical_data)
+
+    # Predict prices
+    predicted_prices = pricer.predict(None)
+
+    # Assert that predicted prices are within bounds
+    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
+    assert predicted_prices.max() <= 250.0, "Predicted prices are above maximum bound"
+    # distribution check (not so strict)
+    assert len(set(predicted_prices)) > 1, "Predicted prices are not varied enough"
+    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
+
+def test_elasticity_based_pricer_fit_and_predict():
+    # Sample historical data
+    historical_data = pd.DataFrame({
+        'productId': [1, 2, 3],
+        'elasticity': [-1.5, -0.5, -2.0],
+        'base_price': [100.0, 150.0, 200.0],
+        'mean_demand': [10, 20, 15]
+    })
+
+    # Initialize and fit ElasticityBasedPricer
+    pricer = ElasticityBasedPricer(alpha=0.1, price_floor=50.0, price_ceil=300.0)
+    pricer.fit(historical_data)
+
+    # Create a mock state space with demand deviations
+    class MockStateSpace:
+        def __init__(self, demand):
+            self.demand = demand
+
+    # Simulate demand higher than mean for all products
+    state_space = MockStateSpace(demand=[15, 25, 20])
+
+    # Predict prices
+    predicted_prices = pricer.predict(state_space)
+
+    # Assert that predicted prices are within bounds
+    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
+    assert predicted_prices.max() <= 300.0, "Predicted prices are above maximum bound"
+    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
+
+    # now we gotta check semantic validity
+    # since demand is higher than mean, prices should generally increase
+    for i, row in historical_data.iterrows():
+        base_price = row['base_price']
+        elasticity = row['elasticity']
+        expected_increase = base_price * (1 + 0.1 * abs(elasticity) * ((state_space.demand[i] - row['mean_demand']) / row['mean_demand']))
+        assert predicted_prices[i] >= base_price, f"Predicted price for product {row['productId']} did not increase as expected"
+        assert abs(predicted_prices[i] - expected_increase) < 1e-5, f"Predicted price for product {row['productId']} does not match expected calculation within 1e-5 tolerance"

experiments/pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+pythonpath = .
+testpaths = procesing/tests agents
+python_files = test*.py
+python_classes = Test*
+python_functions = test_*
+asyncio_mode = auto
+asyncio_default_fixture_loop_scope = function

lib/model_registry.py

@@ -0,0 +1,180 @@
+import redis
+import pickle
+import json
+import pandas as pd
+from typing import Optional, Dict, Any
+import os
+import logging
+log = logging.getLogger(__name__)
+
+class ModelRegistry:
+    """
+    Lightweight model registry using Redis for storing pricing models and elasticity data.
+    Models are serialized using pickle, metadata stored as JSON.
+    """
+
+    def __init__(self, redis_host: str = None, redis_port: int = None):
+        host = redis_host or os.getenv('REDIS_HOST', 'localhost')
+        port = redis_port or int(os.getenv('REDIS_PORT', '6378'))
+
+        self.redis_client = redis.Redis(
+            host=host,
+            port=port,
+            db=0,
+            decode_responses=False
+        )
+        self.metadata_prefix = "model:meta:"
+        self.data_prefix = "model:data:"
+        self.elasticity_prefix = "elasticity:"
+        self.prices_prefix = "prices:"
+
+    def publish_elasticity(self,
+                          elasticity_df: pd.DataFrame,
+                          model_name: str = 'latest',
+                          metadata: Optional[Dict[str, Any]] = None):
+        """
+        Store elasticity estimates in registry.
+
+        Args:
+            elasticity_df: df with [productId, elasticity, std_error, n_obs]
+            model_name: identifier for this elasticity snapshot
+            metadata: additional info (timestamp, window_size, etc)
+        """
+        key = f"{self.elasticity_prefix}{model_name}"
+
+        # serialize dataframe as JSON
+        data_json = elasticity_df.to_json(orient='records')
+
+        # store data
+        self.redis_client.set(key, data_json)
+
+        # store metadata
+        meta = metadata or {}
+        meta.update({
+            'n_products': len(elasticity_df),
+            'mean_elasticity': float(elasticity_df['elasticity'].mean()),
+            'model_type': 'elasticity_snapshot'
+        })
+
+        meta_key = f"{self.metadata_prefix}{model_name}"
+        self.redis_client.set(meta_key, json.dumps(meta))
+
+        log.info(f"Published elasticity model '{model_name}' with {len(elasticity_df)} products")
+
+    def get_elasticity(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
+        """Retrieve elasticity estimates from registry."""
+        key = f"{self.elasticity_prefix}{model_name}"
+        data_json = self.redis_client.get(key)
+
+        if data_json is None:
+            return None
+
+        # decode bytes to string if needed
+        if isinstance(data_json, bytes):
+            data_json = data_json.decode('utf-8')
+
+        return pd.read_json(data_json, orient='records')
+
+    def publish_pricing_model(self,
+                              pricing_function,
+                              model_name: str = 'latest',
+                              metadata: Optional[Dict[str, Any]] = None):
+        """
+        Store a fitted pricing function object.
+
+        Args:
+            pricing_function: fitted PricingFunction instance
+            model_name: identifier
+            metadata: additional info
+        """
+        key = f"{self.data_prefix}{model_name}"
+
+        # serialize object
+        model_bytes = pickle.dumps(pricing_function)
+        self.redis_client.set(key, model_bytes)
+
+        # store metadata
+        meta = metadata or {}
+        meta.update({
+            'model_class': pricing_function.__class__.__name__,
+            'model_type': 'pricing_function'
+        })
+
+        meta_key = f"{self.metadata_prefix}{model_name}"
+        self.redis_client.set(meta_key, json.dumps(meta))
+
+        log.info(f"Published pricing model '{model_name}' ({meta['model_class']})")
+
+    def get_pricing_model(self, model_name: str = 'latest'):
+        """Retrieve a pricing function from registry."""
+        key = f"{self.data_prefix}{model_name}"
+        model_bytes = self.redis_client.get(key)
+
+        if model_bytes is None:
+            return None
+
+        return pickle.loads(model_bytes)
+
+    def list_models(self) -> Dict[str, Any]:
+        """List all registered models with metadata."""
+        models = {}
+
+        for key in self.redis_client.scan_iter(f"{self.metadata_prefix}*"):
+            key_str = key.decode('utf-8') if isinstance(key, bytes) else key
+            model_name = key_str.replace(self.metadata_prefix, '')
+            meta_json = self.redis_client.get(key)
+
+            if meta_json:
+                if isinstance(meta_json, bytes):
+                    meta_json = meta_json.decode('utf-8')
+                models[model_name] = json.loads(meta_json)
+
+        return models
+
+    def publish_prices(self,
+                      prices_df: pd.DataFrame,
+                      model_name: str = 'latest',
+                      metadata: Optional[Dict[str, Any]] = None):
+        """Store predicted prices in registry.
+
+        Args:
+            prices_df: df with [productId, predicted_price, ...]
+            model_name: identifier for this price snapshot
+            metadata: additional info
+        """
+        key = f"{self.prices_prefix}{model_name}"
+        data_json = prices_df.to_json(orient='records')
+
+        self.redis_client.set(key, data_json)
+
+        meta = metadata or {}
+        meta.update({
+            'n_products': len(prices_df),
+            'model_type': 'predicted_prices'
+        })
+
+        meta_key = f"{self.metadata_prefix}prices_{model_name}"
+        self.redis_client.set(meta_key, json.dumps(meta))
+
+        log.info(f"Published prices '{model_name}' for {len(prices_df)} products")
+
+    def get_prices(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
+        """Retrieve predicted prices from registry."""
+        key = f"{self.prices_prefix}{model_name}"
+        data_json = self.redis_client.get(key)
+
+        if data_json is None:
+            return None
+
+        if isinstance(data_json, bytes):
+            data_json = data_json.decode('utf-8')
+
+        return pd.read_json(data_json, orient='records')
+
+    def health_check(self) -> bool:
+        """Check if Redis connection is alive."""
+        try:
+            self.redis_client.ping()
+            return True
+        except:
+            return False

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
+pythonpath = experiments
 testpaths = experiments
 python_files = test*.py
 python_classes = Test*

requirements.txt

@@ -11,3 +11,4 @@ pytest-asyncio
 uv
 scikit-learn
 supabase
+pymc

tests/e2e/__init__.py

@@ -0,0 +1 @@
+"""E2E test suite for PHANTOM dynamic pricing pipeline."""

tests/e2e/fixtures.ts

@@ -0,0 +1,17 @@
+import { test as base } from '@playwright/test';
+
+type TestFixtures = {
+  backendUrl: string;
+  pricingUrl: string;
+};
+
+export const test = base.extend<TestFixtures>({
+  backendUrl: async ({}, use) => {
+    await use(process.env.BACKEND_URL || 'http://localhost:5000');
+  },
+  pricingUrl: async ({}, use) => {
+    await use(process.env.PRICING_PROVIDER_URL || 'http://localhost:5001');
+  },
+});
+
+export { expect } from '@playwright/test';

tests/e2e/helpers/api.ts

@@ -0,0 +1,69 @@
+interface PriceResponse {
+  price: number;
+  base_price: number;
+  markup: number;
+  model_version?: string;
+}
+
+export async function fetchPrice(
+  baseUrl: string,
+  productId: string,
+  mode: string = 'simple_surge',
+  sessionId?: string
+): Promise<PriceResponse> {
+  const params = new URLSearchParams();
+  if (sessionId) params.set('sessionId', sessionId);
+
+  const url = `${baseUrl}/api/pricing?mode=${mode}&productId=${productId}&${params}`;
+  const resp = await fetch(url);
+
+  if (!resp.ok) {
+    throw new Error(`Price fetch failed: ${resp.status}`);
+  }
+
+  return resp.json();
+}
+
+export async function waitForPriceChange(
+  baseUrl: string,
+  productId: string,
+  baselinePrice: number,
+  mode: string,
+  sessionId?: string,
+  maxRetries: number = 10,
+  pollInterval: number = 500
+): Promise<PriceResponse> {
+  for (let i = 0; i < maxRetries; i++) {
+    const priceResp = await fetchPrice(baseUrl, productId, mode, sessionId);
+    if (Math.abs(priceResp.price - baselinePrice) > 0.01) {
+      return priceResp;
+    }
+    await new Promise(r => setTimeout(r, pollInterval));
+  }
+
+  throw new Error(`Price did not change after ${maxRetries} retries`);
+}
+
+export async function ingestEvent(
+  baseUrl: string,
+  sessionId: string,
+  event: string,
+  productId?: string,
+  metadata?: Record<string, any>
+): Promise<void> {
+  const resp = await fetch(`${baseUrl}/api/ingest`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      sessionId,
+      event,
+      productId,
+      timestamp: new Date().toISOString(),
+      metadata,
+    }),
+  });
+
+  if (!resp.ok) {
+    throw new Error(`Event ingest failed: ${resp.status}`);
+  }
+}

tests/e2e/helpers/interactions.ts

@@ -0,0 +1,219 @@
+import { Page } from '@playwright/test';
+
+export async function getSessionId(page: Page): Promise<string | null> {
+  const cookies = await page.context().cookies();
+  const sessionCookie = cookies.find(c => c.name === 'phantom_session_id');
+  return sessionCookie?.value || null;
+}
+
+export async function verifySessionConsistency(page: Page, expectedSessionId: string): Promise<boolean> {
+  const currentSessionId = await getSessionId(page);
+  return currentSessionId === expectedSessionId;
+}
+
+export async function createFreshSession(page: Page, storeType: 'hotel' | 'airline' = 'hotel'): Promise<string> {
+  await page.context().clearCookies();
+  await page.goto('/');
+  await page.waitForLoadState('networkidle');
+  await page.waitForTimeout(500);
+
+  const sid = await getSessionId(page);
+  if (!sid) throw new Error('Session not created');
+  return sid;
+}
+
+interface SearchParams {
+  destination?: string;
+  checkIn?: string;
+  guests?: number;
+  rooms?: number;
+  origin?: string;
+  departure?: string;
+  adults?: number;
+}
+
+export async function performSearch(page: Page, params: SearchParams, storeType: 'hotel' | 'airline' = 'hotel' ): Promise<void> {
+  await page.waitForLoadState('networkidle');
+
+  if (storeType === 'hotel') {
+    const destInput = page.locator('input#destination');
+    await destInput.fill(params.destination || 'New York');
+
+    const checkInInput = page.locator('input#checkIn');
+    const checkInDate = params.checkIn || new Date(Date.now() + 7 * 86400000).toISOString().split('T')[0];
+    await checkInInput.fill(checkInDate);
+
+    const searchBtn = page.locator('button:has-text("Search Rooms")');
+    await searchBtn.click();
+  } else {
+    const originDropdown = page.locator('button:has-text("Select origin")').or(
+      page.locator('[id="origin"]').locator('button').first()
+    );
+    await originDropdown.click();
+    await page.waitForTimeout(200);
+    const originOption = page.locator(`button:has-text("${params.origin || 'JFK'}")`).first();
+    await originOption.click();
+    await page.waitForTimeout(200);
+
+    const destDropdown = page.locator('button:has-text("Select destination")').or(
+      page.locator('[id="destination"]').locator('button').first()
+    );
+    await destDropdown.click();
+    await page.waitForTimeout(200);
+    const destOption = page.locator(`button:has-text("${params.destination || 'LAX'}")`).first();
+    await destOption.click();
+    await page.waitForTimeout(200);
+
+    const departInput = page.locator('input#departDate');
+    const departDate = params.departure || new Date(Date.now() + 7 * 86400000).toISOString().split('T')[0];
+    await departInput.fill(departDate);
+
+    const searchBtn = page.locator('button:has-text("Search Flights")');
+    await searchBtn.click();
+  }
+
+  await page.waitForLoadState('networkidle');
+}
+
+export async function selectRandomProduct(page: Page, storeType: 'hotel' | 'airline' = 'hotel'): Promise<string> {
+  await page.waitForLoadState('networkidle');
+
+  const cardClass = storeType === 'hotel' ? '.hotel-card' : '.flight-card';
+  const productCards = page.locator(cardClass);
+
+  const count = await productCards.count();
+  if (count === 0) throw new Error('No products found on listing page');
+
+  const randomIdx = Math.floor(Math.random() * count);
+  return randomIdx.toString();
+}
+
+export async function openProductFromListing(page: Page, productId?: string): Promise<string> {
+  await page.waitForLoadState('networkidle');
+
+  const hotelCards = page.locator('.hotel-card');
+  const flightCards = page.locator('.flight-card');
+
+  const hotelCount = await hotelCards.count();
+  const flightCount = await flightCards.count();
+
+  let productCards;
+  if (hotelCount > 0) {
+    productCards = hotelCards;
+  } else if (flightCount > 0) {
+    productCards = flightCards;
+  } else {
+    throw new Error('No products found on listing page');
+  }
+
+  const count = await productCards.count();
+  const randomIdx = productId ? 0 : Math.floor(Math.random() * count);
+  await productCards.nth(randomIdx).click();
+
+  await page.waitForLoadState('networkidle');
+
+  const url = page.url();
+  const match = url.match(/\/products\/([^/?]+)/);
+  if (!match) throw new Error('Cannot parse product ID from URL after navigation');
+
+  return match[1];
+}
+
+export async function getPriceFromDOM(page: Page): Promise<number> {
+  await page.waitForLoadState('networkidle');
+
+  await page.waitForSelector('.price-amount', { timeout: 15000 }).catch(() => null);
+
+  const priceSelectors = [
+    '.price-amount',
+    '.price-display',
+    '[data-testid="price"]',
+    '[data-price]',
+  ];
+
+  for (const selector of priceSelectors) {
+    const priceEl = page.locator(selector).first();
+    if (await priceEl.count() > 0) {
+      const text = await priceEl.textContent();
+      if (!text) continue;
+
+      const match = text.match(/[\$]?\s*([\d,]+(?:\.\d{2})?)/);
+      if (match) {
+        const priceStr = match[1].replace(/,/g, '');
+        return parseFloat(priceStr);
+      }
+    }
+  }
+
+  const dataPrice = await page.locator('[data-price]').first().getAttribute('data-price').catch(() => null);
+  if (dataPrice) return parseFloat(dataPrice);
+
+  throw new Error('Cannot extract price from DOM');
+}
+
+export async function navigateToProduct(page: Page,productId: string,storeType: 'hotel' | 'airline' = 'hotel'): Promise<void> {
+  await page.goto(`/products/${productId}`);
+  await page.waitForLoadState('networkidle');
+}
+
+export async function viewProductViaFlow(page: Page, storeType: 'hotel' | 'airline' = 'hotel', searchParams?: SearchParams): Promise<string> {
+  const params = new URLSearchParams();
+  params.set('dateIndex', '7');
+
+  if (storeType === 'hotel') {
+    params.set('destination', searchParams?.destination || 'New York');
+    params.set('adults', '2');
+    params.set('rooms', '1');
+  } else {
+    params.set('origin', searchParams?.origin || 'JFK');
+    params.set('destination', searchParams?.destination || 'LAX');
+    params.set('adults', '1');
+    params.set('children', '0');
+    params.set('infants', '0');
+  }
+
+  await page.goto(`/products?${params.toString()}`);
+  await page.waitForLoadState('networkidle');
+
+  const productId = await openProductFromListing(page);
+  await page.waitForTimeout(500);
+  return productId;
+}
+
+export async function rapidViewProductViaFlow(page: Page, count: number, delayMs: number = 100, storeType: 'hotel' | 'airline' = 'hotel'): Promise<string[]> {
+  const productIds: string[] = [];
+
+  for (let i = 0; i < count; i++) {
+    const productId = await viewProductViaFlow(page, storeType);
+    productIds.push(productId);
+
+    await page.waitForTimeout(delayMs);
+  }
+
+  return productIds;
+}
+
+export async function humanLikeViewProduct(page: Page, storeType: 'hotel' | 'airline' = 'hotel'
+): Promise<string> {
+  const productId = await viewProductViaFlow(page, storeType);
+
+  await page.hover('h1');
+  await page.waitForTimeout(800 + Math.random() * 400);
+
+  await page.mouse.wheel(0, 200);
+  await page.waitForTimeout(500 + Math.random() * 300);
+
+  const paragraphs = await page.locator('p').all();
+  if (paragraphs.length > 0) {
+    await paragraphs[0].hover();
+    await page.waitForTimeout(600 + Math.random() * 400);
+  }
+
+  return productId;
+}
+
+export async function addToCart(page: Page): Promise<void> {
+  const addBtn = page.locator('button:has-text("Add to Cart")');
+  await addBtn.click();
+  await page.waitForTimeout(500);
+}

tests/e2e/helpers/kafka.ts

@@ -0,0 +1,39 @@
+interface InteractionEvent {
+  sessionId: string;
+  event: string;
+  productId?: string;
+  timestamp: string;
+  metadata?: Record<string, any>;
+}
+
+const dumpKafkaTopic = async (backendUrl: string, topic: string) => {
+  const resp = await fetch(`${backendUrl}/api/kafka/dump?topic=${topic}`);
+  if (!resp.ok) throw new Error(`Kafka dump failed: ${resp.status}`);
+  const { messages = [] } = await resp.json();
+  return messages as any[];
+};
+
+export const waitForInteractionEvent = async (
+  backendUrl: string,
+  sessionId: string,
+  eventType: string,
+  maxRetries = 10,
+  pollInterval = 500
+): Promise<InteractionEvent | null> => {
+  for (let i = 0; i < maxRetries; i++) {
+    const msgs = await dumpKafkaTopic(backendUrl, "user-interactions");
+    const hit = msgs.find(m => m.sessionId === sessionId && m.event === eventType);
+    if (hit) return hit as InteractionEvent;
+    await new Promise<void>(r => setTimeout(r, pollInterval));
+  }
+  return null;
+};
+
+export const countProductViews = async (backendUrl: string, productId: string) =>
+  (await dumpKafkaTopic(backendUrl, "user-interactions")).reduce(
+    (n, m) => n + (m.productId === productId && m.event === "view_item_page" ? 1 : 0),
+    0
+  );
+
+export const getSessionEvents = async (backendUrl: string, sessionId: string) =>
+  (await dumpKafkaTopic(backendUrl, "user-interactions")).filter(m => m.sessionId === sessionId);

tests/e2e/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "e2e",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "playwright test",
+    "test:ui": "playwright test --ui",
+    "test:debug": "playwright test --debug"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "@playwright/test": "^1.57.0",
+    "@types/node": "^25.0.6",
+    "typescript": "^5.9.3"
+  }
+}

tests/e2e/playwright.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './scenarios',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  workers: 5,
+  reporter: 'list',
+  use: {
+    baseURL: process.env.WEB_URL || 'http://localhost:3000',
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  timeout: 60000,
+  expect: {
+    timeout: 10000,
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});

tests/e2e/scenarios/session-aware.spec.ts

@@ -0,0 +1,156 @@
+import { test, expect } from '../fixtures';
+import {
+  createFreshSession,
+  viewProductViaFlow,
+  rapidViewProductViaFlow,
+  humanLikeViewProduct,
+  getPriceFromDOM,
+  verifySessionConsistency,
+  addToCart,
+} from '../helpers/interactions';
+import { getSessionEvents } from '../helpers/kafka';
+
+test.describe('SessionAwarePricer E2E', () => {
+  const STORE_TYPE = 'hotel';
+
+  test('baseline: human-like behavior maintains base price', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId1 = await humanLikeViewProduct(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+
+    await page.waitForTimeout(1500);
+
+    const productId2 = await humanLikeViewProduct(page, STORE_TYPE);
+    const secondPrice = await getPriceFromDOM(page);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+
+    expect(Math.abs(secondPrice - baselinePrice) / baselinePrice).toBeLessThan(0.1);
+  });
+
+  test('agent detection: rapid robot-like behavior increases price', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await page.waitForTimeout(500);
+
+    await rapidViewProductViaFlow(page, 8, 100, STORE_TYPE);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+
+    await page.waitForTimeout(2500);
+
+    const events = await getSessionEvents(backendUrl, sessionId);
+    expect(events.length).toBeGreaterThanOrEqual(8);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const agentPrice = await getPriceFromDOM(page);
+
+    expect(agentPrice).toBeGreaterThan(baselinePrice);
+    expect((agentPrice - baselinePrice) / baselinePrice).toBeGreaterThan(0.01);
+  });
+
+  test('velocity threshold: high event rate triggers detection', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    const startTime = Date.now();
+    await rapidViewProductViaFlow(page, 10, 80, STORE_TYPE);
+    const duration = (Date.now() - startTime) / 1000;
+
+    const eventsPerSec = 10 / duration;
+    expect(eventsPerSec).toBeGreaterThan(2.0);
+
+    await page.waitForTimeout(2000);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const agentPrice = await getPriceFromDOM(page);
+
+    expect(agentPrice).toBeGreaterThan(baselinePrice);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('cart ratio: high cart/view ratio signals intent', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await page.waitForTimeout(500);
+    await addToCart(page);
+
+    await page.waitForTimeout(2000);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const cartPrice = await getPriceFromDOM(page);
+
+    expect(cartPrice).toBeGreaterThanOrEqual(baselinePrice);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('mixed behavior: occasional fast actions tolerated', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId1 = await humanLikeViewProduct(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await page.waitForTimeout(1200);
+
+    await rapidViewProductViaFlow(page, 2, 150, STORE_TYPE);
+
+    await page.waitForTimeout(1500);
+    await humanLikeViewProduct(page, STORE_TYPE);
+    const finalPrice = await getPriceFromDOM(page);
+
+    expect(Math.abs(finalPrice - baselinePrice) / baselinePrice).toBeLessThan(0.3);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('session isolation: agent behavior in one session does not affect others', async ({
+    page,
+    context,
+    backendUrl,
+  }) => {
+    const sessionIdA = await createFreshSession(page, STORE_TYPE);
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const basePrice = await getPriceFromDOM(page);
+
+    await rapidViewProductViaFlow(page, 10, 100, STORE_TYPE);
+    await page.waitForTimeout(2000);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const agentPrice = await getPriceFromDOM(page);
+    expect(agentPrice).toBeGreaterThan(basePrice * 0.99);
+
+    const page2 = await context.newPage();
+    const sessionIdB = await createFreshSession(page2, STORE_TYPE);
+
+    await page2.goto(`/products/${productId}`);
+    await page2.waitForLoadState('networkidle');
+    const cleanPrice = await getPriceFromDOM(page2);
+
+    expect(Math.abs(cleanPrice - basePrice) / basePrice).toBeLessThan(0.1);
+    expect(sessionIdA).not.toBe(sessionIdB);
+  });
+
+  test('session persistence: session ID maintained across views', async ({ page }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    await viewProductViaFlow(page, STORE_TYPE);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+
+    await viewProductViaFlow(page, STORE_TYPE);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+
+    await viewProductViaFlow(page, STORE_TYPE);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+});

tests/e2e/scenarios/surge-pricing.spec.ts

@@ -0,0 +1,111 @@
+import { test, expect } from '../fixtures';
+import {
+  createFreshSession,
+  viewProductViaFlow,
+  rapidViewProductViaFlow,
+  getPriceFromDOM,
+  verifySessionConsistency,
+} from '../helpers/interactions';
+import { waitForInteractionEvent, countProductViews } from '../helpers/kafka';
+
+test.describe('SimpleSurgePricer E2E', () => {
+  const STORE_TYPE = 'hotel';
+
+  test('baseline: initial price equals base price', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const price = await getPriceFromDOM(page);
+
+    expect(price).toBeGreaterThan(0);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('surge: rapid views trigger price increase', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await rapidViewProductViaFlow(page, 5, 200, STORE_TYPE);
+
+    await page.waitForTimeout(2000);
+
+    const evt = await waitForInteractionEvent(backendUrl, sessionId, 'view_item_page');
+    expect(evt).not.toBeNull();
+
+    const viewCount = await countProductViews(backendUrl, productId);
+    expect(viewCount).toBeGreaterThanOrEqual(5);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const surgedPrice = await getPriceFromDOM(page);
+
+    expect(surgedPrice).toBeGreaterThan(baselinePrice);
+    expect((surgedPrice - baselinePrice) / baselinePrice).toBeGreaterThan(0.01);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('threshold: price unchanged below threshold', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await rapidViewProductViaFlow(page, 2, 300, STORE_TYPE);
+
+    await page.waitForTimeout(1500);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const currentPrice = await getPriceFromDOM(page);
+
+    expect(Math.abs(currentPrice - baselinePrice) / baselinePrice).toBeLessThan(0.05);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('window: surge decays after window expires', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productId = await viewProductViaFlow(page, STORE_TYPE);
+    const baselinePrice = await getPriceFromDOM(page);
+
+    await rapidViewProductViaFlow(page, 5, 150, STORE_TYPE);
+
+    await page.waitForTimeout(1500);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const surgedPrice = await getPriceFromDOM(page);
+    expect(surgedPrice).toBeGreaterThan(baselinePrice);
+
+    await page.waitForTimeout(12000);
+
+    await page.goto(`/products/${productId}`);
+    await page.waitForLoadState('networkidle');
+    const decayedPrice = await getPriceFromDOM(page);
+    expect(decayedPrice).toBeLessThan(surgedPrice);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+
+  test('isolation: different products have independent surge', async ({ page, backendUrl }) => {
+    const sessionId = await createFreshSession(page, STORE_TYPE);
+
+    const productIdA = await viewProductViaFlow(page, STORE_TYPE);
+    const basePriceA = await getPriceFromDOM(page);
+
+    await rapidViewProductViaFlow(page, 5, 200, STORE_TYPE);
+    await page.waitForTimeout(2000);
+
+    await page.goto(`/products/${productIdA}`);
+    await page.waitForLoadState('networkidle');
+    const surgedPriceA = await getPriceFromDOM(page);
+
+    const productIdB = await viewProductViaFlow(page, STORE_TYPE);
+    const priceB = await getPriceFromDOM(page);
+
+    expect(surgedPriceA).toBeGreaterThan(basePriceA * 0.99);
+    expect(productIdA).not.toBe(productIdB);
+    expect(await verifySessionConsistency(page, sessionId)).toBeTruthy();
+  });
+});

tests/e2e/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "commonjs",
+    "lib": ["ES2022"],
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "types": ["node", "@playwright/test"]
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["node_modules"]
+}

web/package-lock.json

@@ -10,7 +10,7 @@
       "dependencies": {
         "@supabase/ssr": "^0.7.0",
         "@supabase/supabase-js": "^2.81.1",
-        "next": "16.0.0",
+        "next": "^16.0.0",
         "react": "19.2.0",
         "react-dom": "19.2.0",
         "zod": "^4.1.12"
@@ -526,15 +526,15 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.0.tgz",
-      "integrity": "sha512-s5j2iFGp38QsG1LWRQaE2iUY3h1jc014/melHFfLdrsMJPqxqDQwWNwyQTcNoUSGZlCVZuM7t7JDMmSyRilsnA==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.7.tgz",
+      "integrity": "sha512-gpaNgUh5nftFKRkRQGnVi5dpcYSKGcZZkQffZ172OrG/XkrnS7UBTQ648YY+8ME92cC4IojpI2LqTC8sTDhAaw==",
       "license": "MIT"
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.0.tgz",
-      "integrity": "sha512-/CntqDCnk5w2qIwMiF0a9r6+9qunZzFmU0cBX4T82LOflE72zzH6gnOjCwUXYKOBlQi8OpP/rMj8cBIr18x4TA==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.7.tgz",
+      "integrity": "sha512-LlDtCYOEj/rfSnEn/Idi+j1QKHxY9BJFmxx7108A6D8K0SB+bNgfYQATPk/4LqOl4C0Wo3LACg2ie6s7xqMpJg==",
       "cpu": [
         "arm64"
       ],
@@ -548,9 +548,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.0.tgz",
-      "integrity": "sha512-hB4GZnJGKa8m4efvTGNyii6qs76vTNl+3dKHTCAUaksN6KjYy4iEO3Q5ira405NW2PKb3EcqWiRaL9DrYJfMHg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.7.tgz",
+      "integrity": "sha512-rtZ7BhnVvO1ICf3QzfW9H3aPz7GhBrnSIMZyr4Qy6boXF0b5E3QLs+cvJmg3PsTCG2M1PBoC+DANUi4wCOKXpA==",
       "cpu": [
         "x64"
       ],
@@ -564,9 +564,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.0.tgz",
-      "integrity": "sha512-E2IHMdE+C1k+nUgndM13/BY/iJY9KGCphCftMh7SXWcaQqExq/pJU/1Hgn8n/tFwSoLoYC/yUghOv97tAsIxqg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.7.tgz",
+      "integrity": "sha512-mloD5WcPIeIeeZqAIP5c2kdaTa6StwP4/2EGy1mUw8HiexSHGK/jcM7lFuS3u3i2zn+xH9+wXJs6njO7VrAqww==",
       "cpu": [
         "arm64"
       ],
@@ -580,9 +580,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.0.tgz",
-      "integrity": "sha512-xzgl7c7BVk4+7PDWldU+On2nlwnGgFqJ1siWp3/8S0KBBLCjonB6zwJYPtl4MUY7YZJrzzumdUpUoquu5zk8vg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.7.tgz",
+      "integrity": "sha512-+ksWNrZrthisXuo9gd1XnjHRowCbMtl/YgMpbRvFeDEqEBd523YHPWpBuDjomod88U8Xliw5DHhekBC3EOOd9g==",
       "cpu": [
         "arm64"
       ],
@@ -596,9 +596,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.0.tgz",
-      "integrity": "sha512-sdyOg4cbiCw7YUr0F/7ya42oiVBXLD21EYkSwN+PhE4csJH4MSXUsYyslliiiBwkM+KsuQH/y9wuxVz6s7Nstg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.7.tgz",
+      "integrity": "sha512-4WtJU5cRDxpEE44Ana2Xro1284hnyVpBb62lIpU5k85D8xXxatT+rXxBgPkc7C1XwkZMWpK5rXLXTh9PFipWsA==",
       "cpu": [
         "x64"
       ],
@@ -612,9 +612,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.0.tgz",
-      "integrity": "sha512-IAXv3OBYqVaNOgyd3kxR4L3msuhmSy1bcchPHxDOjypG33i2yDWvGBwFD94OuuTjjTt/7cuIKtAmoOOml6kfbg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.7.tgz",
+      "integrity": "sha512-HYlhqIP6kBPXalW2dbMTSuB4+8fe+j9juyxwfMwCe9kQPPeiyFn7NMjNfoFOfJ2eXkeQsoUGXg+O2SE3m4Qg2w==",
       "cpu": [
         "x64"
       ],
@@ -628,9 +628,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.0.tgz",
-      "integrity": "sha512-bmo3ncIJKUS9PWK1JD9pEVv0yuvp1KPuOsyJTHXTv8KDrEmgV/K+U0C75rl9rhIaODcS7JEb6/7eJhdwXI0XmA==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.7.tgz",
+      "integrity": "sha512-EviG+43iOoBRZg9deGauXExjRphhuYmIOJ12b9sAPy0eQ6iwcPxfED2asb/s2/yiLYOdm37kPaiZu8uXSYPs0Q==",
       "cpu": [
         "arm64"
       ],
@@ -644,9 +644,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.0.tgz",
-      "integrity": "sha512-O1cJbT+lZp+cTjYyZGiDwsOjO3UHHzSqobkPNipdlnnuPb1swfcuY6r3p8dsKU4hAIEO4cO67ZCfVVH/M1ETXA==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.7.tgz",
+      "integrity": "sha512-gniPjy55zp5Eg0896qSrf3yB1dw4F/3s8VK1ephdsZZ129j2n6e1WqCbE2YgcKhW9hPB9TVZENugquWJD5x0ug==",
       "cpu": [
         "x64"
       ],
@@ -1447,12 +1447,12 @@
       }
     },
     "node_modules/next": {
-      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/next/-/next-16.0.0.tgz",
-      "integrity": "sha512-nYohiNdxGu4OmBzggxy9rczmjIGI+TpR5vbKTsE1HqYwNm1B+YSiugSrFguX6omMOKnDHAmBPY4+8TNJk0Idyg==",
+      "version": "16.0.7",
+      "resolved": "https://registry.npmjs.org/next/-/next-16.0.7.tgz",
+      "integrity": "sha512-3mBRJyPxT4LOxAJI6IsXeFtKfiJUbjCLgvXO02fV8Wy/lIhPvP94Fe7dGhUgHXcQy4sSuYwQNcOLhIfOm0rL0A==",
       "license": "MIT",
       "dependencies": {
-        "@next/env": "16.0.0",
+        "@next/env": "16.0.7",
         "@swc/helpers": "0.5.15",
         "caniuse-lite": "^1.0.30001579",
         "postcss": "8.4.31",
@@ -1465,14 +1465,14 @@
         "node": ">=20.9.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "16.0.0",
-        "@next/swc-darwin-x64": "16.0.0",
-        "@next/swc-linux-arm64-gnu": "16.0.0",
-        "@next/swc-linux-arm64-musl": "16.0.0",
-        "@next/swc-linux-x64-gnu": "16.0.0",
-        "@next/swc-linux-x64-musl": "16.0.0",
-        "@next/swc-win32-arm64-msvc": "16.0.0",
-        "@next/swc-win32-x64-msvc": "16.0.0",
+        "@next/swc-darwin-arm64": "16.0.7",
+        "@next/swc-darwin-x64": "16.0.7",
+        "@next/swc-linux-arm64-gnu": "16.0.7",
+        "@next/swc-linux-arm64-musl": "16.0.7",
+        "@next/swc-linux-x64-gnu": "16.0.7",
+        "@next/swc-linux-x64-musl": "16.0.7",
+        "@next/swc-win32-arm64-msvc": "16.0.7",
+        "@next/swc-win32-x64-msvc": "16.0.7",
         "sharp": "^0.34.4"
       },
       "peerDependencies": {

web/package.json

@@ -10,7 +10,7 @@
   "dependencies": {
     "@supabase/ssr": "^0.7.0",
     "@supabase/supabase-js": "^2.81.1",
-    "next": "16.0.0",
+    "next": "^16.0.0",
     "react": "19.2.0",
     "react-dom": "19.2.0",
     "zod": "^4.1.12"

web/src/app/airline/checkout/page.tsx

@@ -0,0 +1,11 @@
+export default function AirlineCheckout() {
+  return (
+    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-sky-50 to-blue-50">
+      <div className="text-center p-8">
+        <h1 className="text-4xl font-light text-gray-800 mb-4">
+          Thank you for flying with us
+        </h1>
+      </div>
+    </div>
+  );
+}

web/src/app/api/ingest/route.ts

@@ -7,7 +7,7 @@ export async function POST(req: NextRequest) {
   try {
     const body = await req.json();
 
-    const storeMode = process.env.STORE_MODE || 'hotel';
+    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || process.env.STORE_MODE || 'hotel';
     const userAgent = req.headers.get('user-agent') || undefined;
 
     const event: EventBase = {

web/src/app/api/pricing/route.ts

@@ -11,7 +11,7 @@ export async function GET(req: NextRequest) {
     const productId = searchParams.get('productId');
     const sessionId = searchParams.get('sessionId');
     const experimentId = searchParams.get('experimentId');
-    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || 'shop';
+    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || process.env.STORE_MODE || 'hotel';
 
     if (!productId) {
         return NextResponse.json(
@@ -20,10 +20,40 @@ export async function GET(req: NextRequest) {
         );
     }
 
-    // stub: call external pricing provider (random for now)
-    const basePrice = 100 + Math.random() * 900; // 100-1000 range
-    const price = Math.round(basePrice * 100) / 100;
     const timestamp = new Date().toISOString();
+    let price: number;
+    let basePrice: number | undefined;
+    let markup: number | undefined;
+    let elasticity: number | undefined;
+
+    // call real pricing provider
+    const providerUrl = process.env.PRICING_PROVIDER_URL || 'http://localhost:5001';
+    try {
+        const queryParams = new URLSearchParams();
+        if (sessionId) queryParams.append('sessionId', sessionId);
+        if (experimentId) queryParams.append('experimentId', experimentId);
+
+        const providerResponse = await fetch(
+            `${providerUrl}/api/${storeMode}/price/${productId}?${queryParams.toString()}`,
+            { headers: { 'Accept': 'application/json' }, cache: 'no-store' }
+        );
+
+        if (!providerResponse.ok) {
+            throw new Error(`Provider returned ${providerResponse.status}`);
+        }
+
+        const providerData = await providerResponse.json();
+        price = providerData.price;
+        basePrice = providerData.base_price;
+        markup = providerData.markup;
+        elasticity = providerData.elasticity;
+
+    } catch (err) {
+        console.error('[pricing-provider-error]', err);
+        // fallback to random pricing if provider unavailable
+        const randomBase = 100 + Math.random() * 900;
+        price = Math.round(randomBase * 100) / 100;
+    }
 
     // log price to kafka for elasticity computation
     if (sessionId) {
@@ -43,19 +73,13 @@ export async function GET(req: NextRequest) {
             });
         } catch (err) {
             console.error('[price-log-error]', err);
-            // don't fail the pricing request if logging fails
         }
     }
 
-    // log in dev
     if (process.env.NODE_ENV === 'development') {
         console.log('[pricing-api]', {
-            productId,
-            sessionId,
-            experimentId,
-            storeMode,
-            price,
-            timestamp,
+            productId, sessionId, experimentId, storeMode,
+            price, basePrice, markup, elasticity, timestamp,
         });
     }
 

web/src/app/cart/page.tsx

@@ -96,7 +96,10 @@ export default function CartPage() {
                                 <span className="text-3xl font-bold">${total.toFixed(2)}</span>
                             </div>
                             <button
-                                onClick={() => dispatchInteraction('checkout_start', undefined, { total, itemCount })}
+                                onClick={() => {
+                                    dispatchInteraction('checkout_start', undefined, { total, itemCount });
+                                    window.location.href = '/checkout';
+                                }}
                                 className="w-full py-3 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors"
                             >
                                 Proceed to Checkout

web/src/app/hotel/checkout/page.tsx

@@ -0,0 +1,11 @@
+export default function HotelCheckout() {
+  return (
+    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-blue-50 to-indigo-50">
+      <div className="text-center p-8">
+        <h1 className="text-4xl font-light text-gray-800 mb-4">
+          Thank you for staying with us
+        </h1>
+      </div>
+    </div>
+  );
+}

web/src/components/feats/airline/AirlineHero.tsx

@@ -2,10 +2,20 @@
 
 import { useState, FormEvent } from 'react';
 import { useRouter } from 'next/navigation';
-import { Button, Label, Input, DateInput, RadioGroup, Dropdown, DropdownCounter } from '@/components/ui';
+import { Button, Label, DateInput, Dropdown, DropdownCounter, SelectDropdown, SelectOption } from '@/components/ui';
 import { dateToDaysFromToday } from '@/lib/airline-utils';
 
-type TripType = 'roundtrip' | 'oneway' | 'multicity';
+const CITIES: SelectOption[] = [
+  { value: 'JFK', label: 'New York (JFK)', sublabel: 'John F. Kennedy International' },
+  { value: 'LAX', label: 'Los Angeles (LAX)', sublabel: 'Los Angeles International' },
+  { value: 'ORD', label: 'Chicago (ORD)', sublabel: "O'Hare International" },
+  { value: 'MIA', label: 'Miami (MIA)', sublabel: 'Miami International' },
+  { value: 'SFO', label: 'San Francisco (SFO)', sublabel: 'San Francisco International' },
+  { value: 'SEA', label: 'Seattle (SEA)', sublabel: 'Seattle-Tacoma International' },
+  { value: 'ATL', label: 'Atlanta (ATL)', sublabel: 'Hartsfield-Jackson International' },
+  { value: 'DFW', label: 'Dallas (DFW)', sublabel: 'Dallas/Fort Worth International' },
+];
+
 
 const PlaneIcon = () => (
   <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
@@ -22,11 +32,9 @@ const LocationIcon = () => (
 
 export default function AirlineHero() {
   const router = useRouter();
-  const [tripType, setTripType] = useState<TripType>('roundtrip');
   const [origin, setOrigin] = useState('');
   const [destination, setDestination] = useState('');
   const [departDate, setDepartDate] = useState('');
-  const [returnDate, setReturnDate] = useState('');
   const [passengers, setPassengers] = useState({ adults: 1, children: 0, infants: 0 });
 
   const handleSearch = (e: FormEvent) => {
@@ -40,8 +48,6 @@ export default function AirlineHero() {
 
     if (origin) params.set('origin', origin);
     if (destination) params.set('destination', destination);
-    if (tripType !== 'roundtrip') params.set('tripType', tripType);
-    if (returnDate && tripType === 'roundtrip') params.set('returnDate', returnDate);
 
     params.set('adults', passengers.adults.toString());
     params.set('children', passengers.children.toString());
@@ -66,28 +72,15 @@ export default function AirlineHero() {
 
         <div className="search-form">
           <form onSubmit={handleSearch}>
-            <div className="mb-6">
-              <RadioGroup
-                name="tripType"
-                value={tripType}
-                onChange={setTripType}
-                options={[
-                  { value: 'roundtrip', label: 'Round-trip' },
-                  { value: 'oneway', label: 'One-way' },
-                  { value: 'multicity', label: 'Multi-city' },
-                ]}
-              />
-            </div>
-
-            <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
+            <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
               <div>
                 <Label htmlFor="origin">From</Label>
-                <Input
-                  type="text"
+                <SelectDropdown
                   id="origin"
                   value={origin}
-                  onChange={(e) => setOrigin(e.target.value)}
-                  placeholder="Airport or city"
+                  onChange={setOrigin}
+                  options={CITIES}
+                  placeholder="Select origin"
                   icon={<PlaneIcon />}
                   required
                 />
@@ -95,12 +88,12 @@ export default function AirlineHero() {
 
               <div>
                 <Label htmlFor="destination">To</Label>
-                <Input
-                  type="text"
+                <SelectDropdown
                   id="destination"
                   value={destination}
-                  onChange={(e) => setDestination(e.target.value)}
-                  placeholder="Airport or city"
+                  onChange={setDestination}
+                  options={CITIES}
+                  placeholder="Select destination"
                   icon={<LocationIcon />}
                   required
                 />
@@ -115,20 +108,6 @@ export default function AirlineHero() {
                   required
                 />
               </div>
-
-              <div>
-                <Label htmlFor="returnDate">Return</Label>
-                {tripType === 'roundtrip' ? (
-                  <DateInput
-                    id="returnDate"
-                    value={returnDate}
-                    onChange={(e) => setReturnDate(e.target.value)}
-                    required
-                  />
-                ) : (
-                  <DateInput id="returnDate" disabled />
-                )}
-              </div>
             </div>
 
             <div className="grid grid-cols-4 sm:grid-cols-3 lg:grid-cols-4 gap-4 mt-4">

web/src/components/feats/hotel/HotelCard.tsx

@@ -21,7 +21,7 @@ const AmenityIcon = ({ name }: { name: string }) => {
         breakfast: 'Breakfast',
         spa: 'Spa',
     };
-    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name}</span>;
+    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name.replaceAll("_", " ")}</span>;
 };
 
 export default function HotelCard({ hotel }: { hotel: Hotel }) {
@@ -47,18 +47,31 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
         window.location.href = `/hotel/products/${hotel.id}`;
     };
 
+    const imageUrl = `https://images.unsplash.com/photo-1551882547-ff40c63fe5fa?w=400&h=300&fit=crop`;
+
     return (
         <div
             className="hotel-card cursor-pointer"
             onClick={handleCardClick}
         >
-            <div className="hotel-image bg-gray-200 flex items-center justify-center">
-                <span className="text-gray-400 text-sm">Image</span>
+            <div className="hotel-image relative overflow-hidden">
+                <img
+                    src={imageUrl}
+                    alt={hotel.name}
+                    className="w-full h-full object-cover"
+                    onError={(e) => {
+                        e.currentTarget.style.display = 'none';
+                        const fallback = e.currentTarget.nextElementSibling as HTMLElement;
+                        if (fallback) fallback.style.display = 'flex';
+                    }}
+                />
+                <div className="absolute inset-0 bg-gray-200 flex items-center justify-center" style={{ display: 'none' }}>
+                    <span className="text-gray-400 text-sm">Image</span>
+                </div>
             </div>
 
             <div className="hotel-info">
                 <h3 ref={titleRef} className="hotel-name">{hotel.name}</h3>
-                <div className="hotel-location text-sm mb-2">{hotel.roomType}</div>
                 <div className="text-sm text-[var(--text-secondary)] mb-2">
                     {hotel.checkIn} - {hotel.checkOut}
                 </div>
@@ -67,9 +80,6 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
                         <AmenityIcon key={a} name={a} />
                     ))}
                 </div>
-                {hotel.refundable && (
-                    <div className="free-cancellation mt-2">Free cancellation</div>
-                )}
             </div>
 
             <div className="hotel-pricing">

web/src/components/feats/hotel/HotelDetails.tsx

@@ -1,6 +1,8 @@
 'use client';
 
+import { useState, useEffect } from 'react';
 import type { Hotel } from '@/lib/hotel-utils';
+import PriceDisplay from '@/components/ui/PriceDisplay';
 
 interface HotelDetailsProps {
   product: Hotel;
@@ -8,19 +10,63 @@ interface HotelDetailsProps {
   addedToCart: boolean;
 }
 
+const PriceTotalDisplay = ({ productId, nights }: { productId: string; nights: number }) => {
+  const [price, setPrice] = useState<number | null>(null);
+
+  useEffect(() => {
+    const fetchPrice = async () => {
+      try {
+        const sessionRes = await fetch('/api/session');
+        const sessionData = await sessionRes.json();
+        const params = new URLSearchParams({
+          productId,
+          sessionId: sessionData.sessionId || '',
+          experimentId: sessionData.experimentId || '',
+        });
+        const res = await fetch(`/api/pricing?${params.toString()}`);
+        const data = await res.json();
+        setPrice(data.price);
+      } catch (err) {
+        console.error('failed to fetch price for total:', err);
+      }
+    };
+    fetchPrice();
+  }, [productId]);
+
+  if (!price) return <span className="text-4xl font-bold text-gray-900">Loading...</span>;
+
+  return (
+    <span className="text-4xl font-bold text-gray-900">
+      ${(price * nights).toFixed(2)}
+    </span>
+  );
+};
+
 export default function HotelDetails({ product, onAddToCart, addedToCart }: HotelDetailsProps) {
+  const imageUrl = `https://images.unsplash.com/photo-1566073771259-6a8506099945?w=800&h=600&fit=crop`;
+
   return (
     <div className="w-full flex flex-col lg:flex-row gap-12 py-8">
-      {/* Image Section - Larger and cleaner */}
-      <div className="w-full lg:w-1/2 bg-gray-100 rounded-lg aspect-[4/3] flex items-center justify-center shrink-0">
-        <span className="text-gray-400 text-lg font-medium">Hotel Image</span>
+      <div className="w-full lg:w-1/2 rounded-lg aspect-[4/3] overflow-hidden shrink-0">
+        <img
+          src={imageUrl}
+          alt={product.name}
+          className="w-full h-full object-cover"
+          onError={(e) => {
+            e.currentTarget.style.display = 'none';
+            if (e.currentTarget.nextElementSibling) {
+              (e.currentTarget.nextElementSibling as HTMLElement).style.display = 'flex';
+            }
+          }}
+        />
+        <div className="w-full h-full bg-gray-100 rounded-lg flex items-center justify-center" style={{ display: 'none' }}>
+          <span className="text-gray-400 text-lg font-medium">Hotel Image</span>
+        </div>
       </div>
 
-      {/* Details Section - Full height/width usage */}
       <div className="flex-1 flex flex-col">
         <div className="border-b pb-6 mb-6">
           <h1 className="text-4xl font-bold text-gray-900 mb-2">{product.name}</h1>
-          <p className="text-xl text-gray-500">{product.roomType}</p>
         </div>
 
         <div className="grid grid-cols-2 gap-8 mb-8">
@@ -39,24 +85,17 @@ export default function HotelDetails({ product, onAddToCart, addedToCart }: Hote
           <div className="flex flex-wrap gap-3">
             {product.amenities.map(a => (
               <span key={a} className="px-3 py-1.5 bg-gray-100 text-gray-700 rounded-md text-sm font-medium">
-                {a}
+                  {a.replaceAll('_', ' ')}
               </span>
             ))}
           </div>
         </div>
 
-        {product.refundable && (
-          <div className="mb-8 p-4 bg-green-50 text-green-800 rounded-md inline-block">
-            <span className="font-medium">Free cancellation available</span>
-          </div>
-        )}
-
         <div className="mt-auto pt-6 border-t flex items-center justify-between">
           <div>
-            <p className="text-sm text-gray-500 mb-1">Total for {product.nights} night{product.nights > 1 ? 's' : ''}</p>
-            <div className="flex items-baseline gap-2">
-              <span className="text-4xl font-bold text-gray-900">${product.pricePerNight * product.nights}</span>
-              <span className="text-gray-500">/ {product.nights} nights</span>
+            <p className="text-sm text-gray-500 mb-1">Price per night</p>
+            <div className="mb-3">
+              <PriceDisplay productId={product.id} className="!text-2xl" />
             </div>
           </div>
 

web/src/components/ui/DateInput.tsx

@@ -1,7 +1,29 @@
-import { InputHTMLAttributes } from 'react';
+import { InputHTMLAttributes, useMemo } from 'react';
 
 interface DateInpProps extends Omit<InputHTMLAttributes<HTMLInputElement>, 'type'> {}
 
 export default function DateInput({ className = '', ...props }: DateInpProps) {
-  return <input type="date" className={`input-field ${className}`.trim()} {...props} />;
+  const { minDate, maxDate } = useMemo(() => {
+    const today = new Date();
+    const tomorrow = new Date(today);
+    tomorrow.setDate(today.getDate() + 1);
+
+    const tenDaysOut = new Date(tomorrow);
+    tenDaysOut.setDate(tomorrow.getDate() + 9); // tomorrow + 9 = 10 days total
+
+    return {
+      minDate: tomorrow.toISOString().split('T')[0],
+      maxDate: tenDaysOut.toISOString().split('T')[0]
+    };
+  }, []);
+
+  return (
+    <input
+      type="date"
+      className={`input-field ${className}`.trim()}
+      min={minDate}
+      max={maxDate}
+      {...props}
+    />
+  );
 }

web/src/components/ui/Navigation.tsx

@@ -20,7 +20,7 @@ const NavLink = ({ href, children }: { href: string; children: React.ReactNode }
       href={href}
       className={`px-4 py-2 rounded-md transition-colors ${
         isActive
-          ? 'bg-[var(--accent-primary)] text-white font-semibold'
+          ? 'bg-[var(--accent-primary)] font-semibold'
           : 'hover:bg-[var(--accent-primary-light)] text-[var(--text-primary)]'
       }`}
     >
@@ -37,9 +37,7 @@ export default function Navigation() {
           <div className="flex items-center space-x-1">
             <NavLink href="/">Home</NavLink>
             <NavLink href="/products">Products</NavLink>
-            <NavLink href="/search">Search</NavLink>
             <NavLink href="/cart">Cart</NavLink>
-            <NavLink href="/checkout">Checkout</NavLink>
           </div>
         </div>
       </div>

web/src/components/ui/SelectDropdown.tsx

@@ -0,0 +1,119 @@
+'use client';
+
+import { useState, useRef, useEffect, ReactNode } from 'react';
+
+export interface SelectOption {
+  value: string;
+  label: string;
+  sublabel?: string;
+}
+
+interface SelectDropdownProps {
+  value: string;
+  onChange: (value: string) => void;
+  options: SelectOption[];
+  placeholder?: string;
+  icon?: ReactNode;
+  required?: boolean;
+  id?: string;
+}
+
+export default function SelectDropdown({
+  value,
+  onChange,
+  options,
+  placeholder = 'Select...',
+  icon,
+  required,
+  id,
+}: SelectDropdownProps) {
+  const [open, setOpen] = useState(false);
+  const [filter, setFilter] = useState('');
+  const ref = useRef<HTMLDivElement>(null);
+  const inputRef = useRef<HTMLInputElement>(null);
+
+  useEffect(() => {
+    const handleClick = (e: MouseEvent) => {
+      if (ref.current && !ref.current.contains(e.target as Node)) {
+        setOpen(false);
+        setFilter('');
+      }
+    };
+    document.addEventListener('mousedown', handleClick);
+    return () => document.removeEventListener('mousedown', handleClick);
+  }, []);
+
+  const selectedOption = options.find((o) => o.value === value);
+  const filtered = options.filter(
+    (o) =>
+      o.label.toLowerCase().includes(filter.toLowerCase()) ||
+      o.value.toLowerCase().includes(filter.toLowerCase()) ||
+      o.sublabel?.toLowerCase().includes(filter.toLowerCase())
+  );
+
+  const handleSelect = (opt: SelectOption) => {
+    onChange(opt.value);
+    setOpen(false);
+    setFilter('');
+  };
+
+  return (
+    <div className="relative" ref={ref}>
+      <div
+        className="input-field flex items-center gap-2 cursor-pointer box-border"
+        onClick={() => {
+          setOpen(true);
+          setTimeout(() => inputRef.current?.focus(), 0);
+        }}
+      >
+        {icon && <span className="text-[var(--text-secondary)]">{icon}</span>}
+        {open ? (
+          <input
+            ref={inputRef}
+            type="text"
+            id={id}
+            value={filter}
+            onChange={(e) => setFilter(e.target.value)}
+            placeholder={placeholder}
+            className="flex-1 bg-transparent outline-none text-sm text-[var(--text-primary)]"
+          />
+        ) : (
+          <span className={`flex-1 text-sm ${value ? 'text-[var(--text-primary)]' : 'text-[var(--text-secondary)]'}`}>
+            {selectedOption ? selectedOption.label : placeholder}
+          </span>
+        )}
+        <svg
+          className={`w-4 h-4 text-[var(--text-secondary)] transition-transform ${open ? 'rotate-180' : ''}`}
+          fill="none"
+          stroke="currentColor"
+          viewBox="0 0 24 24"
+        >
+          <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" />
+        </svg>
+      </div>
+      {open && (
+        <div className="absolute z-20 mt-1 w-full bg-[var(--bg-primary)] border-2 border-[var(--accent-primary)] rounded-md shadow-lg max-h-60 overflow-y-auto">
+          {filtered.length === 0 ? (
+            <div className="px-4 py-3 text-sm text-[var(--text-secondary)]">No results</div>
+          ) : (
+            filtered.map((opt) => (
+              <div
+                key={opt.value}
+                onClick={() => handleSelect(opt)}
+                className={`px-4 py-2 cursor-pointer transition-colors hover:bg-[var(--accent-primary-light)] ${
+                  opt.value === value ? 'bg-[var(--accent-primary-light)]' : ''
+                }`}
+              >
+                <div className="text-sm font-medium text-[var(--text-primary)]">{opt.label}</div>
+                {opt.sublabel && <div className="text-xs text-[var(--text-secondary)]">{opt.sublabel}</div>}
+              </div>
+            ))
+          )}
+        </div>
+      )}
+      {required && !value && (
+        <input type="text" required className="sr-only" tabIndex={-1} value="" onChange={() => {}} />
+      )}
+    </div>
+  );
+}

web/src/components/ui/index.ts

@@ -5,3 +5,5 @@ export { default as DateInput } from './DateInput';
 export { default as RadioGroup } from './RadioGroup';
 export { default as Dropdown, DropdownCounter } from './Dropdown';
 export { default as Navigation } from './Navigation';
+export { default as SelectDropdown } from './SelectDropdown';
+export type { SelectOption } from './SelectDropdown';

web/src/lib/config.ts

@@ -16,7 +16,7 @@ const envSchema = z.object({
 // parse and validate env at module load, fail fast with descriptive errors
 const parseEnv = (): Env => {
     const result = envSchema.safeParse({
-        STORE_MODE: process.env.STORE_MODE,
+        STORE_MODE: process.env.NEXT_PUBLIC_STORE_MODE || process.env.STORE_MODE,
         NEXT_PUBLIC_API_BASE: process.env.NEXT_PUBLIC_API_BASE,
         NEXT_PUBLIC_APP_ENV: process.env.NEXT_PUBLIC_APP_ENV,
     });

web/src/lib/hotel-utils.ts

@@ -21,7 +21,6 @@ export interface Hotel {
   checkOut: string;
   dateIndex: number;
   amenities: string[];
-  refundable: boolean;
   pricePerNight: number;
   nights: number;
 }
@@ -30,19 +29,37 @@ const EPOCH = new Date(0);
 
 export const transformProduct = (p: HotelProduct): Hotel => {
   const { id, room_type, date_index, metadata } = p;
-  const checkIn = new Date(EPOCH.getTime() + date_index * 86400000);
+
+  // DB stores date_index as days since epoch
+  // but if value is small (<1000), treat as days from today for backward compat
+  let checkIn: Date;
+  if (date_index < 1000) {
+    // legacy: treat as offset from today
+    const today = new Date();
+    today.setHours(0, 0, 0, 0);
+    checkIn = new Date(today.getTime() + date_index * 86400000);
+  } else {
+    // proper: days since epoch
+    checkIn = new Date(EPOCH.getTime() + date_index * 86400000);
+  }
+
   const nights = 1;
   const checkOut = new Date(checkIn.getTime() + nights * 86400000);
 
+  const formatOpts: Intl.DateTimeFormatOptions = {
+    month: 'short',
+    day: 'numeric',
+    year: checkIn.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined
+  };
+
   return {
     id,
     name: metadata?.name || room_type,
     roomType: room_type,
-    checkIn: checkIn.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
-    checkOut: checkOut.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
+    checkIn: checkIn.toLocaleDateString('en-US', formatOpts),
+    checkOut: checkOut.toLocaleDateString('en-US', formatOpts),
     dateIndex: date_index,
     amenities: metadata?.amenities || [],
-    refundable: metadata?.refundable || false,
     pricePerNight: metadata?.base_price || 100,
     nights,
   };

web/src/styles/airline.css

@@ -278,6 +278,8 @@
   padding: 12px;
   transition: border-color 0.2s ease;
   width: 100%;
+  min-height: 48px;
+  box-sizing: border-box;
 }
 
 [data-mode="airline"] .input-field:focus {