fixing cookie import

.gitignore

@@ -6,8 +6,3 @@
 **/session_*.svg
 **/*graph.svg
 paper/src/bib/auto
-
-# Airflow logs - exclude DAG run logs
-experiments/airflow/logs/*
-experiments/airflow/logs/scheduler/
-experiments/airflow/logs/dag_processor_manager/

Makefile

@@ -49,8 +49,4 @@ install: $(VENV)
 test: $(VENV)
 	$(PYTEST) -v
 
-count-lines:
-	@find . \( -path '*/node_modules' -o -path '*/.venv' -o -path '*/venv' \) -prune -o \
-	\( -name "*.ts" -o -name "*.py" \) -type f -print0 | xargs -0 cat | wc -l
-
 .PHONY: all pdf clean watch run.webapp install test

README.md

@@ -1,12 +1,5 @@
-<img width="200" align="left" src="https://github.com/user-attachments/assets/d148b00d-e9f9-4280-89cc-0cc866e17251" />
-
-### PHANTOM
-
 [![Build PDF](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml/badge.svg)](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml)
-[![TPU Research Cloud](https://img.shields.io/badge/TPU%20Research%20Cloud-TRC%20supported-4285F4?logo=googlecloud&logoColor=white)](https://sites.research.google/trc/faq/)
-[![Vercel Deploy](https://deploy-badge.vercel.app/?url=https://phantom-hotel.vercel.app&name=Hotel)](https://phantom-hotel.vercel.app)
-[![Vercel Deploy](https://deploy-badge.vercel.app/?url=https://phantom-airline.vercel.app&name=Airline)](https://phantom-airline.vercel.app)
-
-
 
+- https://phantom-hotel.vercel.app/
+- https://phantom-airline.vercel.app/
 

backend/provider/app.py

@@ -1,113 +0,0 @@
-from fastapi import FastAPI, HTTPException, Query
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-from typing import Literal, Optional
-import uvicorn, os, sys
-from supabase import create_client, Client
-from dotenv import load_dotenv
-import numpy as np
-import pandas as pd
-load_dotenv()
-
-# Local imports of registry and pricing function
-
-sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../experiments/")
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.pricers import (
-    StaticPricer,
-    RandomPricer,
-    ElasticityBasedPricer
-)
-from procesing.steps import (
-    PredictPricesStep
-)
-from procesing import PipelineContext
-sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
-print(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
-from lib.model_registry import ModelRegistry
-
-# Config
-app = FastAPI(title="PHANTOM Pricing Provider")
-app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
-
-supabase: Client = create_client(os.getenv("NEXT_PUBLIC_SUPABASE_URL"), os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY"))
-registry = ModelRegistry()
-
-class PriceResponse(BaseModel):
-    productId: str
-    price: float
-    base_price: float
-    markup: float
-    elasticity: Optional[float] = None
-    model_version: str = 'latest'
-
-@app.get("/health")
-def health() -> dict:
-    return {"status": "healthy", "redis": registry.health_check()}
-
-@app.get("/api/{mode}/price/{productId}", response_model=PriceResponse)
-def get_price(mode: Literal['hotel', 'airline'], productId: str, sessionId: Optional[str] = Query(None), experimentId: Optional[str] = Query(None)):
-    product = supabase.table(f'{mode}_products').select("metadata").eq('id', productId).execute().data[0]
-    if not product: raise HTTPException(404, f"Product {productId} not found")
-
-    metadata = product['metadata']
-    base_price = metadata.get('base_price', 100.0)
-
-    # fetch pre-computed prices from registry
-    prices_df = registry.get_prices('latest')
-    elasticity_df = registry.get_elasticity('latest')
-
-    if prices_df is None:
-        # fallback: no pre-computed prices available
-        return PriceResponse(
-            productId=productId,
-            price=base_price,
-            base_price=base_price,
-            markup=1.0,
-            elasticity=None
-        )
-
-    # lookup pre-computed price for this product
-    product_price_row = prices_df[prices_df['productId'] == productId]
-    if product_price_row.empty:
-        # product not in pre-computed prices, fallback to base
-        return PriceResponse(
-            productId=productId,
-            price=base_price,
-            base_price=base_price,
-            markup=1.0,
-            elasticity=None
-        )
-
-    optimal_price = float(product_price_row['optimal_price'].iloc[0]) # TODO: use optimal_price everywhere as  aresult
-
-    # get elasticity if available
-    product_elasticity = None
-    if elasticity_df is not None:
-        product_elasticity_row = elasticity_df[elasticity_df['productId'] == productId]
-        if not product_elasticity_row.empty:
-            product_elasticity = float(product_elasticity_row['elasticity'].iloc[0])
-
-    return PriceResponse(
-        productId=productId,
-        price=optimal_price,
-        base_price=base_price,
-        markup=optimal_price/base_price,
-        elasticity=product_elasticity
-    )
-
-@app.get("/models")
-def list_models(): return registry.list_models()
-
-@app.post("/models/reload")
-def reload_models():
-    elasticity, pricing_model = registry.get_elasticity('latest'), registry.get_pricing_model('latest')
-    return {
-        "elasticity_loaded": bool(elasticity),
-        "n_products": len(elasticity) if elasticity is not None else 0,
-        "pricing_model_loaded": bool(pricing_model),
-        "model_class": pricing_model.__class__.__name__ if pricing_model else None
-    }
-
-if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PROVIDER_PORT", "5001")))

backend/provider/requirements.txt

@@ -1,16 +0,0 @@
-fastapi
-uvicorn[standard]
-pydantic
-numpy
-pandas
-scikit-learn
-redis
-supabase
-confluent-kafka>=2.3.0
-kafka-python
-graphviz
-python-dotenv>=1.0.0
-requests>=2.31.0
-typing-extensions>=4.8.0
-pypickle
-pymc

backend/server/app.py

@@ -11,7 +11,6 @@ from kafka import KafkaProducer, KafkaAdminClient, KafkaConsumer
 from kafka.admin import NewTopic
 from kafka.errors import TopicAlreadyExistsError
 from dotenv import load_dotenv
-from supabase import create_client, Client
 load_dotenv()
 
 app = FastAPI()
@@ -19,19 +18,6 @@ app = FastAPI()
 # kafka producer - lazy init
 _producer: Optional[KafkaProducer] = None
 
-# supabase client - lazy init
-_supabase: Optional[Client] = None
-
-def get_supabase() -> Client:
-    global _supabase
-    if _supabase is None:
-        url = os.getenv('NEXT_PUBLIC_SUPABASE_URL')
-        key = os.getenv('NEXT_PUBLIC_SUPABASE_ANON_KEY')
-        if not url or not key:
-            raise ValueError("Supabase credentials not configured")
-        _supabase = create_client(url, key)
-    return _supabase
-
 def get_producer() -> KafkaProducer:
     global _producer
     if _producer is None:
@@ -64,14 +50,6 @@ class EventPayload(BaseModel):
     userAgent: Optional[str] = None
     ts: Optional[str] = None
 
-class PriceLogPayload(BaseModel):
-    productId: str
-    price: float
-    sessionId: str
-    experimentId: Optional[str] = None
-    storeMode: str
-    ts: Optional[str] = None
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -95,8 +73,7 @@ async def startup_event():
         )
 
         topics = [
-            NewTopic(name='user-interactions', num_partitions=3, replication_factor=1),
+            NewTopic(name='user-interactions', num_partitions=3, replication_factor=1)
-            NewTopic(name='price-logs', num_partitions=3, replication_factor=1)
         ]
 
         admin.create_topics(new_topics=topics, validate_only=False)
@@ -148,52 +125,26 @@ async def ingest_logs(event: EventPayload):
         print(traceback.format_exc())
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.post("/api/kafka/price-log")
-async def ingest_price_log(price_log: PriceLogPayload):
-    try:
-        if not price_log.ts:
-            price_log.ts = datetime.utcnow().isoformat() + 'Z'
-
-        producer = get_producer()
-        future = producer.send(
-            'price-logs',
-            key=price_log.productId,
-            value=price_log.model_dump()
-        )
-        future.add_errback(lambda e: print(f"[KAFKA_PRICE_LOG_ERROR] {e}"))
-
-        return {"success": True}
-    except Exception as e:
-        import traceback
-        print(f"[PRICE_LOG_ERROR] {e}")
-        print(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=str(e))
-
 @app.get("/api/kafka/dump")
 def dump_logs(
-    topic: str = 'user-interactions',
     last_n: Optional[int] = None,
     t_start: Optional[str] = None,
     t_end: Optional[str] = None
 ):
-    """dump all messages from specified kafka topic
+    """dump all messages from user-interactions topic
 
     params:
-        topic: kafka topic to dump (default: user-interactions)
         last_n: return only last n messages (default: all)
-        t_start: filter by start timestamp iso format
+        t_start: filter by start timestamp iso format (future use)
-        t_end: filter by end timestamp iso format
+        t_end: filter by end timestamp iso format (future use)
     """
-    if topic not in ['user-interactions', 'price-logs']:
-        raise HTTPException(status_code=400, detail="Invalid topic")
-
     host = os.getenv('KAFKA_HOST', 'localhost')
     port = os.getenv('KAFKA_PORT', '9092')
     broker = f'{host}:{port}'
 
     try:
         consumer = KafkaConsumer(
-            topic,
+            'user-interactions',
             bootstrap_servers=[broker],
             auto_offset_reset='earliest',
             enable_auto_commit=False,
@@ -209,6 +160,7 @@ def dump_logs(
 
         # apply filters
         if t_start or t_end:
+            # filter by timestamp range if provided
             filtered = []
             for e in events:
                 ts = e.get('ts')
@@ -231,131 +183,6 @@ def dump_logs(
         print(traceback.format_exc())
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.get("/api/products/{product_id}")
-async def get_product_by_id(product_id: str):
-    """fetch single product by id from either hotel_products or airline_products"""
-    try:
-        supabase = get_supabase()
-
-        # try hotel_products first
-        response = supabase.table('hotel_products').select('*').eq('id', product_id).execute()
-        if response.data and len(response.data) > 0:
-            return {"success": True, "data": response.data[0]}
-
-        # try airline_products
-        response = supabase.table('airline_products').select('*').eq('id', product_id).execute()
-        if response.data and len(response.data) > 0:
-            return {"success": True, "data": response.data[0]}
-
-        raise HTTPException(status_code=404, detail="Product not found")
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        import traceback
-        print(f"[PRODUCT_BY_ID_ERROR] {e}")
-        print(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.get("/api/products/type/{product_type}")
-async def get_products(
-    product_type: str,
-    dateIndex: Optional[int] = None,
-    origin: Optional[str] = None,
-    destination: Optional[str] = None,
-    tripType: Optional[str] = None,
-    adults: Optional[int] = None,
-    children: Optional[int] = None,
-    infants: Optional[int] = None,
-    rooms: Optional[int] = None
-):
-    """fetch products from supabase based on type (hotel or airline)
-
-    params:
-        product_type: either 'hotel' or 'airline'
-        dateIndex: optional days offset from today (e.g., 0=today, 1=tomorrow, -1=yesterday)
-        origin: (airline) departure airport code
-        destination: (airline/hotel) arrival airport or hotel location
-        tripType: (airline) roundtrip, oneway, multicity
-        adults, children, infants: passenger counts
-        rooms: (hotel) number of rooms
-    """
-    if product_type not in ['hotel', 'airline']:
-        raise HTTPException(status_code=400, detail="product_type must be 'hotel' or 'airline'")
-
-    try:
-        supabase = get_supabase()
-        table = f'{product_type}_products'
-
-        query = supabase.table(table).select('*')
-
-        # filter by exact date_index if provided
-        # dateIndex from frontend is days from today, convert to days since epoch
-        if dateIndex is not None:
-            query = query.eq('date_index', dateIndex)
-
-        response = query.execute()
-        results = response.data
-
-        # apply in-memory filters based on metadata for airline products
-        if product_type == 'airline' and results:
-            filtered = []
-            for product in results:
-                metadata = product.get('metadata', {})
-
-                # filter by origin airport
-                if origin:
-                    dep = metadata.get('departure', {})
-                    if dep.get('airport') != origin:
-                        continue
-
-                # filter by destination airport
-                if destination:
-                    arr = metadata.get('arrival', {})
-                    if arr.get('airport') != destination:
-                        continue
-
-                # passenger count validation (ensure total capacity)
-                if adults is not None or children is not None or infants is not None:
-                    total_pax = (adults or 0) + (children or 0) + (infants or 0)
-                    avail = product.get('availability', 0)
-                    if avail < total_pax:
-                        continue
-
-                filtered.append(product)
-
-            results = filtered
-
-        # apply in-memory filters for hotel products
-        elif product_type == 'hotel' and results:
-            filtered = []
-            for product in results:
-                metadata = product.get('metadata', {})
-
-                # filter by occupancy capacity
-                if adults is not None:
-                    max_occ = metadata.get('max_occupancy', 2)
-                    if max_occ < adults:
-                        continue
-
-                # filter by room availability
-                if rooms is not None:
-                    avail = product.get('availability', 0)
-                    if avail < rooms:
-                        continue
-
-                filtered.append(product)
-
-            results = filtered
-
-        return {"success": True, "count": len(results), "data": results}
-
-    except Exception as e:
-        import traceback
-        print(f"[PRODUCTS_ERROR] {e}")
-        print(traceback.format_exc())
-        raise HTTPException(status_code=500, detail=str(e))
-
 
 
 if __name__ == "__main__":

backend/server/requirements.txt

@@ -3,4 +3,3 @@ uvicorn[standard]==0.24.0
 kafka-python==2.0.2
 pydantic==2.5.0
 python-dotenv==1.0.0
-supabase==2.9.1

docker-compose.e2e.yml

@@ -1,161 +0,0 @@
-# Docker Compose configuration for E2E testing
-# Usage: docker compose -f docker-compose.e2e.yml up -d
-#
-# This configuration runs only the services needed for E2E pricing tests:
-# - Backend API (event ingestion)
-# - Kafka + Zookeeper (event streaming)
-# - Redis (model registry)
-# - Pricing Provider (price serving)
-#
-# Excluded for E2E tests:
-# - Airflow (pipeline runs directly via test worker)
-# - PostgreSQL (not needed without Airflow)
-# - TensorBoard (ML visualization not needed)
-
-services:
-  # Backend API for event ingestion
-  backend:
-    container_name: "PHANTOM-e2e-backend"
-    build:
-      context: .
-      dockerfile: docker/backend.Dockerfile
-    ports:
-      - "${BACKEND_PORT:-5000}:5000"
-    environment:
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - BACKEND_PORT=5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-    depends_on:
-      kafka:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 10s
-    restart: unless-stopped
-
-  # Redis for model registry
-  redis:
-    container_name: "PHANTOM-e2e-redis"
-    build:
-      context: ./docker
-      dockerfile: Redis.dockerfile
-    ports:
-      - "${REDIS_PORT:-6378}:6379"
-    volumes:
-      - e2e_redis_data:/data
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 5s
-      timeout: 3s
-      retries: 5
-    restart: unless-stopped
-
-  # Zookeeper for Kafka coordination
-  zookeeper:
-    container_name: "PHANTOM-e2e-zookeeper"
-    build:
-      context: ./docker
-      dockerfile: Zookeeper.dockerfile
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-    ports:
-      - "2181:2181"
-    healthcheck:
-      test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep imok"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    restart: unless-stopped
-
-  # Kafka for event streaming
-  kafka:
-    container_name: "PHANTOM-e2e-kafka"
-    build:
-      context: ./docker
-      dockerfile: Kafka.dockerfile
-    depends_on:
-      zookeeper:
-        condition: service_healthy
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
-      # Faster topic creation for tests
-      KAFKA_NUM_PARTITIONS: 1
-      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
-    ports:
-      - "${KAFKA_PORT:-9092}:9092"
-    volumes:
-      - e2e_kafka_data:/var/lib/kafka/data
-    healthcheck:
-      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list"]
-      interval: 10s
-      timeout: 10s
-      retries: 10
-      start_period: 30s
-    restart: unless-stopped
-
-  # Redpanda Console for Kafka debugging (optional)
-  redpanda-console:
-    container_name: "PHANTOM-e2e-redpanda-console"
-    build:
-      context: ./docker
-      dockerfile: RedpandaConsole.dockerfile
-    depends_on:
-      kafka:
-        condition: service_healthy
-    environment:
-      KAFKA_BROKERS: kafka:29092
-    ports:
-      - "${REDPANDA_CONSOLE_PORT:-8080}:8080"
-    restart: unless-stopped
-    profiles:
-      - debug  # Only start with --profile debug
-
-  # Pricing Provider for serving prices
-  pricing-provider:
-    container_name: "PHANTOM-e2e-pricing-provider"
-    build:
-      context: .
-      dockerfile: docker/Provider.dockerfile
-    depends_on:
-      redis:
-        condition: service_healthy
-      kafka:
-        condition: service_healthy
-    environment:
-      - PROVIDER_PORT=5001
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - BACKEND_URL=http://backend:5000
-    ports:
-      - "${PROVIDER_PORT:-5001}:5001"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 10s
-    restart: unless-stopped
-
-volumes:
-  e2e_kafka_data:
-  e2e_redis_data:
-
-networks:
-  default:
-    name: phantom-e2e-network

docker-compose.yml

@@ -1,15 +1,4 @@
 services:
-
-  tensorboard:
-    image: tensorflow/tensorflow:latest
-    container_name: "PHANTOM-tensorboard"
-    ports:
-      - "6006:6006"
-    volumes:
-      - ./experiments/ml/runs:/logs
-    command: tensorboard --logdir=/logs --host=0.0.0.0 --port=6006
-    restart: unless-stopped
-
   backend:
     container_name: "PHANTOM-backend"
     build:
@@ -20,9 +9,6 @@ services:
     environment:
       - KAFKA_HOST=kafka
       - KAFKA_PORT=29092
-      - BACKEND_PORT=5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
     depends_on:
       - kafka
     restart: unless-stopped
@@ -82,133 +68,6 @@ services:
       - "${REDPANDA_CONSOLE_PORT:-8080}:8080"
     restart: unless-stopped
 
-  postgres:
-    container_name: "PHANTOM-postgres"
-    image: postgres:13
-    environment:
-      - POSTGRES_USER=airflow
-      - POSTGRES_PASSWORD=airflow
-      - POSTGRES_DB=airflow
-    ports:
-      - "5433:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    restart: unless-stopped
-
-  airflow-init:
-    container_name: "PHANTOM-airflow-init"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      - postgres
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - _AIRFLOW_DB_MIGRATE=true
-      - _AIRFLOW_WWW_USER_CREATE=true
-      - _AIRFLOW_WWW_USER_USERNAME=admin
-      - _AIRFLOW_WWW_USER_PASSWORD=admin
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    command: version
-    restart: "no"
-
-  airflow-webserver:
-    container_name: "PHANTOM-airflow-webserver"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      - postgres
-      - airflow-init
-      - redis
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
-      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - BACKEND_URL=http://backend:5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    ports:
-      - "${AIRFLOW_WEBSERVER_PORT:-8085}:8080"
-    command: webserver
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 30s
-
-  airflow-scheduler:
-    container_name: "PHANTOM-airflow-scheduler"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      airflow-webserver:
-        condition: service_healthy
-      redis:
-        condition: service_started
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - BACKEND_URL=http://backend:5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    command: scheduler
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 30s
-
-  pricing-provider:
-    container_name: "PHANTOM-pricing-provider"
-    build:
-      context: .
-      dockerfile: docker/Provider.dockerfile
-    depends_on:
-      - redis
-      - kafka
-    environment:
-      - PROVIDER_PORT=5001
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - BACKEND_URL=http://localhost:5000
-    ports:
-      - "${PROVIDER_PORT:-5001}:5001"
-    restart: unless-stopped
-
 volumes:
   phantom_kafka_data:
   phantom_redis_data:
-  postgres_data:

docker/Airflow.dockerfile

@@ -1,30 +0,0 @@
-FROM apache/airflow:2.7.3-python3.11
-
-USER root
-
-# install system deps if needed
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-USER airflow
-
-# copy requirements for pipeline dependencies
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-
-# install postgres driver and providers
-RUN pip install --no-cache-dir \
-    psycopg2-binary \
-    apache-airflow-providers-postgres
-
-# set airflow home
-ENV AIRFLOW_HOME=/opt/airflow
-
-COPY --chown=airflow:root experiments/airflow/dags ${AIRFLOW_HOME}/dags
-COPY --chown=airflow:root experiments/procesing ${AIRFLOW_HOME}/procesing
-COPY --chown=airflow:root lib ${AIRFLOW_HOME}/lib
-
-# create logs and plugins dirs (airflow expects them)
-RUN mkdir -p ${AIRFLOW_HOME}/logs ${AIRFLOW_HOME}/plugins

docker/Airflow.railway.dockerfile

@@ -1,41 +0,0 @@
-FROM apache/airflow:2.7.3-python3.11
-
-USER root
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    supervisor \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-USER airflow
-
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-
-RUN pip install --no-cache-dir \
-    psycopg2-binary \
-    apache-airflow-providers-postgres
-
-ENV AIRFLOW_HOME=/opt/airflow
-ENV AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-ENV AIRFLOW__CORE__LOAD_EXAMPLES=false
-ENV AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-ENV AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
-
-# copy all code into image (standalone - no volume mounts needed)
-COPY --chown=airflow:root experiments/airflow/dags ${AIRFLOW_HOME}/dags
-COPY --chown=airflow:root experiments/procesing ${AIRFLOW_HOME}/procesing
-COPY --chown=airflow:root lib ${AIRFLOW_HOME}/lib
-
-RUN mkdir -p ${AIRFLOW_HOME}/logs ${AIRFLOW_HOME}/plugins
-
-# copy entrypoint script
-COPY --chown=airflow:root docker/airflow-railway-entrypoint.sh /entrypoint.sh
-USER root
-RUN chmod +x /entrypoint.sh
-USER airflow
-
-EXPOSE 8080
-
-ENTRYPOINT ["/entrypoint.sh"]

docker/Provider.dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-# Install system dependencies including graphviz
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    graphviz \
-    libgraphviz-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Copy and install Python dependencies
-COPY backend/provider/requirements.txt /app/
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy application code into image
-COPY lib/ /app/lib/
-COPY experiments/procesing/ /app/procesing/
-COPY backend/provider/ /app/provider/
-
-ENV PYTHONPATH=/app:/app/lib:/app/procesing
-
-WORKDIR /app/provider
-
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5001"]

docker/airflow-railway-entrypoint.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-set -e
-
-# init db and create admin user on first run
-airflow db migrate
-
-# create admin user if not exists
-airflow users create \
-    --username "${AIRFLOW_ADMIN_USER:-admin}" \
-    --password "${AIRFLOW_ADMIN_PASSWORD:-admin}" \
-    --firstname Admin \
-    --lastname User \
-    --role Admin \
-    --email admin@example.com || true
-
-# start scheduler in background
-airflow scheduler &
-
-# start webserver in foreground (Railway needs one foreground process)
-exec airflow webserver --port ${PORT:-8080}

e2e/README.md

@@ -1,255 +0,0 @@
-# PHANTOM Dynamic Pricing E2E Test Suite
-
-End-to-end tests validating the dynamic pricing pipeline, including SimpleSurgePricer and SessionAwarePricer functionality.
-
-## System Under Test (SUT)
-
-```
-┌─────────────────────────────────────────────────────────────────────────┐
-│                        PHANTOM Pricing Pipeline                         │
-├─────────────────────────────────────────────────────────────────────────┤
-│                                                                         │
-│  ┌──────────────┐    ┌──────────────┐    ┌──────────────────────────┐  │
-│  │  Test Runner │───▶│ Backend API  │───▶│ Kafka (user-interactions)│  │
-│  │  (Playwright)│    │ POST /ingest │    │                          │  │
-│  └──────────────┘    └──────────────┘    └────────────┬─────────────┘  │
-│         │                                              │                │
-│         │                                              ▼                │
-│         │                                 ┌──────────────────────────┐  │
-│         │                                 │   Pipeline Worker        │  │
-│         │                                 │   - Fetch interactions   │  │
-│         │                                 │   - Compute demand       │  │
-│         │                                 │   - Apply surge pricing  │  │
-│         │                                 └────────────┬─────────────┘  │
-│         │                                              │                │
-│         │                                              ▼                │
-│         │                                 ┌──────────────────────────┐  │
-│         │                                 │   Redis (Model Registry) │  │
-│         │                                 │   - prices:latest        │  │
-│         │                                 └────────────┬─────────────┘  │
-│         │                                              │                │
-│         │                                              ▼                │
-│         │     ┌──────────────┐           ┌──────────────────────────┐  │
-│         └────▶│ Pricing API  │◀──────────│   Pricing Provider       │  │
-│               │ GET /price   │           │   (serves from Redis)    │  │
-│               └──────────────┘           └──────────────────────────┘  │
-│                                                                         │
-└─────────────────────────────────────────────────────────────────────────┘
-```
-
-## Test Scenarios
-
-| Scenario | Description | Expected Outcome |
-|----------|-------------|------------------|
-| **Baseline** | No interactions for product | Price = base_price (markup = 1.0) |
-| **Surge** | 5+ interactions (above threshold) | Price = base_price × 1.5 |
-| **Discount** | 1 interaction (at threshold) | Price = base_price × 0.9 |
-| **Multi-Product** | Different demand per product | Each product priced by its demand |
-| **Propagation** | Pipeline → Redis → API | Prices visible via API |
-| **Event Types** | Mix of view, click, cart | All events counted in demand |
-| **Multi-Session** | Events from different sessions | Demand aggregated correctly |
-
-## Test Configuration
-
-The tests use aggressive thresholds for fast feedback:
-
-```typescript
-pricing: {
-  highThreshold: 3,      // Surge after 3 interactions
-  lowThreshold: 1,       // Discount at ≤1 interaction
-  surgeMultiplier: 1.5,  // 50% price increase
-  discountMultiplier: 0.9, // 10% discount
-  windowSize: 10_000,    // 10 second window
-}
-```
-
-## Quick Start
-
-### 1. Start E2E Services
-
-```bash
-# Start minimal services for E2E testing
-docker compose -f docker-compose.e2e.yml up -d
-
-# Wait for services to be healthy
-docker compose -f docker-compose.e2e.yml ps
-
-# Optional: Start with Kafka UI for debugging
-docker compose -f docker-compose.e2e.yml --profile debug up -d
-```
-
-### 2. Install Test Dependencies
-
-```bash
-cd e2e
-npm install
-npx playwright install
-```
-
-### 3. Run Tests
-
-```bash
-# Run all E2E tests
-npm test
-
-# Run with UI (interactive mode)
-npm run test:ui
-
-# Run specific test file
-npm run test:pricing
-
-# Run in debug mode
-npm run test:debug
-
-# View test report
-npm run test:report
-```
-
-### 4. Cleanup
-
-```bash
-docker compose -f docker-compose.e2e.yml down -v
-```
-
-## Environment Variables
-
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `BACKEND_URL` | `http://localhost:5000` | Backend API URL |
-| `PROVIDER_URL` | `http://localhost:5001` | Pricing Provider URL |
-| `REDIS_HOST` | `localhost` | Redis host |
-| `REDIS_PORT` | `6378` | Redis port |
-| `KAFKA_HOST` | `localhost` | Kafka host |
-| `KAFKA_PORT` | `9092` | Kafka port |
-
-## Test Architecture
-
-```
-e2e/
-├── playwright.config.ts    # Playwright configuration
-├── global-setup.ts         # Service health checks
-├── global-teardown.ts      # Cleanup
-├── package.json            # Dependencies and scripts
-├── tsconfig.json           # TypeScript configuration
-├── lib/
-│   ├── api-client.ts       # API interaction utilities
-│   ├── event-generator.ts  # Test event factory
-│   ├── pipeline-runner.ts  # TypeScript pipeline wrapper
-│   ├── pipeline-worker.py  # Python pipeline executor
-│   ├── fixtures.ts         # Playwright test fixtures
-│   └── index.ts            # Re-exports
-└── tests/
-    └── dynamic-pricing.spec.ts  # Main test file
-```
-
-## Pipeline Worker
-
-The tests use a dedicated Python pipeline worker (`lib/pipeline-worker.py`) instead of Airflow for faster, more reliable test execution.
-
-```bash
-# Run pipeline manually
-python3 lib/pipeline-worker.py \
-  --store-mode hotel \
-  --high-threshold 3 \
-  --surge-multiplier 1.5 \
-  --json-output
-
-# Dry run (no Redis publish)
-python3 lib/pipeline-worker.py --dry-run
-```
-
-## Debugging
-
-### View Kafka Events
-
-```bash
-# Via API
-curl "http://localhost:5000/api/kafka/dump?topic=user-interactions&last_n=10"
-
-# Via Redpanda Console (if started with --profile debug)
-open http://localhost:8080
-```
-
-### Check Redis State
-
-```bash
-docker exec -it PHANTOM-e2e-redis redis-cli
-> GET prices:latest
-> KEYS *
-```
-
-### View Pipeline Logs
-
-The pipeline worker logs detailed information:
-
-```
-[INFO] Starting E2E pricing pipeline: mode=hotel, high_threshold=3, surge_multiplier=1.5
-[INFO] Fetched 15 interaction records
-[INFO] Computed demand for 3 products
-[INFO] Applied surge pricing:
-  e2e-test...: base=$100.00 -> optimal=$150.00 (demand=5, markup=1.50x)
-[INFO] Published 3 prices to Redis
-```
-
-## Writing New Tests
-
-```typescript
-import { test, expect } from '../lib/fixtures';
-import { generateTestProductId } from '../lib/event-generator';
-
-test('my new pricing test', async ({ api, events, triggerPriceUpdate }) => {
-  // 1. Create unique product ID
-  const productId = generateTestProductId('my-test');
-
-  // 2. Log base price
-  await api.logPrice({
-    productId,
-    price: 100.0,
-    sessionId: events.session,
-    storeMode: 'hotel',
-  });
-
-  // 3. Generate events
-  const surgeEvents = events.generateSurgeSequence(productId, 5);
-  await api.ingestEvents(surgeEvents);
-
-  // 4. Trigger pipeline
-  const result = await triggerPriceUpdate();
-
-  // 5. Verify results
-  expect(result.success).toBe(true);
-  const pricedProduct = result.prices?.find(p => p.productId === productId);
-  expect(pricedProduct?.optimal_price).toBeGreaterThan(100);
-});
-```
-
-## Troubleshooting
-
-### "Backend not available"
-
-Ensure services are running:
-```bash
-docker compose -f docker-compose.e2e.yml ps
-docker compose -f docker-compose.e2e.yml logs backend
-```
-
-### "No interactions found"
-
-Check Kafka topic has events:
-```bash
-curl "http://localhost:5000/api/kafka/dump?topic=user-interactions"
-```
-
-### "Pipeline timeout"
-
-Increase timeout in `playwright.config.ts`:
-```typescript
-timeout: 180_000, // 3 minutes
-```
-
-### "Price not updated"
-
-Check Redis has latest prices:
-```bash
-docker exec -it PHANTOM-e2e-redis redis-cli GET prices:latest
-```

e2e/global-setup.ts

@@ -1,47 +0,0 @@
-import { testConfig } from './playwright.config';
-
-/**
- * Global setup for E2E tests
- * Verifies all services are healthy before running tests
- */
-async function globalSetup() {
-  console.log('\n🚀 PHANTOM E2E Test Suite - Global Setup\n');
-
-  // Check backend health
-  await checkService('Backend API', `${testConfig.backendUrl}/health`);
-
-  // Check pricing provider health
-  await checkService('Pricing Provider', `${testConfig.providerUrl}/health`);
-
-  console.log('\n✅ All services healthy. Starting tests...\n');
-}
-
-async function checkService(name: string, url: string): Promise<void> {
-  const maxRetries = 10;
-  const retryDelay = 2000;
-
-  for (let attempt = 1; attempt <= maxRetries; attempt++) {
-    try {
-      const response = await fetch(url);
-      if (response.ok) {
-        const data = await response.json();
-        console.log(`✅ ${name}: healthy`);
-        if (data.redis !== undefined) {
-          console.log(`   └─ Redis: ${data.redis ? 'connected' : 'disconnected'}`);
-        }
-        if (data.kafka !== undefined) {
-          console.log(`   └─ Kafka: ${data.kafka}`);
-        }
-        return;
-      }
-    } catch (error) {
-      if (attempt === maxRetries) {
-        throw new Error(`❌ ${name} is not available at ${url} after ${maxRetries} attempts`);
-      }
-      console.log(`⏳ Waiting for ${name} (attempt ${attempt}/${maxRetries})...`);
-      await new Promise(resolve => setTimeout(resolve, retryDelay));
-    }
-  }
-}
-
-export default globalSetup;

e2e/global-teardown.ts

@@ -1,10 +0,0 @@
-/**
- * Global teardown for E2E tests
- * Cleans up test data and resources
- */
-async function globalTeardown() {
-  console.log('\n🧹 PHANTOM E2E Test Suite - Global Teardown\n');
-  console.log('✅ Cleanup complete\n');
-}
-
-export default globalTeardown;

e2e/lib/api-client.ts

@@ -1,191 +0,0 @@
-import { testConfig } from '../playwright.config';
-
-/**
- * Event payload structure matching the backend API
- */
-export interface EventPayload {
-  sessionId: string;
-  experimentId?: string;
-  eventName: string;
-  page: string;
-  productId?: string;
-  metadata?: Record<string, unknown>;
-  storeMode: 'hotel' | 'airline';
-  userAgent?: string;
-  ts?: string;
-}
-
-/**
- * Price log payload structure
- */
-export interface PriceLogPayload {
-  productId: string;
-  price: number;
-  sessionId: string;
-  experimentId?: string;
-  storeMode: 'hotel' | 'airline';
-  ts?: string;
-}
-
-/**
- * Price response from the pricing provider
- */
-export interface PriceResponse {
-  productId: string;
-  price: number;
-  base_price: number;
-  markup: number;
-  elasticity: number | null;
-  model_version: string;
-}
-
-/**
- * API client for interacting with PHANTOM services
- */
-export class PhantomApiClient {
-  private backendUrl: string;
-  private providerUrl: string;
-
-  constructor(
-    backendUrl: string = testConfig.backendUrl,
-    providerUrl: string = testConfig.providerUrl
-  ) {
-    this.backendUrl = backendUrl;
-    this.providerUrl = providerUrl;
-  }
-
-  /**
-   * Send a user interaction event to the ingestion API
-   */
-  async ingestEvent(event: EventPayload): Promise<{ success: boolean }> {
-    const payload: EventPayload = {
-      ...event,
-      ts: event.ts || new Date().toISOString(),
-    };
-
-    const response = await fetch(`${this.backendUrl}/api/kafka/ingest`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(payload),
-    });
-
-    if (!response.ok) {
-      throw new Error(`Failed to ingest event: ${response.status} ${await response.text()}`);
-    }
-
-    return response.json();
-  }
-
-  /**
-   * Send multiple events in rapid succession
-   */
-  async ingestEvents(events: EventPayload[], delayMs: number = 100): Promise<void> {
-    for (const event of events) {
-      await this.ingestEvent(event);
-      if (delayMs > 0) {
-        await new Promise(resolve => setTimeout(resolve, delayMs));
-      }
-    }
-  }
-
-  /**
-   * Log a price observation
-   */
-  async logPrice(priceLog: PriceLogPayload): Promise<{ success: boolean }> {
-    const payload: PriceLogPayload = {
-      ...priceLog,
-      ts: priceLog.ts || new Date().toISOString(),
-    };
-
-    const response = await fetch(`${this.backendUrl}/api/kafka/price-log`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(payload),
-    });
-
-    if (!response.ok) {
-      throw new Error(`Failed to log price: ${response.status} ${await response.text()}`);
-    }
-
-    return response.json();
-  }
-
-  /**
-   * Get the current price for a product from the pricing provider
-   */
-  async getPrice(
-    mode: 'hotel' | 'airline',
-    productId: string,
-    sessionId?: string
-  ): Promise<PriceResponse> {
-    const params = new URLSearchParams();
-    if (sessionId) {
-      params.set('sessionId', sessionId);
-    }
-
-    const url = `${this.providerUrl}/api/${mode}/price/${productId}${params.toString() ? '?' + params.toString() : ''}`;
-    const response = await fetch(url);
-
-    if (!response.ok) {
-      throw new Error(`Failed to get price: ${response.status} ${await response.text()}`);
-    }
-
-    return response.json();
-  }
-
-  /**
-   * Dump events from Kafka topic for debugging
-   */
-  async dumpKafkaEvents(
-    topic: 'user-interactions' | 'price-logs' = 'user-interactions',
-    lastN?: number
-  ): Promise<{ success: boolean; count: number; data: unknown[] }> {
-    const params = new URLSearchParams({ topic });
-    if (lastN) {
-      params.set('last_n', String(lastN));
-    }
-
-    const response = await fetch(`${this.backendUrl}/api/kafka/dump?${params.toString()}`);
-
-    if (!response.ok) {
-      throw new Error(`Failed to dump Kafka events: ${response.status}`);
-    }
-
-    return response.json();
-  }
-
-  /**
-   * Check health of backend service
-   */
-  async checkBackendHealth(): Promise<{ status: string; kafka: string }> {
-    const response = await fetch(`${this.backendUrl}/health`);
-    return response.json();
-  }
-
-  /**
-   * Check health of pricing provider
-   */
-  async checkProviderHealth(): Promise<{ status: string; redis: boolean }> {
-    const response = await fetch(`${this.providerUrl}/health`);
-    return response.json();
-  }
-
-  /**
-   * List registered models in the pricing provider
-   */
-  async listModels(): Promise<Record<string, unknown>> {
-    const response = await fetch(`${this.providerUrl}/models`);
-    return response.json();
-  }
-
-  /**
-   * Reload models in the pricing provider
-   */
-  async reloadModels(): Promise<{ elasticity_loaded: boolean; pricing_model_loaded: boolean }> {
-    const response = await fetch(`${this.providerUrl}/models/reload`, { method: 'POST' });
-    return response.json();
-  }
-}
-
-// Singleton instance for convenience
-export const apiClient = new PhantomApiClient();

e2e/lib/event-generator.ts

@@ -1,249 +0,0 @@
-import { EventPayload, PriceLogPayload } from './api-client';
-import { v4 as uuidv4 } from 'uuid';
-
-/**
- * Canonical event names matching the frontend
- */
-export const EventNames = {
-  // Navigation events
-  PAGE_VIEW: 'page_view',
-  VIEW_ITEM_PAGE: 'view_item_page',
-  LEARN_MORE: 'learn_more_about_item',
-
-  // Cart events
-  ADD_TO_CART: 'add_item_to_cart',
-  REMOVE_FROM_CART: 'remove_item',
-  CHECKOUT_START: 'checkout_start',
-  PURCHASE_COMPLETE: 'purchase_complete',
-
-  // Search/Filter events
-  SEARCH: 'search',
-  FILTER_DATE: 'filter_for_date',
-  FILTER_AMENITIES: 'filter_for_amenities',
-  FILTER_PRICE: 'filter_for_price',
-  SORT_CHANGE: 'sort_change',
-
-  // Dwell signals (engagement)
-  HOVER_TITLE: 'hover_over_title',
-  HOVER_PARAGRAPH: 'hover_over_paragraph',
-  HOVER_LINK: 'hover_over_link',
-  HOVER_BUTTON: 'hover_over_button',
-
-  // Session
-  SESSION_START: 'session_start',
-} as const;
-
-export type EventName = typeof EventNames[keyof typeof EventNames];
-
-/**
- * Test product configuration
- */
-export interface TestProduct {
-  id: string;
-  basePrice: number;
-  storeMode: 'hotel' | 'airline';
-  name?: string;
-}
-
-/**
- * Generates test events for dynamic pricing E2E tests
- */
-export class EventGenerator {
-  private sessionId: string;
-  private experimentId: string;
-  private storeMode: 'hotel' | 'airline';
-
-  constructor(options?: {
-    sessionId?: string;
-    experimentId?: string;
-    storeMode?: 'hotel' | 'airline';
-  }) {
-    this.sessionId = options?.sessionId || uuidv4();
-    this.experimentId = options?.experimentId || uuidv4();
-    this.storeMode = options?.storeMode || 'hotel';
-  }
-
-  get session(): string {
-    return this.sessionId;
-  }
-
-  get experiment(): string {
-    return this.experimentId;
-  }
-
-  /**
-   * Create a new session for isolation between test scenarios
-   */
-  newSession(): string {
-    this.sessionId = uuidv4();
-    return this.sessionId;
-  }
-
-  /**
-   * Generate a single event
-   */
-  createEvent(
-    eventName: EventName,
-    productId: string,
-    metadata?: Record<string, unknown>
-  ): EventPayload {
-    return {
-      sessionId: this.sessionId,
-      experimentId: this.experimentId,
-      eventName,
-      page: `/${this.storeMode}/products/${productId}`,
-      productId,
-      metadata: metadata || {},
-      storeMode: this.storeMode,
-      userAgent: 'PHANTOM-E2E-Test/1.0',
-      ts: new Date().toISOString(),
-    };
-  }
-
-  /**
-   * Generate a product view event
-   */
-  viewProduct(productId: string): EventPayload {
-    return this.createEvent(EventNames.VIEW_ITEM_PAGE, productId, {
-      referrer: `/${this.storeMode}/products`,
-      viewport: { width: 1920, height: 1080 },
-    });
-  }
-
-  /**
-   * Generate a "learn more" event (high intent signal)
-   */
-  learnMore(productId: string): EventPayload {
-    return this.createEvent(EventNames.LEARN_MORE, productId, {
-      section: 'details',
-    });
-  }
-
-  /**
-   * Generate a hover event (engagement signal)
-   */
-  hover(productId: string, element: 'title' | 'paragraph' | 'button' = 'title'): EventPayload {
-    const eventMap = {
-      title: EventNames.HOVER_TITLE,
-      paragraph: EventNames.HOVER_PARAGRAPH,
-      button: EventNames.HOVER_BUTTON,
-    };
-    return this.createEvent(eventMap[element], productId, {
-      duration_ms: Math.floor(Math.random() * 2000) + 500,
-    });
-  }
-
-  /**
-   * Generate an add-to-cart event
-   */
-  addToCart(productId: string, quantity: number = 1): EventPayload {
-    return this.createEvent(EventNames.ADD_TO_CART, productId, {
-      quantity,
-      cart_size: quantity,
-    });
-  }
-
-  /**
-   * Generate a sequence of high-velocity events for surge pricing trigger
-   * This simulates rapid user interest in a product
-   */
-  generateSurgeSequence(productId: string, count: number): EventPayload[] {
-    const events: EventPayload[] = [];
-
-    for (let i = 0; i < count; i++) {
-      // Mix of different event types to simulate realistic behavior
-      events.push(this.viewProduct(productId));
-
-      if (i % 2 === 0) {
-        events.push(this.learnMore(productId));
-      }
-
-      if (i % 3 === 0) {
-        events.push(this.hover(productId, 'title'));
-      }
-    }
-
-    return events;
-  }
-
-  /**
-   * Generate a normal browsing session (not triggering surge)
-   */
-  generateNormalSession(productId: string): EventPayload[] {
-    return [
-      this.viewProduct(productId),
-      this.hover(productId, 'title'),
-    ];
-  }
-
-  /**
-   * Generate high-velocity agent-like behavior
-   * This should trigger SessionAwarePricer's agent detection
-   */
-  generateAgentBehavior(productIds: string[]): EventPayload[] {
-    const events: EventPayload[] = [];
-
-    // Rapid-fire product views across multiple products
-    for (const productId of productIds) {
-      events.push(this.viewProduct(productId));
-      // Very quick succession - agent-like behavior
-    }
-
-    return events;
-  }
-
-  /**
-   * Generate a price log entry
-   */
-  createPriceLog(productId: string, price: number): PriceLogPayload {
-    return {
-      productId,
-      price,
-      sessionId: this.sessionId,
-      experimentId: this.experimentId,
-      storeMode: this.storeMode,
-      ts: new Date().toISOString(),
-    };
-  }
-}
-
-/**
- * Pre-configured test products for E2E tests
- * These should match products in your test database
- */
-export const TestProducts = {
-  // Hotel products with known base prices
-  hotel1: {
-    id: 'e2e-test-hotel-001',
-    basePrice: 150.00,
-    storeMode: 'hotel' as const,
-    name: 'E2E Test Hotel 1',
-  },
-  hotel2: {
-    id: 'e2e-test-hotel-002',
-    basePrice: 200.00,
-    storeMode: 'hotel' as const,
-    name: 'E2E Test Hotel 2',
-  },
-  hotel3: {
-    id: 'e2e-test-hotel-003',
-    basePrice: 100.00,
-    storeMode: 'hotel' as const,
-    name: 'E2E Test Hotel 3',
-  },
-
-  // Airline products
-  airline1: {
-    id: 'e2e-test-airline-001',
-    basePrice: 350.00,
-    storeMode: 'airline' as const,
-    name: 'E2E Test Flight 1',
-  },
-};
-
-/**
- * Generate a unique test product ID for isolation
- */
-export function generateTestProductId(prefix: string = 'e2e-test'): string {
-  return `${prefix}-${uuidv4().slice(0, 8)}`;
-}

e2e/lib/fixtures.ts

@@ -1,143 +0,0 @@
-import { test as base, expect } from '@playwright/test';
-import { PhantomApiClient, apiClient } from './api-client';
-import { EventGenerator, TestProducts } from './event-generator';
-import { runPricingPipeline, waitForPriceUpdate, PipelineResult } from './pipeline-runner';
-import { testConfig } from '../playwright.config';
-
-/**
- * Extended test fixtures for PHANTOM E2E tests
- */
-export interface PhantomTestFixtures {
-  /** API client for interacting with PHANTOM services */
-  api: PhantomApiClient;
-
-  /** Event generator for creating test events */
-  events: EventGenerator;
-
-  /** Run the pricing pipeline and wait for updates */
-  triggerPriceUpdate: (options?: {
-    storeMode?: 'hotel' | 'airline';
-    highThreshold?: number;
-    lowThreshold?: number;
-    surgeMultiplier?: number;
-    discountMultiplier?: number;
-  }) => Promise<PipelineResult>;
-
-  /** Wait for a specific price condition */
-  waitForPrice: (
-    productId: string,
-    condition: (price: number, basePrice: number) => boolean,
-    storeMode?: 'hotel' | 'airline'
-  ) => Promise<{ price: number; basePrice: number; markup: number }>;
-
-  /** Test configuration */
-  config: typeof testConfig;
-}
-
-/**
- * Custom test with PHANTOM fixtures
- */
-export const test = base.extend<PhantomTestFixtures>({
-  api: async ({}, use) => {
-    await use(apiClient);
-  },
-
-  events: async ({}, use) => {
-    // Create a new event generator with a fresh session for each test
-    const generator = new EventGenerator({
-      storeMode: 'hotel',
-    });
-    await use(generator);
-  },
-
-  triggerPriceUpdate: async ({}, use) => {
-    const trigger = async (options = {}) => {
-      const result = await runPricingPipeline({
-        storeMode: 'hotel',
-        highThreshold: testConfig.pricing.highThreshold,
-        lowThreshold: testConfig.pricing.lowThreshold,
-        surgeMultiplier: testConfig.pricing.surgeMultiplier,
-        discountMultiplier: testConfig.pricing.discountMultiplier,
-        ...options,
-      });
-
-      // Wait a moment for Redis to be fully updated
-      await new Promise(resolve => setTimeout(resolve, 500));
-
-      return result;
-    };
-
-    await use(trigger);
-  },
-
-  waitForPrice: async ({ api }, use) => {
-    const waiter = async (
-      productId: string,
-      condition: (price: number, basePrice: number) => boolean,
-      storeMode: 'hotel' | 'airline' = 'hotel'
-    ) => {
-      let lastPrice = 0;
-      let lastBasePrice = 0;
-
-      const updated = await waitForPriceUpdate(async () => {
-        const priceResponse = await api.getPrice(storeMode, productId);
-        lastPrice = priceResponse.price;
-        lastBasePrice = priceResponse.base_price;
-        return condition(priceResponse.price, priceResponse.base_price);
-      });
-
-      if (!updated) {
-        throw new Error(
-          `Price condition not met within timeout. Last price: ${lastPrice}, base: ${lastBasePrice}`
-        );
-      }
-
-      return {
-        price: lastPrice,
-        basePrice: lastBasePrice,
-        markup: lastPrice / lastBasePrice,
-      };
-    };
-
-    await use(waiter);
-  },
-
-  config: async ({}, use) => {
-    await use(testConfig);
-  },
-});
-
-export { expect };
-
-/**
- * Helper assertions for pricing tests
- */
-export const PricingAssertions = {
-  /**
-   * Assert that a price has surge markup applied
-   */
-  isSurged: (price: number, basePrice: number, expectedMultiplier: number, tolerance = 0.01) => {
-    const actualMarkup = price / basePrice;
-    const minExpected = expectedMultiplier * (1 - tolerance);
-    const maxExpected = expectedMultiplier * (1 + tolerance);
-    return actualMarkup >= minExpected && actualMarkup <= maxExpected;
-  },
-
-  /**
-   * Assert that a price has discount applied
-   */
-  isDiscounted: (price: number, basePrice: number, expectedMultiplier: number, tolerance = 0.01) => {
-    const actualMarkup = price / basePrice;
-    const minExpected = expectedMultiplier * (1 - tolerance);
-    const maxExpected = expectedMultiplier * (1 + tolerance);
-    return actualMarkup >= minExpected && actualMarkup <= maxExpected;
-  },
-
-  /**
-   * Assert that a price is at base (no surge/discount)
-   */
-  isBase: (price: number, basePrice: number, tolerance = 0.01) => {
-    const actualMarkup = price / basePrice;
-    return actualMarkup >= (1 - tolerance) && actualMarkup <= (1 + tolerance);
-  },
-};

e2e/lib/index.ts

@@ -1,6 +0,0 @@
-// Re-export all test utilities
-
-export * from './api-client';
-export * from './event-generator';
-export * from './pipeline-runner';
-export * from './fixtures';

e2e/lib/pipeline-runner.ts

@@ -1,152 +0,0 @@
-import { spawn } from 'child_process';
-import path from 'path';
-import { testConfig } from '../playwright.config';
-
-/**
- * Pipeline execution result
- */
-export interface PipelineResult {
-  success: boolean;
-  interactions_count: number;
-  products_count: number;
-  prices_published: boolean;
-  prices?: Array<{
-    productId: string;
-    current_price: number;
-    base_price: number;
-    optimal_price: number;
-    demand_score: number;
-  }>;
-  timestamp?: string;
-  message?: string;
-  error?: string;
-}
-
-/**
- * Pipeline configuration options
- */
-export interface PipelineOptions {
-  storeMode?: 'hotel' | 'airline';
-  highThreshold?: number;
-  lowThreshold?: number;
-  surgeMultiplier?: number;
-  discountMultiplier?: number;
-  dryRun?: boolean;
-}
-
-/**
- * Execute the pricing pipeline to update prices based on current events
- */
-export async function runPricingPipeline(options: PipelineOptions = {}): Promise<PipelineResult> {
-  const {
-    storeMode = 'hotel',
-    highThreshold = testConfig.pricing.highThreshold,
-    lowThreshold = testConfig.pricing.lowThreshold,
-    surgeMultiplier = testConfig.pricing.surgeMultiplier,
-    discountMultiplier = testConfig.pricing.discountMultiplier,
-    dryRun = false,
-  } = options;
-
-  const workerPath = path.join(__dirname, 'pipeline-worker.py');
-
-  const args = [
-    workerPath,
-    '--store-mode', storeMode,
-    '--high-threshold', String(highThreshold),
-    '--low-threshold', String(lowThreshold),
-    '--surge-multiplier', String(surgeMultiplier),
-    '--discount-multiplier', String(discountMultiplier),
-    '--json-output',
-  ];
-
-  if (dryRun) {
-    args.push('--dry-run');
-  }
-
-  return new Promise((resolve, reject) => {
-    const python = spawn('python3', args, {
-      env: {
-        ...process.env,
-        BACKEND_URL: testConfig.backendUrl,
-        REDIS_HOST: testConfig.redisHost,
-        REDIS_PORT: String(testConfig.redisPort),
-        KAFKA_HOST: testConfig.kafkaHost,
-        KAFKA_PORT: String(testConfig.kafkaPort),
-      },
-    });
-
-    let stdout = '';
-    let stderr = '';
-
-    python.stdout.on('data', (data) => {
-      stdout += data.toString();
-    });
-
-    python.stderr.on('data', (data) => {
-      stderr += data.toString();
-      // Log pipeline output for debugging
-      console.log('[Pipeline]', data.toString().trim());
-    });
-
-    python.on('close', (code) => {
-      if (code === 0) {
-        try {
-          // Find JSON output in stdout (last JSON object)
-          const jsonMatch = stdout.match(/\{[\s\S]*\}$/);
-          if (jsonMatch) {
-            const result = JSON.parse(jsonMatch[0]);
-            resolve(result);
-          } else {
-            resolve({
-              success: true,
-              interactions_count: 0,
-              products_count: 0,
-              prices_published: false,
-              message: 'Pipeline completed but no JSON output',
-            });
-          }
-        } catch (parseError) {
-          resolve({
-            success: true,
-            interactions_count: 0,
-            products_count: 0,
-            prices_published: false,
-            message: 'Pipeline completed but output not parseable',
-          });
-        }
-      } else {
-        reject(new Error(`Pipeline exited with code ${code}: ${stderr}`));
-      }
-    });
-
-    python.on('error', (error) => {
-      reject(new Error(`Failed to start pipeline: ${error.message}`));
-    });
-  });
-}
-
-/**
- * Wait for prices to be updated in Redis and available via the pricing API
- */
-export async function waitForPriceUpdate(
-  checkFn: () => Promise<boolean>,
-  maxWaitMs: number = testConfig.timing.maxPriceWait,
-  intervalMs: number = testConfig.timing.priceCheckInterval
-): Promise<boolean> {
-  const startTime = Date.now();
-
-  while (Date.now() - startTime < maxWaitMs) {
-    try {
-      const updated = await checkFn();
-      if (updated) {
-        return true;
-      }
-    } catch (error) {
-      // Ignore errors during polling
-    }
-
-    await new Promise(resolve => setTimeout(resolve, intervalMs));
-  }
-
-  return false;
-}

e2e/lib/pipeline-worker.py

@@ -1,245 +0,0 @@
-#!/usr/bin/env python3
-"""
-E2E Test Pipeline Worker
-
-A lightweight worker that runs the surge pricing pipeline for E2E tests.
-This bypasses Airflow for faster, more reliable test execution.
-
-Usage:
-    python pipeline-worker.py --store-mode hotel --high-threshold 3 --surge-multiplier 1.5
-"""
-
-import argparse
-import json
-import logging
-import os
-import sys
-from typing import Optional
-from datetime import datetime
-
-# Add project paths
-project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.insert(0, project_root)
-sys.path.insert(0, os.path.join(project_root, 'experiments'))
-sys.path.insert(0, os.path.join(project_root, 'lib'))
-
-from procesing.context import PipelineContext
-from procesing.providers import BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    ComputeDemandStep,
-    AggregatePriceLogsStep,
-    JoinProductFeaturesStep,
-)
-from procesing.pricers.simple import SimpleSurgePricer
-from lib.model_registry import ModelRegistry
-
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s [%(levelname)s] %(message)s'
-)
-log = logging.getLogger(__name__)
-
-
-class E2ETestProvider(BackendAPIProvider):
-    """Provider configured for E2E test environment"""
-
-    def __init__(self, backend_url: str = None):
-        self.backend_url = backend_url or os.getenv('BACKEND_URL', 'http://localhost:5000')
-        super().__init__()
-
-
-def run_pricing_pipeline(
-    store_mode: str = 'hotel',
-    high_threshold: int = 3,
-    low_threshold: int = 1,
-    surge_multiplier: float = 1.5,
-    discount_multiplier: float = 0.9,
-    dry_run: bool = False
-) -> dict:
-    """
-    Execute the surge pricing pipeline and publish results to Redis.
-
-    Args:
-        store_mode: 'hotel' or 'airline'
-        high_threshold: Demand threshold for surge pricing
-        low_threshold: Demand threshold for discount pricing
-        surge_multiplier: Price multiplier for high demand
-        discount_multiplier: Price multiplier for low demand
-        dry_run: If True, don't publish to Redis
-
-    Returns:
-        dict with pipeline results and statistics
-    """
-    log.info(f"Starting E2E pricing pipeline: mode={store_mode}, "
-             f"high_threshold={high_threshold}, surge_multiplier={surge_multiplier}")
-
-    # Initialize provider and context
-    provider = E2ETestProvider()
-    context = PipelineContext(provider=provider, store_mode=store_mode)
-
-    # Step 1: Fetch interactions from Kafka
-    log.info("Fetching interactions from Kafka...")
-    fetch_interactions = FetchInteractionsStep(context)
-    interactions_df = fetch_interactions.transform(None)
-    log.info(f"Fetched {len(interactions_df)} interaction records")
-
-    if interactions_df.empty:
-        log.warning("No interactions found. Pipeline will produce no price updates.")
-        return {
-            'success': True,
-            'interactions_count': 0,
-            'products_count': 0,
-            'prices_published': False,
-            'message': 'No interactions to process'
-        }
-
-    # Step 2: Fetch price logs from Kafka
-    log.info("Fetching price logs from Kafka...")
-    fetch_prices = FetchPriceLogsStep(context)
-    price_logs_df = fetch_prices.transform(None)
-    log.info(f"Fetched {len(price_logs_df)} price log records")
-
-    # Step 3: Compute demand scores
-    log.info("Computing demand scores...")
-    compute_demand = ComputeDemandStep(context)
-    demand_df = compute_demand.transform(interactions_df)
-    log.info(f"Computed demand for {len(demand_df)} products")
-
-    if demand_df.empty:
-        log.warning("No demand data computed.")
-        return {
-            'success': True,
-            'interactions_count': len(interactions_df),
-            'products_count': 0,
-            'prices_published': False,
-            'message': 'No demand data to process'
-        }
-
-    # Step 4: Aggregate price logs
-    log.info("Aggregating price logs...")
-    aggregate_prices = AggregatePriceLogsStep(context)
-    price_agg_df = aggregate_prices.transform(price_logs_df)
-    log.info(f"Aggregated prices for {len(price_agg_df)} products")
-
-    # Step 5: Join product features
-    log.info("Joining product features...")
-    join_features = JoinProductFeaturesStep(context)
-    features_df = join_features.transform((demand_df, price_agg_df))
-    log.info(f"Joined features for {len(features_df)} products")
-
-    if features_df.empty:
-        log.warning("No product features after join.")
-        return {
-            'success': True,
-            'interactions_count': len(interactions_df),
-            'products_count': 0,
-            'prices_published': False,
-            'message': 'No product features to price'
-        }
-
-    # Step 6: Apply surge pricing
-    log.info(f"Applying surge pricing (high={high_threshold}, surge={surge_multiplier}x)...")
-
-    # Rename columns for pricer compatibility
-    data = features_df.rename(columns={'demand_score': 'demand'})
-
-    surge_pricer = SimpleSurgePricer(
-        high_threshold=high_threshold,
-        low_threshold=low_threshold,
-        surge_multiplier=surge_multiplier,
-        discount_multiplier=discount_multiplier
-    )
-    surge_pricer.fit(data)
-    data['optimal_price'] = surge_pricer.predict()
-
-    # Prepare output DataFrame
-    prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
-        'price': 'current_price',
-        'demand': 'demand_score'
-    })
-
-    log.info(f"Generated optimal prices for {len(prices_df)} products")
-
-    # Log pricing decisions
-    for _, row in prices_df.iterrows():
-        markup = row['optimal_price'] / row['base_price'] if row['base_price'] > 0 else 1.0
-        log.info(f"  {row['productId'][:8]}...: base=${row['base_price']:.2f} "
-                 f"-> optimal=${row['optimal_price']:.2f} (demand={row['demand_score']:.0f}, markup={markup:.2f}x)")
-
-    # Step 7: Publish to Redis
-    if not dry_run:
-        log.info("Publishing prices to Redis registry...")
-        registry = ModelRegistry()
-
-        metadata = {
-            'timestamp': datetime.utcnow().isoformat(),
-            'store_mode': store_mode,
-            'pipeline': 'e2e_test_worker',
-            'high_threshold': high_threshold,
-            'low_threshold': low_threshold,
-            'surge_multiplier': surge_multiplier,
-            'discount_multiplier': discount_multiplier,
-        }
-
-        registry.publish_prices(prices_df, model_name='latest', metadata=metadata)
-        log.info(f"✅ Published {len(prices_df)} prices to Redis")
-    else:
-        log.info("Dry run - skipping Redis publish")
-
-    return {
-        'success': True,
-        'interactions_count': len(interactions_df),
-        'products_count': len(prices_df),
-        'prices_published': not dry_run,
-        'prices': prices_df.to_dict(orient='records'),
-        'timestamp': datetime.utcnow().isoformat()
-    }
-
-
-def main():
-    parser = argparse.ArgumentParser(description='E2E Test Pipeline Worker')
-    parser.add_argument('--store-mode', choices=['hotel', 'airline'], default='hotel',
-                        help='Store mode (hotel or airline)')
-    parser.add_argument('--high-threshold', type=int, default=3,
-                        help='Demand threshold for surge pricing')
-    parser.add_argument('--low-threshold', type=int, default=1,
-                        help='Demand threshold for discount pricing')
-    parser.add_argument('--surge-multiplier', type=float, default=1.5,
-                        help='Price multiplier for high demand')
-    parser.add_argument('--discount-multiplier', type=float, default=0.9,
-                        help='Price multiplier for low demand')
-    parser.add_argument('--dry-run', action='store_true',
-                        help='Run without publishing to Redis')
-    parser.add_argument('--json-output', action='store_true',
-                        help='Output results as JSON')
-
-    args = parser.parse_args()
-
-    try:
-        result = run_pricing_pipeline(
-            store_mode=args.store_mode,
-            high_threshold=args.high_threshold,
-            low_threshold=args.low_threshold,
-            surge_multiplier=args.surge_multiplier,
-            discount_multiplier=args.discount_multiplier,
-            dry_run=args.dry_run
-        )
-
-        if args.json_output:
-            print(json.dumps(result, indent=2))
-        else:
-            log.info(f"Pipeline completed: {result['products_count']} products priced")
-
-        sys.exit(0 if result['success'] else 1)
-
-    except Exception as e:
-        log.error(f"Pipeline failed: {e}")
-        if args.json_output:
-            print(json.dumps({'success': False, 'error': str(e)}))
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

e2e/package.json

@@ -1,27 +0,0 @@
-{
-  "name": "phantom-e2e-tests",
-  "version": "1.0.0",
-  "description": "E2E tests for PHANTOM Dynamic Pricing Pipeline",
-  "scripts": {
-    "test": "playwright test",
-    "test:ui": "playwright test --ui",
-    "test:headed": "playwright test --headed",
-    "test:debug": "playwright test --debug",
-    "test:report": "playwright show-report",
-    "test:pricing": "playwright test dynamic-pricing",
-    "test:health": "playwright test --grep 'health'",
-    "pipeline:run": "python3 lib/pipeline-worker.py --store-mode hotel --high-threshold 3 --surge-multiplier 1.5",
-    "pipeline:dry-run": "python3 lib/pipeline-worker.py --dry-run --json-output",
-    "services:check": "curl -s http://localhost:5000/health && curl -s http://localhost:5001/health"
-  },
-  "devDependencies": {
-    "@playwright/test": "^1.49.0",
-    "@types/node": "^20.0.0",
-    "typescript": "^5.0.0",
-    "uuid": "^9.0.0",
-    "@types/uuid": "^9.0.0"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  }
-}

e2e/playwright.config.ts

@@ -1,84 +0,0 @@
-import { defineConfig, devices } from '@playwright/test';
-
-/**
- * Playwright configuration for PHANTOM Dynamic Pricing E2E Tests
- *
- * Tests validate the entire pricing pipeline:
- * Frontend Events → Kafka → Pipeline Processing → Redis → Pricing API
- */
-export default defineConfig({
-  testDir: './tests',
-  fullyParallel: false, // Run tests sequentially to avoid race conditions in shared state
-  forbidOnly: !!process.env.CI,
-  retries: process.env.CI ? 2 : 0,
-  workers: 1, // Single worker for E2E tests to ensure isolation
-  reporter: [
-    ['html', { outputFolder: 'playwright-report' }],
-    ['list']
-  ],
-
-  // Global timeout for each test
-  timeout: 120_000, // 2 minutes per test (includes pipeline processing time)
-
-  // Expect timeout for assertions
-  expect: {
-    timeout: 30_000, // 30 seconds for price updates to propagate
-  },
-
-  use: {
-    // Base URL for the backend API
-    baseURL: process.env.BACKEND_URL || 'http://localhost:5000',
-
-    // Collect trace on first retry
-    trace: 'on-first-retry',
-
-    // Screenshot on failure
-    screenshot: 'only-on-failure',
-  },
-
-  // Global setup and teardown
-  globalSetup: require.resolve('./global-setup'),
-  globalTeardown: require.resolve('./global-teardown'),
-
-  projects: [
-    {
-      name: 'dynamic-pricing',
-      testMatch: /.*\.spec\.ts/,
-    },
-  ],
-
-  // Environment configuration
-  // These can be overridden via environment variables
-});
-
-// Export test configuration constants
-export const testConfig = {
-  // API endpoints
-  backendUrl: process.env.BACKEND_URL || 'http://localhost:5000',
-  providerUrl: process.env.PROVIDER_URL || 'http://localhost:5001',
-
-  // Redis configuration
-  redisHost: process.env.REDIS_HOST || 'localhost',
-  redisPort: parseInt(process.env.REDIS_PORT || '6378'),
-
-  // Kafka configuration
-  kafkaHost: process.env.KAFKA_HOST || 'localhost',
-  kafkaPort: parseInt(process.env.KAFKA_PORT || '9092'),
-
-  // Pricing thresholds for tests (aggressive settings for fast feedback)
-  pricing: {
-    highThreshold: 3,     // Trigger surge after 3 interactions
-    lowThreshold: 1,      // Trigger discount at 1 or fewer interactions
-    surgeMultiplier: 1.5, // 50% price increase on surge
-    discountMultiplier: 0.9, // 10% discount on low demand
-    windowSize: 10_000,   // 10 second window for demand calculation
-  },
-
-  // Timing configuration
-  timing: {
-    eventDelay: 100,           // Delay between events (ms)
-    pipelineWait: 5_000,       // Wait for pipeline processing (ms)
-    priceCheckInterval: 1_000, // Interval between price checks (ms)
-    maxPriceWait: 30_000,      // Max wait for price update (ms)
-  },
-};

e2e/tests/dynamic-pricing.spec.ts

@@ -1,497 +0,0 @@
-/**
- * PHANTOM Dynamic Pricing E2E Test Suite
- *
- * Validates that SimpleSurgePricer and SessionAwarePricer correctly adjust
- * product prices in real-time based on high-velocity user interactions.
- *
- * System Under Test (SUT):
- * - Frontend (interaction generation via API calls)
- * - Backend API (POST /api/ingest → Kafka)
- * - Kafka (user-interactions topic)
- * - Pipeline Worker (demand calculation → surge pricing)
- * - Redis (model registry)
- * - Pricing Provider (GET /api/{mode}/price/{productId})
- *
- * Test Configuration:
- * - high_threshold: 3 (trigger surge after 3 demand signals)
- * - surge_multiplier: 1.5x (50% price increase)
- * - low_threshold: 1 (trigger discount at 1 or fewer)
- * - discount_multiplier: 0.9x (10% discount)
- * - window_size: 10s (fast feedback loop)
- */
-
-import { test, expect, PricingAssertions } from '../lib/fixtures';
-import { EventNames, generateTestProductId } from '../lib/event-generator';
-
-test.describe('Dynamic Pricing Pipeline', () => {
-  test.describe.configure({ mode: 'serial' });
-
-  /**
-   * Scenario 1: Baseline Pricing
-   *
-   * Precondition: Clean state with no recent interactions for the product
-   * Expected: Price should equal base_price (markup = 1.0)
-   */
-  test('should return base price when no interactions exist', async ({ api, config }) => {
-    // Use a unique product ID to ensure no prior interactions
-    const productId = generateTestProductId('baseline');
-
-    // Get price from provider - should be base price (fallback)
-    // Note: This tests the fallback behavior when product isn't in Redis
-    const priceResponse = await api.getPrice('hotel', productId).catch(() => null);
-
-    // For unknown products, the API returns 404 or falls back to base
-    // This validates the fallback mechanism works
-    test.info().annotations.push({
-      type: 'info',
-      description: `Tested baseline pricing for product: ${productId}`,
-    });
-  });
-
-  /**
-   * Scenario 2: Surge Pricing Trigger
-   *
-   * Precondition: Fresh product with no interactions
-   * Action: Generate 5+ high-velocity interactions (above high_threshold=3)
-   * Expected: Price increases by surge_multiplier (1.5x)
-   */
-  test('should apply surge pricing when demand exceeds threshold', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    // Step 1: Create a fresh session
-    const sessionId = events.newSession();
-    const productId = generateTestProductId('surge');
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing surge pricing for product: ${productId}`,
-    });
-
-    // Step 2: Log initial price for this product (establish baseline)
-    await api.logPrice({
-      productId,
-      price: 100.0, // Base price
-      sessionId,
-      storeMode: 'hotel',
-    });
-
-    // Step 3: Generate high-velocity interactions (5 events > threshold of 3)
-    console.log(`\n📊 Generating ${5} surge events for product ${productId.slice(0, 8)}...`);
-
-    const surgeEvents = events.generateSurgeSequence(productId, 5);
-
-    for (const event of surgeEvents) {
-      await api.ingestEvent(event);
-      await new Promise(r => setTimeout(r, config.timing.eventDelay));
-    }
-
-    console.log(`✅ Ingested ${surgeEvents.length} events`);
-
-    // Step 4: Trigger the pricing pipeline
-    console.log('\n⚙️ Triggering pricing pipeline...');
-    const pipelineResult = await triggerPriceUpdate({
-      storeMode: 'hotel',
-      highThreshold: config.pricing.highThreshold,
-      surgeMultiplier: config.pricing.surgeMultiplier,
-    });
-
-    console.log(`📈 Pipeline processed ${pipelineResult.products_count} products`);
-
-    // Step 5: Verify surge pricing was applied
-    if (pipelineResult.prices && pipelineResult.prices.length > 0) {
-      const pricedProduct = pipelineResult.prices.find(p => p.productId === productId);
-
-      if (pricedProduct) {
-        const markup = pricedProduct.optimal_price / pricedProduct.base_price;
-
-        console.log(`\n💰 Price Result for ${productId.slice(0, 8)}:`);
-        console.log(`   Base Price: $${pricedProduct.base_price.toFixed(2)}`);
-        console.log(`   Optimal Price: $${pricedProduct.optimal_price.toFixed(2)}`);
-        console.log(`   Demand Score: ${pricedProduct.demand_score}`);
-        console.log(`   Markup: ${markup.toFixed(2)}x`);
-
-        // Verify surge was applied
-        expect(pricedProduct.demand_score).toBeGreaterThanOrEqual(config.pricing.highThreshold);
-        expect(markup).toBeCloseTo(config.pricing.surgeMultiplier, 1);
-      }
-    }
-
-    // Annotations for test report
-    test.info().annotations.push({
-      type: 'result',
-      description: `Pipeline processed ${pipelineResult.products_count} products`,
-    });
-  });
-
-  /**
-   * Scenario 3: Discount Pricing Trigger
-   *
-   * Precondition: Product with very low interaction count
-   * Action: Generate only 1 interaction (at or below low_threshold=1)
-   * Expected: Price decreases by discount_multiplier (0.9x)
-   */
-  test('should apply discount pricing when demand is below threshold', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    const sessionId = events.newSession();
-    const productId = generateTestProductId('discount');
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing discount pricing for product: ${productId}`,
-    });
-
-    // Step 1: Log initial price
-    await api.logPrice({
-      productId,
-      price: 100.0,
-      sessionId,
-      storeMode: 'hotel',
-    });
-
-    // Step 2: Generate minimal interaction (1 event = low_threshold)
-    console.log(`\n📊 Generating 1 low-demand event for product ${productId.slice(0, 8)}...`);
-
-    const event = events.viewProduct(productId);
-    await api.ingestEvent(event);
-
-    console.log('✅ Ingested 1 event');
-
-    // Step 3: Trigger pipeline
-    console.log('\n⚙️ Triggering pricing pipeline...');
-    const pipelineResult = await triggerPriceUpdate({
-      storeMode: 'hotel',
-      lowThreshold: config.pricing.lowThreshold,
-      discountMultiplier: config.pricing.discountMultiplier,
-    });
-
-    // Step 4: Verify discount pricing
-    if (pipelineResult.prices && pipelineResult.prices.length > 0) {
-      const pricedProduct = pipelineResult.prices.find(p => p.productId === productId);
-
-      if (pricedProduct) {
-        const markup = pricedProduct.optimal_price / pricedProduct.base_price;
-
-        console.log(`\n💰 Price Result for ${productId.slice(0, 8)}:`);
-        console.log(`   Base Price: $${pricedProduct.base_price.toFixed(2)}`);
-        console.log(`   Optimal Price: $${pricedProduct.optimal_price.toFixed(2)}`);
-        console.log(`   Demand Score: ${pricedProduct.demand_score}`);
-        console.log(`   Markup: ${markup.toFixed(2)}x`);
-
-        // Verify discount was applied
-        expect(pricedProduct.demand_score).toBeLessThanOrEqual(config.pricing.lowThreshold);
-        expect(markup).toBeCloseTo(config.pricing.discountMultiplier, 1);
-      }
-    }
-  });
-
-  /**
-   * Scenario 4: Multi-Product Differential Pricing
-   *
-   * Precondition: Multiple products with different interaction levels
-   * Action:
-   *   - Product A: 5 interactions (surge)
-   *   - Product B: 1 interaction (discount)
-   *   - Product C: 2 interactions (neutral)
-   * Expected: Each product priced according to its demand
-   */
-  test('should price multiple products differentially based on demand', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    const sessionId = events.newSession();
-
-    // Create 3 test products with different demand patterns
-    const products = {
-      surge: { id: generateTestProductId('multi-surge'), eventCount: 5, expectedMarkup: config.pricing.surgeMultiplier },
-      discount: { id: generateTestProductId('multi-discount'), eventCount: 1, expectedMarkup: config.pricing.discountMultiplier },
-      neutral: { id: generateTestProductId('multi-neutral'), eventCount: 2, expectedMarkup: 1.0 },
-    };
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing multi-product pricing: surge=${products.surge.id.slice(0, 8)}, discount=${products.discount.id.slice(0, 8)}, neutral=${products.neutral.id.slice(0, 8)}`,
-    });
-
-    // Step 1: Log base prices for all products
-    for (const [name, product] of Object.entries(products)) {
-      await api.logPrice({
-        productId: product.id,
-        price: 100.0,
-        sessionId,
-        storeMode: 'hotel',
-      });
-    }
-
-    // Step 2: Generate different interaction levels for each product
-    console.log('\n📊 Generating differentiated events:');
-
-    for (const [name, product] of Object.entries(products)) {
-      console.log(`   ${name}: ${product.eventCount} events`);
-
-      for (let i = 0; i < product.eventCount; i++) {
-        const event = events.viewProduct(product.id);
-        await api.ingestEvent(event);
-        await new Promise(r => setTimeout(r, 50));
-      }
-    }
-
-    console.log('✅ All events ingested');
-
-    // Step 3: Trigger pipeline
-    console.log('\n⚙️ Triggering pricing pipeline...');
-    const pipelineResult = await triggerPriceUpdate();
-
-    // Step 4: Verify differential pricing
-    console.log('\n💰 Multi-Product Pricing Results:');
-
-    if (pipelineResult.prices) {
-      for (const [name, product] of Object.entries(products)) {
-        const pricedProduct = pipelineResult.prices.find(p => p.productId === product.id);
-
-        if (pricedProduct) {
-          const markup = pricedProduct.optimal_price / pricedProduct.base_price;
-
-          console.log(`   ${name} (${product.id.slice(0, 8)}):`);
-          console.log(`      Demand: ${pricedProduct.demand_score}, Markup: ${markup.toFixed(2)}x (expected: ${product.expectedMarkup}x)`);
-
-          // Verify markup is in expected range (with tolerance)
-          expect(markup).toBeCloseTo(product.expectedMarkup, 1);
-        }
-      }
-    }
-  });
-
-  /**
-   * Scenario 5: Price Update Propagation
-   *
-   * Validates that price updates flow correctly from the pipeline
-   * through Redis to the Pricing Provider API.
-   */
-  test('should propagate prices from pipeline to pricing API', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    const sessionId = events.newSession();
-    const productId = generateTestProductId('propagation');
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing price propagation for product: ${productId}`,
-    });
-
-    // Step 1: Log base price
-    await api.logPrice({
-      productId,
-      price: 150.0, // Different base price for this test
-      sessionId,
-      storeMode: 'hotel',
-    });
-
-    // Step 2: Generate surge-level interactions
-    console.log(`\n📊 Generating surge events for propagation test...`);
-
-    const surgeEvents = events.generateSurgeSequence(productId, 6);
-    await api.ingestEvents(surgeEvents, config.timing.eventDelay);
-
-    console.log(`✅ Ingested ${surgeEvents.length} events`);
-
-    // Step 3: Trigger pipeline
-    console.log('\n⚙️ Triggering pricing pipeline...');
-    const pipelineResult = await triggerPriceUpdate();
-
-    expect(pipelineResult.success).toBe(true);
-    expect(pipelineResult.prices_published).toBe(true);
-
-    console.log(`📈 Pipeline published ${pipelineResult.products_count} prices to Redis`);
-
-    // Step 4: Wait for Redis propagation
-    await new Promise(r => setTimeout(r, 1000));
-
-    // Step 5: Verify via Pricing Provider API
-    // Note: This requires the product to exist in Supabase
-    // For pure E2E testing, we verify the pipeline output instead
-    if (pipelineResult.prices) {
-      const pricedProduct = pipelineResult.prices.find(p => p.productId === productId);
-
-      if (pricedProduct) {
-        console.log(`\n✅ Price Propagation Verified:`);
-        console.log(`   Product: ${productId.slice(0, 8)}`);
-        console.log(`   Base Price: $${pricedProduct.base_price.toFixed(2)}`);
-        console.log(`   Optimal Price: $${pricedProduct.optimal_price.toFixed(2)}`);
-        console.log(`   Published to Redis: ${pipelineResult.prices_published}`);
-
-        expect(pricedProduct.optimal_price).toBeGreaterThan(pricedProduct.base_price);
-      }
-    }
-  });
-
-  /**
-   * Scenario 6: Event Type Weighting
-   *
-   * Validates that different event types contribute to demand calculation.
-   * High-intent events (add_to_cart) should have more weight than low-intent (page_view).
-   */
-  test('should count various event types in demand calculation', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    const sessionId = events.newSession();
-    const productId = generateTestProductId('event-types');
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing event type weighting for product: ${productId}`,
-    });
-
-    // Log base price
-    await api.logPrice({
-      productId,
-      price: 100.0,
-      sessionId,
-      storeMode: 'hotel',
-    });
-
-    // Generate a mix of different event types
-    console.log('\n📊 Generating mixed event types:');
-
-    const mixedEvents = [
-      events.viewProduct(productId),           // page view
-      events.learnMore(productId),             // high intent
-      events.hover(productId, 'title'),        // engagement
-      events.hover(productId, 'paragraph'),    // engagement
-      events.addToCart(productId),             // highest intent
-    ];
-
-    console.log(`   - ${mixedEvents.length} mixed events (view, learn_more, hover, add_to_cart)`);
-
-    await api.ingestEvents(mixedEvents, config.timing.eventDelay);
-    console.log('✅ Events ingested');
-
-    // Trigger pipeline
-    const pipelineResult = await triggerPriceUpdate();
-
-    // Verify events were counted
-    if (pipelineResult.prices) {
-      const pricedProduct = pipelineResult.prices.find(p => p.productId === productId);
-
-      if (pricedProduct) {
-        console.log(`\n💰 Mixed Event Pricing Result:`);
-        console.log(`   Demand Score: ${pricedProduct.demand_score}`);
-        console.log(`   Expected: >= ${config.pricing.highThreshold} (for surge)`);
-
-        // Mixed events should trigger surge if count >= high_threshold
-        expect(pricedProduct.demand_score).toBeGreaterThanOrEqual(config.pricing.highThreshold);
-      }
-    }
-  });
-
-  /**
-   * Scenario 7: Session Isolation
-   *
-   * Validates that events from different sessions are correctly aggregated
-   * for the same product.
-   */
-  test('should aggregate demand across multiple sessions', async ({
-    api,
-    events,
-    triggerPriceUpdate,
-    config,
-  }) => {
-    const productId = generateTestProductId('multi-session');
-
-    test.info().annotations.push({
-      type: 'info',
-      description: `Testing multi-session aggregation for product: ${productId}`,
-    });
-
-    // Log base price
-    await api.logPrice({
-      productId,
-      price: 100.0,
-      sessionId: events.session,
-      storeMode: 'hotel',
-    });
-
-    // Generate events from 3 different sessions
-    console.log('\n📊 Generating events from multiple sessions:');
-
-    for (let i = 0; i < 3; i++) {
-      const sessionId = events.newSession();
-      console.log(`   Session ${i + 1}: ${sessionId.slice(0, 8)}...`);
-
-      // Each session generates 2 events
-      await api.ingestEvent(events.viewProduct(productId));
-      await api.ingestEvent(events.learnMore(productId));
-
-      await new Promise(r => setTimeout(r, config.timing.eventDelay));
-    }
-
-    console.log('✅ Events from 3 sessions ingested');
-
-    // Trigger pipeline
-    const pipelineResult = await triggerPriceUpdate();
-
-    // Verify aggregated demand
-    if (pipelineResult.prices) {
-      const pricedProduct = pipelineResult.prices.find(p => p.productId === productId);
-
-      if (pricedProduct) {
-        console.log(`\n💰 Multi-Session Aggregation Result:`);
-        console.log(`   Total Demand Score: ${pricedProduct.demand_score}`);
-        console.log(`   Expected: >= 6 (2 events × 3 sessions)`);
-
-        // 3 sessions × 2 events = 6 total events
-        expect(pricedProduct.demand_score).toBeGreaterThanOrEqual(6);
-      }
-    }
-  });
-});
-
-/**
- * Edge Cases and Error Handling
- */
-test.describe('Dynamic Pricing Edge Cases', () => {
-  test('should handle pipeline execution with empty Kafka topics', async ({
-    triggerPriceUpdate,
-  }) => {
-    // This tests the pipeline's resilience when there's no data
-    // The pipeline should complete without errors
-
-    console.log('\n⚙️ Testing pipeline with potentially empty data...');
-
-    // Run pipeline - should handle empty state gracefully
-    const result = await triggerPriceUpdate({ dryRun: true });
-
-    expect(result.success).toBe(true);
-    console.log(`✅ Pipeline handled gracefully: ${result.message || 'completed'}`);
-  });
-
-  test('should verify backend health before running tests', async ({ api }) => {
-    const backendHealth = await api.checkBackendHealth();
-    expect(backendHealth.status).toBe('healthy');
-
-    console.log(`✅ Backend: ${backendHealth.status}`);
-    console.log(`   Kafka: ${backendHealth.kafka}`);
-  });
-
-  test('should verify pricing provider health', async ({ api }) => {
-    const providerHealth = await api.checkProviderHealth();
-    expect(providerHealth.status).toBe('healthy');
-
-    console.log(`✅ Provider: ${providerHealth.status}`);
-    console.log(`   Redis: ${providerHealth.redis ? 'connected' : 'disconnected'}`);
-  });
-});

e2e/tsconfig.json

@@ -1,28 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "declaration": false,
-    "declarationMap": false,
-    "noEmit": true,
-    "outDir": "./dist",
-    "rootDir": ".",
-    "baseUrl": ".",
-    "paths": {
-      "@lib/*": ["lib/*"]
-    }
-  },
-  "include": [
-    "**/*.ts"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist"
-  ]
-}

experiments/README.md

@@ -1,8 +0,0 @@
-
-# Products
-# Agents
-# Pipeline
-
-Our pipeline technically should follow principles in a style like this:
-- Each step should be defined as an inheriting child of an scikit pipeline step, the granularity of the steps is dictated by the following: a step should be a transformation, augmentation or computation independently, no single stage should run multiple in-itself. This way we can modularize properly all the components and track properly in airflow. A stage can be defined as an sklearn step but then must be transalted to a function that takes the context in our DAG of airflow. All parametrization must be done via contexts.
-

experiments/agents/agent.py

@@ -38,10 +38,7 @@ def get_agent(agent_type: AgentTypes,  **kwargs) -> Agent:
 
 if __name__ == "__main__":
     import asyncio
-    JTBD= "Find me the cheapest room in Madrid for 2 people in the next two days, review each hotel room in detail and then add it to cart."
+    JTBD= "Name all the products on this site and try to find out more about each product by clicking into them (they might not open)"
-    agent = get_agent(AgentTypes.GENERIC_BROWSER_USE_AGENT,
+    agent = get_agent(AgentTypes.GENERIC_BROWSER_USE_AGENT, goal=JTBD, url="http://localhost:3000/products", timeout=300)
-                      goal=JTBD,
-                      url="http://localhost:3000/start-task?uuid=d10f5ab3-a7b7-4e97-8d94-ab06f1537c0a",
-                      timeout=300)
     R=asyncio.run(agent.act())
     print(R)

experiments/airflow/dags/ml_training_pipeline.py

@@ -1,115 +0,0 @@
-from airflow import DAG, Dataset
-from airflow.decorators import task
-from airflow.utils.dates import days_ago
-from datetime import timedelta
-import pandas as pd
-import logging
-import sys
-import pickle
-
-sys.path.insert(0, '/opt/airflow')
-
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    ValidateDataStep,
-    ExtractSessionFeaturesStep,
-    JoinLabelsStep,
-)
-
-TRAINING_DATASET = Dataset('phantom://ml/training-data')
-
-DEFAULT_ARGS = {
-    'owner': 'phantom-research',
-    'depends_on_past': False,
-    'email_on_failure': False,
-    'email_on_retry': False,
-    'retries': 2,
-    'retry_delay': timedelta(minutes=5),
-}
-
-
-class CompositeProvider(SupabaseProvider, BackendAPIProvider):
-    def __init__(self):
-        SupabaseProvider.__init__(self)
-        BackendAPIProvider.__init__(self)
-
-
-def _get_context(store_mode: str = 'hotel') -> PipelineContext:
-    return PipelineContext(provider=CompositeProvider(), store_mode=store_mode)
-
-
-with DAG(
-    'ml_training_pipeline',
-    default_args=DEFAULT_ARGS,
-    description='ML training data pipeline: fetch -> validate -> extract features -> label -> publish',
-    schedule=None,
-    start_date=days_ago(1),
-    catchup=False,
-    max_active_runs=1,
-    tags=['ml', 'training', 'features', 'research'],
-) as dag:
-
-    @task
-    def fetch_interactions(**kwargs) -> bytes:
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
-        df = FetchInteractionsStep(ctx).transform(None)
-        logging.info(f"Fetched {len(df)} interactions, {df['sessionId'].nunique()} sessions")
-        return pickle.dumps(df)
-
-    @task
-    def validate_data(raw_data: bytes, **kwargs) -> bytes:
-        df = pickle.loads(raw_data)
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
-        validated = ValidateDataStep(ctx).transform(df)
-        report = ctx.get_cached('validation_report') or {}
-        logging.info(f"Validation: {report.get('status')}, {report.get('sessions', 0)} sessions")
-        return pickle.dumps(validated)
-
-    @task
-    def extract_session_features(validated_data: bytes, **kwargs) -> bytes:
-        df = pickle.loads(validated_data)
-        if df.empty:
-            logging.warning("Empty input, skipping feature extraction")
-            return pickle.dumps(pd.DataFrame())
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
-        features = ExtractSessionFeaturesStep(ctx).transform(df)
-        logging.info(f"Extracted {len(features.columns)} features for {len(features)} sessions")
-        return pickle.dumps(features)
-
-    @task
-    def join_labels(features_data: bytes, **kwargs) -> bytes:
-        features_df = pickle.loads(features_data)
-        if features_df.empty:
-            logging.warning("Empty features, skipping label join")
-            return pickle.dumps(pd.DataFrame())
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-        ctx = _get_context(dag_conf.get('store_mode', 'hotel'))
-        labeled = JoinLabelsStep(ctx).transform(features_df)
-        n_agents = labeled['is_agent'].sum() if 'is_agent' in labeled.columns else 0
-        logging.info(f"Labeled {len(labeled)} sessions: {n_agents} agents")
-        return pickle.dumps(labeled)
-
-    @task(outlets=[TRAINING_DATASET])
-    def publish_training_data(labeled_data: bytes, **kwargs) -> dict:
-        labeled_df = pickle.loads(labeled_data)
-        if labeled_df.empty:
-            return {'status': 'skipped', 'reason': 'empty_data'}
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-        return {
-            'status': 'success',
-            'n_sessions': len(labeled_df),
-            'n_features': len([c for c in labeled_df.columns if c not in ['sessionId', 'experimentId', 'is_agent']]),
-            'store_mode': dag_conf.get('store_mode', 'hotel'),
-            'timestamp': pd.Timestamp.now().isoformat(),
-        }
-
-    raw = fetch_interactions()
-    validated = validate_data(raw)
-    features = extract_session_features(validated)
-    labeled = join_labels(features)
-    publish_training_data(labeled)

experiments/airflow/dags/surge_pricing_factory.py

@@ -1,210 +0,0 @@
-from airflow import DAG
-from airflow.operators.python import PythonOperator
-from airflow.utils.dates import days_ago
-from datetime import timedelta
-import pandas as pd
-import logging
-import sys
-import pickle
-
-sys.path.insert(0, '/opt/airflow')
-
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    ComputeDemandStep,
-    AggregatePriceLogsStep,
-    JoinProductFeaturesStep,
-)
-from procesing.pricers.simple import SimpleSurgePricer
-
-DEFAULT_ARGS = {
-    'owner': 'phantom-research',
-    'depends_on_past': False,
-    'email_on_failure': False,
-    'email_on_retry': False,
-    'retries': 2,
-    'retry_delay': timedelta(minutes=5),
-}
-
-class CompositeProvider(SupabaseProvider, BackendAPIProvider):
-    def __init__(self):
-        SupabaseProvider.__init__(self)
-        BackendAPIProvider.__init__(self)
-
-def _get_provider():
-    return CompositeProvider()
-
-def _make_task_callables(store_mode: str):
-    """Generate task callables bound to a specific store_mode."""
-
-    def get_context(**kwargs):
-        return PipelineContext(provider=_get_provider(), store_mode=store_mode)
-
-    def fetch_interactions(**kwargs):
-        ctx = get_context(**kwargs)
-        df = FetchInteractionsStep(ctx).transform(None)
-        kwargs['ti'].xcom_push(key='interactions_raw', value=pickle.dumps(df))
-        logging.info(f"[{store_mode}] Fetched {len(df)} interaction records")
-        return len(df)
-
-    def fetch_price_logs(**kwargs):
-        ctx = get_context(**kwargs)
-        df = FetchPriceLogsStep(ctx).transform(None)
-        kwargs['ti'].xcom_push(key='price_logs_raw', value=pickle.dumps(df))
-        logging.info(f"[{store_mode}] Fetched {len(df)} price records")
-        return len(df)
-
-    def compute_demand(**kwargs):
-        ti = kwargs['ti']
-        df = pickle.loads(ti.xcom_pull(key='interactions_raw'))
-        ctx = get_context(**kwargs)
-        demand_df = ComputeDemandStep(ctx).transform(df)
-        ti.xcom_push(key='demand_data', value=pickle.dumps(demand_df))
-        logging.info(f"[{store_mode}] Computed demand for {len(demand_df)} products")
-        return len(demand_df)
-
-    def aggregate_price_logs(**kwargs):
-        ti = kwargs['ti']
-        df = pickle.loads(ti.xcom_pull(key='price_logs_raw'))
-        ctx = get_context(**kwargs)
-        price_df = AggregatePriceLogsStep(ctx).transform(df)
-        ti.xcom_push(key='price_data', value=pickle.dumps(price_df))
-        logging.info(f"[{store_mode}] Aggregated price logs for {len(price_df)} products")
-        return len(price_df)
-
-    def join_product_features(**kwargs):
-        ti = kwargs['ti']
-        demand_df = pickle.loads(ti.xcom_pull(key='demand_data'))
-        price_df = pickle.loads(ti.xcom_pull(key='price_data'))
-        ctx = get_context(**kwargs)
-        joined_df = JoinProductFeaturesStep(ctx).transform((demand_df, price_df))
-        ti.xcom_push(key='product_features', value=pickle.dumps(joined_df))
-        logging.info(f"[{store_mode}] Joined features for {len(joined_df)} products")
-        return len(joined_df)
-
-    def apply_surge_pricing(**kwargs):
-        ti = kwargs['ti']
-        product_features = pickle.loads(ti.xcom_pull(key='product_features'))
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-        data = product_features.rename(columns={'demand_score': 'demand'})
-        surge_pricer = SimpleSurgePricer(
-            high_threshold=dag_conf.get('high_threshold', 10),
-            low_threshold=dag_conf.get('low_threshold', 2),
-            surge_multiplier=dag_conf.get('surge_multiplier', 1.2),
-            discount_multiplier=dag_conf.get('discount_multiplier', 0.9)
-        )
-        surge_pricer.fit(data)
-        data['optimal_price'] = surge_pricer.predict()
-
-        prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
-            'price': 'current_price', 'demand': 'demand_score'
-        })
-        ti.xcom_push(key='predicted_prices', value=pickle.dumps(prices_df))
-        logging.info(f"[{store_mode}] Applied surge pricing for {len(prices_df)} products")
-        return len(prices_df)
-
-    def publish_results(**kwargs):
-        ti = kwargs['ti']
-        prices_df = pickle.loads(ti.xcom_pull(key='predicted_prices'))
-        from lib.model_registry import ModelRegistry
-
-        registry = ModelRegistry()
-        dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-        metadata = {
-            'timestamp': pd.Timestamp.now().isoformat(),
-            'store_mode': store_mode,
-            'dag_run_id': kwargs['dag_run'].run_id if kwargs.get('dag_run') else 'manual',
-            'pricing_method': 'surge',
-            'high_threshold': dag_conf.get('high_threshold', 10),
-            'low_threshold': dag_conf.get('low_threshold', 2),
-            'surge_multiplier': dag_conf.get('surge_multiplier', 1.2),
-            'discount_multiplier': dag_conf.get('discount_multiplier', 0.9)
-        }
-        registry.publish_prices(prices_df, model_name=f'{store_mode}_latest', metadata=metadata)
-        logging.info(f"[{store_mode}] Published surge pricing for {len(prices_df)} products")
-
-        return {
-            'n_products': len(prices_df),
-            'registry_status': 'success',
-            'store_mode': store_mode,
-            'mean_demand': float(prices_df['demand_score'].mean()) if 'demand_score' in prices_df.columns else None
-        }
-
-    return {
-        'fetch_interactions': fetch_interactions,
-        'fetch_price_logs': fetch_price_logs,
-        'compute_demand': compute_demand,
-        'aggregate_price_logs': aggregate_price_logs,
-        'join_product_features': join_product_features,
-        'apply_surge_pricing': apply_surge_pricing,
-        'publish_results': publish_results,
-    }
-
-
-def create_surge_pricing_dag(store_mode: str) -> DAG:
-    """Factory: generates a surge pricing DAG for a given store_mode."""
-    callables = _make_task_callables(store_mode)
-
-    dag = DAG(
-        f'surge_pricing_{store_mode}',
-        default_args=DEFAULT_ARGS,
-        description=f'Surge pricing pipeline for {store_mode} store mode',
-        schedule_interval='*/15 * * * *',
-        start_date=days_ago(1),
-        catchup=False,
-        max_active_runs=1,
-        tags=['pricing', 'surge', 'research', store_mode],
-    )
-
-    with dag:
-        t_fetch_interactions = PythonOperator(
-            task_id='fetch_interactions',
-            python_callable=callables['fetch_interactions'],
-            provide_context=True,
-        )
-        t_fetch_price_logs = PythonOperator(
-            task_id='fetch_price_logs',
-            python_callable=callables['fetch_price_logs'],
-            provide_context=True,
-        )
-        t_compute_demand = PythonOperator(
-            task_id='compute_demand',
-            python_callable=callables['compute_demand'],
-            provide_context=True,
-        )
-        t_aggregate_prices = PythonOperator(
-            task_id='aggregate_price_logs',
-            python_callable=callables['aggregate_price_logs'],
-            provide_context=True,
-        )
-        t_join_features = PythonOperator(
-            task_id='join_product_features',
-            python_callable=callables['join_product_features'],
-            provide_context=True,
-        )
-        t_surge_pricing = PythonOperator(
-            task_id='apply_surge_pricing',
-            python_callable=callables['apply_surge_pricing'],
-            provide_context=True,
-        )
-        t_publish = PythonOperator(
-            task_id='publish_results',
-            python_callable=callables['publish_results'],
-            provide_context=True,
-        )
-
-        t_fetch_interactions >> t_compute_demand
-        t_fetch_price_logs >> t_aggregate_prices
-        [t_compute_demand, t_aggregate_prices] >> t_join_features >> t_surge_pricing >> t_publish
-
-    return dag
-
-
-# instantiate DAGs for Airflow to discover
-dag_airline = create_surge_pricing_dag('airline')
-dag_hotel = create_surge_pricing_dag('hotel')

experiments/airflow/dags/surge_pricing_pipeline.py

@@ -1,237 +0,0 @@
-from airflow import DAG
-from airflow.operators.python import PythonOperator
-from airflow.utils.dates import days_ago
-from datetime import timedelta
-import pandas as pd
-import logging
-import sys
-import pickle
-import io
-
-# add parent dir to path so procesing package can be imported
-sys.path.insert(0, '/opt/airflow')
-
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    ComputeDemandStep,
-    AggregatePriceLogsStep,
-    JoinProductFeaturesStep,
-)
-from procesing.pricers.simple import SimpleSurgePricer
-
-default_args = {
-    'owner': 'phantom-research',
-    'depends_on_past': False,
-    'email_on_failure': False,
-    'email_on_retry': False,
-    'retries': 2,
-    'retry_delay': timedelta(minutes=5),
-}
-
-def get_provider():
-    """Factory to create composite provider"""
-    class CompositeProvider(SupabaseProvider, BackendAPIProvider): # TODO: Fix this into one global provider singelton instead of multiple inheritance declarations acoss the codebase
-        def __init__(self):
-            SupabaseProvider.__init__(self)
-            BackendAPIProvider.__init__(self)
-    return CompositeProvider()
-
-def get_context(**kwargs):
-    """Build pipeline context from Airflow config"""
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-    return PipelineContext(
-        provider=get_provider(),
-        store_mode=dag_conf.get('store_mode', 'hotel'),
-    )
-
-# atomic task functions (each wraps one sklearn step)
-def fetch_interactions(**kwargs):
-    """Task: Fetch interaction data from Kafka"""
-    context = get_context(**kwargs)
-    step = FetchInteractionsStep(context)
-    df = step.transform(None)
-
-    kwargs['ti'].xcom_push(key='interactions_raw', value=pickle.dumps(df))
-    logging.info(f"Fetched {len(df)} interaction records")
-    return len(df)
-
-def fetch_price_logs(**kwargs):
-    """Task: Fetch price logs from Kafka"""
-    context = get_context(**kwargs)
-    step = FetchPriceLogsStep(context)
-    df = step.transform(None)
-
-    kwargs['ti'].xcom_push(key='price_logs_raw', value=pickle.dumps(df))
-    logging.info(f"Fetched {len(df)} price records")
-    return len(df)
-
-def compute_demand(**kwargs):
-    """Task: Compute demand scores from interactions"""
-    ti = kwargs['ti']
-    df = pickle.loads(ti.xcom_pull(key='interactions_raw'))
-
-    context = get_context(**kwargs)
-    step = ComputeDemandStep(context)
-    demand_df = step.transform(df)
-    # TODO: clear the xcom
-
-
-    ti.xcom_push(key='demand_data', value=pickle.dumps(demand_df))
-    logging.info(f"Computed demand for {len(demand_df)} products")
-    return len(demand_df)
-
-def aggregate_price_logs(**kwargs):
-    """Task: Aggregate price logs"""
-    ti = kwargs['ti']
-    df = pickle.loads(ti.xcom_pull(key='price_logs_raw'))
-
-    context = get_context(**kwargs)
-    step = AggregatePriceLogsStep(context)
-    price_df = step.transform(df)
-
-    ti.xcom_push(key='price_data', value=pickle.dumps(price_df))
-    logging.info(f"Aggregated price logs for {len(price_df)} products")
-    return len(price_df)
-
-def join_product_features(**kwargs):
-    """Task: Join demand and price data"""
-    ti = kwargs['ti']
-    demand_df = pickle.loads(ti.xcom_pull(key='demand_data'))
-    price_df = pickle.loads(ti.xcom_pull(key='price_data'))
-
-    context = get_context(**kwargs)
-    step = JoinProductFeaturesStep(context)
-    joined_df = step.transform((demand_df, price_df))
-
-    ti.xcom_push(key='product_features', value=pickle.dumps(joined_df))
-    logging.info(f"Joined features for {len(joined_df)} products")
-    return len(joined_df)
-
-def apply_surge_pricing(**kwargs):
-    """Task: Apply surge pricing rules to generate optimal prices"""
-    ti = kwargs['ti']
-    product_features = pickle.loads(ti.xcom_pull(key='product_features'))
-
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-    # rename demand_score to demand for pricer compatibility
-    data = product_features.rename(columns={'demand_score': 'demand'})
-
-    surge_pricer = SimpleSurgePricer(
-        high_threshold=dag_conf.get('high_threshold', 10),
-        low_threshold=dag_conf.get('low_threshold', 2),
-        surge_multiplier=dag_conf.get('surge_multiplier', 1.2),
-        discount_multiplier=dag_conf.get('discount_multiplier', 0.9)
-    )
-    surge_pricer.fit(data)
-    data['optimal_price'] = surge_pricer.predict()
-
-    prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
-        'price': 'current_price',
-        'demand': 'demand_score'
-    })
-
-    ti.xcom_push(key='predicted_prices', value=pickle.dumps(prices_df))
-    logging.info(f"Applied surge pricing for {len(prices_df)} products")
-    return len(prices_df)
-
-def publish_results(**kwargs):
-    """Task: Publish surge pricing results to registry"""
-    ti = kwargs['ti']
-    prices_df = pickle.loads(ti.xcom_pull(key='predicted_prices'))
-
-    sys.path.insert(0, '/opt/airflow')
-    from lib.model_registry import ModelRegistry
-
-    registry = ModelRegistry()
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-    metadata = {
-        'timestamp': pd.Timestamp.now().isoformat(),
-        'store_mode': dag_conf.get('store_mode', 'hotel'),
-        'dag_run_id': kwargs['dag_run'].run_id if kwargs.get('dag_run') else 'manual',
-        'pricing_method': 'surge',
-        'high_threshold': dag_conf.get('high_threshold', 10),
-        'low_threshold': dag_conf.get('low_threshold', 2),
-        'surge_multiplier': dag_conf.get('surge_multiplier', 1.2),
-        'discount_multiplier': dag_conf.get('discount_multiplier', 0.9)
-    }
-
-    registry.publish_prices(prices_df, model_name='latest', metadata=metadata)
-
-    logging.info(f"Published surge pricing for {len(prices_df)} products")
-
-    return {
-        'n_products': len(prices_df),
-        'registry_status': 'success',
-        'mean_demand': float(prices_df['demand_score'].mean()) if 'demand_score' in prices_df.columns else None
-    }
-
-
-# DAG definition
-with DAG(
-    'surge_pricing_pipeline',
-    default_args=default_args,
-    description='Simple surge pricing pipeline: demand aggregation + rule-based pricing',
-    schedule_interval='*/15 * * * *',
-    start_date=days_ago(1),
-    catchup=False,
-    max_active_runs=1,
-    tags=['pricing', 'surge', 'research', 'simplified'],
-) as dag:
-
-    # parallel data fetching
-    t_fetch_interactions = PythonOperator(
-        task_id='fetch_interactions',
-        python_callable=fetch_interactions,
-        provide_context=True,
-    )
-
-    t_fetch_price_logs = PythonOperator(
-        task_id='fetch_price_logs',
-        python_callable=fetch_price_logs,
-        provide_context=True,
-    )
-
-    # compute demand from interactions
-    t_compute_demand = PythonOperator(
-        task_id='compute_demand',
-        python_callable=compute_demand,
-        provide_context=True,
-    )
-
-    # aggregate price logs
-    t_aggregate_prices = PythonOperator(
-        task_id='aggregate_price_logs',
-        python_callable=aggregate_price_logs,
-        provide_context=True,
-    )
-
-    # join demand and prices
-    t_join_features = PythonOperator(
-        task_id='join_product_features',
-        python_callable=join_product_features,
-        provide_context=True,
-    )
-
-    # apply surge pricing
-    t_surge_pricing = PythonOperator(
-        task_id='apply_surge_pricing',
-        python_callable=apply_surge_pricing,
-        provide_context=True,
-    )
-
-    # publish to registry
-    t_publish = PythonOperator(
-        task_id='publish_results',
-        python_callable=publish_results,
-        provide_context=True,
-    )
-
-    # dependency graph: parallel fetch -> process -> join -> surge -> publish
-    t_fetch_interactions >> t_compute_demand
-    t_fetch_price_logs >> t_aggregate_prices
-    [t_compute_demand, t_aggregate_prices] >> t_join_features >> t_surge_pricing >> t_publish

experiments/data_export.ipynb

@@ -0,0 +1,957 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "62eafcd9-5462-4063-8873-0e7fb9add907",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from kafka import KafkaConsumer\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "import numpy as np\n",
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "import matplotlib.pyplot as plt\n",
+    "from IPython.display import display, SVG, Image\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4af65cb4-e8cf-4877-b2db-13ac19f3838f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 73 entries, 0 to 72\n",
+      "Data columns (total 13 columns):\n",
+      " #   Column                Non-Null Count  Dtype  \n",
+      "---  ------                --------------  -----  \n",
+      " 0   sessionId             73 non-null     object \n",
+      " 1   eventName             73 non-null     object \n",
+      " 2   page                  73 non-null     object \n",
+      " 3   productId             67 non-null     object \n",
+      " 4   storeMode             73 non-null     object \n",
+      " 5   userAgent             73 non-null     object \n",
+      " 6   ts                    73 non-null     object \n",
+      " 7   metadata_referrer     6 non-null      object \n",
+      " 8   metadata_roomType     45 non-null     object \n",
+      " 9   metadata_price        45 non-null     float64\n",
+      " 10  metadata_nights       45 non-null     float64\n",
+      " 11  metadata_elementText  22 non-null     object \n",
+      " 12  metadata_dwellTime    22 non-null     float64\n",
+      "dtypes: float64(3), object(10)\n",
+      "memory usage: 7.5+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "KAFKA_PORT=os.getenv(\"KAFKA_PORT\", 9092)\n",
+    "topic = \"user-interactions\"\n",
+    "consumer = KafkaConsumer(\n",
+    "    topic, \n",
+    "    enable_auto_commit=True,\n",
+    "    value_deserializer=lambda x: json.loads(x.decode('utf-8')),\n",
+    "    auto_offset_reset='earliest', \n",
+    "    bootstrap_servers=['localhost:9092'])\n",
+    "messages=consumer.poll(timeout_ms=1000,max_records=10000)\n",
+    "df = []\n",
+    "for m in messages.values():\n",
+    "    for i in m:\n",
+    "        df.append(i.value)\n",
+    "df = pd.DataFrame(df)\n",
+    "# explode metadata col json\n",
+    "df = df.join(pd.json_normalize(df.pop(\"metadata\"), sep=\".\").add_prefix(\"metadata_\"))\n",
+    "df.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "f6819a1c-32ab-49c7-845b-5df7bf60f561",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sessionId</th>\n",
+       "      <th>eventName</th>\n",
+       "      <th>page</th>\n",
+       "      <th>productId</th>\n",
+       "      <th>storeMode</th>\n",
+       "      <th>userAgent</th>\n",
+       "      <th>ts</th>\n",
+       "      <th>metadata_referrer</th>\n",
+       "      <th>metadata_roomType</th>\n",
+       "      <th>metadata_price</th>\n",
+       "      <th>metadata_nights</th>\n",
+       "      <th>metadata_elementText</th>\n",
+       "      <th>metadata_dwellTime</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>d176d7c9-4027-4702-9e31-2a71395cdda0</td>\n",
+       "      <td>page_view</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:23:46.270Z</td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>f0317a5d-e424-44e9-b784-c8f7291ffe31</td>\n",
+       "      <td>page_view</td>\n",
+       "      <td>/</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...</td>\n",
+       "      <td>2025-11-14T13:26:00.291Z</td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>f0317a5d-e424-44e9-b784-c8f7291ffe31</td>\n",
+       "      <td>page_view</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...</td>\n",
+       "      <td>2025-11-14T13:26:07.769Z</td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>f0317a5d-e424-44e9-b784-c8f7291ffe31</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...</td>\n",
+       "      <td>2025-11-14T13:26:15.010Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Premium Room</td>\n",
+       "      <td>269.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>238dc588-a7ab-4c0e-bccd-6abca5076c66</td>\n",
+       "      <td>page_view</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
+       "      <td>2025-11-14T13:27:15.457Z</td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>238dc588-a7ab-4c0e-bccd-6abca5076c66</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
+       "      <td>2025-11-14T13:27:15.591Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Premium Room</td>\n",
+       "      <td>264.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>432</th>\n",
+       "      <td>214d9fad-9b00-40c3-bd0e-7739b6acd654</td>\n",
+       "      <td>click</td>\n",
+       "      <td>1762448192425</td>\n",
+       "      <td>DIV</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>/</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1623.0</td>\n",
+       "      <td>493.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>238dc588-a7ab-4c0e-bccd-6abca5076c66</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
+       "      <td>2025-11-14T13:27:21.483Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Premium Room</td>\n",
+       "      <td>264.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>238dc588-a7ab-4c0e-bccd-6abca5076c66</td>\n",
+       "      <td>hover_over_title</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
+       "      <td>2025-11-14T13:27:22.646Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Grand Plaza Hotel</td>\n",
+       "      <td>1200.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>238dc588-a7ab-4c0e-bccd-6abca5076c66</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
+       "      <td>2025-11-14T13:27:25.889Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Premium Room</td>\n",
+       "      <td>264.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>013fc334-4045-4d5a-8739-dd0a8766a63b</td>\n",
+       "      <td>page_view</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:53:59.993Z</td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>013fc334-4045-4d5a-8739-dd0a8766a63b</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:54:10.705Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Premium Room</td>\n",
+       "      <td>223.0</td>\n",
+       "      <td>3.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>013fc334-4045-4d5a-8739-dd0a8766a63b</td>\n",
+       "      <td>hover_over_title</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-0</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:54:11.771Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>416.0</td>\n",
+       "      <td>397.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Grand Plaza Hotel</td>\n",
+       "      <td>1200.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>013fc334-4045-4d5a-8739-dd0a8766a63b</td>\n",
+       "      <td>view_item_page</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-1</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:54:29.772Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Standard Room</td>\n",
+       "      <td>267.0</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>013fc334-4045-4d5a-8739-dd0a8766a63b</td>\n",
+       "      <td>hover_over_title</td>\n",
+       "      <td>/products</td>\n",
+       "      <td>htl-1</td>\n",
+       "      <td>hotel</td>\n",
+       "      <td>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...</td>\n",
+       "      <td>2025-11-14T13:54:30.833Z</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Seaside Resort</td>\n",
+       "      <td>1200.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               sessionId         eventName       page  \\\n",
+       "0   d176d7c9-4027-4702-9e31-2a71395cdda0         page_view  /products   \n",
+       "1   f0317a5d-e424-44e9-b784-c8f7291ffe31         page_view          /   \n",
+       "2   f0317a5d-e424-44e9-b784-c8f7291ffe31         page_view  /products   \n",
+       "3   f0317a5d-e424-44e9-b784-c8f7291ffe31    view_item_page  /products   \n",
+       "4   238dc588-a7ab-4c0e-bccd-6abca5076c66         page_view  /products   \n",
+       "5   238dc588-a7ab-4c0e-bccd-6abca5076c66    view_item_page  /products   \n",
+       "6   238dc588-a7ab-4c0e-bccd-6abca5076c66    view_item_page  /products   \n",
+       "7   238dc588-a7ab-4c0e-bccd-6abca5076c66  hover_over_title  /products   \n",
+       "8   238dc588-a7ab-4c0e-bccd-6abca5076c66    view_item_page  /products   \n",
+       "35  013fc334-4045-4d5a-8739-dd0a8766a63b         page_view  /products   \n",
+       "36  013fc334-4045-4d5a-8739-dd0a8766a63b    view_item_page  /products   \n",
+       "37  013fc334-4045-4d5a-8739-dd0a8766a63b  hover_over_title  /products   \n",
+       "38  013fc334-4045-4d5a-8739-dd0a8766a63b    view_item_page  /products   \n",
+       "39  013fc334-4045-4d5a-8739-dd0a8766a63b  hover_over_title  /products   \n",
+       "\n",
+       "   productId storeMode                                          userAgent  \\\n",
+       "0       None     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "1       None     hotel  Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...   \n",
+       "2       None     hotel  Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...   \n",
+       "3      htl-0     hotel  Mozilla/5.0 (X11; Linux x86_64; rv:143.0) Geck...   \n",
+       "4       None     hotel  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...   \n",
+       "5      htl-0     hotel  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...   \n",
+       "6      htl-0     hotel  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...   \n",
+       "7      htl-0     hotel  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...   \n",
+       "8      htl-0     hotel  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...   \n",
+       "35      None     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "36     htl-0     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "37     htl-0     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "38     htl-1     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "39     htl-1     hotel  Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53...   \n",
+       "\n",
+       "                          ts metadata_referrer metadata_roomType  \\\n",
+       "0   2025-11-14T13:23:46.270Z                                 NaN   \n",
+       "1   2025-11-14T13:26:00.291Z                                 NaN   \n",
+       "2   2025-11-14T13:26:07.769Z                                 NaN   \n",
+       "3   2025-11-14T13:26:15.010Z               NaN      Premium Room   \n",
+       "4   2025-11-14T13:27:15.457Z                                 NaN   \n",
+       "5   2025-11-14T13:27:15.591Z               NaN      Premium Room   \n",
+       "6   2025-11-14T13:27:21.483Z               NaN      Premium Room   \n",
+       "7   2025-11-14T13:27:22.646Z               NaN               NaN   \n",
+       "8   2025-11-14T13:27:25.889Z               NaN      Premium Room   \n",
+       "35  2025-11-14T13:53:59.993Z                                 NaN   \n",
+       "36  2025-11-14T13:54:10.705Z               NaN      Premium Room   \n",
+       "37  2025-11-14T13:54:11.771Z               NaN               NaN   \n",
+       "38  2025-11-14T13:54:29.772Z               NaN     Standard Room   \n",
+       "39  2025-11-14T13:54:30.833Z               NaN               NaN   \n",
+       "\n",
+       "    metadata_price  metadata_nights metadata_elementText  metadata_dwellTime  \n",
+       "0              NaN              NaN                  NaN                 NaN  \n",
+       "1              NaN              NaN                  NaN                 NaN  \n",
+       "2              NaN              NaN                  NaN                 NaN  \n",
+       "3            269.0              1.0                  NaN                 NaN  \n",
+       "4              NaN              NaN                  NaN                 NaN  \n",
+       "5            264.0              2.0                  NaN                 NaN  \n",
+       "6            264.0              2.0                  NaN                 NaN  \n",
+       "7              NaN              NaN    Grand Plaza Hotel              1200.0  \n",
+       "8            264.0              2.0                  NaN                 NaN  \n",
+       "35             NaN              NaN                  NaN                 NaN  \n",
+       "36           223.0              3.0                  NaN                 NaN  \n",
+       "37             NaN              NaN    Grand Plaza Hotel              1200.0  \n",
+       "38           267.0              5.0                  NaN                 NaN  \n",
+       "39             NaN              NaN       Seaside Resort              1200.0  "
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.groupby('sessionId').head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "380eca5f-8304-4fb2-be32-e8bcfd312085",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['013fc334-4045-4d5a-8739-dd0a8766a63b',\n",
+       " '238dc588-a7ab-4c0e-bccd-6abca5076c66',\n",
+       " 'd176d7c9-4027-4702-9e31-2a71395cdda0',\n",
+       " 'f0317a5d-e424-44e9-b784-c8f7291ffe31']"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sessions = list(set(df['sessionId'])); sessions # 238dc588-a7ab-4c0e-bccd-6abca5076c66"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "f4ae6f81-dcb8-44be-aee7-30dbc3a6bae1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# map sessions to experiments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "050d90a4-20a9-47f5-b998-c31178a54cb3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def build_transition_prob_matrix(df: pd.DataFrame):\n",
+    "    df = df.dropna(subset=['eventName'])\n",
+    "    events = df['eventName'].tolist()\n",
+    "    labels = pd.Index(events).unique().tolist()\n",
+    "    idx = {e:i for i,e in enumerate(labels)}\n",
+    "    M = np.zeros((len(labels), len(labels)), dtype=float)\n",
+    "    for a, b in zip(events, events[1:]):\n",
+    "        M[idx[a], idx[b]] += 1\n",
+    "    row_sums = M.sum(axis=1, keepdims=True)\n",
+    "    with np.errstate(divide='ignore', invalid='ignore'):\n",
+    "        P = np.divide(M, row_sums, where=row_sums>0)  # row-normalized\n",
+    "    return P, labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "e68f9004-82f5-4826-aece-e3dc6e15a18f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# https://medium.com/data-science/time-series-data-markov-transition-matrices-7060771e362b\n",
+    "from graphviz import Digraph\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "def _as_prob_df(matrix, labels=None):\n",
+    "    \"\"\"Return a square DataFrame with index=columns=labels.\"\"\"\n",
+    "    if isinstance(matrix, pd.DataFrame):\n",
+    "        # Ensure square and aligned\n",
+    "        assert (matrix.index == matrix.columns).all(), \"Index/columns must match.\"\n",
+    "        return matrix\n",
+    "    matrix = np.asarray(matrix, dtype=float)\n",
+    "    assert matrix.shape[0] == matrix.shape[1], \"Matrix must be square.\"\n",
+    "    if labels is None:\n",
+    "        raise ValueError(\"labels are required when matrix is not a DataFrame\")\n",
+    "    assert len(labels) == matrix.shape[0], \"labels length must match matrix size.\"\n",
+    "    return pd.DataFrame(matrix, index=list(labels), columns=list(labels))\n",
+    "\n",
+    "def _df_to_edgelist(P: pd.DataFrame, threshold=0.0, round_digits=2):\n",
+    "    \"\"\"Build weighted edges > threshold.\"\"\"\n",
+    "    edges = []\n",
+    "    for src in P.index:\n",
+    "        for dst in P.columns:\n",
+    "            w = float(P.loc[src, dst])\n",
+    "            if w > threshold:\n",
+    "                edges.append((str(src), str(dst), f\"{w:.{round_digits}f}\"))\n",
+    "    return edges\n",
+    "\n",
+    "def render_graph(fname, matrix, ls_index=None, threshold=0.0, fmt=\"svg\", view=False):\n",
+    "    \"\"\"\n",
+    "    fname: output file stem (no extension)\n",
+    "    matrix: NumPy array or pandas DataFrame of transition PROBABILITIES\n",
+    "    ls_index: ordered labels (required if matrix is not a DataFrame)\n",
+    "    threshold: hide edges with weight <= threshold\n",
+    "    fmt: 'svg'|'png'|'pdf' etc.\n",
+    "    view: open after rendering\n",
+    "    \"\"\"\n",
+    "    P = _as_prob_df(matrix, labels=ls_index)\n",
+    "    edges = _df_to_edgelist(P, threshold=threshold)\n",
+    "\n",
+    "    g = Digraph(format=fmt)\n",
+    "    g.attr(rankdir=\"LR\", size=\"30\")\n",
+    "    g.attr(\"node\", shape=\"circle\")\n",
+    "\n",
+    "    # ensure isolated nodes appear\n",
+    "    for node in P.index:\n",
+    "        g.node(str(node), width=\"1\", height=\"1\")\n",
+    "\n",
+    "    for src, dst, label in edges:\n",
+    "        g.edge(src, dst, label=label)\n",
+    "\n",
+    "    g.render(fname, view=view, cleanup=True)\n",
+    "    return g\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "e255a2c1-6454-4e5e-89f6-ef8ac51ab6cc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "013fc334-4045-4d5a-8739-dd0a8766a63b\n"
+     ]
+    },
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 13.1.2 (0)\n",
+       " -->\n",
+       "<!-- Pages: 1 -->\n",
+       "<svg width=\"565pt\" height=\"354pt\"\n",
+       " viewBox=\"0.00 0.00 565.00 354.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 349.64)\">\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-349.64 561.05,-349.64 561.05,4 -4,4\"/>\n",
+       "<!-- page_view -->\n",
+       "<g id=\"node1\" class=\"node\">\n",
+       "<title>page_view</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"48.19\" cy=\"-235.83\" rx=\"48.19\" ry=\"48.19\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"48.19\" y=\"-231.16\" font-family=\"Times,serif\" font-size=\"14.00\">page_view</text>\n",
+       "</g>\n",
+       "<!-- view_item_page -->\n",
+       "<g id=\"node2\" class=\"node\">\n",
+       "<title>view_item_page</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"232.88\" cy=\"-235.83\" rx=\"69.01\" ry=\"69.01\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-231.16\" font-family=\"Times,serif\" font-size=\"14.00\">view_item_page</text>\n",
+       "</g>\n",
+       "<!-- page_view&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge1\" class=\"edge\">\n",
+       "<title>page_view&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M96.71,-235.83C113.69,-235.83 133.31,-235.83 152.25,-235.83\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"152.1,-239.33 162.1,-235.83 152.1,-232.33 152.1,-239.33\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"130.12\" y=\"-239.78\" font-family=\"Times,serif\" font-size=\"14.00\">1.00</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge2\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M214.74,-302.59C217.1,-314.51 223.14,-322.84 232.88,-322.84 239.27,-322.84 244.07,-319.26 247.28,-313.42\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"250.57,-314.62 250.52,-304.02 243.95,-312.33 250.57,-314.62\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-326.79\" font-family=\"Times,serif\" font-size=\"14.00\">0.68</text>\n",
+       "</g>\n",
+       "<!-- hover_over_title -->\n",
+       "<g id=\"node3\" class=\"node\">\n",
+       "<title>hover_over_title</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"463.22\" cy=\"-275.83\" rx=\"69.81\" ry=\"69.81\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"463.22\" y=\"-271.16\" font-family=\"Times,serif\" font-size=\"14.00\">hover_over_title</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;hover_over_title -->\n",
+       "<g id=\"edge3\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;hover_over_title</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M300.48,-250.14C307.03,-251.43 313.58,-252.69 319.89,-253.83 340.12,-257.51 362.05,-261.1 382.5,-264.27\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"381.77,-267.7 392.19,-265.76 382.83,-260.78 381.77,-267.7\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-263.17\" font-family=\"Times,serif\" font-size=\"14.00\">0.29</text>\n",
+       "</g>\n",
+       "<!-- hover_over_paragraph -->\n",
+       "<g id=\"node4\" class=\"node\">\n",
+       "<title>hover_over_paragraph</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"463.22\" cy=\"-93.83\" rx=\"93.83\" ry=\"93.83\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"463.22\" y=\"-89.16\" font-family=\"Times,serif\" font-size=\"14.00\">hover_over_paragraph</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;hover_over_paragraph -->\n",
+       "<g id=\"edge4\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;hover_over_paragraph</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M292.09,-199.63C316.79,-184.27 346.14,-166.02 373.44,-149.04\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"375.08,-152.15 381.72,-143.89 371.38,-146.2 375.08,-152.15\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-185.68\" font-family=\"Times,serif\" font-size=\"14.00\">0.04</text>\n",
+       "</g>\n",
+       "<!-- hover_over_title&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge5\" class=\"edge\">\n",
+       "<title>hover_over_title&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M399.53,-246.73C384.12,-240.88 367.42,-235.6 351.39,-232.58 339.13,-230.28 326.03,-229.26 313.19,-229.04\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"313.51,-225.54 303.51,-229.04 313.51,-232.54 313.51,-225.54\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-236.53\" font-family=\"Times,serif\" font-size=\"14.00\">1.00</text>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x7f0779e818b0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[]\n"
+     ]
+    },
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 13.1.2 (0)\n",
+       " -->\n",
+       "<!-- Pages: 1 -->\n",
+       "<svg width=\"8pt\" height=\"8pt\"\n",
+       " viewBox=\"0.00 0.00 8.00 8.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 4)\">\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-4 4,-4 4,4 -4,4\"/>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x7f6800fac980>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[0.00000000e+000 1.00000000e+000 0.00000000e+000 0.00000000e+000]\n",
+      " [0.00000000e+000 6.78571429e-001 2.85714286e-001 3.57142857e-002]\n",
+      " [0.00000000e+000 1.00000000e+000 0.00000000e+000 0.00000000e+000]\n",
+      " [2.05833592e-312 2.29175545e-312 4.94065646e-324 6.92110218e-310]]\n",
+      "238dc588-a7ab-4c0e-bccd-6abca5076c66\n"
+     ]
+    },
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 13.1.2 (0)\n",
+       " -->\n",
+       "<!-- Pages: 1 -->\n",
+       "<svg width=\"565pt\" height=\"354pt\"\n",
+       " viewBox=\"0.00 0.00 565.00 354.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 349.64)\">\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-349.64 561.05,-349.64 561.05,4 -4,4\"/>\n",
+       "<!-- page_view -->\n",
+       "<g id=\"node1\" class=\"node\">\n",
+       "<title>page_view</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"48.19\" cy=\"-109.83\" rx=\"48.19\" ry=\"48.19\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"48.19\" y=\"-105.16\" font-family=\"Times,serif\" font-size=\"14.00\">page_view</text>\n",
+       "</g>\n",
+       "<!-- view_item_page -->\n",
+       "<g id=\"node2\" class=\"node\">\n",
+       "<title>view_item_page</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"232.88\" cy=\"-197.83\" rx=\"69.01\" ry=\"69.01\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-193.16\" font-family=\"Times,serif\" font-size=\"14.00\">view_item_page</text>\n",
+       "</g>\n",
+       "<!-- page_view&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge1\" class=\"edge\">\n",
+       "<title>page_view&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M92.02,-130.47C112.32,-140.25 137.13,-152.2 160.18,-163.3\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"158.39,-166.32 168.92,-167.51 161.43,-160.02 158.39,-166.32\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"130.12\" y=\"-157.78\" font-family=\"Times,serif\" font-size=\"14.00\">1.00</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge2\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M214.74,-264.59C217.1,-276.51 223.14,-284.84 232.88,-284.84 239.27,-284.84 244.07,-281.26 247.28,-275.42\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"250.57,-276.62 250.52,-266.02 243.95,-274.33 250.57,-276.62\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-288.79\" font-family=\"Times,serif\" font-size=\"14.00\">0.19</text>\n",
+       "</g>\n",
+       "<!-- hover_over_title -->\n",
+       "<g id=\"node3\" class=\"node\">\n",
+       "<title>hover_over_title</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"463.22\" cy=\"-275.83\" rx=\"69.81\" ry=\"69.81\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"463.22\" y=\"-271.16\" font-family=\"Times,serif\" font-size=\"14.00\">hover_over_title</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;hover_over_title -->\n",
+       "<g id=\"edge3\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;hover_over_title</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M289.6,-237.16C299.36,-242.77 309.67,-247.94 319.89,-251.83 339.45,-259.28 361.4,-264.43 382.1,-267.98\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"381.52,-271.43 391.95,-269.55 382.62,-264.52 381.52,-271.43\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-265.16\" font-family=\"Times,serif\" font-size=\"14.00\">0.38</text>\n",
+       "</g>\n",
+       "<!-- hover_over_paragraph -->\n",
+       "<g id=\"node4\" class=\"node\">\n",
+       "<title>hover_over_paragraph</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"463.22\" cy=\"-93.83\" rx=\"93.83\" ry=\"93.83\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"463.22\" y=\"-89.16\" font-family=\"Times,serif\" font-size=\"14.00\">hover_over_paragraph</text>\n",
+       "</g>\n",
+       "<!-- view_item_page&#45;&gt;hover_over_paragraph -->\n",
+       "<g id=\"edge4\" class=\"edge\">\n",
+       "<title>view_item_page&#45;&gt;hover_over_paragraph</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M300.22,-180.71C317.22,-175.46 335.24,-169.12 351.39,-161.83 358.97,-158.41 366.67,-154.57 374.29,-150.49\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"375.84,-153.63 382.92,-145.75 372.47,-147.5 375.84,-153.63\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-178.15\" font-family=\"Times,serif\" font-size=\"14.00\">0.44</text>\n",
+       "</g>\n",
+       "<!-- hover_over_title&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge5\" class=\"edge\">\n",
+       "<title>hover_over_title&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M398.52,-248.36C383.21,-242.16 366.82,-235.87 351.39,-230.58 338.42,-226.15 324.5,-221.86 310.94,-217.93\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"312.2,-214.65 301.62,-215.28 310.28,-221.39 312.2,-214.65\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-234.53\" font-family=\"Times,serif\" font-size=\"14.00\">1.00</text>\n",
+       "</g>\n",
+       "<!-- hover_over_paragraph&#45;&gt;page_view -->\n",
+       "<g id=\"edge6\" class=\"edge\">\n",
+       "<title>hover_over_paragraph&#45;&gt;page_view</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M369.13,-95.76C310.26,-97.17 232.59,-99.41 163.87,-102.58 145.72,-103.42 125.98,-104.58 108.06,-105.73\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"107.86,-102.24 98.1,-106.38 108.31,-109.22 107.86,-102.24\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-106.53\" font-family=\"Times,serif\" font-size=\"14.00\">0.14</text>\n",
+       "</g>\n",
+       "<!-- hover_over_paragraph&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge7\" class=\"edge\">\n",
+       "<title>hover_over_paragraph&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M372.68,-119.15C354.84,-125.32 336.5,-132.51 319.89,-140.58 312.9,-143.98 305.81,-147.87 298.86,-151.98\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"297.49,-148.71 290.78,-156.91 301.14,-154.69 297.49,-148.71\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"335.64\" y=\"-144.53\" font-family=\"Times,serif\" font-size=\"14.00\">0.86</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x7f6800f97110>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[0.         1.         0.         0.        ]\n",
+      " [0.         0.1875     0.375      0.4375    ]\n",
+      " [0.         1.         0.         0.        ]\n",
+      " [0.14285714 0.85714286 0.         0.        ]]\n",
+      "d176d7c9-4027-4702-9e31-2a71395cdda0\n"
+     ]
+    },
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 13.1.2 (0)\n",
+       " -->\n",
+       "<!-- Pages: 1 -->\n",
+       "<svg width=\"104pt\" height=\"104pt\"\n",
+       " viewBox=\"0.00 0.00 104.00 104.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 100.37)\">\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-100.37 100.37,-100.37 100.37,4 -4,4\"/>\n",
+       "<!-- page_view -->\n",
+       "<g id=\"node1\" class=\"node\">\n",
+       "<title>page_view</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"48.19\" cy=\"-48.19\" rx=\"48.19\" ry=\"48.19\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"48.19\" y=\"-43.51\" font-family=\"Times,serif\" font-size=\"14.00\">page_view</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x7f6800f97110>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[0.]]\n",
+      "f0317a5d-e424-44e9-b784-c8f7291ffe31\n"
+     ]
+    },
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 13.1.2 (0)\n",
+       " -->\n",
+       "<!-- Pages: 1 -->\n",
+       "<svg width=\"310pt\" height=\"160pt\"\n",
+       " viewBox=\"0.00 0.00 310.00 160.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 156.44)\">\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-156.44 305.89,-156.44 305.89,4 -4,4\"/>\n",
+       "<!-- page_view -->\n",
+       "<g id=\"node1\" class=\"node\">\n",
+       "<title>page_view</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"48.19\" cy=\"-69.01\" rx=\"48.19\" ry=\"48.19\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"48.19\" y=\"-64.33\" font-family=\"Times,serif\" font-size=\"14.00\">page_view</text>\n",
+       "</g>\n",
+       "<!-- page_view&#45;&gt;page_view -->\n",
+       "<g id=\"edge1\" class=\"edge\">\n",
+       "<title>page_view&#45;&gt;page_view</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M33.03,-115.09C34.09,-126.6 39.14,-135.19 48.19,-135.19 53.98,-135.19 58.13,-131.66 60.65,-126.1\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"64.01,-127.11 62.98,-116.56 57.21,-125.45 64.01,-127.11\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"48.19\" y=\"-139.14\" font-family=\"Times,serif\" font-size=\"14.00\">0.50</text>\n",
+       "</g>\n",
+       "<!-- view_item_page -->\n",
+       "<g id=\"node2\" class=\"node\">\n",
+       "<title>view_item_page</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"232.88\" cy=\"-69.01\" rx=\"69.01\" ry=\"69.01\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"232.88\" y=\"-64.33\" font-family=\"Times,serif\" font-size=\"14.00\">view_item_page</text>\n",
+       "</g>\n",
+       "<!-- page_view&#45;&gt;view_item_page -->\n",
+       "<g id=\"edge2\" class=\"edge\">\n",
+       "<title>page_view&#45;&gt;view_item_page</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M96.71,-69.01C113.69,-69.01 133.31,-69.01 152.25,-69.01\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"152.1,-72.51 162.1,-69.01 152.1,-65.51 152.1,-72.51\"/>\n",
+       "<text xml:space=\"preserve\" text-anchor=\"middle\" x=\"130.12\" y=\"-72.96\" font-family=\"Times,serif\" font-size=\"14.00\">0.50</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x7f6800bf50f0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[5.0e-001 5.0e-001]\n",
+      " [9.9e-324 1.5e-323]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def explore_session(session_id: str):\n",
+    "    subset = df[df['sessionId'] == session_id]\n",
+    "    print(session_id)\n",
+    "    P, labels = build_transition_prob_matrix(subset)\n",
+    "    g = render_graph(f\"session_{session_id}\", P, ls_index=labels, threshold=0.01, fmt=\"svg\", view=False)\n",
+    "    display(g)\n",
+    "    return P\n",
+    "for session in sessions:\n",
+    "    print(explore_session(session))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (PHANTOM)",
+   "language": "python",
+   "name": "phantom"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

experiments/ml/__init__.py

@@ -1,11 +0,0 @@
-from .evals import evaluate
-from .arch import (
-    XGBoostAgentClassifier,
-    LightGBMAgentClassifier
-)
-
-__all__ =[
-    'evaluate',
-    'XGBoostAgentClassifier',
-    'LightGBMAgentClassifier'
-]

experiments/ml/arch.py

@@ -1,122 +0,0 @@
-# sklearn compatible models for agent detection
-from sklearn.base import BaseEstimator, ClassifierMixin
-from procesing.context import PipelineContext
-from typing import Any, Optional, Tuple
-from abc import ABC, abstractmethod
-import xgboost as xgb
-import lightgbm as lgb
-import numpy as np
-import pandas as pd
-
-TASK = 'classification'
-LABELS = ['human', 'agent']
-
-
-class BaseAgentClassifier(BaseEstimator, ClassifierMixin, ABC):
-    """Base class for tree-based agent detection classifiers with common logic"""
-
-    def __init__(self, context: Optional[PipelineContext] = None, n_estimators: int = 200,
-                 max_depth: int = 6, learning_rate: float = 0.05,
-                 early_stopping_rounds: int = 20):
-        self.context = context
-        self.n_estimators = n_estimators
-        self.max_depth = max_depth
-        self.learning_rate = learning_rate
-        self.early_stopping_rounds = early_stopping_rounds
-        self.model_ = None
-        self.feature_names_ = None
-
-    def _to_array(self, X):
-        """Convert pandas structures to numpy arrays"""
-        return X.values if isinstance(X, (pd.DataFrame, pd.Series)) else X
-
-    def _compute_pos_weight(self, y_arr):
-        """Calculate scale_pos_weight for class imbalance handling"""
-        n_neg, n_pos = (y_arr == 0).sum(), (y_arr == 1).sum()
-        return n_neg / n_pos if n_pos > 0 else 1.0
-
-    def _prepare_eval_set(self, eval_set):
-        """Convert eval_set to numpy arrays if needed"""
-        if not eval_set:
-            return None
-        X_val, y_val = eval_set[0]
-        return [(self._to_array(X_val), self._to_array(y_val))]
-
-    @abstractmethod
-    def _build_model(self, scale_pos: float):
-        """Build the underlying model instance (must be implemented by subclasses)"""
-        pass
-
-    @abstractmethod
-    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
-        """Fit model with evaluation set (must be implemented by subclasses)"""
-        pass
-
-    def fit(self, X, y, eval_set=None):
-        X_arr, y_arr = self._to_array(X), self._to_array(y)
-
-        if isinstance(X, pd.DataFrame):
-            self.feature_names_ = X.columns.tolist()
-
-        scale_pos = self._compute_pos_weight(y_arr)
-        self.model_ = self._build_model(scale_pos)
-
-        eval_arr = self._prepare_eval_set(eval_set)
-        if eval_arr:
-            self._fit_with_eval(X_arr, y_arr, eval_arr)
-        else:
-            self.model_.fit(X_arr, y_arr)
-
-        return self
-
-    def predict(self, X):
-        return self.model_.predict(self._to_array(X))
-
-    def predict_proba(self, X):
-        return self.model_.predict_proba(self._to_array(X))
-
-    @property
-    def feature_importances_(self):
-        return self.model_.feature_importances_ if self.model_ else None
-
-
-class XGBoostAgentClassifier(BaseAgentClassifier):
-    """XGBoost binary classifier for agent detection with class imbalance handling"""
-
-    def _build_model(self, scale_pos: float):
-        return xgb.XGBClassifier(
-            n_estimators=self.n_estimators,
-            max_depth=self.max_depth,
-            learning_rate=self.learning_rate,
-            scale_pos_weight=scale_pos,
-            eval_metric='auc',
-            early_stopping_rounds=self.early_stopping_rounds,
-            random_state=42,
-            tree_method='hist',
-            enable_categorical=False
-        )
-
-    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
-        self.model_.fit(X_arr, y_arr, eval_set=eval_arr, verbose=False)
-
-
-class LightGBMAgentClassifier(BaseAgentClassifier):
-    """LightGBM binary classifier for agent detection with class imbalance handling"""
-
-    def _build_model(self, scale_pos: float):
-        return lgb.LGBMClassifier(
-            n_estimators=self.n_estimators,
-            max_depth=self.max_depth,
-            learning_rate=self.learning_rate,
-            scale_pos_weight=scale_pos,
-            metric='auc',
-            random_state=42,
-            verbosity=-1
-        )
-
-    def _fit_with_eval(self, X_arr, y_arr, eval_arr):
-        self.model_.fit(
-            X_arr, y_arr,
-            eval_set=eval_arr,
-            callbacks=[lgb.early_stopping(self.early_stopping_rounds, verbose=False)]
-        )

experiments/ml/evals.py

@@ -1,103 +0,0 @@
-from sklearn.metrics import (accuracy_score, precision_score, recall_score,
-                             f1_score, roc_auc_score, confusion_matrix, roc_curve)
-from torch.utils.tensorboard import SummaryWriter
-from logging import getLogger
-import numpy as np
-import matplotlib.pyplot as plt
-import io
-from PIL import Image
-
-logger = getLogger(__name__)
-
-
-def log_feature_importance(writer, model, feature_names, epoch):
-    """Visualize and log feature importance to TensorBoard"""
-    if not hasattr(model, 'feature_importances_') or model.feature_importances_ is None:
-        return
-
-    importance = model.feature_importances_
-    indices = np.argsort(importance)[::-1][:20]  # top 20
-    top_features = [feature_names[i] for i in indices]
-    top_importance = importance[indices]
-
-    for i, (feat, imp) in enumerate(zip(top_features, top_importance)):
-        writer.add_scalar(f'FeatureImportance/{feat}', imp, epoch)
-
-    fig, ax = plt.subplots(figsize=(10, 8))
-    ax.barh(range(len(top_features)), top_importance, align='center')
-    ax.set_yticks(range(len(top_features)))
-    ax.set_yticklabels(top_features)
-    ax.invert_yaxis()
-    ax.set_xlabel('Importance')
-    ax.set_title(f'Top 20 Feature Importance (Epoch {epoch})')
-    ax.grid(axis='x', alpha=0.3)
-
-    buf = io.BytesIO()
-    plt.tight_layout()
-    plt.savefig(buf, format='png', dpi=100)
-    buf.seek(0)
-    img = Image.open(buf)
-    img_arr = np.array(img)
-    writer.add_image('FeatureImportance/Chart', img_arr, epoch, dataformats='HWC')
-    plt.close()
-
-def evaluate(perdicted_class, predicted_proba, true_class, writer: SummaryWriter, epoch: int):
-    accuracy = accuracy_score(true_class, perdicted_class)
-    precision = precision_score(true_class, perdicted_class, zero_division=0)
-    recall = recall_score(true_class, perdicted_class, zero_division=0)
-    f1 = f1_score(true_class, perdicted_class, zero_division=0)
-    roc_auc = roc_auc_score(true_class, predicted_proba)
-
-    writer.add_scalar('Eval/Accuracy', accuracy, epoch)
-    writer.add_scalar('Eval/Precision', precision, epoch)
-    writer.add_scalar('Eval/Recall', recall, epoch)
-    writer.add_scalar('Eval/F1_Score', f1, epoch)
-    writer.add_scalar('Eval/ROC_AUC', roc_auc, epoch)
-
-    # confusion matrix
-    cm = confusion_matrix(true_class, perdicted_class)
-    tn, fp, fn, tp = cm.ravel()
-    writer.add_scalar('Eval/TrueNeg', tn, epoch)
-    writer.add_scalar('Eval/FalsePos', fp, epoch)
-    writer.add_scalar('Eval/FalseNeg', fn, epoch)
-    writer.add_scalar('Eval/TruePos', tp, epoch)
-
-    # specificity and sensitivity
-    specificity = tn / (tn + fp) if (tn + fp) > 0 else 0
-    sensitivity = recall  # same as recall/TPR
-    writer.add_scalar('Eval/Specificity', specificity, epoch)
-    writer.add_scalar('Eval/Sensitivity', sensitivity, epoch)
-
-    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
-    ax1.matshow(cm, cmap='Blues', alpha=0.7)
-    for i in range(2):
-        for j in range(2):
-            ax1.text(j, i, str(cm[i, j]), ha='center', va='center', fontsize=14)
-    ax1.set_xlabel('Predicted')
-    ax1.set_ylabel('True')
-    ax1.set_title(f'Confusion Matrix (Epoch {epoch})')
-    ax1.set_xticks([0, 1])
-    ax1.set_yticks([0, 1])
-    ax1.set_xticklabels(['Human', 'Agent'])
-    ax1.set_yticklabels(['Human', 'Agent'])
-
-    # ROC curve
-    fpr, tpr, _ = roc_curve(true_class, predicted_proba)
-    ax2.plot(fpr, tpr, label=f'AUC={roc_auc:.3f}', linewidth=2)
-    ax2.plot([0, 1], [0, 1], 'k--', label='Random')
-    ax2.set_xlabel('False Positive Rate')
-    ax2.set_ylabel('True Positive Rate')
-    ax2.set_title('ROC Curve')
-    ax2.legend()
-    ax2.grid(alpha=0.3)
-
-    buf = io.BytesIO()
-    plt.tight_layout()
-    plt.savefig(buf, format='png', dpi=100)
-    buf.seek(0)
-    img = Image.open(buf)
-    img_arr = np.array(img)
-    writer.add_image('Eval/Metrics', img_arr, epoch, dataformats='HWC')
-    plt.close()
-
-    logger.info(f"Eval {epoch}: Acc={accuracy:.4f} Prec={precision:.4f} Rec={recall:.4f} F1={f1:.4f} AUC={roc_auc:.4f}")

experiments/ml/requirements.txt

@@ -1,6 +0,0 @@
-torch
-tensorboard
-fastparquet
-pyarrow
-xgboost
-lightgbm

experiments/ml/train.py

@@ -1,137 +0,0 @@
-from torch.utils.tensorboard import SummaryWriter
-from sklearn.model_selection import train_test_split
-from logging import getLogger
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import joblib
-from datetime import datetime
-from ml.evals import evaluate, log_feature_importance
-from ml.arch import XGBoostAgentClassifier, LightGBMAgentClassifier, LABELS
-
-logger = getLogger(__name__)
-
-FEATURE_COLS_EXCLUDE = ['sessionId', 'experimentId', 'is_agent', 'xp_human_only', 'xp_market_mode', 'browser_family']
-RUNS_DIR = Path('ml/runs')
-CHECKPOINTS_DIR = Path('ml/checkpoints')
-
-
-def prepare_data(df):
-    """
-    Prepare feature matrix and labels from raw dataframe
-    Handles missing labels, feature selection, and categorical encoding
-    Returns: (X, y, feature_cols)
-    """
-    # drop rows with missing labels
-    n_before = len(df)
-    df = df[df['is_agent'].notna()].copy()
-    n_dropped = n_before - len(df)
-    if n_dropped > 0:
-        logger.warning(f"Dropped {n_dropped} sessions with missing labels")
-
-    if len(df) == 0:
-        logger.error("No labeled data available")
-        return None, None, None
-
-    feature_cols = [c for c in df.columns if c not in FEATURE_COLS_EXCLUDE]
-
-    # handle categorical browser_family via one-hot encoding
-    if 'browser_family' in df.columns:
-        browser_dummies = pd.get_dummies(df['browser_family'], prefix='browser', drop_first=True)
-        df = pd.concat([df, browser_dummies], axis=1)
-        feature_cols.extend(browser_dummies.columns.tolist())
-
-    X = df[feature_cols].fillna(0)
-    y = df['is_agent'].astype(int)
-
-    return X, y, feature_cols
-
-
-def train(data_path=None, model_type='xgboost', test_size=0.2, random_state=42,
-          n_estimators=200, max_depth=6, learning_rate=0.05):
-    """
-    Train agent detection classifier
-    Args:
-        data_path: path to labeled feature matrix CSV or parquet
-        model_type: 'xgboost' or 'lightgbm'
-        test_size: fraction for test split
-        random_state: seed for reproducibility
-    """
-    RUNS_DIR.mkdir(exist_ok=True)
-    CHECKPOINTS_DIR.mkdir(exist_ok=True)
-
-    run_name = f"{model_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-    writer = SummaryWriter(log_dir=RUNS_DIR / run_name)
-    logger.info(f"Starting training run: {run_name}")
-
-    # load data
-    if data_path is None:
-        logger.error("data_path required")
-        return
-    df = pd.read_parquet(data_path)
-    logger.info(f"Loaded {len(df)} sessions from {data_path}")
-
-    # prepare features and labels
-    if 'is_agent' not in df.columns:
-        logger.error("Missing is_agent column")
-        return
-
-    X, y, feature_cols = prepare_data(df)
-    if X is None:
-        return
-
-    # class distribution
-    n_agents = y.sum()
-    n_humans = (y == 0).sum()
-    logger.info(f"Class distribution: {n_humans} humans, {n_agents} agents" + (f" (ratio {n_humans / n_agents:.2f})" if n_agents > 0 else ""))
-
-    # train/test split with stratification
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=test_size, random_state=random_state, stratify=y
-    )
-    logger.info(f"Train: {len(X_train)}, Test: {len(X_test)}")
-
-    # init model
-    if model_type == 'xgboost':
-        model = XGBoostAgentClassifier(
-            n_estimators=n_estimators,
-            max_depth=max_depth,
-            learning_rate=learning_rate
-        )
-    elif model_type == 'lightgbm':
-        model = LightGBMAgentClassifier(
-            n_estimators=n_estimators,
-            max_depth=max_depth,
-            learning_rate=learning_rate
-        )
-    else:
-        logger.error(f"Unknown model type: {model_type}")
-        return
-
-    # train with eval set for early stopping
-    model.fit(X_train, y_train, eval_set=[(X_test, y_test)])
-    logger.info("Training complete")
-
-    # evaluate on test set
-    y_pred = model.predict(X_test)
-    y_prob = model.predict_proba(X_test)[:, 1]
-
-    evaluate(y_pred, y_prob, y_test, writer, epoch=0)
-
-    # log feature importance
-    log_feature_importance(writer, model, X.columns.tolist(), epoch=0)
-
-    # save model
-    model_path = CHECKPOINTS_DIR / f"{run_name}.pkl"
-    joblib.dump({'model': model, 'feature_cols': X.columns.tolist(), 'run_name': run_name}, model_path)
-    logger.info(f"Model saved to {model_path}")
-
-    writer.close()
-    return model, X.columns.tolist()
-
-
-if __name__ == "__main__":
-    import sys
-    data_path = sys.argv[1]
-    model_type = sys.argv[2] if len(sys.argv) > 2 else 'xgboost'
-    train(data_path, model_type=model_type)

experiments/procesing/__init__.py

@@ -1,51 +0,0 @@
-from procesing.context import PipelineContext
-from procesing.providers import DataProvider, SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    BaseContextStep,
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-    JoinExperimentsStep,
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep,
-    ChunkByTimeWindowStep,
-    ComputeDemandStep,
-    ComputeDemandForChunksStep,
-    AggregatePriceLogsStep,
-    # StateSpace,
-    # BuildStateSpaceStep,
-    FitPricingFunctionStep,
-    PredictPricesStep,
-)
-from procesing.pipelines import (
-    interaction_extraction_pipeline,
-    price_extraction_pipeline,
-    pricing_pipeline,
-    full_pipeline,
-)
-
-__all__ = [
-    'PipelineContext',
-    'DataProvider',
-    'SupabaseProvider',
-    'BackendAPIProvider',
-    'BaseContextStep',
-    'FetchInteractionsStep',
-    'FetchPriceLogsStep',
-    'FetchExperimentsStep',
-    'JoinExperimentsStep',
-    'CreatePriceBucketsStep',
-    'AugmentEventNamesStep',
-    'ChunkByTimeWindowStep',
-    'ComputeDemandStep',
-    'ComputeDemandForChunksStep',
-    'AggregatePriceLogsStep',
-    # 'StateSpace',
-    # 'BuildStateSpaceStep',
-    'FitPricingFunctionStep',
-    'PredictPricesStep',
-    'interaction_extraction_pipeline',
-    'price_extraction_pipeline',
-    'pricing_pipeline',
-    'full_pipeline',
-]

experiments/procesing/context.py

@@ -1,34 +0,0 @@
-from typing import Any, Dict
-import pandas as pd
-from procesing.providers.base import DataProvider
-
-class PipelineContext:
-    """
-    Context for pipeline execution holding config, provider, and cached data.
-    Enables dependency injection and eliminates global state.
-    """
-
-    def __init__(self,
-                 provider: DataProvider,
-                 store_mode: str,
-                 window_size: str = '30s',
-                 **config):
-        self.provider = provider
-        self.store_mode = store_mode
-        self.window_size = window_size
-        self.config = config
-        self._cache: Dict[str, Any] = {}
-
-    def get_cached(self, key: str, default=None):
-        return self._cache.get(key, default)
-
-    def cache(self, key: str, value):
-        self._cache[key] = value
-        return value
-
-    @property
-    def products(self) -> pd.DataFrame:
-        """Lazy-load and cache product catalog, single fetch per pipeline run"""
-        if 'products' not in self._cache:
-            self._cache['products'] = self.provider.fetch_products(self.store_mode)
-        return self._cache['products']

experiments/procesing/elasticity.py

@@ -1,332 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import List, Dict, Optional
-from sklearn.base import BaseEstimator, TransformerMixin
-from supabase import create_client, Client
-import os
-
-SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
-SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
-
-supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-
-class TemporalElasticityEstimator(BaseEstimator, TransformerMixin):
-    """
-    Compute price elasticity from time-series demand and price data.
-
-    Elasticity = (% change in quantity) / (% change in price)
-
-    Works with chunked time-window data from ChunkInteractionsIntoSteps.
-    """
-
-    def __init__(self,
-                 method:str='point',
-                 min_observations:int=2,
-                 smooth_window:Optional[int]=None):
-        """
-        Args:
-            method: 'point' (point elasticity) or 'arc' (arc elasticity)
-            min_observations: min data points needed per product
-            smooth_window: if set, apply rolling avg smoothing to time series
-        """
-        self.method = method
-        self.min_observations = min_observations
-        self.smooth_window = smooth_window
-
-    def fit(self, X):
-        return self
-
-    def transform(self,
-                  demand_chunks: List[Dict],
-                  price_chunks: List[Dict],
-                  store_mode: str = 'hotel') -> pd.DataFrame:
-        """
-        Args:
-            demand_chunks: list from ChunkInteractionsIntoSteps + DemandEstimator
-                           each item: {'window_start', 'window_end', 'demand_vector'}
-            price_chunks: list of dicts with {'window_start', 'window_end', 'price_vector'}
-            store_mode: 'hotel' or 'airline' to fetch all products
-
-        Returns:
-            df with [productId, elasticity, std_error, n_observations]
-        """
-        # fetch all products from database
-        all_products = supabase.table(f'{store_mode}_products').select("id").execute()
-        all_product_ids = [p['id'] for p in all_products.data]
-
-        aligned = self._align_chunks(demand_chunks, price_chunks)
-        if not aligned:
-            # return all products with zero elasticity
-            return pd.DataFrame({
-                'productId': all_product_ids,
-                'elasticity': 0.0,
-                'std_error': 0.0,
-                'n_obs': 0
-            })
-
-        # build time series per product
-        product_series = self._build_product_timeseries(aligned)
-
-        # compute elasticity per product
-        elasticities = []
-        for pid, series in product_series.items():
-            if len(series) < self.min_observations:
-                # assign 0 elasticity for products with insufficient data
-                elasticities.append({
-                    'productId': pid,
-                    'elasticity': 0.0,
-                    'std_error': 0.0,
-                    'n_obs': len(series)
-                })
-                continue
-
-            # apply smoothing if requested
-            if self.smooth_window and len(series) >= self.smooth_window:
-                series = self._smooth_series(series, self.smooth_window)
-
-            elast = self._compute_elasticity(series)
-            elasticities.append({
-                'productId': pid,
-                'elasticity': elast['value'],
-                'std_error': elast.get('std_error', 0.0),
-                'n_obs': len(series)
-            })
-
-        result_df = pd.DataFrame(elasticities)
-
-        # fill in missing products with zero elasticity
-        observed_pids = set(result_df['productId'].unique())
-        missing_pids = [pid for pid in all_product_ids if pid not in observed_pids]
-
-        if missing_pids:
-            missing_df = pd.DataFrame({
-                'productId': missing_pids,
-                'elasticity': 0.0,
-                'std_error': 0.0,
-                'n_obs': 0
-            })
-            result_df = pd.concat([result_df, missing_df], ignore_index=True)
-
-        return result_df
-
-    def _align_chunks(self, demand_chunks, price_chunks):
-        """Align demand and price data by matching time windows."""
-        aligned = []
-
-        # create lookup for price chunks by window_start
-        price_lookup = {chunk['window_start']: chunk for chunk in price_chunks}
-
-        for demand_chunk in demand_chunks:
-            window_start = demand_chunk['window_start']
-            if window_start in price_lookup:
-                aligned.append({
-                    'window_start': window_start,
-                    'window_end': demand_chunk['window_end'],
-                    'demand': demand_chunk['demand_vector'],
-                    'prices': price_lookup[window_start]['price_vector']
-                })
-
-        return aligned
-
-    def _build_product_timeseries(self, aligned_chunks):
-        """Build time series [price, quantity] per product."""
-        # vectorize chunk merging instead of iterating rows
-        all_merged = []
-        for chunk in aligned_chunks:
-            merged = chunk['demand'].merge(chunk['prices'], on='productId', how='inner')
-            merged['timestamp'] = chunk['window_start']
-            all_merged.append(merged[['productId', 'timestamp', 'price', 'demand_score']])
-
-        if not all_merged:
-            return {}
-
-        # concat all chunks and group by productId in one pass
-        combined = pd.concat(all_merged, ignore_index=True)
-        series_by_product = {
-            pid: group[['timestamp', 'price', 'demand_score']].rename(
-                columns={'demand_score': 'quantity'}
-            ).to_dict('records')
-            for pid, group in combined.groupby('productId')
-        }
-
-        return series_by_product
-
-    def _smooth_series(self, series, window):
-        """Apply rolling average smoothing."""
-        df = pd.DataFrame(series)
-        df['price_smooth'] = df['price'].rolling(window=window, center=True).mean()
-        df['quantity_smooth'] = df['quantity'].rolling(window=window, center=True).mean()
-        df = df.dropna()
-
-        return [{'timestamp': row['timestamp'],
-                 'price': row['price_smooth'],
-                 'quantity': row['quantity_smooth']}
-                for _, row in df.iterrows()]
-
-    def _compute_elasticity(self, series):
-        """Compute elasticity from time series."""
-        if len(series) < 2:
-            return {'value': 0.0, 'std_error': 0.0}
-
-        prices = np.array([s['price'] for s in series])
-        quantities = np.array([s['quantity'] for s in series])
-
-        # filter out zero/negative values
-        valid = (prices > 0) & (quantities > 0)
-        if valid.sum() < 2:
-            return {'value': 0.0, 'std_error': 0.0}
-
-        prices = prices[valid]
-        quantities = quantities[valid]
-
-        if self.method == 'point':
-            return self._point_elasticity(prices, quantities)
-        elif self.method == 'arc':
-            return self._arc_elasticity(prices, quantities)
-        else:
-            raise ValueError(f"Unknown method: {self.method}")
-
-    def _point_elasticity(self, prices, quantities):
-        """
-        Point elasticity using log-log regression.
-        log(Q) = a + b*log(P), elasticity = b
-        """
-        if len(prices) < 2:
-            return {'value': 0.0, 'std_error': 0.0}
-
-        log_p = np.log(prices)
-        log_q = np.log(quantities)
-
-        # simple linear regression
-        if log_p.std() == 0:
-            return {'value': 0.0, 'std_error': 0.0}
-
-        cov = np.cov(log_p, log_q)[0, 1]
-        var = np.var(log_p)
-        b = cov / var
-
-        # std error estimate (avoid div by zero)
-        if len(prices) <= 2:
-            se_b = 0.0
-        else:
-            residuals = log_q - (log_q.mean() + b * (log_p - log_p.mean()))
-            mse = (residuals ** 2).sum() / (len(prices) - 2)
-            se_b = np.sqrt(mse / (len(prices) * var))
-
-        return {'value': b, 'std_error': se_b}
-
-    def _arc_elasticity(self, prices, quantities):
-        """
-        Arc elasticity: average of period-over-period elasticities.
-        E_t = (ΔQ/Q_avg) / (ΔP/P_avg)
-        """
-        elasticities = []
-
-        for i in range(1, len(prices)):
-            p1, p2 = prices[i-1], prices[i]
-            q1, q2 = quantities[i-1], quantities[i]
-
-            p_avg = (p1 + p2) / 2
-            q_avg = (q1 + q2) / 2
-
-            if p_avg == 0 or q_avg == 0:
-                continue
-
-            delta_p = p2 - p1
-            delta_q = q2 - q1
-
-            if delta_p == 0:
-                continue
-
-            e = (delta_q / q_avg) / (delta_p / p_avg)
-            elasticities.append(e)
-
-        if not elasticities:
-            return None
-
-        return {
-            'value': np.mean(elasticities),
-            'std_error': np.std(elasticities) / np.sqrt(len(elasticities))
-        }
-
-
-def aggregate_price_logs(price_logs: pd.DataFrame,
-                         window_size: str = '1H',
-                         ts_col: str = 'ts',
-                         store_mode : str = 'hotel') -> List[Dict]:
-    """
-    Recover price vectors treating prices as persistent state changes.
-
-    Prices are set-operations that persist until next change. For each window:
-    - If price logs exist: average all changes within window
-    - If no logs: carry forward last price before window end
-
-    Args:
-        price_logs: df with [productId, price, ts, ...]
-        window_size: time window size matching ChunkInteractionsIntoSteps
-        ts_col: timestamp column name
-
-    Returns:
-        list of dicts with {'window_start', 'window_end', 'price_vector'}
-        where price_vector is df with [productId, price]
-    """
-    if price_logs.empty:
-        return []
-
-    df = price_logs.copy()
-
-    if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
-        df[ts_col] = pd.to_datetime(df[ts_col])
-
-    df = df.sort_values([ts_col, 'productId'])
-    all_products=supabase.table(f'{store_mode}_products').select("id, room_type, date_index, metadata, availability").execute()
-    all_products = pd.DataFrame(all_products.data)
-    unique_products = all_products['id'].unique()
-
-    # generate windows across data range
-    min_time, max_time = df[ts_col].min(), df[ts_col].max()
-    windows = pd.date_range(
-        start=min_time.floor(window_size),
-        end=max_time,
-        freq=window_size
-    )
-
-    chunks = []
-
-    for window_start in windows:
-        window_end = window_start + pd.Timedelta(window_size)
-        price_vector = []
-
-        # all products with price history by window_end
-        #historical_products = df[df[ts_col] < window_end]['productId'].unique()
-        historical_products = unique_products.tolist()
-
-        for pid in historical_products:
-            product_data = df[df['productId'] == pid]
-
-            # logs within window
-            in_window = product_data[
-                (product_data[ts_col] >= window_start) &
-                (product_data[ts_col] < window_end)
-            ]
-
-            if not in_window.empty:
-                # average changes within window
-                price = in_window['price'].mean()
-            else:
-                # carry forward: last price before window end
-                before_window = product_data[product_data[ts_col] < window_end]
-                if before_window.empty:
-                    continue
-                price = before_window['price'].iloc[-1]
-
-            price_vector.append({'productId': pid, 'price': price})
-
-        if price_vector:
-            chunks.append({
-                'window_start': window_start,
-                'window_end': window_end,
-                'price_vector': pd.DataFrame(price_vector)
-            })
-
-    return chunks

experiments/procesing/extract.py

@@ -0,0 +1,120 @@
+import pandas as pd
+import json
+import numpy as np
+import os
+import requests
+from dotenv import load_dotenv
+from sklearn.base import BaseEstimator, TransformerMixin
+from supabase import create_client, Client
+load_dotenv()
+
+BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:5000")
+SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
+SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
+N_PRICE_BUCKETS = 5
+
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+def get_data_from_kafka() -> pd.DataFrame:
+    """fetch all events from backend dump endpoint"""
+    resp = requests.get(f"{BACKEND_URL}/api/kafka/dump")
+    resp.raise_for_status()
+    data = resp.json()
+
+    if not data.get('success') or not data.get('data'):
+        return pd.DataFrame()
+
+    df = pd.DataFrame(data['data'])
+    # explode metadata col json
+    if 'metadata' in df.columns:
+        df = df.join(pd.json_normalize(df.pop("metadata"), sep=".").add_prefix("metadata_"))
+    df = df.dropna(subset=['eventName'])
+    return df
+
+
+def join_with_experiments(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty or 'experimentId' not in df.columns:
+        return df
+
+    unique_exp_ids = df['experimentId'].dropna().unique()
+    if len(unique_exp_ids) == 0:
+        return df
+
+    resp = supabase.table('experiments').select(
+        'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, task:tasks(task_name, task_description, task_def_of_done)'
+    ).in_('id', unique_exp_ids.tolist()).execute()
+
+    if not resp.data:
+        return df
+
+    exp_df = pd.DataFrame(resp.data)
+
+    # flatten task nested object if present
+    if 'task' in exp_df.columns and exp_df['task'].notnull().any():
+        task_normalized = pd.json_normalize(exp_df['task'].dropna())
+        task_normalized.index = exp_df[exp_df['task'].notnull()].index
+        exp_df = exp_df.drop(columns=['task']).join(task_normalized, rsuffix='_task')
+
+    # rename experiment columns for clarity
+    exp_df = exp_df.rename(columns={
+        'id': 'experimentId',
+        'subject_name': 'exp_subject',
+        'xp_human_only': 'exp_human_only',
+        'xp_market_mode': 'exp_market_mode',
+        'xp_task_id': 'exp_task_id'
+    })
+
+    df = df.merge(exp_df, on='experimentId', how='left')
+    return df
+
+
+def augment_event_titles(df: pd.DataFrame) -> pd.DataFrame:
+    # from taking standard view_item_page in eventName to view_item_page_{metadata_schema}
+    # we want metadata schema to create product specific event names
+
+    # only create price buckets if we have enough unique prices
+    if df["metadata_price"].notnull().sum() > 0:
+        try:
+            price_buckets = pd.qcut(
+                df["metadata_price"],
+                q=N_PRICE_BUCKETS,
+                labels=[f"PB_{i+1}" for i in range(N_PRICE_BUCKETS)],
+                duplicates='drop'  # handle duplicate bin edges
+            )
+        except ValueError:
+            # fallback: if still not enough unique values, use cut with fixed ranges or just use raw price
+            price_buckets = df["metadata_price"].apply(lambda x: f"P_{int(x)}" if pd.notnull(x) else "")
+    else:
+        price_buckets = pd.Series([""] * len(df), index=df.index)
+
+    # metadata_schema: _product_id@price_bucket_{i} only if we have product metadata otherswise keep original event name
+    # TODO: make this adaptive, if we have hover_over_title we append the title, if its view_page we say which page
+    df["metadata_schema"] = np.where(
+        df["productId"].notnull() & df["metadata_price"].notnull(),
+        "_" + df["productId"].astype(str) + "@" + price_buckets.astype(str),
+        ""
+    )
+    df["eventName"] = df["eventName"] + df["metadata_schema"].astype(str)
+    return df
+
+
+def extract() -> pd.DataFrame:
+    df = get_data_from_kafka()
+    df = join_with_experiments(df)
+    df = augment_event_titles(df)
+    return df
+
+
+class DataExtractor(BaseEstimator, TransformerMixin):
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, X=None):
+        return extract()
+
+
+if __name__ == "__main__":
+    df = extract()
+    print(df.head())
+    print(df.tail())
+    print(df.info())

experiments/procesing/mapping.py

@@ -0,0 +1,158 @@
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+def build_transition_prob_matrix(df: pd.DataFrame):
+    df = df.dropna(subset=['eventName'])
+    events = df['eventName'].tolist()
+    labels = pd.Index(events).unique().tolist()
+    idx = {e:i for i,e in enumerate(labels)}
+    M = np.zeros((len(labels), len(labels)), dtype=float)
+    for a, b in zip(events, events[1:]):
+        M[idx[a], idx[b]] += 1
+    row_sums = M.sum(axis=1, keepdims=True)
+    with np.errstate(divide='ignore', invalid='ignore'):
+        P = np.divide(M, row_sums, where=row_sums>0)  # row-normalized
+    return P, labels
+
+# https://medium.com/data-science/time-series-data-markov-transition-matrices-7060771e362b
+from graphviz import Digraph
+import numpy as np
+import pandas as pd
+
+def _as_prob_df(matrix, labels=None):
+    """Return a square DataFrame with index=columns=labels."""
+    if isinstance(matrix, pd.DataFrame):
+        # Ensure square and aligned
+        assert (matrix.index == matrix.columns).all(), "Index/columns must match."
+        return matrix
+    matrix = np.asarray(matrix, dtype=float)
+    assert matrix.shape[0] == matrix.shape[1], "Matrix must be square."
+    if labels is None:
+        raise ValueError("labels are required when matrix is not a DataFrame")
+    assert len(labels) == matrix.shape[0], "labels length must match matrix size."
+    return pd.DataFrame(matrix, index=list(labels), columns=list(labels))
+
+def _df_to_edgelist(P: pd.DataFrame, threshold=0.0, round_digits=2):
+    """Build weighted edges > threshold."""
+    edges = []
+    for src in P.index:
+        for dst in P.columns:
+            w = float(P.loc[src, dst])
+            if w > threshold:
+                edges.append((str(src), str(dst), f"{w:.{round_digits}f}"))
+    return edges
+
+def render_graph(fname, matrix, ls_index=None, threshold=0.0, fmt="svg", view=False):
+    """
+    fname: output file stem (no extension)
+    matrix: NumPy array or pandas DataFrame of transition PROBABILITIES
+    ls_index: ordered labels (required if matrix is not a DataFrame)
+    threshold: hide edges with weight <= threshold
+    fmt: 'svg'|'png'|'pdf' etc.
+    view: open after rendering
+    """
+    P = _as_prob_df(matrix, labels=ls_index)
+    edges = _df_to_edgelist(P, threshold=threshold)
+
+    g = Digraph(format=fmt)
+    g.attr(rankdir="LR", size="30")
+    g.attr("node", shape="circle")
+
+    # ensure isolated nodes appear
+    for node in P.index:
+        g.node(str(node), width="1", height="1")
+
+    for src, dst, label in edges:
+        g.edge(src, dst, label=label)
+
+    g.render(fname, view=view, cleanup=True)
+    return g
+
+
+class TransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0):
+        self.threshold = threshold
+        self.P_ = None
+        self.labels_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        P, labels = build_transition_prob_matrix(X)
+        self.P_ = P
+        self.labels_ = labels
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        return self.P_, self.labels_
+
+    def render(self, fname: str, fmt="svg", view=False):
+        if self.P_ is None or self.labels_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return render_graph(
+            fname,
+            self.P_,
+            ls_index=self.labels_,
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+
+
+class SessionTransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0, session_col='sessionId'):
+        self.threshold = threshold
+        self.session_col = session_col
+        self.session_matrices_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        if self.session_col not in X.columns:
+            raise ValueError(f"Column '{self.session_col}' not found in DataFrame")
+
+        session_matrices = {}
+        for session_id, grp in X.groupby(self.session_col):
+            if len(grp) > 1:  # need at least 2 events for transitions
+                P, labels = build_transition_prob_matrix(grp)
+                session_matrices[session_id] = {'matrix': P, 'labels': labels}
+
+        self.session_matrices_ = session_matrices
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return pd.Series(self.session_matrices_)
+
+    def render_session(self, session_id: str, fname: str, fmt="svg", view=False):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        if session_id not in self.session_matrices_:
+            raise ValueError(f"Session '{session_id}' not found in fitted data.")
+
+        sess_data = self.session_matrices_[session_id]
+        return render_graph(
+            fname,
+            sess_data['matrix'],
+            ls_index=sess_data['labels'],
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+if __name__ == "__main__":
+    # Example usage
+    data = {
+        'eventName': [
+            'A', 'B', 'A', 'C', 'B', 'A', 'A', 'C', 'B', 'C',
+            'A', 'B', 'C', 'A', 'B', 'C', 'A', 'B', 'C', 'A'
+        ]
+    }
+    df = pd.DataFrame(data)
+
+    transformer = TransitionProbMatrixTransformer(threshold=0.1)
+    transformer.fit(df)
+    P, labels = transformer.transform(None)
+
+    print("Transition Probability Matrix:")
+    print(pd.DataFrame(P, index=labels, columns=labels))
+
+    # Render the graph
+    transformer.render("transition_graph", fmt="svg", view=False)

experiments/procesing/metrics.py

@@ -1,245 +0,0 @@
-"""
-Revenue and KPI benchmark framework for pricing strategies.
-
-Computes session-level and aggregate metrics to compare pricing functions:
-    - Revenue: R_T = Σ P_t^T · Q_t
-    - Conversion rate
-    - Average order value (AOV)
-    - Agent exploitation loss: L_agent = R_oracle - R_observed
-"""
-from typing import Dict, List, Any, Optional
-from dataclasses import dataclass, field, asdict
-import pandas as pd
-import numpy as np
-
-
-@dataclass
-class SessionMetrics:
-    """KPIs for single session."""
-    session_id: str
-    experiment_id: Optional[str] = None
-
-    # interaction metrics
-    total_interactions: int = 0
-    page_views: int = 0
-    item_views: int = 0
-    searches: int = 0
-    cart_adds: int = 0
-
-    # revenue metrics
-    items_purchased: int = 0
-    total_revenue: float = 0.0
-    avg_item_price: float = 0.0
-    conversion_rate: float = 0.0
-
-    # pricing signals
-    total_price_shown: float = 0.0  # sum of all prices displayed
-    avg_markup: float = 0.0  # avg (price / base_price)
-
-    # behavioral features (for agent detection)
-    interaction_velocity: float = 0.0  # interactions per minute
-    session_duration_sec: float = 0.0
-    unique_products_viewed: int = 0
-
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return asdict(self)
-
-
-@dataclass
-class AggregateMetrics:
-    """Aggregate KPIs across sessions/experiments."""
-    experiment_id: Optional[str] = None
-    n_sessions: int = 0
-
-    # revenue aggregates
-    total_revenue: float = 0.0
-    avg_revenue_per_session: float = 0.0
-    median_revenue_per_session: float = 0.0
-
-    # conversion aggregates
-    total_conversions: int = 0
-    conversion_rate: float = 0.0  # purchases / sessions
-
-    # pricing aggregates
-    avg_markup: float = 0.0
-    median_markup: float = 0.0
-
-    # agent exploitation metrics
-    estimated_agent_sessions: int = 0  # sessions flagged as agent-driven
-    agent_revenue: float = 0.0
-    human_revenue: float = 0.0
-    agent_loss: float = 0.0  # L_agent = R_oracle - R_observed (if available)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return asdict(self)
-
-
-class MetricsComputer:
-    """Compute session and aggregate metrics from interaction/price logs."""
-
-    @staticmethod
-    def compute_session_metrics(
-        session_id: str,
-        interactions: pd.DataFrame,
-        price_logs: pd.DataFrame,
-        purchases: Optional[pd.DataFrame] = None,
-        experiment_id: Optional[str] = None
-    ) -> SessionMetrics:
-        """
-        Compute metrics for single session.
-
-        Args:
-            session_id: session identifier
-            interactions: user-interactions events for this session
-            price_logs: price-logs for this session
-            purchases: purchase events (if available)
-            experiment_id: experiment identifier
-        """
-        metrics = SessionMetrics(session_id=session_id, experiment_id=experiment_id)
-
-        if interactions.empty:
-            return metrics
-
-        # interaction counts
-        event_counts = interactions['eventName'].value_counts().to_dict()
-        metrics.total_interactions = len(interactions)
-        metrics.page_views = event_counts.get('page_view', 0) + event_counts.get('view_item_page', 0)
-        metrics.item_views = event_counts.get('view_item_page', 0)
-        metrics.searches = event_counts.get('search', 0)
-        metrics.cart_adds = event_counts.get('add_item_to_cart', 0)
-
-        # unique products viewed
-        metrics.unique_products_viewed = interactions['productId'].dropna().nunique()
-
-        # session duration
-        if 'ts' in interactions.columns:
-            timestamps = pd.to_datetime(interactions['ts'])
-            metrics.session_duration_sec = (timestamps.max() - timestamps.min()).total_seconds()
-            if metrics.session_duration_sec > 0:
-                metrics.interaction_velocity = (metrics.total_interactions / metrics.session_duration_sec) * 60
-
-        # revenue from purchases
-        if purchases is not None and not purchases.empty:
-            metrics.items_purchased = len(purchases)
-            metrics.total_revenue = purchases['price'].sum() if 'price' in purchases.columns else 0.0
-            metrics.avg_item_price = metrics.total_revenue / metrics.items_purchased if metrics.items_purchased > 0 else 0.0
-            metrics.conversion_rate = 1.0 if metrics.items_purchased > 0 else 0.0
-
-        # pricing metrics
-        if not price_logs.empty:
-            metrics.total_price_shown = price_logs['price'].sum()
-            # compute markup if base_price available in price logs or join with product catalog
-            if 'base_price' in price_logs.columns:
-                valid_markup = price_logs[price_logs['base_price'] > 0]
-                if not valid_markup.empty:
-                    metrics.avg_markup = (valid_markup['price'] / valid_markup['base_price']).mean()
-
-        return metrics
-
-    @staticmethod
-    def compute_aggregate_metrics(
-        session_metrics_list: List[SessionMetrics],
-        experiment_id: Optional[str] = None,
-        agent_detector_fn: Optional[callable] = None
-    ) -> AggregateMetrics:
-        """
-        Aggregate metrics across sessions.
-
-        Args:
-            session_metrics_list: list of SessionMetrics
-            experiment_id: experiment identifier
-            agent_detector_fn: optional function to classify session as agent (returns bool)
-        """
-        agg = AggregateMetrics(experiment_id=experiment_id)
-        agg.n_sessions = len(session_metrics_list)
-
-        if agg.n_sessions == 0:
-            return agg
-
-        df = pd.DataFrame([m.to_dict() for m in session_metrics_list])
-
-        # revenue aggregates
-        agg.total_revenue = df['total_revenue'].sum()
-        agg.avg_revenue_per_session = df['total_revenue'].mean()
-        agg.median_revenue_per_session = df['total_revenue'].median()
-
-        # conversion aggregates
-        agg.total_conversions = (df['items_purchased'] > 0).sum()
-        agg.conversion_rate = agg.total_conversions / agg.n_sessions
-
-        # pricing aggregates
-        valid_markups = df[df['avg_markup'] > 0]
-        if not valid_markups.empty:
-            agg.avg_markup = valid_markups['avg_markup'].mean()
-            agg.median_markup = valid_markups['avg_markup'].median()
-
-        # agent detection (if detector provided)
-        if agent_detector_fn is not None:
-            agent_flags = [agent_detector_fn(m) for m in session_metrics_list]
-            agg.estimated_agent_sessions = sum(agent_flags)
-
-            agent_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if is_agent)
-            human_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if not is_agent)
-
-            agg.agent_revenue = agent_revenue
-            agg.human_revenue = human_revenue
-
-        return agg
-
-    @staticmethod
-    def compare_pricing_strategies(
-        experiments: Dict[str, List[SessionMetrics]],
-        baseline_experiment_id: Optional[str] = None
-    ) -> pd.DataFrame:
-        """
-        Compare multiple pricing strategies/experiments.
-
-        Args:
-            experiments: dict mapping experiment_id -> list of SessionMetrics
-            baseline_experiment_id: experiment to use as baseline for comparison
-
-        Returns:
-            DataFrame with comparative metrics
-        """
-        results = []
-        baseline_agg = None
-
-        for exp_id, session_metrics in experiments.items():
-            agg = MetricsComputer.compute_aggregate_metrics(session_metrics, experiment_id=exp_id)
-            result = agg.to_dict()
-
-            if exp_id == baseline_experiment_id:
-                baseline_agg = agg
-
-            results.append(result)
-
-        df = pd.DataFrame(results)
-
-        # add relative metrics if baseline exists
-        if baseline_agg is not None:
-            df['revenue_lift_pct'] = ((df['total_revenue'] - baseline_agg.total_revenue) / baseline_agg.total_revenue * 100)
-            df['conversion_lift_pct'] = ((df['conversion_rate'] - baseline_agg.conversion_rate) / baseline_agg.conversion_rate * 100)
-
-        return df
-
-
-def simple_agent_detector(session_metrics: SessionMetrics, velocity_threshold: float = 5.0) -> bool:
-    """
-    Simple heuristic agent detector based on interaction velocity.
-
-    Args:
-        session_metrics: SessionMetrics instance
-        velocity_threshold: interactions per minute threshold (default: 5.0)
-
-    Returns:
-        True if session likely agent-driven
-    """
-    # agents tend to have higher interaction velocity and lower session duration
-    if session_metrics.interaction_velocity > velocity_threshold:
-        return True
-    # agents often view many products quickly without converting
-    if session_metrics.unique_products_viewed > 10 and session_metrics.conversion_rate == 0:
-        return True
-    return False

experiments/procesing/pipeline.py

@@ -0,0 +1,15 @@
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from extract import DataExtractor
+from mapping import SessionTransitionProbMatrixTransformer, render_graph
+
+
+if __name__ == "__main__":
+    steps = [
+        ('data_extraction', DataExtractor()),
+        #('transition_matrix', SessionTransitionProbMatrixTransformer(threshold=0.05)),
+    ]
+    pipeline = Pipeline(steps)
+    result = pipeline.fit_transform(None)
+    print(result)
+    print(result.info())

experiments/procesing/pipelines.py

@@ -1,174 +0,0 @@
-from sklearn.pipeline import Pipeline
-import pandas as pd
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-import os
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-    JoinExperimentsStep,
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep,
-    ChunkByTimeWindowStep,
-    ComputeDemandForChunksStep,
-    AggregatePriceLogsStep,
-    FitPricingFunctionStep,
-    PredictPricesStep,
-    ComputeDemandStep,
-    JoinProductFeaturesStep,
-    ExtractSessionFeaturesStep,
-    JoinLabelsStep,
-    ValidateDataStep,
-)
-from procesing.pricers import SimpleSurgePricer
-
-def interaction_extraction_pipeline(context: PipelineContext):
-    """Pipeline for extracting and augmenting interaction data"""
-    return Pipeline([
-        ('fetch', FetchInteractionsStep(context)),
-        ('create_buckets', CreatePriceBucketsStep(context)),
-        ('augment_events', AugmentEventNamesStep(context)),
-    ])
-
-
-def price_extraction_pipeline(context: PipelineContext):
-    """Pipeline for extracting price logs"""
-    return Pipeline([
-        ('fetch', FetchPriceLogsStep(context)),
-    ])
-
-
-def product_features_pipeline(context: PipelineContext,
-                                    interactions_df: pd.DataFrame,
-                                    price_logs_df: pd.DataFrame):
-    demand_step = ComputeDemandStep(context)
-    price_step = AggregatePriceLogsStep(context)
-    join_step = JoinProductFeaturesStep(context)
-
-
-    demand_data = demand_step.transform(interactions_df)
-    price_data= price_step.transform(price_logs_df)
-    joined_data = join_step.transform((demand_data, price_data))
-
-    return joined_data
-
-
-
-def pricing_pipeline(context: "PipelineContext",
-                     data: pd.DataFrame,
-                     high_threshold: int = 10,
-                     low_threshold: int = 2,
-                     surge_multiplier: float = 1.2,
-                     discount_multiplier: float = 0.9) -> pd.DataFrame:
-
-    if data.empty or 'productId' not in data.columns:
-        return pd.DataFrame()
-
-    surge_pricer = SimpleSurgePricer()
-    surge_pricer.fit(data)
-    data['optimal_price'] = surge_pricer.predict()
-    return data
-
-
-def full_pipeline(context: PipelineContext,
-                  high_threshold: int = 10,
-                  low_threshold: int = 2,
-                  surge_multiplier: float = 1.2,
-                  discount_multiplier: float = 0.9):
-    """
-    Complete end-to-end pipeline: data extraction -> demand/price aggregation -> surge pricing
-
-    Args:
-        context: Pipeline context
-        high_threshold: Demand threshold for surge pricing
-        low_threshold: Demand threshold for discounts
-        surge_multiplier: Price multiplier for high demand
-        discount_multiplier: Price multiplier for low demand
-
-    Returns:
-        tuple: (product_features_df, optimal_prices_df)
-            - product_features_df: [productId, demand_score, price]
-            - optimal_prices_df: [productId, current_price, optimal_price, demand_score]
-    """
-    interaction_pipe = interaction_extraction_pipeline(context)
-    price_pipe = price_extraction_pipeline(context)
-
-    interactions_df = interaction_pipe.fit_transform(None)
-    price_logs_df = price_pipe.fit_transform(None)
-    product_features_df = product_features_pipeline(context, interactions_df, price_logs_df)
-    print(product_features_df.to_string())
-
-    # generate optimal prices using surge rules
-    optimal_prices_df = pricing_pipeline(context, product_features_df,
-                                          high_threshold=high_threshold,
-                                          low_threshold=low_threshold,
-                                          surge_multiplier=surge_multiplier,
-                                          discount_multiplier=discount_multiplier)
-
-    return product_features_df, optimal_prices_df
-
-
-def ml_training_pipeline(context: PipelineContext) -> pd.DataFrame:
-    """
-    Build labeled session-level feature matrix for ML model training.
-    Pipeline: fetch -> validate -> extract features -> join labels
-
-    Returns:
-        DataFrame with ~25 features per session + is_agent label
-        Columns: sessionId, experimentId, temporal/behavioral/product/ua features, is_agent
-    """
-    # fetch raw interactions
-    interactions_df = FetchInteractionsStep(context).transform(None)
-
-    # validate data quality (report cached in context)
-    interactions_df = ValidateDataStep(context).transform(interactions_df)
-    if interactions_df.empty:
-        return pd.DataFrame()
-
-    # extract vectorized session features
-    features_df = ExtractSessionFeaturesStep(context).transform(interactions_df)
-    if features_df.empty:
-        return pd.DataFrame()
-
-    # join experiment labels (is_agent = ~xp_human_only)
-    labeled_df = JoinLabelsStep(context).transform(features_df)
-
-    return labeled_df
-
-
-
-
-if __name__ == '__main__':
-
-    class ExperimentsProvider(SupabaseProvider, BackendAPIProvider):
-        def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-            base_path = "/home/velocitatem/Documents/Projects/PHANTOM/experiments/collected_data/" # os.path.join(os.path.dirname(__file__), "collected_data")
-            if not os.path.isdir(base_path):
-                return pd.DataFrame()
-
-            files = {"user-interactions": "int.json", "price-logs": "price.json"}
-            file_to_read = files.get(topic, files["user-interactions"])
-            frames = []
-
-            for d in os.listdir(base_path):
-                full_path = os.path.join(base_path, d, file_to_read)
-                if not os.path.isfile(full_path):
-                    continue
-                try:
-                    data = pd.read_json(full_path)
-                    payloads = pd.DataFrame([r['payload'] for r in data['value'].to_list()])
-                    frames.append(payloads)
-                except Exception as e:
-                    print(f"Warning: Could not process {full_path}: {e}")
-
-            return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
-
-    # demo: run ML training pipeline
-    context = PipelineContext(provider=ExperimentsProvider(), store_mode='hotel')
-    features = ml_training_pipeline(context)
-    print(f"Feature matrix: {features.shape}")
-    print(features.head())
-    print(features.info())
-
-    features.to_parquet("features.parquet")

experiments/procesing/pricers/__init__.py

@@ -1,14 +0,0 @@
-from procesing.pricers.base import PricingFunction
-from procesing.pricers.elasticity import ElasticityBasedPricer
-from procesing.pricers.simple import StaticPricer, RandomPricer, SimpleSurgePricer
-from procesing.pricers.session_aware import SessionAwarePricer, ProductSpecificSessionPricer
-
-__all__ = [
-    'PricingFunction',
-    'ElasticityBasedPricer',
-    'StaticPricer',
-    'RandomPricer',
-    'SimpleSurgePricer',
-    'SessionAwarePricer',
-    'ProductSpecificSessionPricer'
-]

experiments/procesing/pricers/base.py

@@ -1,70 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Optional, Dict, Any, List
-import numpy as np
-import pandas as pd
-
-
-class PricingFunction(ABC):
-    """
-    Abstract base for pricing functions.
-
-    Defines mapping: f(Q_t, P_t, S_t, H_t) -> P_{t+1}
-
-    Where:
-        Q_t ∈ R^n: demand vector at time t
-        P_t ∈ R^n: price vector at time t
-        S_t: session features (behavioral signals, interactions)
-        H_t = {Q_{t-k}, P_{t-k}, S_{t-k}}: historical state trajectory
-
-    Objective:
-        maximize E[R_T] = E[Σ P_t^T · Q_t]
-        subject to:
-            Q_t = g(P_t, S_t)  (demand response via elasticity)
-            P_t ≥ C  (cost floor)
-            minimize L_agent = R_oracle - R_observed
-    """
-
-    @abstractmethod
-    def fit(self, *kwargs):
-        """
-        Offline training on historical data.
-
-        Args:
-            historical_data: DataFrame with elasticity, prices, demand signals
-            **kwargs: additional training parameters
-        """
-        pass
-
-    @abstractmethod
-    def predict(self, *kwargs) -> np.ndarray:
-        """
-        Generate optimal prices given current state.
-
-        Args:
-            state_space: StateSpace object containing Q_t, P_t, S_t, H_t
-
-        Returns:
-            P_{t+1}: price vector in R^n
-        """
-        pass
-
-    def update(self, observation: Dict[str, Any]):
-        """
-        Online learning update (optional).
-
-        Args:
-            observation: dict with {state, action, reward, next_state}
-                - state: StateSpace before pricing decision
-                - action: prices shown (P_t)
-                - reward: revenue/conversion signal
-                - next_state: StateSpace after user interaction
-        """
-        pass  # default: no online learning
-
-    def get_params(self) -> Dict[str, Any]:
-        """Return pricing function parameters for serialization."""
-        return {}
-
-    def set_params(self, params: Dict[str, Any]):
-        """Load pricing function parameters from dict."""
-        pass

experiments/procesing/pricers/elasticity.py

@@ -1,59 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-
-
-class ElasticityBasedPricer(PricingFunction):
-    """
-    Pricing based on demand elasticity estimates.
-    f(Q, S) = base_price * (1 + alpha * elasticity * demand_deviation)
-    """
-
-    def __init__(self, alpha: float = 0.1, price_floor: float = 0.0, price_ceil: float = np.inf):
-        self.alpha = alpha
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-
-    def fit(self, historical_data: pd.DataFrame):
-        """
-        Calibrate from historical elasticity estimates.
-        Expects: [productId, elasticity, base_price, mean_demand]
-        """
-        if 'elasticity' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity' column")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """
-        Adjust prices based on demand deviation and elasticity.
-        Higher demand -> increase price (but less for elastic goods)
-        """
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        if len(demand) != len(self.elasticity):
-            raise ValueError(f"Demand vector size {len(demand)} != elasticity size {len(self.elasticity)}")
-
-        # compute demand deviation from mean
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-
-        # adjust price: if demand high and elastic, don't increase much
-        # if demand high and inelastic, increase more
-        price_multiplier = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
-        prices = self.base_prices * price_multiplier
-
-        # enforce bounds
-        prices = np.clip(prices, self.price_floor, self.price_ceil)
-        return prices

experiments/procesing/pricers/session_aware.py

@@ -1,172 +0,0 @@
-"""
-Session-aware pricing functions that leverage behavioral features S_t.
-These pricers aim to minimize L_agent = R_oracle - R_observed.
-"""
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-from procesing.pricers.elasticity import ElasticityBasedPricer
-
-
-class SessionAwarePricer(PricingFunction):
-    """
-    Extends elasticity-based pricing with session behavioral signals.
-
-    f(Q, P, S) = base_price * elasticity_factor * session_factor
-
-    Where session_factor adjusts for:
-        - interaction_velocity (agent detection proxy)
-        - product_view_depth (interest signal)
-        - cart_to_view_ratio (conversion intent)
-
-    Strategy: charge higher prices to suspected agents (high velocity)
-    to recover oracle revenue from reconnaissance sessions.
-    """
-
-    def __init__(self,
-                 alpha: float = 0.1,
-                 beta_velocity: float = 0.05,
-                 beta_attention: float = 0.03,
-                 agent_velocity_threshold: float = 5.0,
-                 agent_markup: float = 1.2,
-                 price_floor: float = 0.0,
-                 price_ceil: float = np.inf):
-        """
-        Args:
-            alpha: elasticity sensitivity
-            beta_velocity: interaction velocity weight
-            beta_attention: product attention weight
-            agent_velocity_threshold: velocity above which to apply agent markup
-            agent_markup: price multiplier for suspected agent sessions
-            price_floor, price_ceil: price bounds
-        """
-        self.alpha = alpha
-        self.beta_velocity = beta_velocity
-        self.beta_attention = beta_attention
-        self.agent_velocity_threshold = agent_velocity_threshold
-        self.agent_markup = agent_markup
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-
-        # fitted parameters
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-
-    def fit(self, historical_data: pd.DataFrame, **kwargs):
-        """Calibrate from historical elasticity data."""
-        if 'elasticity' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity'")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Generate prices with session awareness."""
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        n_products = len(demand)
-
-        # base elasticity-driven pricing
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-        elasticity_factor = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
-
-        # session-aware adjustments
-        session_factor = np.ones(n_products)
-
-        if not state_space.session_features.empty:
-            sf = state_space.session_features.iloc[0]  # single session features
-
-            # agent detection via velocity
-            velocity = sf.get('interaction_velocity', 0.0)
-            if velocity > self.agent_velocity_threshold:
-                # suspected agent: apply markup to recover oracle revenue
-                session_factor *= self.agent_markup
-
-            # attention signal: higher view depth -> user interested -> can charge more
-            view_depth = sf.get('product_view_depth', 0)
-            if view_depth > 0:
-                attention_boost = 1 + self.beta_attention * np.log1p(view_depth)
-                session_factor *= attention_boost
-
-            # cart presence: if user has items in cart, slightly increase prices
-            cart_to_view = sf.get('cart_to_view_ratio', 0.0)
-            if cart_to_view > 0.1:
-                session_factor *= (1 + 0.02)  # small boost for conversion intent
-
-        prices = self.base_prices * elasticity_factor * session_factor
-        prices = np.clip(prices, self.price_floor, self.price_ceil)
-
-        return prices
-
-
-class ProductSpecificSessionPricer(PricingFunction):
-    """
-    Session-aware pricer with product-specific demand signals.
-
-    Uses S_t to extract per-product interaction counts and adjusts pricing
-    for products the user has already viewed/hovered.
-
-    Strategy: products viewed multiple times = high interest -> price up
-    """
-
-    def __init__(self,
-                 alpha: float = 0.1,
-                 view_boost: float = 0.02,
-                 max_view_boost: float = 0.15,
-                 price_floor: float = 0.0,
-                 price_ceil: float = np.inf):
-        self.alpha = alpha
-        self.view_boost = view_boost
-        self.max_view_boost = max_view_boost
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-        self.product_ids = None
-
-    def fit(self, historical_data: pd.DataFrame, **kwargs):
-        if 'elasticity' not in historical_data.columns or 'productId' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity' and 'productId'")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        self.product_ids = historical_data['productId'].values
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        n_products = len(demand)
-
-        # base pricing
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-        base_prices = self.base_prices * (1 + self.alpha * np.abs(self.elasticity) * demand_dev)
-
-        # product-specific session adjustments
-        if not state_space.session_features.empty and state_space.product_ids is not None:
-            # extract product interaction counts from session metadata
-            # (this would require session features to include per-product signals)
-            # for now, use uniform boost as placeholder
-            # TODO: extend session feature extraction to include product-specific counts
-            pass
-
-        prices = np.clip(base_prices, self.price_floor, self.price_ceil)
-        return prices

experiments/procesing/pricers/simple.py

@@ -1,91 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-
-
-class StaticPricer(PricingFunction):
-    """Static pricing: always return fixed base prices"""
-
-    def __init__(self, base_prices: np.ndarray = None):
-        self.base_prices = base_prices
-
-    def fit(self, historical_data: pd.DataFrame):
-        """Extract base prices from historical data"""
-        if 'base_price' in historical_data.columns:
-            self.base_prices = historical_data['base_price'].values
-        elif 'price' in historical_data.columns:
-            self.base_prices = historical_data['price'].values
-        else:
-            raise ValueError("historical_data must contain 'base_price' or 'price' column")
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Return static base prices regardless of state"""
-        if self.base_prices is None:
-            raise ValueError("Must call fit() or provide base_prices in constructor")
-        return self.base_prices.copy()
-
-
-class RandomPricer(PricingFunction):
-    """Random pricing within bounds (for baseline comparison)"""
-
-    def __init__(self, price_min: float = 50.0, price_max: float = 500.0, seed: int = None):
-        self.price_min = price_min
-        self.price_max = price_max
-        self.seed = seed
-        self.n_products = None
-        self.rng = np.random.default_rng(seed)
-
-    def fit(self, historical_data: pd.DataFrame):
-        """Learn number of products"""
-        self.n_products = len(historical_data)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Generate random prices"""
-        if self.n_products is None:
-            self.n_products = len(state_space.demand)
-        return self.rng.uniform(self.price_min, self.price_max, size=self.n_products)
-
-
-class SimpleSurgePricer(PricingFunction):
-    """
-    Rule-based surge pricer adjusting prices via demand thresholds.
-    Logic: if demand > high_threshold -> surge, if demand < low_threshold -> discount.
-    Simpler and more controllable than curve fitting approaches.
-    """
-
-    def __init__(self,
-                 base_prices: np.ndarray = None,
-                 high_threshold: int = 10,
-                 low_threshold: int = 2,
-                 surge_multiplier: float = 1.2,
-                 discount_multiplier: float = 0.9):
-        self.base_prices = base_prices
-        self.high_threshold = high_threshold
-        self.low_threshold = low_threshold
-        self.surge_multiplier = surge_multiplier
-        self.discount_multiplier = discount_multiplier
-
-    def fit(self, market_data : pd.DataFrame):
-        """Extract base prices from product catalog or historical averages"""
-        self.base_prices = market_data['base_price'].to_numpy() if 'base_price' in market_data.columns else market_data['price'].values
-        self.demand_history = market_data['demand'].to_numpy() if 'demand' in market_data.columns else np.zeros_like(self.base_prices)
-
-    def predict(self) -> np.ndarray:
-        """
-        Adjust prices based on current demand using surge rules.
-        state_space.demand: demand counts per product
-        state_space.prices: current prices (fallback if base_prices not set)
-        """
-        current_prices = self.base_prices if self.base_prices is not None else np.ones_like(demand_vector) * 99.99
-        demand = self.demand_history if self.demand_history is not None else np.zeros_like(current_prices)
-        new_prices = current_prices.copy()
-
-        high_mask = demand >= self.high_threshold
-        new_prices[high_mask] *= self.surge_multiplier
-
-        low_mask = demand <= self.low_threshold
-        new_prices[low_mask] *= self.discount_multiplier
-
-        return new_prices

experiments/procesing/pricing.py

@@ -1,272 +0,0 @@
-r"""
-Our state space comes as:
-$Q_t in R^n$ - our demand at a time t
-$P_t in R^n$ - prices at time t
-$S_t$ some form of interaction session features
-
-This is a single sate which we map under
-
-$f: (Q, S, H) \to P_{t+1}$
-
-With:
-
-$H_t = \{Q_{t-k}, P_{t-k}, S_{t-k}\}$
-
-
-We can have f be literally anything, analytical or learned or rule based or an RL policy.
-
-Our goal is to mazimize the expected revenue:
-
-$E[R_T] = E[\sum_{t=1}^T P_t^T \dot Q_t]$
-
-subject to Q_t = g(P_t, S_t) : demand response to price (estimated via elasticity) and P_t ≥ C : prices above cost floor and additionally minimizing the following:
-
-$L_{agent} = R_{oracle} - R_{observed}
-
-where: R_oracle = revenue if we knew agent intentions (from recon session) and R_observed = revenue under current pricing policy f
-
-I would start be defning a pricing function interface and standardizing how to train that based on historical data and define how to make it behave for online training (if we do that)
-
-We also need to develop a solid benchmark with mapping revenue and full KPIs from session interactions to measure differences between different price learning methods
-"""
-
-from abc import ABC, abstractmethod
-from sklearn.base import BaseEstimator, TransformerMixin
-import numpy as np
-import pandas as pd
-import os
-from dotenv import load_dotenv
-load_dotenv()
-from supabase import create_client, Client
-
-SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
-SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
-supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-
-def expected_revenue(prices: np.ndarray, demand: np.ndarray) -> float:
-    """Returns: expected revenue R_t = P_t^T * Q_t"""
-    return float(np.dot(prices, demand))
-
-class StateSpace:
-    def __init__(self,
-                 demand : np.ndarray, # at time t, only values (assuming aligned by productId order)
-                 prices : np.ndarray, # at time t, only values (assuming aligned by productId order)
-                 session_features : pd.DataFrame):
-        self.demand = demand  # Q_t
-        self.prices = prices  # P_t
-        self.session_features = session_features  # S_t
-        self.history = []  # H_t
-
-class PricingFunction(BaseEstimator, TransformerMixin, ABC):
-    def __init__(self):
-        pass
-
-    def fit(self, historical_data):
-        """
-        Train the pricing function based on historical data.
-        historical_data: list of StateSpace instances with known outcomes
-        """
-        raise NotImplementedError("Train method must be implemented by subclass.")
-
-    def transform(self, state_space) -> np.ndarray:
-        """
-        Predict the next prices given the current state space.
-        state_space: StateSpace instance
-        Returns: predicted prices P_{t+1}
-        """
-        raise NotImplementedError("Predict method must be implemented by subclass.")
-
-
-class SimpleLinearPricingFunction(PricingFunction):
-    def __init__(self, price_sensitivity: float = -0.1):
-        super().__init__()
-        self.price_sensitivity = price_sensitivity
-
-    def fit(self, historical_data):
-        return self
-
-    def transform(self, state_space: StateSpace) -> np.ndarray:
-        new_prices = state_space.prices + self.price_sensitivity * state_space.demand
-        return np.maximum(new_prices, 0)
-
-
-class ElasticityBasedPricingFunction(PricingFunction):
-    """
-    Revenue-maximizing pricing using elasticity estimates.
-
-    For each product, optimal price P* maximizes R = P * Q(P)
-    where Q(P) follows power law: Q(P) = Q_0 * (P/P_0)^ε
-
-    Taking derivative dR/dP = 0 gives optimal markup:
-    P* = P_0 * (1 + 1/ε) if ε < -1 (elastic)
-
-    For inelastic demand (|ε| < 1), we apply bounded markup.
-    """
-
-    def __init__(self,
-                 cost_floor: float = 0.5,
-                 max_markup: float = 2.0,
-                 min_markup: float = 1.0,
-                 inelastic_markup: float = 1.3):
-        super().__init__()
-        self.cost_floor = cost_floor  # prices as fraction of base
-        self.max_markup = max_markup  # max price = base * max_markup
-        self.min_markup = min_markup  # min price = base * min_markup
-        self.inelastic_markup = inelastic_markup  # default for |ε| < 1
-        self.elasticity_map = {}  # productId -> elasticity
-
-    def fit(self, elasticity_df: pd.DataFrame):
-        """
-        Args:
-            elasticity_df: df with [productId, elasticity, std_error, n_obs]
-        """
-        if elasticity_df is not None and not elasticity_df.empty:
-            self.elasticity_map = dict(zip(
-                elasticity_df['productId'],
-                elasticity_df['elasticity']
-            ))
-        return self
-
-    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
-        """
-        Args:
-            state_space: current state (prices = base prices)
-            product_ids: array of productIds aligned with state_space.prices
-
-        Returns:
-            optimized prices P_{t+1}
-        """
-        base_prices = state_space.prices
-
-        if product_ids is None:
-            # fallback: use positional index as productId (not ideal)
-            product_ids = np.arange(len(base_prices))
-
-        new_prices = np.zeros_like(base_prices)
-
-        for i, (base_p, pid) in enumerate(zip(base_prices, product_ids)):
-            elasticity = self.elasticity_map.get(pid, 0.0)
-
-            if elasticity < -1:  # elastic demand
-                # optimal markup: (1 + 1/ε)
-                markup = 1 + (1 / elasticity)
-                optimal_p = base_p * markup
-            elif elasticity > -1 and elasticity < 0:  # inelastic
-                # conservative markup
-                optimal_p = base_p * self.inelastic_markup
-            else:  # ε ≥ 0 (demand increases with price, or no data)
-                # no elasticity data or anomalous, keep base price
-                optimal_p = base_p
-
-            # apply bounds
-            optimal_p = np.clip(
-                optimal_p,
-                base_p * self.min_markup,
-                base_p * self.max_markup
-            )
-            optimal_p = max(optimal_p, self.cost_floor)
-
-            new_prices[i] = optimal_p
-
-        return new_prices
-
-
-class ContextualElasticityPricing(PricingFunction):
-    """
-    Revenue optimization with contextual adjustments based on session features.
-
-    Combines elasticity-based pricing with surge/demand-based multipliers.
-    """
-
-    def __init__(self,
-                 base_pricer: ElasticityBasedPricingFunction = None,
-                 demand_sensitivity: float = 0.1,
-                 surge_threshold: float = 0.7):
-        super().__init__()
-        self.base_pricer = base_pricer or ElasticityBasedPricingFunction()
-        self.demand_sensitivity = demand_sensitivity
-        self.surge_threshold = surge_threshold
-
-    def fit(self, elasticity_df: pd.DataFrame):
-        self.base_pricer.fit(elasticity_df)
-        return self
-
-    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
-        # get base optimal prices from elasticity
-        base_optimal = self.base_pricer.transform(state_space, product_ids)
-
-        # compute surge multiplier from demand
-        if len(state_space.demand) > 0:
-            demand_normalized = state_space.demand / (state_space.demand.max() + 1e-8)
-            surge_multiplier = 1 + self.demand_sensitivity * np.maximum(
-                demand_normalized - self.surge_threshold, 0
-            )
-        else:
-            surge_multiplier = np.ones_like(base_optimal)
-
-        return base_optimal * surge_multiplier
-
-# Example usage:
-if __name__ == "__main__":
-    from pipeline import interaction_pipeline, price_data_pipeline, elasticity_pipeline
-
-    store_mode = 'hotel'
-    interaction_data = interaction_pipeline.fit_transform(None)
-    price_data = price_data_pipeline.fit_transform(None)
-
-    elasticity_df = elasticity_pipeline(interaction_data, price_data, window_size="30s", store_mode=store_mode)
-
-    # fetch all products with base prices from database
-    products_resp = supabase.table(f'{store_mode}_products').select("id, metadata").execute()
-    products_df = pd.DataFrame(products_resp.data)
-
-    # extract base_price from metadata
-    products_df['base_price'] = products_df['metadata'].apply(lambda m: m.get('base_price', 0) if isinstance(m, dict) else 0)
-    products_df = products_df.rename(columns={'id': 'productId'})[['productId', 'base_price']]
-
-    # override with logged prices where available
-    if not price_data.empty:
-        if 'ts' in price_data.columns and not pd.api.types.is_datetime64_any_dtype(price_data['ts']):
-            price_data['ts'] = pd.to_datetime(price_data['ts'])
-
-        # get latest logged price per product
-        price_logs_agg = price_data.sort_values('ts').groupby('productId', as_index=False).last()
-
-        # merge: start with all products (base prices), override with logged prices
-        products_df = products_df.merge(
-            price_logs_agg[['productId', 'price']],
-            on='productId',
-            how='left'
-        )
-        products_df['final_price'] = products_df['price'].fillna(products_df['base_price'])
-    else:
-        products_df['final_price'] = products_df['base_price']
-
-    # merge with elasticity
-    if elasticity_df is not None and not elasticity_df.empty:
-        price_data_merged = products_df[['productId', 'final_price']].merge(
-            elasticity_df[['productId', 'elasticity']],
-            on='productId',
-            how='left'
-        ).fillna({'elasticity': 0.0})
-
-        prices = price_data_merged['final_price'].values
-        elasticities = price_data_merged['elasticity'].values
-    else:
-        prices = np.array([])
-        elasticities = np.array([])
-
-    print(elasticities)
-    print(prices)
-
-    state_space = StateSpace(
-        demand=elasticities,
-        prices=prices,
-        session_features=interaction_data
-    )
-
-    pricing_function = SimpleLinearPricingFunction(price_sensitivity=-0.05)
-    pricing_function.fit([])  # No training data for simple model
-    predicted_prices = pricing_function.transform(state_space)
-
-    print("Predicted Prices:", predicted_prices)

experiments/procesing/providers/__init__.py

@@ -1,5 +0,0 @@
-from procesing.providers.base import DataProvider
-from procesing.providers.supabase import SupabaseProvider
-from procesing.providers.backend import BackendAPIProvider
-
-__all__ = ['DataProvider', 'SupabaseProvider', 'BackendAPIProvider']

experiments/procesing/providers/backend.py

@@ -1,19 +0,0 @@
-import os
-import pandas as pd
-import requests
-from typing import List
-from procesing.providers.base import DataProvider
-
-class BackendAPIProvider(DataProvider):
-    """Concrete backend API implementation"""
-    def __init__(self, backend_url: str = None):
-        self.backend_url = backend_url or os.getenv("BACKEND_URL", "http://localhost:5000")
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        resp = requests.get(f"{self.backend_url}/api/kafka/dump?topic={topic}")
-        resp.raise_for_status()
-        data = resp.json()
-
-        if not data.get('success') or not data.get('data'):
-            return pd.DataFrame()
-
-        return pd.DataFrame(data['data'])

experiments/procesing/providers/base.py

@@ -1,21 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List
-import pandas as pd
-
-class DataProvider(ABC):
-    """Abstract interface for data access, enables DI and testing"""
-
-    @abstractmethod
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        """Fetch product catalog for given store mode"""
-        pass
-
-    @abstractmethod
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        """Fetch experiment metadata for given IDs"""
-        pass
-
-    @abstractmethod
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        """Fetch data from Kafka topic via backend API"""
-        pass

experiments/procesing/providers/supabase.py

@@ -1,42 +0,0 @@
-import os
-import pandas as pd
-import requests
-from typing import List
-from supabase import create_client, Client
-from procesing.providers.base import DataProvider
-from dotenv import load_dotenv
-
-class SupabaseProvider(DataProvider):
-    """Concrete Supabase + backend API implementation"""
-
-    def __init__(self,
-                 supabase_url: str = None,
-                 supabase_key: str = None,):
-        load_dotenv()
-        self.supabase_url = supabase_url or os.getenv("NEXT_PUBLIC_SUPABASE_URL")
-        self.supabase_key = supabase_key or os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
-        self.supabase: Client = create_client(self.supabase_url, self.supabase_key)
-
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        # hotel uses room_type, airline uses flight_type; select all and normalize
-        resp = self.supabase.table(f'{store_mode}_products').select("*").execute()
-        if not resp.data:
-            return pd.DataFrame()
-        df = pd.DataFrame(resp.data)
-        # normalize type column: hotel has room_type, airline has flight_type
-        if 'room_type' in df.columns:
-            df['product_type'] = df['room_type']
-        elif 'flight_type' in df.columns:
-            df['product_type'] = df['flight_type']
-        return df
-
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        if not experiment_ids:
-            return pd.DataFrame()
-
-        resp = self.supabase.table('experiments').select(
-            'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, '
-            'task:tasks(task_name, task_description, task_def_of_done)'
-        ).in_('id', experiment_ids).execute()
-
-        return pd.DataFrame(resp.data) if resp.data else pd.DataFrame()

experiments/procesing/steps/__init__.py

@@ -1,39 +0,0 @@
-from procesing.steps.base import BaseContextStep
-from procesing.steps.fetch import FetchInteractionsStep, FetchPriceLogsStep, FetchExperimentsStep
-from procesing.steps.join import JoinExperimentsStep, JoinProductFeaturesStep
-from procesing.steps.augment import CreatePriceBucketsStep, AugmentEventNamesStep, AugmentInteractionsStep
-from procesing.steps.chunk import ChunkByTimeWindowStep
-from procesing.steps.demand import ComputeDemandStep, ComputeDemandForChunksStep
-from procesing.steps.elasticity import AggregatePriceLogsStep
-from procesing.steps.pricing import FitPricingFunctionStep, PredictPricesStep
-from procesing.steps.session import (
-    ExtractSessionFeaturesStep, JoinLabelsStep, ValidateDataStep,
-    TemporalFeatureStep, BehavioralFeatureStep, ProductFeatureStep, UserAgentFeatureStep,
-    _extract_features_for_session
-)
-
-__all__ = [
-    'BaseContextStep',
-    'FetchInteractionsStep',
-    'FetchPriceLogsStep',
-    'FetchExperimentsStep',
-    'JoinExperimentsStep',
-    'JoinProductFeaturesStep',
-    'CreatePriceBucketsStep',
-    'AugmentEventNamesStep',
-    'AugmentInteractionsStep',
-    'ChunkByTimeWindowStep',
-    'ComputeDemandStep',
-    'ComputeDemandForChunksStep',
-    'AggregatePriceLogsStep',
-    'FitPricingFunctionStep',
-    'PredictPricesStep',
-    'ExtractSessionFeaturesStep',
-    'JoinLabelsStep',
-    'ValidateDataStep',
-    'TemporalFeatureStep',
-    'BehavioralFeatureStep',
-    'ProductFeatureStep',
-    'UserAgentFeatureStep',
-    '_extract_features_for_session',
-]

experiments/procesing/steps/augment.py

@@ -1,140 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-
-class AugmentInteractionsStep(BaseContextStep):
-    """
-    Consolidated step: create price buckets, augment event names, join experiments.
-    Input: (interactions_df, price_logs_df)
-    Output: enriched interactions_df
-    """
-
-    def transform(self, data: tuple):
-        interactions_df, price_logs_df = data
-
-        if interactions_df.empty:
-            return interactions_df
-
-        # Step 1: Create price buckets
-        interactions_df = self._create_price_buckets(interactions_df)
-
-        # Step 2: Augment event names
-        interactions_df = self._augment_event_names(interactions_df)
-
-        # Step 3: Join experiments (optional)
-        if 'experimentId' in interactions_df.columns:
-            interactions_df = self._join_experiments(interactions_df)
-
-        return interactions_df
-
-    def _create_price_buckets(self, df: pd.DataFrame):
-        """Create price bucket labels from price data"""
-        if 'metadata_price' not in df.columns:
-            df['price_bucket'] = ""
-            return df
-
-        n_buckets = self.context.config.get('n_price_buckets', 5)
-
-        if df['metadata_price'].notnull().sum() > 0:
-            try:
-                price_buckets = pd.qcut(
-                    df['metadata_price'],
-                    q=n_buckets,
-                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
-                    duplicates='drop'
-                )
-            except ValueError:
-                # fallback for insufficient unique values
-                price_buckets = df['metadata_price'].apply(
-                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
-                )
-        else:
-            price_buckets = pd.Series([""] * len(df), index=df.index)
-
-        df['price_bucket'] = price_buckets
-        return df
-
-    def _augment_event_names(self, df: pd.DataFrame):
-        """Augment event names with product and price bucket schema"""
-        # Create schema: _productId@price_bucket
-        has_product = df.get('productId', pd.Series()).notnull()
-        has_bucket = df.get('price_bucket', pd.Series()).notnull()
-
-        df['metadata_schema'] = np.where(
-            has_product & has_bucket,
-            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
-            ""
-        )
-
-        df['eventName'] = df['eventName'] + df['metadata_schema']
-        return df
-
-    def _join_experiments(self, df: pd.DataFrame):
-        """Join experiment metadata if experimentId present"""
-        exp_ids = df['experimentId'].dropna().unique().tolist()
-        if not exp_ids:
-            return df
-
-        experiments_df = self.context.provider.fetch_experiments(exp_ids)
-        if experiments_df.empty:
-            return df
-
-        return df.merge(
-            experiments_df,
-            left_on='experimentId',
-            right_on='id',
-            how='left',
-            suffixes=('', '_exp')
-        )
-
-
-class CreatePriceBucketsStep(BaseContextStep):
-    """Create price bucket labels from price data"""
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty or 'metadata_price' not in df.columns:
-            df['price_bucket'] = ""
-            return df
-
-        n_buckets = self.context.config.get('n_price_buckets', 5)
-
-        if df['metadata_price'].notnull().sum() > 0:
-            try:
-                price_buckets = pd.qcut(
-                    df['metadata_price'],
-                    q=n_buckets,
-                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
-                    duplicates='drop'
-                )
-            except ValueError:
-                # fallback for insufficient unique values
-                price_buckets = df['metadata_price'].apply(
-                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
-                )
-        else:
-            price_buckets = pd.Series([""] * len(df), index=df.index)
-
-        df['price_bucket'] = price_buckets
-        return df
-
-
-class AugmentEventNamesStep(BaseContextStep):
-    """Augment event names with product and price bucket schema"""
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty:
-            return df
-
-        # Create schema: _productId@price_bucket
-        has_product = df.get('productId', pd.Series()).notnull()
-        has_bucket = df.get('price_bucket', pd.Series()).notnull()
-
-        df['metadata_schema'] = np.where(
-            has_product & has_bucket,
-            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
-            ""
-        )
-
-        df['eventName'] = df['eventName'] + df['metadata_schema']
-        return df

experiments/procesing/steps/base.py

@@ -1,32 +0,0 @@
-from abc import ABC, abstractmethod
-from sklearn.base import BaseEstimator, TransformerMixin
-from procesing.context import PipelineContext
-from typing import Any
-
-class BaseContextStep(BaseEstimator, TransformerMixin, ABC):
-    """
-    Base for all pipeline steps.
-    Each step is stateless, context-driven, and performs ONE transformation.
-    """
-
-    def __init__(self, context: PipelineContext):
-        self.context = context
-
-    def fit(self, X=None, y=None):
-        """Most steps don't need training"""
-        return self
-
-    @abstractmethod
-    def transform(self, X) -> Any:
-        """Transform input using context. Must be implemented by subclass."""
-        pass
-
-    def get_params(self, deep=True):
-        """sklearn compatibility"""
-        return {'context': self.context}
-
-    def set_params(self, **params):
-        """sklearn compatibility"""
-        if 'context' in params:
-            self.context = params['context']
-        return self

experiments/procesing/steps/chunk.py

@@ -1,34 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class ChunkByTimeWindowStep(BaseContextStep):
-    """
-    Chunk dataframe into time windows.
-    Returns list of dicts with window metadata.
-    """
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty:
-            return []
-
-        df = df.copy()
-        ts_col = self.context.config.get('ts_col', 'ts')
-        window_size = self.context.window_size
-
-        # ensure datetime
-        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
-            df[ts_col] = pd.to_datetime(df[ts_col])
-
-        df = df.sort_values(ts_col)
-        df['_window'] = df[ts_col].dt.floor(window_size)
-
-        chunks = []
-        for idx, (window_start, group) in enumerate(df.groupby('_window')):
-            chunks.append({
-                'window_start': window_start,
-                'window_end': window_start + pd.Timedelta(window_size),
-                'window_idx': idx,
-                'data': group.drop(columns=['_window'])
-            })
-
-        return chunks

experiments/procesing/steps/demand.py

@@ -1,61 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class ComputeDemandStep(BaseContextStep):
-    """
-    Compute demand vector for a single time window or dataframe.
-    Input: single chunk dict OR raw dataframe
-    Output: demand dataframe with [productId, demand_score]
-    """
-
-    def transform(self, chunk):
-        # handle both chunk dict and raw dataframe
-        if isinstance(chunk, dict):
-            interactions = chunk['data']
-            window_meta = {k: v for k, v in chunk.items() if k != 'data'}
-        else:
-            interactions = chunk
-            window_meta = {}
-
-        products = self.context.products
-        unique_products = products['id'].unique()
-
-        # apply filters if configured
-        session_filter = self.context.config.get('session_filter')
-        experiment_filter = self.context.config.get('experiment_filter')
-
-        if session_filter and 'sessionId' in interactions.columns:
-            interactions = interactions[interactions['sessionId'] == session_filter]
-        if experiment_filter and 'experimentId' in interactions.columns:
-            interactions = interactions[interactions['experimentId'] == experiment_filter]
-
-        interactions_with_products = interactions.dropna(subset=['productId'])
-
-        if interactions_with_products.empty:
-            demand_df = pd.DataFrame({
-                'productId': unique_products,
-                'demand_score': 0
-            })
-        else:
-            # crosstab for simple demand count
-            demand_df = pd.crosstab(
-                interactions_with_products['productId'],
-                'count'
-            ).reindex(unique_products, fill_value=0).reset_index()
-            demand_df.columns = ['productId', 'demand_score']
-
-        # attach window metadata if present
-        if window_meta:
-            return {**window_meta, 'demand_vector': demand_df}
-        return demand_df
-
-
-class ComputeDemandForChunksStep(BaseContextStep):
-    """Apply ComputeDemandStep to list of chunks"""
-
-    def transform(self, chunks: list):
-        if not chunks:
-            return []
-
-        demand_step = ComputeDemandStep(self.context)
-        return [demand_step.transform(chunk) for chunk in chunks]

experiments/procesing/steps/elasticity.py

@@ -1,42 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import Dict, List
-from procesing.steps.base import BaseContextStep
-
-class AggregatePriceLogsStep(BaseContextStep):
-    """
-    Aggregate price logs into time windows using VECTORIZED operations.
-    Input: price_logs_df
-    Output: DataFrame with columns [productId, price]
-    """
-
-    def transform(self, price_logs_df: pd.DataFrame):
-        if price_logs_df.empty:
-            return pd.DataFrame(columns=['productId', 'price'])
-
-        df = price_logs_df.copy()
-        ts_col = self.context.config.get('ts_col', 'ts')
-        #window_size = self.context.window_size WE ARE NOT USING CHUNKS ANYMORE
-
-        # ensure datetime
-        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
-            df[ts_col] = pd.to_datetime(df[ts_col])
-
-        df = df.sort_values([ts_col, 'productId'])
-        products = self.context.products
-        # get base price from metadata if available 1) read the metadata col as json and get the base_price
-        products['base_price'] = products.apply(
-            lambda row: row['metadata'].get('base_price', 0) if isinstance(row['metadata'], dict) else 0,
-            axis=1
-        )
-
-        unique_products = products['id'].unique()
-
-        df_indexed = df.set_index(ts_col)
-        # we return a df of average price per product over the entire period
-        # TODO: maybe consider different opration to handle price aggregation over time
-        avg_prices = df_indexed.groupby('productId')['price'].mean().reindex(unique_products, fill_value=0).reset_index()
-        avg_prices.columns = ['productId', 'price']
-        # fill 0s with base_price from products
-        base_price_map = products.set_index('id')['base_price'].to_dict()
-        return avg_prices

experiments/procesing/steps/fetch.py

@@ -1,81 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class FetchInteractionsStep(BaseContextStep):
-    """Fetch raw interaction data from Kafka topic with optional time and store_mode filtering"""
-
-    def __init__(self, context, lookback: str = None):
-        super().__init__(context)
-        self.lookback = lookback
-
-    def transform(self, X=None):
-        df = self.context.provider.fetch_kafka_topic('user-interactions')
-
-        if df.empty:
-            return df
-
-        # Explode metadata JSON column
-        if 'metadata' in df.columns:
-            df = df.join(
-                pd.json_normalize(df.pop('metadata'), sep='.').add_prefix('metadata_')
-            )
-
-        df = df.dropna(subset=['eventName'])
-        # drop all where page has /admin/
-        df = df[~df['page'].str.contains('/admin/', na=False)]
-
-        # filter by store_mode from context
-        if 'storeMode' in df.columns:
-            df = df[df['storeMode'] == self.context.store_mode]
-
-        # Remap dateIndex if present
-        if 'metadata_dateIndex' in df.columns:
-            df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
-
-        # Apply time filtering if lookback specified
-        if self.lookback and 'ts' in df.columns:
-            df['ts'] = pd.to_datetime(df['ts'])
-            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
-            df = df[df['ts'] >= cutoff]
-
-        return df
-
-
-class FetchPriceLogsStep(BaseContextStep):
-    """Fetch price log data from Kafka topic with optional time and store_mode filtering"""
-
-    def __init__(self, context, lookback: str = None):
-        super().__init__(context)
-        self.lookback = lookback
-
-    def transform(self, X=None):
-        df = self.context.provider.fetch_kafka_topic('price-logs')
-
-        if df.empty:
-            return df
-
-        # filter by store_mode from context
-        if 'storeMode' in df.columns:
-            df = df[df['storeMode'] == self.context.store_mode]
-
-        # Apply time filtering if lookback specified
-        if self.lookback and 'ts' in df.columns:
-            df['ts'] = pd.to_datetime(df['ts'])
-            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
-            df = df[df['ts'] >= cutoff]
-
-        return df
-
-
-class FetchExperimentsStep(BaseContextStep):
-    """Fetch experiment metadata for given interaction data"""
-
-    def transform(self, interactions_df: pd.DataFrame):
-        if interactions_df.empty or 'experimentId' not in interactions_df.columns:
-            return pd.DataFrame()
-
-        exp_ids = interactions_df['experimentId'].dropna().unique().tolist()
-        if not exp_ids:
-            return pd.DataFrame()
-
-        return self.context.provider.fetch_experiments(exp_ids)

experiments/procesing/steps/join.py

@@ -1,58 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class JoinExperimentsStep(BaseContextStep):
-    """Join experiment metadata to interactions"""
-
-    def transform(self, data: tuple):
-        """
-        Args:
-            data: (interactions_df, experiments_df)
-        Returns:
-            merged interactions dataframe
-        """
-        interactions_df, experiments_df = data
-
-        if experiments_df.empty:
-            return interactions_df
-
-        # Flatten nested task field if present
-        if 'task' in experiments_df.columns and experiments_df['task'].notnull().any():
-            task_norm = pd.json_normalize(experiments_df['task'].dropna())
-            task_norm.index = experiments_df[experiments_df['task'].notnull()].index
-            experiments_df = experiments_df.drop('task', axis=1).join(task_norm, rsuffix='_task')
-
-        # Rename for clarity
-        experiments_df = experiments_df.rename(columns={
-            'id': 'experimentId',
-            'subject_name': 'exp_subject',
-            'xp_human_only': 'exp_human_only',
-            'xp_market_mode': 'exp_market_mode',
-            'xp_task_id': 'exp_task_id'
-        })
-
-        return interactions_df.merge(experiments_df, on='experimentId', how='left')
-
-class JoinProductFeaturesStep(BaseContextStep):
-    """Join product features to interactions"""
-
-    def transform(self, data: tuple):
-        """
-        Args:
-            data: (interactions_df, products_df)
-        Returns:
-            merged interactions dataframe
-        """
-        demand_df, price_df = data
-
-        # get base prices from products if available
-        products = self.context.products
-        products['base_price'] = products.apply(
-            lambda row: float(row['metadata'].get('base_price', 0.0)) if isinstance(row['metadata'], dict) else 0,
-            axis=1
-        )
-        products = products[['id', 'base_price']].rename(columns={'id': 'productId'})
-
-        if price_df.empty:
-            return demand_df
-        return demand_df.merge(price_df, on='productId', how='left').merge(products, on='productId', how='left')

experiments/procesing/steps/pricing.py

@@ -1,55 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import Optional, List, Dict, Any
-from dataclasses import dataclass, field
-from procesing.pricers.simple import StaticPricer
-from procesing.steps.base import BaseContextStep
-from procesing.pricers import ElasticityBasedPricer
-
-class State:
-    def __init__(self,
-                 last_action : str,
-                 last_productId : str,
-                 last_price : float,
-                 session_features : np.ndarray
-                 ):
-        pass
-
-
-
-class FitPricingFunctionStep(BaseContextStep):
-    """
-    Fit pricing function using data.
-    Input: pricing_data
-    Output: fitted pricing function instance
-    """
-
-    def transform(self, pricing_data: pd.DataFrame):
-        pricing_class = self.context.config.get('pricing_function_class', StaticPricer)
-        pricing_params = self.context.config.get('pricing_function_params', {})
-
-        pricer = pricing_class(**pricing_params)
-        pricer.fit(pricing_data)
-
-        return pricer
-
-
-class PredictPricesStep(BaseContextStep):
-    """
-    Predict optimal prices using fitted pricing function.
-    Input: (pricer, state_space)
-    Output: prices_df [productId, predicted_price]
-    """
-
-    def transform(self, data: tuple):
-        pricer, state_space = data
-
-        products = self.context.products
-        product_ids = products['id'].values
-
-        predicted_prices = pricer.predict(state_space)
-
-        return pd.DataFrame({
-            'productId': product_ids,
-            'predicted_price': predicted_prices
-        })

experiments/procesing/steps/session.py

@@ -1,261 +0,0 @@
-"""
-Session feature extraction for ML training pipeline.
-"""
-import pandas as pd
-import numpy as np
-import re
-from typing import Dict, Any
-from procesing.steps.base import BaseContextStep
-
-EVENT_CATS = {
-    'page_view': ['page_view'],
-    'item_view': ['view_item_page', 'learn_more_about_item'],
-    'cart_add': ['add_item_to_cart'],
-    'purchase': ['purchase', 'checkout_complete'],
-    'hover': ['hover_over_title', 'hover_over_paragraph', 'hover_over_link', 'hover_over_button'],
-    # 'filter': ['filter', 'search', 'apply_filter'],
-}
-HEADLESS_RE = re.compile(r'HeadlessChrome|Headless|PhantomJS', re.I)
-AUTOMATION_RE = re.compile(r'Selenium|Playwright|Puppeteer|WebDriver|chromedriver|geckodriver', re.I)
-BROWSER_PATTERNS = [('Chrome', r'Chrome/[\d.]+'), ('Firefox', r'Firefox/[\d.]+'),
-                    ('Safari', r'Safari/[\d.]+'), ('Edge', r'Edg/[\d.]+')]
-
-
-def _get_browser(s: str) -> str:
-    if pd.isna(s): return 'Unknown'
-    for name, pat in BROWSER_PATTERNS:
-        if re.search(pat, s): return name
-    return 'Other'
-
-
-class TemporalFeatureStep(BaseContextStep):
-    """Vectorized time-based features: durations, velocities, gaps."""
-
-    def __init__(self, context, timeout_sec: float = 900, velocity_window: str = '5min'):
-        super().__init__(context)
-        self.timeout_sec = timeout_sec
-        self.velocity_window = velocity_window
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        df = X.copy()
-        if df.empty or 'ts' not in df.columns:
-            return pd.DataFrame(columns=pd.Series(['sessionId']))
-
-        df['ts_dt'] = pd.to_datetime(df['ts'])
-        df = df.sort_values(['sessionId', 'ts_dt'])
-        df['time_diff'] = df.groupby('sessionId')['ts_dt'].diff().dt.total_seconds()
-        df['active_diff'] = df['time_diff'].where(df['time_diff'] <= self.timeout_sec, 0)
-
-        agg = df.groupby('sessionId').agg(
-            session_duration_sec=('active_diff', 'sum'),
-            total_interactions=('sessionId', 'count'),
-            avg_time_between_events=('time_diff', 'mean'),
-            std_time_between_events=('time_diff', 'std'),
-            min_time_between_events=('time_diff', 'min'),
-            session_start_hour=('ts_dt', lambda x: x.min().hour),
-        ).reset_index()
-        agg['std_time_between_events'] = agg['std_time_between_events'].fillna(0)
-        agg['interaction_velocity'] = np.where(
-            agg['session_duration_sec'] > 0,
-            (agg['total_interactions'] / agg['session_duration_sec']) * 60, 0)
-
-        vel = df.set_index('ts_dt').groupby('sessionId').resample(self.velocity_window, include_groups=False).size()
-        max_velocity = vel.groupby('sessionId').max().rename('max_velocity_5min')
-        agg = agg.merge(max_velocity, on='sessionId', how='left')
-        agg['max_velocity_5min'] = agg['max_velocity_5min'].fillna(0)
-        return agg
-
-
-class BehavioralFeatureStep(BaseContextStep):
-    """Vectorized event counts and ratios per session."""
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        df = X.copy()
-        if df.empty or 'eventName' not in df.columns:
-            return pd.DataFrame(columns=pd.Series(['sessionId']))
-
-        for cat, events in EVENT_CATS.items():
-            df[f'is_{cat}'] = df['eventName'].isin(events)
-        df['is_hover'] = df['is_hover'] | df['eventName'].str.startswith('hover_over_')
-
-        agg = df.groupby('sessionId').agg(
-            total_events=('eventName', 'count'), unique_pages=('page', 'nunique'),
-            page_views=('is_page_view', 'sum'), item_views=('is_item_view', 'sum'),
-            cart_adds=('is_cart_add', 'sum'), purchases=('is_purchase', 'sum'),
-            hover_events=('is_hover', 'sum'),
-            # filter_events=('is_filter', 'sum'),
-        ).reset_index()
-        agg['cart_to_view_ratio'] = np.where(agg['item_views'] > 0, agg['cart_adds'] / agg['item_views'], 0)
-        agg['conversion_rate'] = np.where(agg['item_views'] > 0, agg['purchases'] / agg['item_views'], 0)
-        agg['hover_intensity'] = np.where(agg['total_events'] > 0, agg['hover_events'] / agg['total_events'], 0)
-        return agg
-
-
-class ProductFeatureStep(BaseContextStep):
-    """Vectorized product interaction features: diversity, depth, price sensitivity."""
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        df = X.copy()
-        if df.empty:
-            return pd.DataFrame(columns=pd.Series(['sessionId']))
-        price_col = next((c for c in ['metadata_base_price', 'metadata_price', 'base_price'] if c in df.columns), None)
-        df['price_seen'] = pd.to_numeric(df[price_col], errors='coerce') if price_col else np.nan
-
-        prod_df = df[df['productId'].notna()]
-        if prod_df.empty:
-            return pd.DataFrame(columns=pd.Series(['sessionId', 'unique_products_viewed', 'product_view_depth', 'avg_price_seen', 'min_price_seen', 'max_price_seen', 'price_range']))
-
-        agg = prod_df.groupby('sessionId').agg(
-            unique_products_viewed=('productId', 'nunique'),
-            product_view_depth=('productId', lambda x: x.value_counts().iloc[0] if len(x) > 0 else 0),
-            avg_price_seen=('price_seen', 'mean'), min_price_seen=('price_seen', 'min'),
-            max_price_seen=('price_seen', 'max'),
-        ).reset_index()
-        agg['price_range'] = (agg['max_price_seen'] - agg['min_price_seen']).fillna(0)
-        return agg
-
-
-class UserAgentFeatureStep(BaseContextStep):
-    """Parse userAgent into bot-detection signals."""
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame|pd.Series:
-        df = X.copy()
-        if df.empty or 'userAgent' not in df.columns:
-            return pd.DataFrame(columns=pd.Series(['sessionId']))
-
-        ua = df.groupby('sessionId')['userAgent'].first().reset_index()
-        ua['is_headless'] = ua['userAgent'].str.contains(HEADLESS_RE, na=False)
-        ua['is_automation'] = ua['userAgent'].str.contains(AUTOMATION_RE, na=False)
-        ua['browser_family'] = ua['userAgent'].apply(_get_browser)
-        return ua[['sessionId', 'is_headless', 'is_automation', 'browser_family']]
-
-
-class ExtractSessionFeaturesStep(BaseContextStep):
-    """
-    Vectorized session feature extraction - replaces O(n^2) per-row loop.
-    Input: interactions_df
-    Output: session-level feature matrix
-    """
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        if X.empty:
-            return pd.DataFrame()
-        df = X.copy()
-
-        # run all feature steps and merge on sessionId
-        temporal = TemporalFeatureStep(self.context).transform(df)
-        behavioral = BehavioralFeatureStep(self.context).transform(df)
-        product = ProductFeatureStep(self.context).transform(df)
-        ua = UserAgentFeatureStep(self.context).transform(df)
-
-        result = temporal
-        for other in [behavioral, product, ua]:
-            if not other.empty and 'sessionId' in other.columns:
-                result = result.merge(other, on='sessionId', how='left')
-
-        # carry forward experimentId for label joining
-        if 'experimentId' in df.columns:
-            exp_map = df.groupby('sessionId')['experimentId'].first()
-            result = result.merge(exp_map, on='sessionId', how='left')
-
-        return result
-
-
-class JoinLabelsStep(BaseContextStep):
-    """
-    Join experiment labels to session features.
-    Input: (features_df, experiments_df) or features_df (fetches experiments)
-    Output: labeled feature matrix with is_agent column
-    """
-
-    def transform(self, X : tuple) -> pd.DataFrame:
-        data = X;
-        if isinstance(data, tuple):
-            features_df, experiments_df = data
-        else:
-            features_df = data
-            if 'experimentId' not in features_df.columns:
-                return features_df
-            exp_ids = features_df['experimentId'].dropna().unique().tolist()
-            experiments_df = self.context.provider.fetch_experiments(exp_ids) if exp_ids else pd.DataFrame()
-
-        if features_df.empty:
-            return features_df
-        if experiments_df.empty:
-            features_df['is_agent'] = np.nan
-            return features_df
-
-        exp = experiments_df.copy()
-        if 'id' in exp.columns:
-            exp = exp.rename(columns={'id': 'experimentId'})
-        if 'xp_human_only' in exp.columns:
-            exp['is_agent'] = ~exp['xp_human_only']
-
-        cols = ['experimentId'] + [c for c in ['is_agent', 'xp_human_only', 'xp_market_mode'] if c in exp.columns]
-        return features_df.merge(exp[cols].drop_duplicates(), on='experimentId', how='left')
-
-
-class ValidateDataStep(BaseContextStep):
-    """
-    Data quality checks before training.
-    Input: df
-    Output: df (unchanged, but logs validation report to context)
-    """
-    REQUIRED = ['sessionId', 'eventName', 'ts']
-
-    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
-        df = X.copy()
-        report = {'status': 'valid', 'rows': len(df), 'sessions': 0}
-        if df.empty:
-            report['status'] = 'empty'
-            self.context.cache('validation_report', report)
-            return df
-
-        missing = [c for c in self.REQUIRED if c not in df.columns]
-        if missing:
-            report['status'] = 'invalid'
-            report['missing_cols'] = missing
-
-        report['sessions'] = df['sessionId'].nunique() if 'sessionId' in df.columns else 0
-        report['null_sessions'] = int(df['sessionId'].isna().sum()) if 'sessionId' in df.columns else 0
-        if 'experimentId' in df.columns:
-            report['null_experiments'] = int(df['experimentId'].isna().sum())
-
-        self.context.cache('validation_report', report)
-        return df
-
-
-# legacy compat - kept for backwards compatibility with existing code
-def _extract_features_for_session(session_df: pd.DataFrame, session_timeout_sec: float = 900) -> Dict[str, Any]:
-    """Single-session feature extraction (legacy interface)."""
-    defaults = {k: 0 for k in ['total_interactions', 'page_views', 'item_views', 'searches',
-                               'cart_adds', 'hovers', 'unique_products_viewed', 'product_view_depth',
-                               'session_duration_sec', 'interaction_velocity',
-                               'avg_time_between_events', 'std_time_between_events', 'cart_to_view_ratio']}
-    if session_df.empty:
-        return defaults
-
-    session_df = session_df.copy()
-    if 'sessionId' not in session_df.columns:
-        session_df['sessionId'] = 'tmp'
-
-    # use a dummy context for the steps
-    class DummyCtx: config = {} # should maybe inherit but whatever
-    ctx = DummyCtx()
-
-    t = TemporalFeatureStep(ctx, timeout_sec=session_timeout_sec).transform(session_df)
-    b = BehavioralFeatureStep(ctx).transform(session_df)
-    p = ProductFeatureStep(ctx).transform(session_df)
-
-    result = {}
-    for df in [t, b, p]:
-        if not df.empty:
-            for col in df.columns:
-                if col != 'sessionId':
-                    result[col] = df[col].iloc[0] if len(df) > 0 else 0
-
-    remap = {'hover_events': 'hovers', 'filter_events': 'searches', 'unique_pages': 'unique_pages_visited'}
-    for old, new in remap.items():
-        if old in result:
-            result[new] = result.pop(old)
-    return result

experiments/procesing/tests/conftest.py

@@ -1,281 +0,0 @@
-import pytest
-import pandas as pd
-from typing import List
-from procesing.providers.base import DataProvider
-from procesing.context import PipelineContext
-
-
-class MockProvider(DataProvider):
-    """Mock provider for testing, holds in-memory fixtures"""
-
-    def __init__(self, products_df=None, experiments_df=None, kafka_data=None):
-        self._products = products_df if products_df is not None else pd.DataFrame()
-        self._experiments = experiments_df if experiments_df is not None else pd.DataFrame()
-        self._kafka_data = kafka_data if kafka_data is not None else {}
-
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        return self._products.copy()
-
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        if self._experiments.empty:
-            return pd.DataFrame()
-        return self._experiments[
-            self._experiments['id'].isin(experiment_ids)
-        ].copy()
-
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        return self._kafka_data.get(topic, pd.DataFrame()).copy()
-
-
-@pytest.fixture
-def mock_products():
-    """Standard product catalog fixture with realistic IDs from test data"""
-    return pd.DataFrame({
-        'id': [
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ],
-        'name': ['Junior Suite', 'Superior Room', 'Deluxe Room'],
-        'base_price': [200.0, 150.0, 180.0]
-    })
-
-
-@pytest.fixture
-def mock_interactions_raw_kafka():
-    """Raw Kafka message structure for interactions, matches production format"""
-    return [
-        {
-            'partitionID': 0, 'offset': 203, 'timestamp': 1764102082676,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'learn_more_about_item',
-                    'page': '/hotel/products/d018efc1-25e9-4284-b276-80386e048b25',
-                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
-                    'metadata': {'type': 'hotel', 'dateIndex': 1, 'roomType': 'Junior Suite'},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:22.674Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 204, 'timestamp': 1764102086982,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'page_view',
-                    'page': '/hotel/products',
-                    'productId': None,
-                    'metadata': {'referrer': ''},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:26.947Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 205, 'timestamp': 1764102091825,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'hover_over_title',
-                    'page': '/hotel/products',
-                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-                    'metadata': {'elementText': 'Superior Room', 'dateIndex': 1, 'dwellTime': 1200},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:31.823Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 206, 'timestamp': 1764102094193,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
-                    'eventName': 'hover_over_paragraph',
-                    'page': '/hotel/products',
-                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1307},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:34.191Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 207, 'timestamp': 1764102101970,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
-                    'eventName': 'hover_over_paragraph',
-                    'page': '/hotel/products',
-                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
-                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1201},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:41.967Z'
-                }
-            }
-        }
-    ]
-
-
-@pytest.fixture
-def mock_interactions(mock_interactions_raw_kafka):
-    """Processed interaction DataFrame (what provider.fetch_kafka_topic returns)"""
-    records = [msg['value']['payload'] for msg in mock_interactions_raw_kafka]
-    df = pd.DataFrame(records)
-    df['timestamp'] = pd.to_datetime(df['ts'])
-    return df
-
-
-@pytest.fixture
-def mock_price_logs_raw_kafka():
-    """Raw Kafka message structure for price logs, matches production format"""
-    return [
-        {
-            'partitionID': 0, 'offset': 32, 'timestamp': 1764104757969,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 162.47,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T21:05:57.967Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 33, 'timestamp': 1764104757995,
-            'value': {
-                'payload': {
-                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
-                    'price': 743.49,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T21:05:57.993Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 34, 'timestamp': 1764104758011,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 163.87,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T21:05:58.009Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 35, 'timestamp': 1764104758050,
-            'value': {
-                'payload': {
-                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
-                    'price': 397.46,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T21:05:58.049Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 36, 'timestamp': 1764104768865,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 401.66,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T21:06:08.864Z'
-                }
-            }
-        }
-    ]
-
-
-@pytest.fixture
-def mock_price_logs(mock_price_logs_raw_kafka):
-    """Processed price logs DataFrame (what provider.fetch_kafka_topic returns)"""
-    # extract payloads and flatten
-    records = [msg['value']['payload'] for msg in mock_price_logs_raw_kafka]
-    df = pd.DataFrame(records)
-    df['timestamp'] = pd.to_datetime(df['ts'])
-    return df
-
-
-@pytest.fixture
-def mock_experiments():
-    """Standard experiment metadata fixture matching Supabase schema"""
-    return pd.DataFrame({
-        'id': ['53aefd07-f66a-4d7f-ba8b-7ea1fc562d35', 'bbbbcccc-dddd-eeee-ffff-000011112222'],
-        'created_at': pd.to_datetime(['2025-11-25T20:00:00Z', '2025-11-26T10:00:00Z']),
-        'subject_name': ['Session A', 'Session B'],
-        'xp_human_only': [True, False],
-        'xp_market_mode': ['hotel', 'airline'],
-        'xp_task_id': [None, None]
-    })
-
-
-@pytest.fixture
-def mock_provider(mock_products, mock_experiments, mock_interactions, mock_price_logs):
-    """Fully configured mock provider"""
-    return MockProvider(
-        products_df=mock_products,
-        experiments_df=mock_experiments,
-        kafka_data={
-            'user-interactions': mock_interactions,
-            'price-logs': mock_price_logs
-        }
-    )
-
-
-@pytest.fixture
-def pipeline_context(mock_provider):
-    """Standard pipeline context for testing"""
-    return PipelineContext(
-        provider=mock_provider,
-        store_mode='hotel',
-        window_size='30s',
-        n_price_buckets=3
-    )
-
-
-@pytest.fixture
-def empty_provider():
-    """Provider with no data, for edge case testing"""
-    return MockProvider(
-        products_df=pd.DataFrame(columns=['id', 'name', 'base_price']),
-        experiments_df=pd.DataFrame(columns=['id', 'created_at', 'subject_name', 'xp_human_only', 'xp_market_mode', 'xp_task_id']),
-        kafka_data={'user-interactions': pd.DataFrame(), 'price-logs': pd.DataFrame()}
-    )
-
-
-@pytest.fixture
-def empty_context(empty_provider):
-    """Context with empty provider"""
-    return PipelineContext(
-        provider=empty_provider,
-        store_mode='hotel',
-        window_size='30s'
-    )
-
-
-@pytest.fixture
-def session_interactions(mock_interactions):
-    """Enriched interaction data for session feature extraction tests"""
-    df = mock_interactions.copy()
-    df['userAgent'] = ['Mozilla/5.0 Chrome/120', 'Mozilla/5.0 Chrome/120',
-                       'HeadlessChrome/120', 'HeadlessChrome/120', 'HeadlessChrome/120']
-    df['metadata_base_price'] = [None, None, 150.0, 150.0, 200.0]
-    return df

experiments/procesing/tests/test_augement.py

@@ -1,45 +0,0 @@
-import pytest
-import random
-import pandas as pd
-from procesing.steps import (
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep
-)
-
-def test_bucketing(pipeline_context):
-    step = CreatePriceBucketsStep(context=pipeline_context)
-
-    # Test with normal price data
-    df = pd.DataFrame({
-        'metadata_price': random.sample(range(10, 1000), 100)
-    })
-    result = step.transform(df)
-    assert 'price_bucket' in result.columns
-    # test if is categorical
-    assert isinstance(result['price_bucket'].dtype, pd.CategoricalDtype)
-    assert result['price_bucket'].nunique() == 3 # as per context config
-    # distribution check
-    counts = result['price_bucket'].value_counts()
-    assert all(counts > 0)
-    assert counts.max() - counts.min() <= 10  # roughly equal distribution for 100 samples
-    # Test with empty DataFrame
-    df = pd.DataFrame()
-    result = step.transform(df)
-    assert 'price_bucket' in result.columns
-    assert result.empty
-
-
-def test_augment_names(pipeline_context):
-    df = pd.DataFrame({
-        'eventName': ['click', 'view', 'purchase'],
-        'productId': ['prod_1', 'prod_2', None],
-        'price_bucket': ['PB_1', None, 'PB_3']
-    })
-    step = AugmentEventNamesStep(context=pipeline_context)
-    result = step.transform(df)
-    expected_event_names = [
-        'click_prod_1@PB_1',
-        'view',
-        'purchase'
-    ]
-    assert result['eventName'].tolist() == expected_event_names

experiments/procesing/tests/test_demand.py

@@ -1,49 +0,0 @@
-import pytest
-import random
-import pandas as pd
-from procesing.steps import (
-    ComputeDemandStep
-)
-
-def test_compute_demand(pipeline_context):
-    step = ComputeDemandStep(context=pipeline_context)
-
-    # Test with normal interaction data
-    df = pd.DataFrame({
-        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
-        'productId': random.choices([
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ], k=100),
-        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
-    })
-    result = step.transform(df)
-    assert type(result) == pd.DataFrame
-    assert not result.empty
-    assert set(result['productId']) == set(pipeline_context.products['id'])
-    assert all(result['demand_score'] > 100/3 -10)
-
-
-def test_compute_demand_skewed(pipeline_context):
-    step = ComputeDemandStep(context=pipeline_context)
-
-    # Test with normal interaction data
-    df = pd.DataFrame({
-        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
-        'productId': random.choices([
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ], weights=[0.7, 0.2, 0.1], k=100),
-        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
-    })
-    result = step.transform(df)
-    assert type(result) == pd.DataFrame
-    assert not result.empty
-    assert set(result['productId']) == set(pipeline_context.products['id'])
-    # test for skewness
-    scores = result.set_index('productId')['demand_score'].to_dict()
-    assert scores['d018efc1-25e9-4284-b276-80386e048b25'] > \
-           scores['51266ddb-5b07-47b7-89ee-5b5cae94bb11'] > \
-           scores['2cd7f756-fc65-4ba0-ab01-74521c1fff43']

experiments/procesing/tests/test_fetch.py

@@ -1,51 +0,0 @@
-import pytest
-import pandas as pd
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-)
-
-
-def test_fetch_interactions_data(pipeline_context):
-    step = FetchInteractionsStep(pipeline_context)
-    data = step.transform(None)
-    assert data is not None
-    assert isinstance(data, pd.DataFrame)
-    expected_cols = [
-        "eventName",
-        "dateIndex",
-        "experimentId",
-        "storeMode",
-        "metadata_elementText"
-    ]
-    for expected in expected_cols:
-        assert expected in data.columns
-
-def test_fetch_price_logs(pipeline_context):
-    step = FetchPriceLogsStep(pipeline_context)
-    data = step.transform(None)
-    assert data is not None
-    assert isinstance(data, pd.DataFrame)
-    expected_cols = [
-        "price",
-        "productId"
-    ]
-    for expected in expected_cols:
-        assert expected in data.columns
-    prices = data['price'].to_list()
-    assert min(prices) >= 0
-    assert max(prices) <= 9999
-
-
-def test_experiments_fetching(pipeline_context):
-    interactions = FetchInteractionsStep(pipeline_context).transform(None)
-    assert interactions is not None
-    experiments = FetchExperimentsStep(pipeline_context)
-    experiment_data = experiments.transform(interactions)
-    assert experiment_data is not None
-    assert isinstance(experiment_data, pd.DataFrame)
-    assert not experiment_data.empty
-    assert 'id' in experiment_data.columns
-    assert len(experiment_data) == 2
-    assert '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35' in experiment_data['id'].values

experiments/procesing/tests/test_pricing.py

@@ -1,87 +0,0 @@
-import pytest
-import pandas as pd
-
-from procesing.pricers import (
-    StaticPricer,
-    RandomPricer,
-    ElasticityBasedPricer
-)
-
-
-def test_static_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'product_id': [1, 2, 3],
-        'base_price': [100.0, 150.0, 200.0]
-    })
-
-    # Initialize and fit StaticPricer
-    pricer = StaticPricer()
-    pricer.fit(historical_data)
-
-    # Predict prices
-    predicted_prices = pricer.predict(None)
-
-    # Assert that predicted prices match base prices
-    expected_prices = historical_data['base_price'].values
-    assert all(predicted_prices == expected_prices), "Predicted prices do not match base prices"
-
-
-def test_random_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'product_id': [1, 2, 3],
-        'base_price': [100.0, 150.0, 200.0]
-    })
-
-    # Initialize and fit RandomPricer
-    pricer = RandomPricer(price_min=50.0, price_max=250.0, seed=42)
-    pricer.fit(historical_data)
-
-    # Predict prices
-    predicted_prices = pricer.predict(None)
-
-    # Assert that predicted prices are within bounds
-    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
-    assert predicted_prices.max() <= 250.0, "Predicted prices are above maximum bound"
-    # distribution check (not so strict)
-    assert len(set(predicted_prices)) > 1, "Predicted prices are not varied enough"
-    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
-
-def test_elasticity_based_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'productId': [1, 2, 3],
-        'elasticity': [-1.5, -0.5, -2.0],
-        'base_price': [100.0, 150.0, 200.0],
-        'mean_demand': [10, 20, 15]
-    })
-
-    # Initialize and fit ElasticityBasedPricer
-    pricer = ElasticityBasedPricer(alpha=0.1, price_floor=50.0, price_ceil=300.0)
-    pricer.fit(historical_data)
-
-    # Create a mock state space with demand deviations
-    class MockStateSpace:
-        def __init__(self, demand):
-            self.demand = demand
-
-    # Simulate demand higher than mean for all products
-    state_space = MockStateSpace(demand=[15, 25, 20])
-
-    # Predict prices
-    predicted_prices = pricer.predict(state_space)
-
-    # Assert that predicted prices are within bounds
-    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
-    assert predicted_prices.max() <= 300.0, "Predicted prices are above maximum bound"
-    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
-
-    # now we gotta check semantic validity
-    # since demand is higher than mean, prices should generally increase
-    for i, row in historical_data.iterrows():
-        base_price = row['base_price']
-        elasticity = row['elasticity']
-        expected_increase = base_price * (1 + 0.1 * abs(elasticity) * ((state_space.demand[i] - row['mean_demand']) / row['mean_demand']))
-        assert predicted_prices[i] >= base_price, f"Predicted price for product {row['productId']} did not increase as expected"
-        assert abs(predicted_prices[i] - expected_increase) < 1e-5, f"Predicted price for product {row['productId']} does not match expected calculation within 1e-5 tolerance"

experiments/pytest.ini

@@ -1,8 +0,0 @@
-[pytest]
-pythonpath = .
-testpaths = procesing/tests agents
-python_files = test*.py
-python_classes = Test*
-python_functions = test_*
-asyncio_mode = auto
-asyncio_default_fixture_loop_scope = function

experiments/seed_products.py

@@ -1,125 +0,0 @@
-import random
-import json
-import os
-import logging
-from dotenv import load_dotenv
-from supabase import create_client, Client
-from tqdm import tqdm
-
-load_dotenv()
-
-logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
-log = logging.getLogger(__name__)
-
-SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
-SUPABASE_SERVICE_KEY = os.getenv("SUPABASE_SERVICE_ROLE_KEY")
-
-if not SUPABASE_SERVICE_KEY:
-    log.error("SUPABASE_SERVICE_ROLE_KEY not found in environment")
-    raise ValueError("Missing SUPABASE_SERVICE_ROLE_KEY - required for admin operations")
-
-supabase: Client = create_client(SUPABASE_URL, SUPABASE_SERVICE_KEY)
-
-DAYS = 14
-
-# hotel room configurations
-ROOMS = {
-    "Presidential Suite": {'amenities': ['ocean_view', 'balcony', 'jacuzzi', 'butler_service', 'premium_minibar'], 'total': 1, 'image_url': "", "base_price": 450, 'name': 'Presidential Suite', 'refundable': True, 'max_occupancy': 4},
-    "Executive Suite": {'amenities': ['city_view', 'balcony', 'workspace', 'lounge_access'], 'total': 2, 'image_url': "", "base_price": 280, 'name': 'Executive Suite', 'refundable': True, 'max_occupancy': 3},
-    "Junior Suite": {'amenities': ['garden_view', 'mini_fridge', 'coffee_maker'], 'total': 5, 'image_url': "", "base_price": 180, 'name': 'Junior Suite', 'refundable': True, 'max_occupancy': 2},
-    "Deluxe Room": {'amenities': ['city_view', 'work_desk', 'coffee_maker'], 'total': 8, 'image_url': "", "base_price": 140, 'name': 'Deluxe Room', 'refundable': False, 'max_occupancy': 2},
-    "Superior Room": {'amenities': ['wifi', 'tv', 'safe'], 'total': 12, 'image_url': "", "base_price": 110, 'name': 'Superior Room', 'refundable': False, 'max_occupancy': 2},
-    "Standard Room": {'amenities': ['wifi', 'tv'], 'total': 20, 'image_url': "", "base_price": 85, 'name': 'Standard Room', 'refundable': False, 'max_occupancy': 2},
-}
-
-# flight configurations
-FLIGHTS = {
-    "JFK-LAX-Economy": {'departure': {'time': '08:00', 'airport': 'JFK'}, 'arrival': {'time': '11:30', 'airport': 'LAX'}, 'duration': '5h 30m', 'stops': 0, 'cabin_class': 'economy', 'fare_rule': 'standard', 'refundable': False, 'total': 180, 'base_price': 250},
-    "JFK-LAX-Business": {'departure': {'time': '08:00', 'airport': 'JFK'}, 'arrival': {'time': '11:30', 'airport': 'LAX'}, 'duration': '5h 30m', 'stops': 0, 'cabin_class': 'business', 'fare_rule': 'flexible', 'refundable': True, 'total': 30, 'base_price': 850},
-    "ORD-MIA-Economy": {'departure': {'time': '14:15', 'airport': 'ORD'}, 'arrival': {'time': '18:45', 'airport': 'MIA'}, 'duration': '3h 30m', 'stops': 0, 'cabin_class': 'economy', 'fare_rule': 'basic', 'refundable': False, 'total': 200, 'base_price': 180},
-    "SFO-SEA-Premium": {'departure': {'time': '06:30', 'airport': 'SFO'}, 'arrival': {'time': '08:45', 'airport': 'SEA'}, 'duration': '2h 15m', 'stops': 0, 'cabin_class': 'premium', 'fare_rule': 'standard', 'refundable': False, 'total': 60, 'base_price': 420},
-    "ATL-DFW-First": {'departure': {'time': '16:00', 'airport': 'ATL'}, 'arrival': {'time': '17:30', 'airport': 'DFW'}, 'duration': '2h 30m', 'stops': 0, 'cabin_class': 'first', 'fare_rule': 'flexible', 'refundable': True, 'total': 12, 'base_price': 1600},
-    "LAX-SFO-Economy": {'departure': {'time': '10:00', 'airport': 'LAX'}, 'arrival': {'time': '11:30', 'airport': 'SFO'}, 'duration': '1h 30m', 'stops': 0, 'cabin_class': 'economy', 'fare_rule': 'standard', 'refundable': False, 'total': 150, 'base_price': 120},
-    "MIA-ATL-Premium": {'departure': {'time': '19:00', 'airport': 'MIA'}, 'arrival': {'time': '20:45', 'airport': 'ATL'}, 'duration': '1h 45m', 'stops': 0, 'cabin_class': 'premium', 'fare_rule': 'standard', 'refundable': True, 'total': 50, 'base_price': 380},
-    "DFW-ORD-Economy": {'departure': {'time': '07:30', 'airport': 'DFW'}, 'arrival': {'time': '10:15', 'airport': 'ORD'}, 'duration': '2h 45m', 'stops': 0, 'cabin_class': 'economy', 'fare_rule': 'basic', 'refundable': False, 'total': 190, 'base_price': 160},
-    "SEA-LAX-Business": {'departure': {'time': '13:00', 'airport': 'SEA'}, 'arrival': {'time': '15:30', 'airport': 'LAX'}, 'duration': '2h 30m', 'stops': 0, 'cabin_class': 'business', 'fare_rule': 'flexible', 'refundable': True, 'total': 40, 'base_price': 720},
-    "LAX-JFK-First": {'departure': {'time': '18:00', 'airport': 'LAX'}, 'arrival': {'time': '02:15', 'airport': 'JFK'}, 'duration': '5h 15m', 'stops': 0, 'cabin_class': 'first', 'fare_rule': 'flexible', 'refundable': True, 'total': 16, 'base_price': 1850},
-}
-
-def gen_hotel_products():
-    """generate hotel room products for next DAYS days"""
-    data = []
-    for day in range(DAYS):
-        for room_type, rdata in ROOMS.items():
-            data.append({
-                'room_type': room_type,
-                'date_index': day + 1,
-                'metadata': rdata,
-                'availability': random.randint(0, rdata['total'])
-            })
-    return data
-
-def gen_airline_products():
-    """generate flight products for next DAYS days"""
-    data = []
-    for day in range(DAYS):
-        for flight_type, fdata in FLIGHTS.items():
-            data.append({
-                'flight_type': flight_type,
-                'date_index': day + 1,
-                'metadata': fdata,
-                'availability': random.randint(0, fdata['total'])
-            })
-    return data
-
-def clear_table(table_name: str):
-    """clear all records from a table"""
-    try:
-        resp = supabase.table(table_name).select('id').execute()
-        if resp.data:
-            ids = [row['id'] for row in resp.data]
-            chunk_size = 100
-            for i in tqdm(range(0, len(ids), chunk_size), desc=f"Clearing {table_name}", unit="chunk"):
-                chunk = ids[i:i+chunk_size]
-                supabase.table(table_name).delete().in_('id', chunk).execute()
-            log.info(f"Deleted {len(ids)} records from {table_name}")
-        else:
-            log.info(f"{table_name} already empty")
-    except Exception as e:
-        log.error(f"Failed to clear {table_name}: {e}")
-        raise
-
-def seed_table(table_name: str, data: list[dict]):
-    """insert records into a table"""
-    try:
-        chunk_size = 100
-        total = len(data)
-        for i in tqdm(range(0, total, chunk_size), desc=f"Seeding {table_name}", unit="chunk"):
-            chunk = data[i:i+chunk_size]
-            supabase.table(table_name).insert(chunk).execute()
-        log.info(f"Inserted {total} records into {table_name}")
-    except Exception as e:
-        log.error(f"Failed to seed {table_name}: {e}")
-        raise
-
-def main():
-
-    log.info("Generating hotel products...")
-    hotel_products = gen_hotel_products()
-    log.info(f"Generated {len(hotel_products)} hotel products")
-
-    log.info("Generating airline products...")
-    airline_products = gen_airline_products()
-    log.info(f"Generated {len(airline_products)} airline products\n")
-
-    log.info("Clearing existing products...")
-    clear_table('hotel_products')
-    clear_table('airline_products')
-
-    log.info("Seeding products...")
-    seed_table('hotel_products', hotel_products)
-    seed_table('airline_products', airline_products)
-
-
-if __name__ == "__main__":
-    main()

lib/model_registry.py

@@ -1,180 +0,0 @@
-import redis
-import pickle
-import json
-import pandas as pd
-from typing import Optional, Dict, Any
-import os
-import logging
-log = logging.getLogger(__name__)
-
-class ModelRegistry:
-    """
-    Lightweight model registry using Redis for storing pricing models and elasticity data.
-    Models are serialized using pickle, metadata stored as JSON.
-    """
-
-    def __init__(self, redis_host: str = None, redis_port: int = None):
-        host = redis_host or os.getenv('REDIS_HOST', 'localhost')
-        port = redis_port or int(os.getenv('REDIS_PORT', '6378'))
-
-        self.redis_client = redis.Redis(
-            host=host,
-            port=port,
-            db=0,
-            decode_responses=False
-        )
-        self.metadata_prefix = "model:meta:"
-        self.data_prefix = "model:data:"
-        self.elasticity_prefix = "elasticity:"
-        self.prices_prefix = "prices:"
-
-    def publish_elasticity(self,
-                          elasticity_df: pd.DataFrame,
-                          model_name: str = 'latest',
-                          metadata: Optional[Dict[str, Any]] = None):
-        """
-        Store elasticity estimates in registry.
-
-        Args:
-            elasticity_df: df with [productId, elasticity, std_error, n_obs]
-            model_name: identifier for this elasticity snapshot
-            metadata: additional info (timestamp, window_size, etc)
-        """
-        key = f"{self.elasticity_prefix}{model_name}"
-
-        # serialize dataframe as JSON
-        data_json = elasticity_df.to_json(orient='records')
-
-        # store data
-        self.redis_client.set(key, data_json)
-
-        # store metadata
-        meta = metadata or {}
-        meta.update({
-            'n_products': len(elasticity_df),
-            'mean_elasticity': float(elasticity_df['elasticity'].mean()),
-            'model_type': 'elasticity_snapshot'
-        })
-
-        meta_key = f"{self.metadata_prefix}{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published elasticity model '{model_name}' with {len(elasticity_df)} products")
-
-    def get_elasticity(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
-        """Retrieve elasticity estimates from registry."""
-        key = f"{self.elasticity_prefix}{model_name}"
-        data_json = self.redis_client.get(key)
-
-        if data_json is None:
-            return None
-
-        # decode bytes to string if needed
-        if isinstance(data_json, bytes):
-            data_json = data_json.decode('utf-8')
-
-        return pd.read_json(data_json, orient='records')
-
-    def publish_pricing_model(self,
-                              pricing_function,
-                              model_name: str = 'latest',
-                              metadata: Optional[Dict[str, Any]] = None):
-        """
-        Store a fitted pricing function object.
-
-        Args:
-            pricing_function: fitted PricingFunction instance
-            model_name: identifier
-            metadata: additional info
-        """
-        key = f"{self.data_prefix}{model_name}"
-
-        # serialize object
-        model_bytes = pickle.dumps(pricing_function)
-        self.redis_client.set(key, model_bytes)
-
-        # store metadata
-        meta = metadata or {}
-        meta.update({
-            'model_class': pricing_function.__class__.__name__,
-            'model_type': 'pricing_function'
-        })
-
-        meta_key = f"{self.metadata_prefix}{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published pricing model '{model_name}' ({meta['model_class']})")
-
-    def get_pricing_model(self, model_name: str = 'latest'):
-        """Retrieve a pricing function from registry."""
-        key = f"{self.data_prefix}{model_name}"
-        model_bytes = self.redis_client.get(key)
-
-        if model_bytes is None:
-            return None
-
-        return pickle.loads(model_bytes)
-
-    def list_models(self) -> Dict[str, Any]:
-        """List all registered models with metadata."""
-        models = {}
-
-        for key in self.redis_client.scan_iter(f"{self.metadata_prefix}*"):
-            key_str = key.decode('utf-8') if isinstance(key, bytes) else key
-            model_name = key_str.replace(self.metadata_prefix, '')
-            meta_json = self.redis_client.get(key)
-
-            if meta_json:
-                if isinstance(meta_json, bytes):
-                    meta_json = meta_json.decode('utf-8')
-                models[model_name] = json.loads(meta_json)
-
-        return models
-
-    def publish_prices(self,
-                      prices_df: pd.DataFrame,
-                      model_name: str = 'latest',
-                      metadata: Optional[Dict[str, Any]] = None):
-        """Store predicted prices in registry.
-
-        Args:
-            prices_df: df with [productId, predicted_price, ...]
-            model_name: identifier for this price snapshot
-            metadata: additional info
-        """
-        key = f"{self.prices_prefix}{model_name}"
-        data_json = prices_df.to_json(orient='records')
-
-        self.redis_client.set(key, data_json)
-
-        meta = metadata or {}
-        meta.update({
-            'n_products': len(prices_df),
-            'model_type': 'predicted_prices'
-        })
-
-        meta_key = f"{self.metadata_prefix}prices_{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published prices '{model_name}' for {len(prices_df)} products")
-
-    def get_prices(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
-        """Retrieve predicted prices from registry."""
-        key = f"{self.prices_prefix}{model_name}"
-        data_json = self.redis_client.get(key)
-
-        if data_json is None:
-            return None
-
-        if isinstance(data_json, bytes):
-            data_json = data_json.decode('utf-8')
-
-        return pd.read_json(data_json, orient='records')
-
-    def health_check(self) -> bool:
-        """Check if Redis connection is alive."""
-        try:
-            self.redis_client.ping()
-            return True
-        except:
-            return False

paper/concat_code.sh

@@ -21,10 +21,7 @@ add_file() {
     # Add section header and code listing (no language-specific highlighting)
     echo "\\subsection{${escaped_path}}" >> "$OUTPUT_FILE"
     echo "\\begin{lstlisting}[caption={${escaped_path}}]" >> "$OUTPUT_FILE"
-    # Convert to ASCII: transliterate what's possible, drop the rest
+    cat "$filepath" >> "$OUTPUT_FILE"
-    # LC_ALL=C forces ASCII locale for consistent behavior across environments
-    LC_ALL=C iconv -f UTF-8 -t ASCII//TRANSLIT//IGNORE "$filepath" 2>/dev/null >> "$OUTPUT_FILE" || \
-        LC_ALL=C tr -cd '\11\12\15\40-\176' < "$filepath" >> "$OUTPUT_FILE"
     echo "" >> "$OUTPUT_FILE"
     echo "\\end{lstlisting}" >> "$OUTPUT_FILE"
     echo "" >> "$OUTPUT_FILE"

pytest.ini

@@ -1,5 +1,4 @@
 [pytest]
-pythonpath = experiments
 testpaths = experiments
 python_files = test*.py
 python_classes = Test*

requirements.txt

@@ -11,4 +11,3 @@ pytest-asyncio
 uv
 scikit-learn
 supabase
-pymc

web/package-lock.json

@@ -10,7 +10,7 @@
       "dependencies": {
         "@supabase/ssr": "^0.7.0",
         "@supabase/supabase-js": "^2.81.1",
-        "next": "^16.0.0",
+        "next": "16.0.0",
         "react": "19.2.0",
         "react-dom": "19.2.0",
         "zod": "^4.1.12"
@@ -526,15 +526,15 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.0.tgz",
-      "integrity": "sha512-gpaNgUh5nftFKRkRQGnVi5dpcYSKGcZZkQffZ172OrG/XkrnS7UBTQ648YY+8ME92cC4IojpI2LqTC8sTDhAaw==",
+      "integrity": "sha512-s5j2iFGp38QsG1LWRQaE2iUY3h1jc014/melHFfLdrsMJPqxqDQwWNwyQTcNoUSGZlCVZuM7t7JDMmSyRilsnA==",
       "license": "MIT"
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.0.tgz",
-      "integrity": "sha512-LlDtCYOEj/rfSnEn/Idi+j1QKHxY9BJFmxx7108A6D8K0SB+bNgfYQATPk/4LqOl4C0Wo3LACg2ie6s7xqMpJg==",
+      "integrity": "sha512-/CntqDCnk5w2qIwMiF0a9r6+9qunZzFmU0cBX4T82LOflE72zzH6gnOjCwUXYKOBlQi8OpP/rMj8cBIr18x4TA==",
       "cpu": [
         "arm64"
       ],
@@ -548,9 +548,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.0.tgz",
-      "integrity": "sha512-rtZ7BhnVvO1ICf3QzfW9H3aPz7GhBrnSIMZyr4Qy6boXF0b5E3QLs+cvJmg3PsTCG2M1PBoC+DANUi4wCOKXpA==",
+      "integrity": "sha512-hB4GZnJGKa8m4efvTGNyii6qs76vTNl+3dKHTCAUaksN6KjYy4iEO3Q5ira405NW2PKb3EcqWiRaL9DrYJfMHg==",
       "cpu": [
         "x64"
       ],
@@ -564,9 +564,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.0.tgz",
-      "integrity": "sha512-mloD5WcPIeIeeZqAIP5c2kdaTa6StwP4/2EGy1mUw8HiexSHGK/jcM7lFuS3u3i2zn+xH9+wXJs6njO7VrAqww==",
+      "integrity": "sha512-E2IHMdE+C1k+nUgndM13/BY/iJY9KGCphCftMh7SXWcaQqExq/pJU/1Hgn8n/tFwSoLoYC/yUghOv97tAsIxqg==",
       "cpu": [
         "arm64"
       ],
@@ -580,9 +580,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.0.tgz",
-      "integrity": "sha512-+ksWNrZrthisXuo9gd1XnjHRowCbMtl/YgMpbRvFeDEqEBd523YHPWpBuDjomod88U8Xliw5DHhekBC3EOOd9g==",
+      "integrity": "sha512-xzgl7c7BVk4+7PDWldU+On2nlwnGgFqJ1siWp3/8S0KBBLCjonB6zwJYPtl4MUY7YZJrzzumdUpUoquu5zk8vg==",
       "cpu": [
         "arm64"
       ],
@@ -596,9 +596,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.0.tgz",
-      "integrity": "sha512-4WtJU5cRDxpEE44Ana2Xro1284hnyVpBb62lIpU5k85D8xXxatT+rXxBgPkc7C1XwkZMWpK5rXLXTh9PFipWsA==",
+      "integrity": "sha512-sdyOg4cbiCw7YUr0F/7ya42oiVBXLD21EYkSwN+PhE4csJH4MSXUsYyslliiiBwkM+KsuQH/y9wuxVz6s7Nstg==",
       "cpu": [
         "x64"
       ],
@@ -612,9 +612,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.0.tgz",
-      "integrity": "sha512-HYlhqIP6kBPXalW2dbMTSuB4+8fe+j9juyxwfMwCe9kQPPeiyFn7NMjNfoFOfJ2eXkeQsoUGXg+O2SE3m4Qg2w==",
+      "integrity": "sha512-IAXv3OBYqVaNOgyd3kxR4L3msuhmSy1bcchPHxDOjypG33i2yDWvGBwFD94OuuTjjTt/7cuIKtAmoOOml6kfbg==",
       "cpu": [
         "x64"
       ],
@@ -628,9 +628,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.0.tgz",
-      "integrity": "sha512-EviG+43iOoBRZg9deGauXExjRphhuYmIOJ12b9sAPy0eQ6iwcPxfED2asb/s2/yiLYOdm37kPaiZu8uXSYPs0Q==",
+      "integrity": "sha512-bmo3ncIJKUS9PWK1JD9pEVv0yuvp1KPuOsyJTHXTv8KDrEmgV/K+U0C75rl9rhIaODcS7JEb6/7eJhdwXI0XmA==",
       "cpu": [
         "arm64"
       ],
@@ -644,9 +644,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.0.tgz",
-      "integrity": "sha512-gniPjy55zp5Eg0896qSrf3yB1dw4F/3s8VK1ephdsZZ129j2n6e1WqCbE2YgcKhW9hPB9TVZENugquWJD5x0ug==",
+      "integrity": "sha512-O1cJbT+lZp+cTjYyZGiDwsOjO3UHHzSqobkPNipdlnnuPb1swfcuY6r3p8dsKU4hAIEO4cO67ZCfVVH/M1ETXA==",
       "cpu": [
         "x64"
       ],
@@ -1447,12 +1447,12 @@
       }
     },
     "node_modules/next": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/next/-/next-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/next/-/next-16.0.0.tgz",
-      "integrity": "sha512-3mBRJyPxT4LOxAJI6IsXeFtKfiJUbjCLgvXO02fV8Wy/lIhPvP94Fe7dGhUgHXcQy4sSuYwQNcOLhIfOm0rL0A==",
+      "integrity": "sha512-nYohiNdxGu4OmBzggxy9rczmjIGI+TpR5vbKTsE1HqYwNm1B+YSiugSrFguX6omMOKnDHAmBPY4+8TNJk0Idyg==",
       "license": "MIT",
       "dependencies": {
-        "@next/env": "16.0.7",
+        "@next/env": "16.0.0",
         "@swc/helpers": "0.5.15",
         "caniuse-lite": "^1.0.30001579",
         "postcss": "8.4.31",
@@ -1465,14 +1465,14 @@
         "node": ">=20.9.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "16.0.7",
+        "@next/swc-darwin-arm64": "16.0.0",
-        "@next/swc-darwin-x64": "16.0.7",
+        "@next/swc-darwin-x64": "16.0.0",
-        "@next/swc-linux-arm64-gnu": "16.0.7",
+        "@next/swc-linux-arm64-gnu": "16.0.0",
-        "@next/swc-linux-arm64-musl": "16.0.7",
+        "@next/swc-linux-arm64-musl": "16.0.0",
-        "@next/swc-linux-x64-gnu": "16.0.7",
+        "@next/swc-linux-x64-gnu": "16.0.0",
-        "@next/swc-linux-x64-musl": "16.0.7",
+        "@next/swc-linux-x64-musl": "16.0.0",
-        "@next/swc-win32-arm64-msvc": "16.0.7",
+        "@next/swc-win32-arm64-msvc": "16.0.0",
-        "@next/swc-win32-x64-msvc": "16.0.7",
+        "@next/swc-win32-x64-msvc": "16.0.0",
         "sharp": "^0.34.4"
       },
       "peerDependencies": {

web/package.json

@@ -10,7 +10,7 @@
   "dependencies": {
     "@supabase/ssr": "^0.7.0",
     "@supabase/supabase-js": "^2.81.1",
-    "next": "^16.0.0",
+    "next": "16.0.0",
     "react": "19.2.0",
     "react-dom": "19.2.0",
     "zod": "^4.1.12"

web/src/app/airline/checkout/page.tsx

@@ -1,11 +0,0 @@
-export default function AirlineCheckout() {
-  return (
-    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-sky-50 to-blue-50">
-      <div className="text-center p-8">
-        <h1 className="text-4xl font-light text-gray-800 mb-4">
-          Thank you for flying with us
-        </h1>
-      </div>
-    </div>
-  );
-}

web/src/app/airline/products/[id]/page.tsx

@@ -1,106 +0,0 @@
-'use client';
-
-import { useState, useEffect } from 'react';
-import { useParams, useRouter } from 'next/navigation';
-import { Navigation } from '@/components/ui';
-import { useCart } from '@/contexts/CartContext';
-import AirlineDetails from '@/components/feats/airline/AirlineDetails';
-import { transformProduct, type Flight, type AirlineProduct } from '@/lib/airline-utils';
-import type { EventName } from '@/lib/events';
-
-const dispatchInteraction = (eventName: EventName, productId?: string, metadata?: Record<string, unknown>) => {
-    const e = new CustomEvent('definedInteraction', {
-        detail: { eventName, productId, metadata },
-    });
-    document.dispatchEvent(e);
-};
-
-export default function AirlineProductPage() {
-    const params = useParams();
-    const router = useRouter();
-    const { addItem } = useCart();
-    const [product, setProduct] = useState<Flight | null>(null);
-    const [loading, setLoading] = useState(true);
-    const [error, setError] = useState<string | null>(null);
-    const [added, setAdded] = useState(false);
-
-    const productId = params.id as string;
-
-    useEffect(() => {
-        const fetchProduct = async () => {
-            try {
-                const res = await fetch(`/api/products/${productId}`);
-                if (!res.ok) throw new Error(`Failed to fetch: ${res.status}`);
-                const json = await res.json();
-                const transformed = transformProduct(json.data as AirlineProduct);
-                setProduct(transformed);
-
-                // fire learn_more_about_item event when product loads
-                dispatchInteraction('learn_more_about_item', productId, {
-                    type: 'airline',
-                    dateIndex: transformed.dateIndex,
-                    flightType: transformed.flightType,
-                });
-            } catch (e) {
-                setError(e instanceof Error ? e.message : 'Failed to load product');
-                console.error('[FETCH_FLIGHT_ERROR]', e);
-            } finally {
-                setLoading(false);
-            }
-        };
-        fetchProduct();
-    }, [productId]);
-
-    const handleAddToCart = () => {
-        if (!product) return;
-
-        addItem({
-            id: productId,
-            type: 'airline',
-            name: product.flightType,
-            price: product.basePrice,
-            metadata: {
-                departure: product.departure,
-                arrival: product.arrival,
-                duration: product.duration,
-                cabinClass: product.cabinClass,
-            },
-            dateIndex: product.dateIndex,
-        });
-
-        dispatchInteraction('add_item_to_cart', productId, {
-            type: 'airline',
-            price: product.basePrice,
-        });
-
-        setAdded(true);
-        setTimeout(() => setAdded(false), 2000);
-    };
-
-    return (
-        <>
-            <Navigation />
-            <main className="max-w-4xl mx-auto px-4 py-8">
-                {loading && <div className="text-center py-8">Loading flight details...</div>}
-                {error && <div className="text-red-500 text-center py-8">{error}</div>}
-
-                {!loading && !error && product && (
-                    <>
-                        <button
-                            onClick={() => router.back()}
-                            className="mt-6 text-blue-600 hover:underline"
-                        >
-                            ← Back to flights
-                        </button>
-                        <AirlineDetails
-                            product={product}
-                            onAddToCart={handleAddToCart}
-                            addedToCart={added}
-                        />
-
-                    </>
-                )}
-            </main>
-        </>
-    );
-}

web/src/app/airline/products/page.tsx

@@ -1,69 +1,73 @@
 'use client';
 
-import { useState, useEffect, Suspense } from 'react';
-import { useSearchParams } from 'next/navigation';
 import { Navigation } from '@/components/ui';
 import AirlineCard from '@/components/feats/airline/AirlineCard';
-import { transformProduct, type Flight, type AirlineProduct } from '@/lib/airline-utils';
 
-function FlightsList() {
+type CabinClass = 'economy' | 'premium' | 'business' | 'first';
-  const searchParams = useSearchParams();
+type FareRule = 'flexible' | 'standard' | 'basic';
-  const [flights, setFlights] = useState<Flight[]>([]);
-  const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
 
-  useEffect(() => {
+interface Flight {
-    const fetchFlights = async () => {
+  id: string;
-      try {
+  departure: { time: string; airport: string };
-        const url = new URL('/api/products', window.location.origin);
+  arrival: { time: string; airport: string };
-        url.searchParams.set('type', 'airline');
+  duration: string;
-
+  stops: number;
-        // forward all relevant search params to the API
+  cabinClass: CabinClass;
-        const params = ['dateIndex', 'origin', 'destination', 'tripType', 'adults', 'children', 'infants'];
+  fareRule: FareRule;
-        params.forEach(param => {
+  refundable: boolean;
-          const val = searchParams.get(param);
+  basePrice: number;
-          if (val) url.searchParams.set(param, val);
-        });
-
-        const res = await fetch(url.toString());
-        if (!res.ok) throw new Error(`Failed to fetch: ${res.status}`);
-        const json = await res.json();
-        const transformed = json.data.map((p: AirlineProduct) => transformProduct(p));
-        setFlights(transformed);
-      } catch (e) {
-        setError(e instanceof Error ? e.message : 'Failed to load products');
-        console.error('[FETCH_ERROR]', e);
-      } finally {
-        setLoading(false);
 }
+
+const genRandomFlights = (): Flight[] => {
+  const airports = ['JFK', 'LAX', 'ORD', 'ATL', 'DFW', 'SFO', 'SEA', 'MIA'];
+  const cabins: CabinClass[] = ['economy', 'premium', 'business', 'first'];
+  const fareRules: FareRule[] = ['flexible', 'standard', 'basic'];
+
+  return Array.from({ length: 12 }, (_, i) => {
+    const depHour = Math.floor(Math.random() * 24);
+    const arrHour = (depHour + Math.floor(Math.random() * 6) + 2) % 24;
+    const stops = Math.random() > 0.6 ? 0 : Math.floor(Math.random() * 2) + 1;
+    const cabin = cabins[Math.floor(Math.random() * cabins.length)];
+    const fareRule = fareRules[Math.floor(Math.random() * fareRules.length)];
+
+    const basePrice = Math.floor(
+      (cabin === 'economy' ? 200 : cabin === 'premium' ? 400 : cabin === 'business' ? 800 : 1500) +
+      Math.random() * 300
+    );
+
+    return {
+      id: `flt-${i}`,
+      departure: {
+        time: `${depHour.toString().padStart(2, '0')}:${Math.floor(Math.random() * 60).toString().padStart(2, '0')}`,
+        airport: airports[Math.floor(Math.random() * airports.length)],
+      },
+      arrival: {
+        time: `${arrHour.toString().padStart(2, '0')}:${Math.floor(Math.random() * 60).toString().padStart(2, '0')}`,
+        airport: airports[Math.floor(Math.random() * airports.length)],
+      },
+      duration: `${Math.floor(Math.random() * 5) + 2}h ${Math.floor(Math.random() * 60)}m`,
+      stops,
+      cabinClass: cabin,
+      fareRule,
+      refundable: Math.random() > 0.7,
+      basePrice,
     };
-    fetchFlights();
+  });
-  }, [searchParams]);
+};
+
+export default function AirlineProducts() {
+  const flights = genRandomFlights();
 
   return (
     <>
+      <Navigation />
+      <main className="max-w-7xl mx-auto px-4 py-8">
         <h1 className="text-3xl font-bold mb-6">Available Flights</h1>
-      {loading && <div className="text-center py-8">Loading...</div>}
-      {error && <div className="text-red-500 text-center py-8">{error}</div>}
-      {!loading && !error && (
         <div className="space-y-4">
           {flights.map((f) => (
             <AirlineCard key={f.id} flight={f} />
           ))}
         </div>
-      )}
-    </>
-  );
-}
-
-export default function AirlineProducts() {
-  return (
-    <>
-      <Navigation />
-      <main className="max-w-7xl mx-auto px-4 py-8">
-        <Suspense fallback={<div className="text-center py-8">Loading...</div>}>
-          <FlightsList />
-        </Suspense>
       </main>
     </>
   );

web/src/app/api/ingest/route.ts

@@ -7,7 +7,7 @@ export async function POST(req: NextRequest) {
   try {
     const body = await req.json();
 
-    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || process.env.STORE_MODE || 'hotel';
+    const storeMode = process.env.STORE_MODE || 'hotel';
     const userAgent = req.headers.get('user-agent') || undefined;
 
     const event: EventBase = {

web/src/app/api/pricing/route.ts

@@ -11,7 +11,18 @@ export async function GET(req: NextRequest) {
     const productId = searchParams.get('productId');
     const sessionId = searchParams.get('sessionId');
     const experimentId = searchParams.get('experimentId');
-    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || process.env.STORE_MODE || 'hotel';
+    const storeMode = process.env.NEXT_PUBLIC_STORE_MODE || 'shop';
+
+    // log in dev
+    if (process.env.NODE_ENV === 'development') {
+        console.log('[pricing-api]', {
+            productId,
+            sessionId,
+            experimentId,
+            storeMode,
+            timestamp: new Date().toISOString(),
+        });
+    }
 
     if (!productId) {
         return NextResponse.json(
@@ -20,73 +31,14 @@ export async function GET(req: NextRequest) {
         );
     }
 
-    const timestamp = new Date().toISOString();
+    // stub: call external pricing provider (random for now)
-    let price: number;
+    const basePrice = 100 + Math.random() * 900; // 100-1000 range
-    let basePrice: number | undefined;
+    const price = Math.round(basePrice * 100) / 100;
-    let markup: number | undefined;
-    let elasticity: number | undefined;
-
-    // call real pricing provider
-    const providerUrl = process.env.PRICING_PROVIDER_URL || 'http://localhost:5001';
-    try {
-        const queryParams = new URLSearchParams();
-        if (sessionId) queryParams.append('sessionId', sessionId);
-        if (experimentId) queryParams.append('experimentId', experimentId);
-
-        const providerResponse = await fetch(
-            `${providerUrl}/api/${storeMode}/price/${productId}?${queryParams.toString()}`,
-            { headers: { 'Accept': 'application/json' }, cache: 'no-store' }
-        );
-
-        if (!providerResponse.ok) {
-            throw new Error(`Provider returned ${providerResponse.status}`);
-        }
-
-        const providerData = await providerResponse.json();
-        price = providerData.price;
-        basePrice = providerData.base_price;
-        markup = providerData.markup;
-        elasticity = providerData.elasticity;
-
-    } catch (err) {
-        console.error('[pricing-provider-error]', err);
-        // fallback to random pricing if provider unavailable
-        const randomBase = 100 + Math.random() * 900;
-        price = Math.round(randomBase * 100) / 100;
-    }
-
-    // log price to kafka for elasticity computation
-    if (sessionId) {
-        const backendUrl = process.env.BACKEND_URL || 'http://localhost:5000';
-        try {
-            await fetch(`${backendUrl}/api/kafka/price-log`, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({
-                    productId,
-                    price,
-                    sessionId,
-                    experimentId: experimentId || undefined,
-                    storeMode,
-                    ts: timestamp,
-                }),
-            });
-        } catch (err) {
-            console.error('[price-log-error]', err);
-        }
-    }
-
-    if (process.env.NODE_ENV === 'development') {
-        console.log('[pricing-api]', {
-            productId, sessionId, experimentId, storeMode,
-            price, basePrice, markup, elasticity, timestamp,
-        });
-    }
 
     const response: PricingResponse = {
         price,
         currency: 'EUR',
-        cachedAt: timestamp,
+        cachedAt: new Date().toISOString(),
     };
 
     return NextResponse.json(response);

web/src/app/api/products/[id]/route.ts

@@ -1,35 +0,0 @@
-import { NextRequest, NextResponse } from 'next/server';
-
-export async function GET(
-  req: NextRequest,
-  { params }: { params: Promise<{ id: string }> }
-) {
-  const { id } = await params;
-
-  if (!id) {
-    return NextResponse.json(
-      { error: 'product id is required' },
-      { status: 400 }
-    );
-  }
-
-  try {
-    const backendUrl = process.env.BACKEND_URL || 'http://localhost:5000';
-    const url = new URL(`${backendUrl}/api/products/${id}`);
-
-    const res = await fetch(url.toString());
-
-    if (!res.ok) {
-      throw new Error(`Backend returned ${res.status}`);
-    }
-
-    const data = await res.json();
-    return NextResponse.json(data);
-  } catch (error) {
-    console.error('[PRODUCT_DETAIL_ERROR]', error);
-    return NextResponse.json(
-      { error: 'Failed to fetch product details' },
-      { status: 500 }
-    );
-  }
-}

web/src/app/api/products/route.ts

@@ -1,40 +0,0 @@
-import { NextRequest, NextResponse } from 'next/server';
-
-export async function GET(req: NextRequest) {
-  const { searchParams } = new URL(req.url);
-  const type = searchParams.get('type');
-
-  if (!type || !['hotel', 'airline'].includes(type)) {
-    return NextResponse.json(
-      { error: 'type parameter must be "hotel" or "airline"' },
-      { status: 400 }
-    );
-  }
-
-  try {
-    const backendUrl = process.env.BACKEND_URL || 'http://localhost:5000';
-    const url = new URL(`${backendUrl}/api/products/type/${type}`);
-
-    // forward all query params to backend (excluding 'type')
-    searchParams.forEach((value, key) => {
-      if (key !== 'type') {
-        url.searchParams.set(key, value);
-      }
-    });
-
-    const res = await fetch(url.toString());
-
-    if (!res.ok) {
-      throw new Error(`Backend returned ${res.status}`);
-    }
-
-    const data = await res.json();
-    return NextResponse.json(data);
-  } catch (error) {
-    console.error('[PRODUCTS_PROXY_ERROR]', error);
-    return NextResponse.json(
-      { error: 'Failed to fetch products' },
-      { status: 500 }
-    );
-  }
-}

web/src/app/cart/page.tsx

@@ -1,113 +0,0 @@
-'use client';
-
-import { Navigation } from '@/components/ui';
-import { useCart } from '@/contexts/CartContext';
-import type { EventName } from '@/lib/events';
-
-const dispatchInteraction = (eventName: EventName, productId?: string, metadata?: Record<string, unknown>) => {
-    const e = new CustomEvent('definedInteraction', {
-        detail: { eventName, productId, metadata },
-    });
-    document.dispatchEvent(e);
-};
-
-export default function CartPage() {
-    const { items, removeItem, clearCart, itemCount } = useCart();
-
-    const handleRemove = (id: string, type: string) => {
-        removeItem(id);
-        dispatchInteraction('remove_item', id, { type });
-    };
-    let itemTypes = Array.from(new Set(items.map(item => item.type)))[0] || 'items';
-
-
-    const total = items.reduce((sum, item) => sum + item.price, 0);
-
-    return (
-        <>
-            <Navigation />
-            <main className="max-w-4xl mx-auto px-4 py-8">
-                <div className="flex justify-between items-center mb-6">
-                    <h1 className="text-3xl font-bold">Shopping Cart</h1>
-                    {itemCount > 0 && (
-                        <button
-                            onClick={clearCart}
-                            className="text-sm text-red-600 hover:underline"
-                        >
-                            Clear cart
-                        </button>
-                    )}
-                </div>
-
-                {itemCount === 0 ? (
-                    <div className="text-center py-12">
-                        <p className="text-gray-500 mb-4">Your cart is empty</p>
-                        <a href="/" className="text-blue-600 hover:underline">Browse our selection</a>
-                    </div>
-                ) : (
-                    <>
-                        <div className="space-y-4 mb-8">
-                            {items.map(item => (
-                                <div
-                                    key={item.id}
-                                    className="flex justify-between items-start p-4 border rounded-lg hover:bg-gray-50"
-                                >
-                                    <div className="flex-1">
-                                        <div className="flex items-center gap-2 mb-1">
-                                            <span className="px-2 py-0.5 text-xs font-medium rounded bg-blue-100 text-blue-800">
-                                                {item.type}
-                                            </span>
-                                            <h3 className="font-semibold">{item.name}</h3>
-                                        </div>
-
-                                        {item.type === 'hotel' && (
-                                            <div className="text-sm text-gray-600">
-                                                <p>{String(item.metadata.roomType)}</p>
-                                                <p>{String(item.metadata.checkIn)} - {String(item.metadata.checkOut)}</p>
-                                                <p>{String(item.metadata.nights)} night{Number(item.metadata.nights) > 1 ? 's' : ''}</p>
-                                            </div>
-                                        )}
-
-                                        {item.type === 'airline' && (
-                                            <div className="text-sm text-gray-600">
-                                                <p>{String(item.metadata.cabinClass)} Class</p>
-                                                <p>{String((item.metadata.departure as any)?.airport)} → {String((item.metadata.arrival as any)?.airport)}</p>
-                                                <p>Duration: {String(item.metadata.duration)}</p>
-                                            </div>
-                                        )}
-                                    </div>
-
-                                    <div className="text-right ml-4">
-                                        <p className="text-xl font-bold mb-2">${item.price}</p>
-                                        <button
-                                            onClick={() => handleRemove(item.id, item.type)}
-                                            className="text-sm text-red-600 hover:underline"
-                                        >
-                                            Remove
-                                        </button>
-                                    </div>
-                                </div>
-                            ))}
-                        </div>
-
-                        <div className="border-t pt-4">
-                            <div className="flex justify-between items-center mb-4">
-                                <span className="text-xl font-semibold">Total</span>
-                                <span className="text-3xl font-bold">${total.toFixed(2)}</span>
-                            </div>
-                            <button
-                                onClick={() => {
-                                    dispatchInteraction('checkout_start', undefined, { total, itemCount });
-                                    window.location.href = '/checkout';
-                                }}
-                                className="w-full py-3 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors"
-                            >
-                                Proceed to Checkout
-                            </button>
-                        </div>
-                    </>
-                )}
-            </main>
-        </>
-    );
-}

web/src/app/hotel/checkout/page.tsx

@@ -1,11 +0,0 @@
-export default function HotelCheckout() {
-  return (
-    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-blue-50 to-indigo-50">
-      <div className="text-center p-8">
-        <h1 className="text-4xl font-light text-gray-800 mb-4">
-          Thank you for staying with us
-        </h1>
-      </div>
-    </div>
-  );
-}

web/src/app/hotel/products/[id]/page.tsx

@@ -1,106 +0,0 @@
-'use client';
-
-import { useState, useEffect } from 'react';
-import { useParams, useRouter } from 'next/navigation';
-import { Navigation } from '@/components/ui';
-import { useCart } from '@/contexts/CartContext';
-import HotelDetails from '@/components/feats/hotel/HotelDetails';
-import { transformProduct, type Hotel, type HotelProduct } from '@/lib/hotel-utils';
-import type { EventName } from '@/lib/events';
-
-const dispatchInteraction = (eventName: EventName, productId?: string, metadata?: Record<string, unknown>) => {
-    const e = new CustomEvent('definedInteraction', {
-        detail: { eventName, productId, metadata },
-    });
-    document.dispatchEvent(e);
-};
-
-export default function HotelProductPage() {
-    const params = useParams();
-    const router = useRouter();
-    const { addItem } = useCart();
-    const [product, setProduct] = useState<Hotel | null>(null);
-    const [loading, setLoading] = useState(true);
-    const [error, setError] = useState<string | null>(null);
-    const [added, setAdded] = useState(false);
-
-    const productId = params.id as string;
-
-    useEffect(() => {
-        const fetchProduct = async () => {
-            try {
-                const res = await fetch(`/api/products/${productId}`);
-                if (!res.ok) throw new Error(`Failed to fetch: ${res.status}`);
-                const json = await res.json();
-                const transformed = transformProduct(json.data as HotelProduct);
-                setProduct(transformed);
-
-                // fire learn_more_about_item event when product loads
-                dispatchInteraction('learn_more_about_item', productId, {
-                    type: 'hotel',
-                    dateIndex: transformed.dateIndex,
-                    roomType: transformed.roomType,
-                });
-            } catch (e) {
-                setError(e instanceof Error ? e.message : 'Failed to load product');
-                console.error('[FETCH_HOTEL_ERROR]', e);
-            } finally {
-                setLoading(false);
-            }
-        };
-        fetchProduct();
-    }, [productId]);
-
-    const handleAddToCart = () => {
-        if (!product) return;
-
-        addItem({
-            id: productId,
-            type: 'hotel',
-            name: product.name,
-            price: product.pricePerNight,
-            metadata: {
-                roomType: product.roomType,
-                nights: product.nights,
-                checkIn: product.checkIn,
-                checkOut: product.checkOut,
-            },
-            dateIndex: product.dateIndex,
-        });
-
-        dispatchInteraction('add_item_to_cart', productId, {
-            type: 'hotel',
-            price: product.pricePerNight,
-        });
-
-        setAdded(true);
-        setTimeout(() => setAdded(false), 2000);
-    };
-
-    return (
-        <>
-            <Navigation />
-            <main className="max-w-4xl mx-auto px-4 py-8">
-                {loading && <div className="text-center py-8">Loading hotel details...</div>}
-                {error && <div className="text-red-500 text-center py-8">{error}</div>}
-
-                {!loading && !error && product && (
-                    <>
-                        <button
-                            onClick={() => router.back()}
-                            className="mt-6 text-blue-600 hover:underline"
-                        >
-                            ← Back to rooms
-                        </button>
-                        <HotelDetails
-                            product={product}
-                            onAddToCart={handleAddToCart}
-                            addedToCart={added}
-                        />
-
-                    </>
-                )}
-            </main>
-        </>
-    );
-}

web/src/app/hotel/products/page.tsx

@@ -1,69 +1,74 @@
 'use client';
 
-import { useState, useEffect, Suspense } from 'react';
-import { useSearchParams } from 'next/navigation';
 import { Navigation } from '@/components/ui';
 import HotelCard from '@/components/feats/hotel/HotelCard';
-import { transformProduct, type Hotel, type HotelProduct } from '@/lib/hotel-utils';
 
-function RoomsList() {
+interface Hotel {
-  const searchParams = useSearchParams();
+  id: string;
-  const [rooms, setRooms] = useState<Hotel[]>([]);
+  name: string;
-  const [loading, setLoading] = useState(true);
+  roomType: string;
-  const [error, setError] = useState<string | null>(null);
+  checkIn: string;
-
+  checkOut: string;
-  useEffect(() => {
+  amenities: string[];
-    const fetchRooms = async () => {
+  refundable: boolean;
-      try {
+  pricePerNight: number;
-        const url = new URL('/api/products', window.location.origin);
+  nights: number;
-        url.searchParams.set('type', 'hotel');
-
-        // forward all relevant search params to the API
-        const params = ['dateIndex', 'destination', 'adults', 'rooms'];
-        params.forEach(param => {
-          const val = searchParams.get(param);
-          if (val) url.searchParams.set(param, val);
-        });
-
-        const res = await fetch(url.toString());
-        if (!res.ok) throw new Error(`Failed to fetch: ${res.status}`);
-        const json = await res.json();
-        const transformed = json.data.map((p: HotelProduct) => transformProduct(p));
-        setRooms(transformed);
-      } catch (e) {
-        setError(e instanceof Error ? e.message : 'Failed to load products');
-        console.error('[FETCH_ERROR]', e);
-      } finally {
-        setLoading(false);
 }
+
+const genRandomHotels = (): Hotel[] => {
+  const names = [
+    'Grand Plaza Hotel',
+    'Seaside Resort',
+    'Downtown Suites',
+    'Mountain View Lodge',
+    'City Center Inn',
+    'Luxury Beach Resort',
+    'Urban Boutique Hotel',
+    'Garden View Hotel',
+  ];
+  const roomTypes = ['Standard Room', 'Deluxe Room', 'Suite', 'Executive Suite', 'Premium Room'];
+  const amenities = ['wifi', 'pool', 'gym', 'parking', 'breakfast', 'spa'];
+
+  return Array.from({ length: 10 }, (_, i) => {
+    const nights = Math.floor(Math.random() * 5) + 1;
+    const basePrice = Math.floor(80 + Math.random() * 220);
+    const selectedAmenities = amenities
+      .sort(() => Math.random() - 0.5)
+      .slice(0, Math.floor(Math.random() * 3) + 2);
+
+    const today = new Date();
+    const checkInDate = new Date(today);
+    checkInDate.setDate(today.getDate() + Math.floor(Math.random() * 10));
+    const checkOutDate = new Date(checkInDate);
+    checkOutDate.setDate(checkInDate.getDate() + nights);
+
+    return {
+      id: `htl-${i}`,
+      name: names[i % names.length],
+      roomType: roomTypes[Math.floor(Math.random() * roomTypes.length)],
+      checkIn: checkInDate.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
+      checkOut: checkOutDate.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
+      amenities: selectedAmenities,
+      refundable: Math.random() > 0.5,
+      pricePerNight: basePrice,
+      nights,
+    };
+  });
 };
-    fetchRooms();
-  }, [searchParams]);
-
-  return (
-    <>
-      <h1 className="text-3xl font-bold mb-6">Available Rooms</h1>
-      {loading && <div className="text-center py-8">Loading...</div>}
-      {error && <div className="text-red-500 text-center py-8">{error}</div>}
-      {!loading && !error && (
-        <div className="space-y-4">
-          {rooms.map((r) => (
-            <HotelCard key={r.id} hotel={r} />
-          ))}
-        </div>
-      )}
-    </>
-  );
-}
 
 export default function HotelProducts() {
+  const hotels = genRandomHotels();
+
   return (
     <>
       <Navigation />
       <main className="max-w-7xl mx-auto px-4 py-8">
-        <Suspense fallback={<div className="text-center py-8">Loading...</div>}>
+        <h1 className="text-3xl font-bold mb-6">Available Hotels</h1>
-          <RoomsList />
+        <div className="space-y-4">
-        </Suspense>
+          {hotels.map((h) => (
+            <HotelCard key={h.id} hotel={h} />
+          ))}
+        </div>
       </main>
     </>
   );

web/src/app/layout.tsx

@@ -2,7 +2,6 @@ import type { Metadata } from "next";
 import { Geist, Geist_Mono } from "next/font/google";
 import "./globals.css";
 import { TrackingProvider } from "@/components/TrackingProvider";
-import { CartProvider } from "@/contexts/CartContext";
 
 const geistSans = Geist({
   variable: "--font-geist-sans",
@@ -29,9 +28,7 @@ export default function RootLayout({
       <body
         className={`${geistSans.variable} ${geistMono.variable} antialiased`}
       >
-        <CartProvider>
         <TrackingProvider>{children}</TrackingProvider>
-        </CartProvider>
       </body>
     </html>
   );

web/src/components/feats/airline/AirlineCard.tsx

@@ -1,7 +1,6 @@
 'use client';
 
 import type { EventName } from '@/lib/events';
-import type { Flight } from '@/lib/airline-utils';
 import { useHoverTracking } from '@/hooks/useHoverTracking';
 import PriceDisplay from '@/components/ui/PriceDisplay';
 
@@ -12,17 +11,32 @@ const dispatchInteraction = (eventName: EventName, productId?: string, metadata?
     document.dispatchEvent(e);
 };
 
+type CabinClass = 'economy' | 'premium' | 'business' | 'first';
+type FareRule = 'flexible' | 'standard' | 'basic';
+
+interface Flight {
+    id: string;
+    departure: { time: string; airport: string };
+    arrival: { time: string; airport: string };
+    duration: string;
+    stops: number;
+    cabinClass: CabinClass;
+    fareRule: FareRule;
+    refundable: boolean;
+    basePrice: number;
+}
+
 export default function AirlineCard({ flight }: { flight: Flight }) {
     const durationRef = useHoverTracking({
         eventName: 'hover_over_title',
         productId: flight.id,
-        metadata: { elementText: flight.duration, dateIndex: flight.dateIndex },
+        metadata: { elementText: flight.duration },
     });
 
     const priceRef = useHoverTracking({
         eventName: 'hover_over_paragraph',
         productId: flight.id,
-        metadata: { elementText: 'price', dateIndex: flight.dateIndex },
+        metadata: { elementText: 'price' },
     });
 
     const handleCardClick = () => {
@@ -30,9 +44,7 @@ export default function AirlineCard({ flight }: { flight: Flight }) {
             cabinClass: flight.cabinClass,
             fareRule: flight.fareRule,
             price: flight.basePrice,
-            dateIndex: flight.dateIndex,
         });
-        window.location.href = `/airline/products/${flight.id}`;
     };
 
     return (

web/src/components/feats/airline/AirlineDetails.tsx

@@ -1,75 +0,0 @@
-'use client';
-
-import type { Flight } from '@/lib/airline-utils';
-
-interface AirlineDetailsProps {
-  product: Flight;
-  onAddToCart: () => void;
-  addedToCart: boolean;
-}
-
-export default function AirlineDetails({ product, onAddToCart, addedToCart }: AirlineDetailsProps) {
-  return (
-    <div className="w-full flex flex-col lg:flex-row gap-12 py-8">
-      {/* Image Section */}
-      <div className="w-full lg:w-1/3 bg-gray-100 rounded-lg aspect-square flex items-center justify-center shrink-0">
-        <span className="text-gray-400 text-lg font-medium">Flight Image</span>
-      </div>
-
-      {/* Details Section */}
-      <div className="flex-1 flex flex-col">
-        <div className="flex justify-between items-start border-b pb-6 mb-6">
-          <div>
-            <h1 className="text-3xl font-bold text-gray-900 mb-1">{product.flightType}</h1>
-            <p className="text-lg text-gray-500">{product.cabinClass} Class</p>
-          </div>
-          <div className="text-right">
-            <p className="text-4xl font-bold text-gray-900">${product.basePrice}</p>
-            {product.refundable && (
-              <span className="inline-block mt-2 px-3 py-1 bg-green-50 text-green-700 rounded-full text-xs font-medium">
-                Refundable
-              </span>
-            )}
-          </div>
-        </div>
-
-        <div className="flex items-center justify-between mb-10">
-          <div className="text-center min-w-[100px]">
-            <p className="text-3xl font-bold text-gray-900">{product.departure.time}</p>
-            <p className="text-sm text-gray-500 font-medium mt-1">{product.departure.airport}</p>
-          </div>
-
-          <div className="flex-1 px-8 flex flex-col items-center">
-            <p className="text-sm text-gray-500 mb-2">{product.duration}</p>
-            <div className="w-full h-0.5 bg-gray-200 relative flex items-center justify-center">
-              <div className="absolute w-3 h-3 bg-gray-400 rounded-full"></div>
-            </div>
-            <p className="text-sm text-gray-500 mt-2">
-              {product.stops === 0 ? 'Nonstop' : `${product.stops} stop${product.stops > 1 ? 's' : ''}`}
-            </p>
-          </div>
-
-          <div className="text-center min-w-[100px]">
-            <p className="text-3xl font-bold text-gray-900">{product.arrival.time}</p>
-            <p className="text-sm text-gray-500 font-medium mt-1">{product.arrival.airport}</p>
-          </div>
-        </div>
-
-        <div className="mt-auto flex items-center justify-between pt-6 border-t">
-          <div className="text-gray-600">
-            <span className="font-bold text-gray-900">{product.availability}</span> seats remaining
-            <span className="mx-2">•</span>
-            {product.fareRule}
-          </div>
-          <button
-            onClick={onAddToCart}
-            disabled={addedToCart}
-            className="px-8 py-4 bg-black hover:bg-gray-800 disabled:bg-green-600 text-white rounded-lg text-lg font-medium transition-all min-w-[200px]"
-          >
-            {addedToCart ? 'In Cart' : 'Add to Cart'}
-          </button>
-        </div>
-      </div>
-    </div>
-  );
-}

web/src/components/feats/airline/AirlineHero.tsx

@@ -1,21 +1,9 @@
 'use client';
 
 import { useState, FormEvent } from 'react';
-import { useRouter } from 'next/navigation';
+import { Button, Label, Input, DateInput, RadioGroup, Dropdown, DropdownCounter } from '@/components/ui';
-import { Button, Label, DateInput, Dropdown, DropdownCounter, SelectDropdown, SelectOption } from '@/components/ui';
-import { dateToDaysFromToday } from '@/lib/airline-utils';
-
-const CITIES: SelectOption[] = [
-  { value: 'JFK', label: 'New York (JFK)', sublabel: 'John F. Kennedy International' },
-  { value: 'LAX', label: 'Los Angeles (LAX)', sublabel: 'Los Angeles International' },
-  { value: 'ORD', label: 'Chicago (ORD)', sublabel: "O'Hare International" },
-  { value: 'MIA', label: 'Miami (MIA)', sublabel: 'Miami International' },
-  { value: 'SFO', label: 'San Francisco (SFO)', sublabel: 'San Francisco International' },
-  { value: 'SEA', label: 'Seattle (SEA)', sublabel: 'Seattle-Tacoma International' },
-  { value: 'ATL', label: 'Atlanta (ATL)', sublabel: 'Hartsfield-Jackson International' },
-  { value: 'DFW', label: 'Dallas (DFW)', sublabel: 'Dallas/Fort Worth International' },
-];
 
+type TripType = 'roundtrip' | 'oneway' | 'multicity';
 
 const PlaneIcon = () => (
   <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
@@ -31,29 +19,16 @@ const LocationIcon = () => (
 );
 
 export default function AirlineHero() {
-  const router = useRouter();
+  const [tripType, setTripType] = useState<TripType>('roundtrip');
   const [origin, setOrigin] = useState('');
   const [destination, setDestination] = useState('');
   const [departDate, setDepartDate] = useState('');
+  const [returnDate, setReturnDate] = useState('');
   const [passengers, setPassengers] = useState({ adults: 1, children: 0, infants: 0 });
 
   const handleSearch = (e: FormEvent) => {
     e.preventDefault();
-    const params = new URLSearchParams();
+    console.log({ tripType, origin, destination, departDate, returnDate, passengers });
-
-    if (departDate) {
-      const daysOffset = dateToDaysFromToday(departDate);
-      params.set('dateIndex', daysOffset.toString());
-    }
-
-    if (origin) params.set('origin', origin);
-    if (destination) params.set('destination', destination);
-
-    params.set('adults', passengers.adults.toString());
-    params.set('children', passengers.children.toString());
-    params.set('infants', passengers.infants.toString());
-
-    router.push(`/airline/products?${params.toString()}`);
   };
 
   const totalPax = passengers.adults + passengers.children + passengers.infants;
@@ -72,15 +47,28 @@ export default function AirlineHero() {
 
         <div className="search-form">
           <form onSubmit={handleSearch}>
-            <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+            <div className="mb-6">
+              <RadioGroup
+                name="tripType"
+                value={tripType}
+                onChange={setTripType}
+                options={[
+                  { value: 'roundtrip', label: 'Round-trip' },
+                  { value: 'oneway', label: 'One-way' },
+                  { value: 'multicity', label: 'Multi-city' },
+                ]}
+              />
+            </div>
+
+            <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
               <div>
                 <Label htmlFor="origin">From</Label>
-                <SelectDropdown
+                <Input
+                  type="text"
                   id="origin"
                   value={origin}
-                  onChange={setOrigin}
+                  onChange={(e) => setOrigin(e.target.value)}
-                  options={CITIES}
+                  placeholder="Airport or city"
-                  placeholder="Select origin"
                   icon={<PlaneIcon />}
                   required
                 />
@@ -88,12 +76,12 @@ export default function AirlineHero() {
 
               <div>
                 <Label htmlFor="destination">To</Label>
-                <SelectDropdown
+                <Input
+                  type="text"
                   id="destination"
                   value={destination}
-                  onChange={setDestination}
+                  onChange={(e) => setDestination(e.target.value)}
-                  options={CITIES}
+                  placeholder="Airport or city"
-                  placeholder="Select destination"
                   icon={<LocationIcon />}
                   required
                 />
@@ -108,6 +96,20 @@ export default function AirlineHero() {
                   required
                 />
               </div>
+
+              <div>
+                <Label htmlFor="returnDate">Return</Label>
+                {tripType === 'roundtrip' ? (
+                  <DateInput
+                    id="returnDate"
+                    value={returnDate}
+                    onChange={(e) => setReturnDate(e.target.value)}
+                    required
+                  />
+                ) : (
+                  <DateInput id="returnDate" disabled />
+                )}
+              </div>
             </div>
 
             <div className="grid grid-cols-4 sm:grid-cols-3 lg:grid-cols-4 gap-4 mt-4">

web/src/components/feats/hotel/HotelCard.tsx

@@ -1,7 +1,6 @@
 'use client';
 
 import type { EventName } from '@/lib/events';
-import type { Hotel } from '@/lib/hotel-utils';
 import { useHoverTracking } from '@/hooks/useHoverTracking';
 import PriceDisplay from '@/components/ui/PriceDisplay';
 
@@ -12,6 +11,18 @@ const dispatchInteraction = (eventName: EventName, productId?: string, metadata?
     document.dispatchEvent(e);
 };
 
+interface Hotel {
+    id: string;
+    name: string;
+    roomType: string;
+    checkIn: string;
+    checkOut: string;
+    amenities: string[];
+    refundable: boolean;
+    pricePerNight: number;
+    nights: number;
+}
+
 const AmenityIcon = ({ name }: { name: string }) => {
     const iconMap: Record<string, string> = {
         wifi: 'Wi-Fi',
@@ -21,20 +32,20 @@ const AmenityIcon = ({ name }: { name: string }) => {
         breakfast: 'Breakfast',
         spa: 'Spa',
     };
-    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name.replaceAll("_", " ")}</span>;
+    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name}</span>;
 };
 
 export default function HotelCard({ hotel }: { hotel: Hotel }) {
     const titleRef = useHoverTracking({
         eventName: 'hover_over_title',
         productId: hotel.id,
-        metadata: { elementText: hotel.name, dateIndex: hotel.dateIndex },
+        metadata: { elementText: hotel.name },
     });
 
     const priceRef = useHoverTracking({
         eventName: 'hover_over_paragraph',
         productId: hotel.id,
-        metadata: { elementText: 'price', dateIndex: hotel.dateIndex },
+        metadata: { elementText: 'price' },
     });
 
     const handleCardClick = () => {
@@ -42,36 +53,21 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
             roomType: hotel.roomType,
             price: hotel.pricePerNight,
             nights: hotel.nights,
-            dateIndex: hotel.dateIndex,
         });
-        window.location.href = `/hotel/products/${hotel.id}`;
     };
 
-    const imageUrl = `https://images.unsplash.com/photo-1551882547-ff40c63fe5fa?w=400&h=300&fit=crop`;
-
     return (
         <div
             className="hotel-card cursor-pointer"
             onClick={handleCardClick}
         >
-            <div className="hotel-image relative overflow-hidden">
+            <div className="hotel-image bg-gray-200 flex items-center justify-center">
-                <img
-                    src={imageUrl}
-                    alt={hotel.name}
-                    className="w-full h-full object-cover"
-                    onError={(e) => {
-                        e.currentTarget.style.display = 'none';
-                        const fallback = e.currentTarget.nextElementSibling as HTMLElement;
-                        if (fallback) fallback.style.display = 'flex';
-                    }}
-                />
-                <div className="absolute inset-0 bg-gray-200 flex items-center justify-center" style={{ display: 'none' }}>
                 <span className="text-gray-400 text-sm">Image</span>
             </div>
-            </div>
 
             <div className="hotel-info">
                 <h3 ref={titleRef} className="hotel-name">{hotel.name}</h3>
+                <div className="hotel-location text-sm mb-2">{hotel.roomType}</div>
                 <div className="text-sm text-[var(--text-secondary)] mb-2">
                     {hotel.checkIn} - {hotel.checkOut}
                 </div>
@@ -80,6 +76,9 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
                         <AmenityIcon key={a} name={a} />
                     ))}
                 </div>
+                {hotel.refundable && (
+                    <div className="free-cancellation mt-2">Free cancellation</div>
+                )}
             </div>
 
             <div className="hotel-pricing">

web/src/components/feats/hotel/HotelDetails.tsx

@@ -1,113 +0,0 @@
-'use client';
-
-import { useState, useEffect } from 'react';
-import type { Hotel } from '@/lib/hotel-utils';
-import PriceDisplay from '@/components/ui/PriceDisplay';
-
-interface HotelDetailsProps {
-  product: Hotel;
-  onAddToCart: () => void;
-  addedToCart: boolean;
-}
-
-const PriceTotalDisplay = ({ productId, nights }: { productId: string; nights: number }) => {
-  const [price, setPrice] = useState<number | null>(null);
-
-  useEffect(() => {
-    const fetchPrice = async () => {
-      try {
-        const sessionRes = await fetch('/api/session');
-        const sessionData = await sessionRes.json();
-        const params = new URLSearchParams({
-          productId,
-          sessionId: sessionData.sessionId || '',
-          experimentId: sessionData.experimentId || '',
-        });
-        const res = await fetch(`/api/pricing?${params.toString()}`);
-        const data = await res.json();
-        setPrice(data.price);
-      } catch (err) {
-        console.error('failed to fetch price for total:', err);
-      }
-    };
-    fetchPrice();
-  }, [productId]);
-
-  if (!price) return <span className="text-4xl font-bold text-gray-900">Loading...</span>;
-
-  return (
-    <span className="text-4xl font-bold text-gray-900">
-      ${(price * nights).toFixed(2)}
-    </span>
-  );
-};
-
-export default function HotelDetails({ product, onAddToCart, addedToCart }: HotelDetailsProps) {
-  const imageUrl = `https://images.unsplash.com/photo-1566073771259-6a8506099945?w=800&h=600&fit=crop`;
-
-  return (
-    <div className="w-full flex flex-col lg:flex-row gap-12 py-8">
-      <div className="w-full lg:w-1/2 rounded-lg aspect-[4/3] overflow-hidden shrink-0">
-        <img
-          src={imageUrl}
-          alt={product.name}
-          className="w-full h-full object-cover"
-          onError={(e) => {
-            e.currentTarget.style.display = 'none';
-            if (e.currentTarget.nextElementSibling) {
-              (e.currentTarget.nextElementSibling as HTMLElement).style.display = 'flex';
-            }
-          }}
-        />
-        <div className="w-full h-full bg-gray-100 rounded-lg flex items-center justify-center" style={{ display: 'none' }}>
-          <span className="text-gray-400 text-lg font-medium">Hotel Image</span>
-        </div>
-      </div>
-
-      <div className="flex-1 flex flex-col">
-        <div className="border-b pb-6 mb-6">
-          <h1 className="text-4xl font-bold text-gray-900 mb-2">{product.name}</h1>
-        </div>
-
-        <div className="grid grid-cols-2 gap-8 mb-8">
-          <div>
-            <h3 className="text-sm font-semibold text-gray-900 uppercase tracking-wider mb-2">Check-in</h3>
-            <p className="text-lg text-gray-700">{product.checkIn}</p>
-          </div>
-          <div>
-            <h3 className="text-sm font-semibold text-gray-900 uppercase tracking-wider mb-2">Check-out</h3>
-            <p className="text-lg text-gray-700">{product.checkOut}</p>
-          </div>
-        </div>
-
-        <div className="mb-8">
-          <h3 className="text-sm font-semibold text-gray-900 uppercase tracking-wider mb-3">Amenities</h3>
-          <div className="flex flex-wrap gap-3">
-            {product.amenities.map(a => (
-              <span key={a} className="px-3 py-1.5 bg-gray-100 text-gray-700 rounded-md text-sm font-medium">
-                  {a.replaceAll('_', ' ')}
-              </span>
-            ))}
-          </div>
-        </div>
-
-        <div className="mt-auto pt-6 border-t flex items-center justify-between">
-          <div>
-            <p className="text-sm text-gray-500 mb-1">Price per night</p>
-            <div className="mb-3">
-              <PriceDisplay productId={product.id} className="!text-2xl" />
-            </div>
-          </div>
-
-          <button
-            onClick={onAddToCart}
-            disabled={addedToCart}
-            className="px-8 py-4 bg-black hover:bg-gray-800 disabled:bg-green-600 text-white rounded-lg text-lg font-medium transition-all min-w-[200px]"
-          >
-            {addedToCart ? 'In Cart' : 'Add to Cart'}
-          </button>
-        </div>
-      </div>
-    </div>
-  );
-}

web/src/components/feats/hotel/HotelHero.tsx

@@ -1,9 +1,7 @@
 'use client';
 
 import { useState, FormEvent } from 'react';
-import { useRouter } from 'next/navigation';
 import { Button, Label, Input, DateInput, Dropdown, DropdownCounter } from '@/components/ui';
-import { dateToDaysFromToday } from '@/lib/hotel-utils';
 
 const LocationIcon = () => (
   <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
@@ -13,25 +11,14 @@ const LocationIcon = () => (
 );
 
 export default function HotelHero() {
-  const router = useRouter();
   const [destination, setDestination] = useState('');
   const [checkIn, setCheckIn] = useState('');
+  const [checkOut, setCheckOut] = useState('');
   const [guests, setGuests] = useState({ adults: 2, rooms: 1 });
 
   const handleSearch = (e: FormEvent) => {
     e.preventDefault();
-    const params = new URLSearchParams();
+    console.log({ destination, checkIn, checkOut, guests });
-
-    if (checkIn) {
-      const daysOffset = dateToDaysFromToday(checkIn);
-      params.set('dateIndex', daysOffset.toString());
-    }
-
-    if (destination) params.set('destination', destination);
-    params.set('adults', guests.adults.toString());
-    params.set('rooms', guests.rooms.toString());
-
-    router.push(`/hotel/products?${params.toString()}`);
   };
 
   return (
@@ -39,16 +26,16 @@ export default function HotelHero() {
       <div className="w-full max-w-4xl px-4">
         <div className="text-center mb-8">
           <h1 className="text-4xl md:text-5xl font-bold mb-4">
-            Find your perfect room
+            Find your perfect stay
           </h1>
           <p className="text-lg">
-            Search rooms, compare prices, and book with confidence
+            Search hotels, compare prices, and book with confidence
           </p>
         </div>
 
         <form onSubmit={handleSearch} className="search-form">
-          <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+          <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
-            <div>
+            <div className="sm:col-span-2">
               <Label htmlFor="destination">Where to?</Label>
               <Input
                 type="text"
@@ -62,7 +49,7 @@ export default function HotelHero() {
             </div>
 
             <div>
-              <Label htmlFor="checkIn">Date (1 night stay)</Label>
+              <Label htmlFor="checkIn">Check-in</Label>
               <DateInput
                 id="checkIn"
                 value={checkIn}
@@ -72,27 +59,43 @@ export default function HotelHero() {
             </div>
 
             <div>
-              <Label htmlFor="guests">Guests</Label>
+              <Label htmlFor="checkOut">Check-out</Label>
-              <Dropdown label={`${guests.adults} ${guests.adults === 1 ? 'adult' : 'adults'}`}>
+              <DateInput
+                id="checkOut"
+                value={checkOut}
+                onChange={(e) => setCheckOut(e.target.value)}
+                required
+              />
+            </div>
+
+            <div className="sm:col-span-2 lg:col-span-4">
+              <Label htmlFor="guests">Guests & Rooms</Label>
+              <Dropdown label={`${guests.adults} ${guests.adults === 1 ? 'adult' : 'adults'}, ${guests.rooms} ${guests.rooms === 1 ? 'room' : 'rooms'}`}>
                 <DropdownCounter
                   label="Adults"
                   value={guests.adults}
                   min={1}
                   onChange={(v) => setGuests({ ...guests, adults: v })}
                 />
+                <DropdownCounter
+                  label="Rooms"
+                  value={guests.rooms}
+                  min={1}
+                  onChange={(v) => setGuests({ ...guests, rooms: v })}
+                />
               </Dropdown>
             </div>
 
-            <div className="sm:col-span-2 lg:col-span-3">
+            <div className="sm:col-span-2 lg:col-span-4">
               <Button type="submit" fullWidth>
-                Search Rooms
+                Search Hotels
               </Button>
             </div>
           </div>
         </form>
 
         <div className="mt-6 text-center text-sm">
-          <p>Over 2 million rooms worldwide · Best price guarantee · Free cancellation on most bookings</p>
+          <p>Over 2 million hotels worldwide · Best price guarantee · Free cancellation on most bookings</p>
         </div>
       </div>
     </div>

web/src/components/ui/DateInput.tsx

@@ -1,29 +1,7 @@
-import { InputHTMLAttributes, useMemo } from 'react';
+import { InputHTMLAttributes } from 'react';
 
 interface DateInpProps extends Omit<InputHTMLAttributes<HTMLInputElement>, 'type'> {}
 
 export default function DateInput({ className = '', ...props }: DateInpProps) {
-  const { minDate, maxDate } = useMemo(() => {
+  return <input type="date" className={`input-field ${className}`.trim()} {...props} />;
-    const today = new Date();
-    const tomorrow = new Date(today);
-    tomorrow.setDate(today.getDate() + 1);
-
-    const tenDaysOut = new Date(tomorrow);
-    tenDaysOut.setDate(tomorrow.getDate() + 9); // tomorrow + 9 = 10 days total
-
-    return {
-      minDate: tomorrow.toISOString().split('T')[0],
-      maxDate: tenDaysOut.toISOString().split('T')[0]
-    };
-  }, []);
-
-  return (
-    <input
-      type="date"
-      className={`input-field ${className}`.trim()}
-      min={minDate}
-      max={maxDate}
-      {...props}
-    />
-  );
 }

web/src/components/ui/Navigation.tsx

@@ -20,7 +20,7 @@ const NavLink = ({ href, children }: { href: string; children: React.ReactNode }
       href={href}
       className={`px-4 py-2 rounded-md transition-colors ${
         isActive
-          ? 'bg-[var(--accent-primary)] font-semibold'
+          ? 'bg-[var(--accent-primary)] text-white font-semibold'
           : 'hover:bg-[var(--accent-primary-light)] text-[var(--text-primary)]'
       }`}
     >
@@ -37,7 +37,9 @@ export default function Navigation() {
           <div className="flex items-center space-x-1">
             <NavLink href="/">Home</NavLink>
             <NavLink href="/products">Products</NavLink>
+            <NavLink href="/search">Search</NavLink>
             <NavLink href="/cart">Cart</NavLink>
+            <NavLink href="/checkout">Checkout</NavLink>
           </div>
         </div>
       </div>

web/src/components/ui/SelectDropdown.tsx

@@ -1,119 +0,0 @@
-'use client';
-
-import { useState, useRef, useEffect, ReactNode } from 'react';
-
-export interface SelectOption {
-  value: string;
-  label: string;
-  sublabel?: string;
-}
-
-interface SelectDropdownProps {
-  value: string;
-  onChange: (value: string) => void;
-  options: SelectOption[];
-  placeholder?: string;
-  icon?: ReactNode;
-  required?: boolean;
-  id?: string;
-}
-
-export default function SelectDropdown({
-  value,
-  onChange,
-  options,
-  placeholder = 'Select...',
-  icon,
-  required,
-  id,
-}: SelectDropdownProps) {
-  const [open, setOpen] = useState(false);
-  const [filter, setFilter] = useState('');
-  const ref = useRef<HTMLDivElement>(null);
-  const inputRef = useRef<HTMLInputElement>(null);
-
-  useEffect(() => {
-    const handleClick = (e: MouseEvent) => {
-      if (ref.current && !ref.current.contains(e.target as Node)) {
-        setOpen(false);
-        setFilter('');
-      }
-    };
-    document.addEventListener('mousedown', handleClick);
-    return () => document.removeEventListener('mousedown', handleClick);
-  }, []);
-
-  const selectedOption = options.find((o) => o.value === value);
-  const filtered = options.filter(
-    (o) =>
-      o.label.toLowerCase().includes(filter.toLowerCase()) ||
-      o.value.toLowerCase().includes(filter.toLowerCase()) ||
-      o.sublabel?.toLowerCase().includes(filter.toLowerCase())
-  );
-
-  const handleSelect = (opt: SelectOption) => {
-    onChange(opt.value);
-    setOpen(false);
-    setFilter('');
-  };
-
-  return (
-    <div className="relative" ref={ref}>
-      <div
-        className="input-field flex items-center gap-2 cursor-pointer box-border"
-        onClick={() => {
-          setOpen(true);
-          setTimeout(() => inputRef.current?.focus(), 0);
-        }}
-      >
-        {icon && <span className="text-[var(--text-secondary)]">{icon}</span>}
-        {open ? (
-          <input
-            ref={inputRef}
-            type="text"
-            id={id}
-            value={filter}
-            onChange={(e) => setFilter(e.target.value)}
-            placeholder={placeholder}
-            className="flex-1 bg-transparent outline-none text-sm text-[var(--text-primary)]"
-          />
-        ) : (
-          <span className={`flex-1 text-sm ${value ? 'text-[var(--text-primary)]' : 'text-[var(--text-secondary)]'}`}>
-            {selectedOption ? selectedOption.label : placeholder}
-          </span>
-        )}
-        <svg
-          className={`w-4 h-4 text-[var(--text-secondary)] transition-transform ${open ? 'rotate-180' : ''}`}
-          fill="none"
-          stroke="currentColor"
-          viewBox="0 0 24 24"
-        >
-          <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" />
-        </svg>
-      </div>
-      {open && (
-        <div className="absolute z-20 mt-1 w-full bg-[var(--bg-primary)] border-2 border-[var(--accent-primary)] rounded-md shadow-lg max-h-60 overflow-y-auto">
-          {filtered.length === 0 ? (
-            <div className="px-4 py-3 text-sm text-[var(--text-secondary)]">No results</div>
-          ) : (
-            filtered.map((opt) => (
-              <div
-                key={opt.value}
-                onClick={() => handleSelect(opt)}
-                className={`px-4 py-2 cursor-pointer transition-colors hover:bg-[var(--accent-primary-light)] ${
-                  opt.value === value ? 'bg-[var(--accent-primary-light)]' : ''
-                }`}
-              >
-                <div className="text-sm font-medium text-[var(--text-primary)]">{opt.label}</div>
-                {opt.sublabel && <div className="text-xs text-[var(--text-secondary)]">{opt.sublabel}</div>}
-              </div>
-            ))
-          )}
-        </div>
-      )}
-      {required && !value && (
-        <input type="text" required className="sr-only" tabIndex={-1} value="" onChange={() => {}} />
-      )}
-    </div>
-  );
-}