feature: e2e pricing pipeline with inference

.gitignore

@@ -6,8 +6,3 @@
 **/session_*.svg
 **/*graph.svg
 paper/src/bib/auto
-
-# Airflow logs - exclude DAG run logs
-experiments/airflow/logs/*
-experiments/airflow/logs/scheduler/
-experiments/airflow/logs/dag_processor_manager/

Makefile

@@ -49,8 +49,4 @@ install: $(VENV)
 test: $(VENV)
 	$(PYTEST) -v
 
-count-lines:
-	@find . \( -path '*/node_modules' -o -path '*/.venv' -o -path '*/venv' \) -prune -o \
-	\( -name "*.ts" -o -name "*.py" \) -type f -print0 | xargs -0 cat | wc -l
-
 .PHONY: all pdf clean watch run.webapp install test

README.md

@@ -1,6 +1,3 @@
-
-<img width="1952" height="2176" alt="nobody_knows" src="https://github.com/user-attachments/assets/d148b00d-e9f9-4280-89cc-0cc866e17251" />
-
 [![Build PDF](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml/badge.svg)](https://github.com/velocitatem/PHANTOM/actions/workflows/latex.yml)
 
 - https://phantom-hotel.vercel.app/

backend/provider/app.py

@@ -1,113 +0,0 @@
-from fastapi import FastAPI, HTTPException, Query
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-from typing import Literal, Optional
-import uvicorn, os, sys
-from supabase import create_client, Client
-from dotenv import load_dotenv
-import numpy as np
-import pandas as pd
-load_dotenv()
-
-# Local imports of registry and pricing function
-
-sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../experiments/")
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.pricers import (
-    StaticPricer,
-    RandomPricer,
-    ElasticityBasedPricer
-)
-from procesing.steps import (
-    PredictPricesStep
-)
-from procesing import PipelineContext
-sys.path.append(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
-print(os.path.dirname(os.path.abspath(__file__))+ "/../../lib/")
-from lib.model_registry import ModelRegistry
-
-# Config
-app = FastAPI(title="PHANTOM Pricing Provider")
-app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
-
-supabase: Client = create_client(os.getenv("NEXT_PUBLIC_SUPABASE_URL"), os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY"))
-registry = ModelRegistry()
-
-class PriceResponse(BaseModel):
-    productId: str
-    price: float
-    base_price: float
-    markup: float
-    elasticity: Optional[float] = None
-    model_version: str = 'latest'
-
-@app.get("/health")
-def health() -> dict:
-    return {"status": "healthy", "redis": registry.health_check()}
-
-@app.get("/api/{mode}/price/{productId}", response_model=PriceResponse)
-def get_price(mode: Literal['hotel', 'airline'], productId: str, sessionId: Optional[str] = Query(None), experimentId: Optional[str] = Query(None)):
-    product = supabase.table(f'{mode}_products').select("metadata").eq('id', productId).execute().data[0]
-    if not product: raise HTTPException(404, f"Product {productId} not found")
-
-    metadata = product['metadata']
-    base_price = metadata.get('base_price', 100.0)
-
-    # fetch pre-computed prices from registry
-    prices_df = registry.get_prices('latest')
-    elasticity_df = registry.get_elasticity('latest')
-
-    if prices_df is None:
-        # fallback: no pre-computed prices available
-        return PriceResponse(
-            productId=productId,
-            price=base_price,
-            base_price=base_price,
-            markup=1.0,
-            elasticity=None
-        )
-
-    # lookup pre-computed price for this product
-    product_price_row = prices_df[prices_df['productId'] == productId]
-    if product_price_row.empty:
-        # product not in pre-computed prices, fallback to base
-        return PriceResponse(
-            productId=productId,
-            price=base_price,
-            base_price=base_price,
-            markup=1.0,
-            elasticity=None
-        )
-
-    optimal_price = float(product_price_row['optimal_price'].iloc[0]) # TODO: use optimal_price everywhere as  aresult
-
-    # get elasticity if available
-    product_elasticity = None
-    if elasticity_df is not None:
-        product_elasticity_row = elasticity_df[elasticity_df['productId'] == productId]
-        if not product_elasticity_row.empty:
-            product_elasticity = float(product_elasticity_row['elasticity'].iloc[0])
-
-    return PriceResponse(
-        productId=productId,
-        price=optimal_price,
-        base_price=base_price,
-        markup=optimal_price/base_price,
-        elasticity=product_elasticity
-    )
-
-@app.get("/models")
-def list_models(): return registry.list_models()
-
-@app.post("/models/reload")
-def reload_models():
-    elasticity, pricing_model = registry.get_elasticity('latest'), registry.get_pricing_model('latest')
-    return {
-        "elasticity_loaded": bool(elasticity),
-        "n_products": len(elasticity) if elasticity is not None else 0,
-        "pricing_model_loaded": bool(pricing_model),
-        "model_class": pricing_model.__class__.__name__ if pricing_model else None
-    }
-
-if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PROVIDER_PORT", "5001")))

backend/provider/requirements.txt

@@ -1,16 +0,0 @@
-fastapi
-uvicorn[standard]
-pydantic
-numpy
-pandas
-scikit-learn
-redis
-supabase
-confluent-kafka>=2.3.0
-kafka-python
-graphviz
-python-dotenv>=1.0.0
-requests>=2.31.0
-typing-extensions>=4.8.0
-pypickle
-pymc

backend/server/app.py

@@ -290,7 +290,6 @@ async def get_products(
         query = supabase.table(table).select('*')
 
         # filter by exact date_index if provided
-        # dateIndex from frontend is days from today, convert to days since epoch
         if dateIndex is not None:
             query = query.eq('date_index', dateIndex)
 

docker-compose.yml

@@ -71,149 +71,6 @@ services:
       - "${REDPANDA_CONSOLE_PORT:-8080}:8080"
     restart: unless-stopped
 
-  postgres:
-    container_name: "PHANTOM-postgres"
-    image: postgres:13
-    environment:
-      - POSTGRES_USER=airflow
-      - POSTGRES_PASSWORD=airflow
-      - POSTGRES_DB=airflow
-    ports:
-      - "5433:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    restart: unless-stopped
-
-  airflow-init:
-    container_name: "PHANTOM-airflow-init"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      - postgres
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - _AIRFLOW_DB_MIGRATE=true
-      - _AIRFLOW_WWW_USER_CREATE=true
-      - _AIRFLOW_WWW_USER_USERNAME=admin
-      - _AIRFLOW_WWW_USER_PASSWORD=admin
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    volumes:
-      - ./experiments/airflow/dags:/opt/airflow/dags
-      - ./experiments/airflow/logs:/opt/airflow/logs
-      - ./experiments/airflow/plugins:/opt/airflow/plugins
-      - ./experiments/procesing:/opt/airflow/procesing
-      - ./lib:/opt/airflow/lib
-    command: version
-    restart: "no"
-
-  airflow-webserver:
-    container_name: "PHANTOM-airflow-webserver"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      - postgres
-      - airflow-init
-      - redis
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - BACKEND_URL=http://backend:5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    ports:
-      - "${AIRFLOW_WEBSERVER_PORT:-8085}:8080"
-    volumes:
-      - ./experiments/airflow/dags:/opt/airflow/dags:ro
-      - ./experiments/airflow/logs:/opt/airflow/logs
-      - ./experiments/airflow/plugins:/opt/airflow/plugins:ro
-      - ./experiments/procesing:/opt/airflow/procesing:ro
-      - ./lib:/opt/airflow/lib:ro
-    command: webserver
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 30s
-
-  airflow-scheduler:
-    container_name: "PHANTOM-airflow-scheduler"
-    build:
-      context: .
-      dockerfile: docker/Airflow.dockerfile
-    depends_on:
-      airflow-webserver:
-        condition: service_healthy
-      redis:
-        condition: service_started
-    environment:
-      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
-      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
-      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
-      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
-      - AIRFLOW__CORE__LOAD_EXAMPLES=false
-      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - BACKEND_URL=http://backend:5000
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-    volumes:
-      - ./experiments/airflow/dags:/opt/airflow/dags:ro
-      - ./experiments/airflow/logs:/opt/airflow/logs
-      - ./experiments/airflow/plugins:/opt/airflow/plugins:ro
-      - ./experiments/procesing:/opt/airflow/procesing:ro
-      - ./lib:/opt/airflow/lib:ro
-    command: scheduler
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 30s
-
-  pricing-provider:
-    container_name: "PHANTOM-pricing-provider"
-    build:
-      context: .
-      dockerfile: docker/Provider.dockerfile
-    depends_on:
-      - redis
-      - kafka
-    environment:
-      - PROVIDER_PORT=5001
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-      - KAFKA_HOST=kafka
-      - KAFKA_PORT=29092
-      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
-      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
-      - BACKEND_URL=http://localhost:5000
-    ports:
-      - "${PROVIDER_PORT:-5001}:5001"
-    restart: unless-stopped
-
 volumes:
   phantom_kafka_data:
   phantom_redis_data:
-  postgres_data:

docker/Airflow.dockerfile

@@ -1,23 +0,0 @@
-FROM apache/airflow:2.7.3-python3.11
-
-USER root
-
-# install system deps if needed
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-USER airflow
-
-# copy requirements for pipeline dependencies
-COPY requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
-
-# install postgres driver and providers
-RUN pip install --no-cache-dir \
-    psycopg2-binary \
-    apache-airflow-providers-postgres
-
-# set airflow home
-ENV AIRFLOW_HOME=/opt/airflow

docker/Provider.dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-# Install system dependencies including graphviz
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    graphviz \
-    libgraphviz-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Copy and install Python dependencies
-COPY backend/provider/requirements.txt /app/
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy application code into image
-COPY lib/ /app/lib/
-COPY experiments/procesing/ /app/procesing/
-COPY backend/provider/ /app/provider/
-
-ENV PYTHONPATH=/app:/app/lib:/app/procesing
-
-WORKDIR /app/provider
-
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5001"]

experiments/README.md

@@ -1,8 +0,0 @@
-
-# Products
-# Agents
-# Pipeline
-
-Our pipeline technically should follow principles in a style like this:
-- Each step should be defined as an inheriting child of an scikit pipeline step, the granularity of the steps is dictated by the following: a step should be a transformation, augmentation or computation independently, no single stage should run multiple in-itself. This way we can modularize properly all the components and track properly in airflow. A stage can be defined as an sklearn step but then must be transalted to a function that takes the context in our DAG of airflow. All parametrization must be done via contexts.
-

experiments/airflow/dags/surge_pricing_pipeline.py

@@ -1,237 +0,0 @@
-from airflow import DAG
-from airflow.operators.python import PythonOperator
-from airflow.utils.dates import days_ago
-from datetime import timedelta
-import pandas as pd
-import logging
-import sys
-import pickle
-import io
-
-# add parent dir to path so procesing package can be imported
-sys.path.insert(0, '/opt/airflow')
-
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    ComputeDemandStep,
-    AggregatePriceLogsStep,
-    JoinProductFeaturesStep,
-)
-from procesing.pricers.simple import SimpleSurgePricer
-
-default_args = {
-    'owner': 'phantom-research',
-    'depends_on_past': False,
-    'email_on_failure': False,
-    'email_on_retry': False,
-    'retries': 2,
-    'retry_delay': timedelta(minutes=5),
-}
-
-def get_provider():
-    """Factory to create composite provider"""
-    class CompositeProvider(SupabaseProvider, BackendAPIProvider): # TODO: Fix this into one global provider singelton instead of multiple inheritance declarations acoss the codebase
-        def __init__(self):
-            SupabaseProvider.__init__(self)
-            BackendAPIProvider.__init__(self)
-    return CompositeProvider()
-
-def get_context(**kwargs):
-    """Build pipeline context from Airflow config"""
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-    return PipelineContext(
-        provider=get_provider(),
-        store_mode=dag_conf.get('store_mode', 'hotel'),
-    )
-
-# atomic task functions (each wraps one sklearn step)
-def fetch_interactions(**kwargs):
-    """Task: Fetch interaction data from Kafka"""
-    context = get_context(**kwargs)
-    step = FetchInteractionsStep(context)
-    df = step.transform(None)
-
-    kwargs['ti'].xcom_push(key='interactions_raw', value=pickle.dumps(df))
-    logging.info(f"Fetched {len(df)} interaction records")
-    return len(df)
-
-def fetch_price_logs(**kwargs):
-    """Task: Fetch price logs from Kafka"""
-    context = get_context(**kwargs)
-    step = FetchPriceLogsStep(context)
-    df = step.transform(None)
-
-    kwargs['ti'].xcom_push(key='price_logs_raw', value=pickle.dumps(df))
-    logging.info(f"Fetched {len(df)} price records")
-    return len(df)
-
-def compute_demand(**kwargs):
-    """Task: Compute demand scores from interactions"""
-    ti = kwargs['ti']
-    df = pickle.loads(ti.xcom_pull(key='interactions_raw'))
-
-    context = get_context(**kwargs)
-    step = ComputeDemandStep(context)
-    demand_df = step.transform(df)
-    # TODO: clear the xcom
-
-
-    ti.xcom_push(key='demand_data', value=pickle.dumps(demand_df))
-    logging.info(f"Computed demand for {len(demand_df)} products")
-    return len(demand_df)
-
-def aggregate_price_logs(**kwargs):
-    """Task: Aggregate price logs"""
-    ti = kwargs['ti']
-    df = pickle.loads(ti.xcom_pull(key='price_logs_raw'))
-
-    context = get_context(**kwargs)
-    step = AggregatePriceLogsStep(context)
-    price_df = step.transform(df)
-
-    ti.xcom_push(key='price_data', value=pickle.dumps(price_df))
-    logging.info(f"Aggregated price logs for {len(price_df)} products")
-    return len(price_df)
-
-def join_product_features(**kwargs):
-    """Task: Join demand and price data"""
-    ti = kwargs['ti']
-    demand_df = pickle.loads(ti.xcom_pull(key='demand_data'))
-    price_df = pickle.loads(ti.xcom_pull(key='price_data'))
-
-    context = get_context(**kwargs)
-    step = JoinProductFeaturesStep(context)
-    joined_df = step.transform((demand_df, price_df))
-
-    ti.xcom_push(key='product_features', value=pickle.dumps(joined_df))
-    logging.info(f"Joined features for {len(joined_df)} products")
-    return len(joined_df)
-
-def apply_surge_pricing(**kwargs):
-    """Task: Apply surge pricing rules to generate optimal prices"""
-    ti = kwargs['ti']
-    product_features = pickle.loads(ti.xcom_pull(key='product_features'))
-
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-    # rename demand_score to demand for pricer compatibility
-    data = product_features.rename(columns={'demand_score': 'demand'})
-
-    surge_pricer = SimpleSurgePricer(
-        high_threshold=dag_conf.get('high_threshold', 10),
-        low_threshold=dag_conf.get('low_threshold', 2),
-        surge_multiplier=dag_conf.get('surge_multiplier', 1.2),
-        discount_multiplier=dag_conf.get('discount_multiplier', 0.9)
-    )
-    surge_pricer.fit(data)
-    data['optimal_price'] = surge_pricer.predict()
-
-    prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
-        'price': 'current_price',
-        'demand': 'demand_score'
-    })
-
-    ti.xcom_push(key='predicted_prices', value=pickle.dumps(prices_df))
-    logging.info(f"Applied surge pricing for {len(prices_df)} products")
-    return len(prices_df)
-
-def publish_results(**kwargs):
-    """Task: Publish surge pricing results to registry"""
-    ti = kwargs['ti']
-    prices_df = pickle.loads(ti.xcom_pull(key='predicted_prices'))
-
-    sys.path.insert(0, '/opt/airflow')
-    from lib.model_registry import ModelRegistry
-
-    registry = ModelRegistry()
-    dag_conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
-
-    metadata = {
-        'timestamp': pd.Timestamp.now().isoformat(),
-        'store_mode': dag_conf.get('store_mode', 'hotel'),
-        'dag_run_id': kwargs['dag_run'].run_id if kwargs.get('dag_run') else 'manual',
-        'pricing_method': 'surge',
-        'high_threshold': dag_conf.get('high_threshold', 10),
-        'low_threshold': dag_conf.get('low_threshold', 2),
-        'surge_multiplier': dag_conf.get('surge_multiplier', 1.2),
-        'discount_multiplier': dag_conf.get('discount_multiplier', 0.9)
-    }
-
-    registry.publish_prices(prices_df, model_name='latest', metadata=metadata)
-
-    logging.info(f"Published surge pricing for {len(prices_df)} products")
-
-    return {
-        'n_products': len(prices_df),
-        'registry_status': 'success',
-        'mean_demand': float(prices_df['demand_score'].mean()) if 'demand_score' in prices_df.columns else None
-    }
-
-
-# DAG definition
-with DAG(
-    'surge_pricing_pipeline',
-    default_args=default_args,
-    description='Simple surge pricing pipeline: demand aggregation + rule-based pricing',
-    schedule_interval='*/15 * * * *',
-    start_date=days_ago(1),
-    catchup=False,
-    max_active_runs=1,
-    tags=['pricing', 'surge', 'research', 'simplified'],
-) as dag:
-
-    # parallel data fetching
-    t_fetch_interactions = PythonOperator(
-        task_id='fetch_interactions',
-        python_callable=fetch_interactions,
-        provide_context=True,
-    )
-
-    t_fetch_price_logs = PythonOperator(
-        task_id='fetch_price_logs',
-        python_callable=fetch_price_logs,
-        provide_context=True,
-    )
-
-    # compute demand from interactions
-    t_compute_demand = PythonOperator(
-        task_id='compute_demand',
-        python_callable=compute_demand,
-        provide_context=True,
-    )
-
-    # aggregate price logs
-    t_aggregate_prices = PythonOperator(
-        task_id='aggregate_price_logs',
-        python_callable=aggregate_price_logs,
-        provide_context=True,
-    )
-
-    # join demand and prices
-    t_join_features = PythonOperator(
-        task_id='join_product_features',
-        python_callable=join_product_features,
-        provide_context=True,
-    )
-
-    # apply surge pricing
-    t_surge_pricing = PythonOperator(
-        task_id='apply_surge_pricing',
-        python_callable=apply_surge_pricing,
-        provide_context=True,
-    )
-
-    # publish to registry
-    t_publish = PythonOperator(
-        task_id='publish_results',
-        python_callable=publish_results,
-        provide_context=True,
-    )
-
-    # dependency graph: parallel fetch -> process -> join -> surge -> publish
-    t_fetch_interactions >> t_compute_demand
-    t_fetch_price_logs >> t_aggregate_prices
-    [t_compute_demand, t_aggregate_prices] >> t_join_features >> t_surge_pricing >> t_publish

experiments/procesing/__init__.py

@@ -1,51 +1,19 @@
-from procesing.context import PipelineContext
+from .extract import (
-from procesing.providers import DataProvider, SupabaseProvider, BackendAPIProvider
+    KafkaDataFetcher,
-from procesing.steps import (
+    ExperimentJoiner,
-    BaseContextStep,
+    EventTitleAugmenter,
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-    JoinExperimentsStep,
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep,
-    ChunkByTimeWindowStep,
-    ComputeDemandStep,
-    ComputeDemandForChunksStep,
-    AggregatePriceLogsStep,
-    # StateSpace,
-    # BuildStateSpaceStep,
-    FitPricingFunctionStep,
-    PredictPricesStep,
-)
-from procesing.pipelines import (
-    interaction_extraction_pipeline,
-    price_extraction_pipeline,
-    pricing_pipeline,
-    full_pipeline,
 )
+from .demand import DemandEstimator
+from .mapping import SessionTransitionProbMatrixTransformer, render_graph
+from .pipeline import etl_pipeline, pricing_pipeline
 
 __all__ = [
-    'PipelineContext',
+    'KafkaDataFetcher',
-    'DataProvider',
+    'ExperimentJoiner',
-    'SupabaseProvider',
+    'EventTitleAugmenter',
-    'BackendAPIProvider',
+    'DemandEstimator',
-    'BaseContextStep',
+    'SessionTransitionProbMatrixTransformer',
-    'FetchInteractionsStep',
+    'render_graph',
-    'FetchPriceLogsStep',
+    'etl_pipeline',
-    'FetchExperimentsStep',
-    'JoinExperimentsStep',
-    'CreatePriceBucketsStep',
-    'AugmentEventNamesStep',
-    'ChunkByTimeWindowStep',
-    'ComputeDemandStep',
-    'ComputeDemandForChunksStep',
-    'AggregatePriceLogsStep',
-    # 'StateSpace',
-    # 'BuildStateSpaceStep',
-    'FitPricingFunctionStep',
-    'PredictPricesStep',
-    'interaction_extraction_pipeline',
-    'price_extraction_pipeline',
     'pricing_pipeline',
-    'full_pipeline',
 ]

experiments/procesing/context.py

@@ -1,34 +0,0 @@
-from typing import Any, Dict
-import pandas as pd
-from procesing.providers.base import DataProvider
-
-class PipelineContext:
-    """
-    Context for pipeline execution holding config, provider, and cached data.
-    Enables dependency injection and eliminates global state.
-    """
-
-    def __init__(self,
-                 provider: DataProvider,
-                 store_mode: str,
-                 window_size: str = '30s',
-                 **config):
-        self.provider = provider
-        self.store_mode = store_mode
-        self.window_size = window_size
-        self.config = config
-        self._cache: Dict[str, Any] = {}
-
-    def get_cached(self, key: str, default=None):
-        return self._cache.get(key, default)
-
-    def cache(self, key: str, value):
-        self._cache[key] = value
-        return value
-
-    @property
-    def products(self) -> pd.DataFrame:
-        """Lazy-load and cache product catalog, single fetch per pipeline run"""
-        if 'products' not in self._cache:
-            self._cache['products'] = self.provider.fetch_products(self.store_mode)
-        return self._cache['products']

experiments/procesing/demand.py

@@ -0,0 +1,119 @@
+from sklearn.base import BaseEstimator, TransformerMixin
+import numpy as np
+import pandas as pd
+from supabase import create_client, Client
+from typing import Optional, Literal
+import os
+import logging
+log = logging.getLogger(__name__)
+
+SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
+SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
+
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+class ChunkInteractionsIntoSteps(BaseEstimator, TransformerMixin):
+    """
+    Split interaction data into time windows for temporal analysis.
+    Returns a list of dataframes, one per time window.
+    """
+    def __init__(self,
+                 window_size:str='1h',
+                 ts_col:str='ts',
+                 return_metadata:bool=True):
+        """
+        Args:
+            window_size: pandas freq string ('1h', '30T', '1D', etc)
+            ts_col: timestamp column name
+            return_metadata: if True, return dict with metadata per chunk
+        """
+        self.window_size = window_size
+        self.ts_col = ts_col
+        self.return_metadata = return_metadata
+
+    def fit(self, X):
+        return self
+
+    def transform(self, interactions: pd.DataFrame):
+        """
+        Returns:
+            if return_metadata=False: list of dataframes, one per window
+            if return_metadata=True: list of dicts with keys:
+                - 'data': dataframe for this window
+                - 'window_start': start timestamp
+                - 'window_end': end timestamp
+                - 'window_idx': integer index
+        """
+        if interactions.empty:
+            return []
+
+        df = interactions.copy()
+
+        # ensure timestamp is datetime
+        if not pd.api.types.is_datetime64_any_dtype(df[self.ts_col]):
+            df[self.ts_col] = pd.to_datetime(df[self.ts_col])
+
+        # sort by time
+        df = df.sort_values(self.ts_col)
+
+        # assign window
+        df['_window'] = df[self.ts_col].dt.floor(self.window_size)
+
+        # group by window
+        chunks = []
+        for idx, (window_start, group) in enumerate(df.groupby('_window')):
+            chunk_data = group.drop(columns=['_window'])
+
+            if self.return_metadata:
+                chunks.append({
+                    'data': chunk_data,
+                    'window_start': window_start,
+                    'window_end': window_start + pd.Timedelta(self.window_size),
+                    'window_idx': idx
+                })
+            else:
+                chunks.append(chunk_data)
+
+        return chunks
+
+
+class DemandEstimator(BaseEstimator, TransformerMixin):
+    def __init__(self,
+                 store_mode:str='hotel',
+                 session_filter:str="",
+                 experiment_filter:str=""):
+        self.store=store_mode
+        self.session_filter=session_filter if len(session_filter)>0 else None
+        self.experiment_filter=experiment_filter if len(experiment_filter)>0 else None
+    def fit(self, X):
+        return self
+
+    def transform(self, interactions : pd.DataFrame):
+        if interactions.empty:
+            return pd.DataFrame(columns=["productId", "demand_score"])
+        if self.session_filter:
+            interactions = interactions[interactions['sessionId'] == self.session_filter]
+        if self.experiment_filter:
+            interactions = interactions[interactions['experimentId'] == self.experiment_filter]
+
+        products=supabase.table(f'{self.store}_products').select("id, room_type, date_index, metadata, availability").execute()
+        products = pd.DataFrame(products.data)
+        unique_products = products['id'].unique()
+        log.info(f"Demand estimator found {len(unique_products)} in data")
+
+        # filter out rows without productId
+        interactions_with_products = interactions.dropna(subset=['productId'])
+
+        if interactions_with_products.empty:
+            # no interactions with products, return all zeros
+            return pd.DataFrame({
+                'productId': unique_products,
+                'demand_score': 0
+            })
+
+        # TODO: improve demand score calculation rather than just counting interactions (use weights..)
+        # while maintaining simplicity of a simple cross tab approach
+        product_demand = pd.crosstab(interactions_with_products['productId'], "no_of_interactions")
+        product_demand = product_demand.reindex(unique_products, fill_value=0).reset_index()
+        product_demand.columns = ['productId', 'demand_score']
+        return product_demand

experiments/procesing/elasticity.py

@@ -130,24 +130,25 @@ class TemporalElasticityEstimator(BaseEstimator, TransformerMixin):
 
     def _build_product_timeseries(self, aligned_chunks):
         """Build time series [price, quantity] per product."""
-        # vectorize chunk merging instead of iterating rows
+        series_by_product = {}
-        all_merged = []
+
         for chunk in aligned_chunks:
-            merged = chunk['demand'].merge(chunk['prices'], on='productId', how='inner')
+            demand_df = chunk['demand']
-            merged['timestamp'] = chunk['window_start']
+            price_df = chunk['prices']
-            all_merged.append(merged[['productId', 'timestamp', 'price', 'demand_score']])
 
-        if not all_merged:
+            # merge on productId
-            return {}
+            merged = demand_df.merge(price_df, on='productId', how='inner')
 
-        # concat all chunks and group by productId in one pass
+            for _, row in merged.iterrows():
-        combined = pd.concat(all_merged, ignore_index=True)
+                pid = row['productId']
-        series_by_product = {
+                if pid not in series_by_product:
-            pid: group[['timestamp', 'price', 'demand_score']].rename(
+                    series_by_product[pid] = []
-                columns={'demand_score': 'quantity'}
+
-            ).to_dict('records')
+                series_by_product[pid].append({
-            for pid, group in combined.groupby('productId')
+                    'timestamp': chunk['window_start'],
-        }
+                    'price': row['price'],
+                    'quantity': row['demand_score']
+                })
 
         return series_by_product
 

experiments/procesing/extract.py

@@ -0,0 +1,207 @@
+import pandas as pd
+import json
+import numpy as np
+import os
+import requests
+from dotenv import load_dotenv
+from sklearn.base import BaseEstimator, TransformerMixin
+from supabase import create_client, Client
+from typing import Tuple, List, Dict
+load_dotenv()
+
+BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:5000")
+SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
+SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
+N_PRICE_BUCKETS = 5
+
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+
+class KafkaDataFetcher(BaseEstimator, TransformerMixin):
+    def __init__(self, topic: str = "user-interactions"):
+        self.topic = topic # also can be price-logs
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, X=None):
+        resp = requests.get(f"{BACKEND_URL}/api/kafka/dump?topic={self.topic}")
+        resp.raise_for_status()
+        data = resp.json()
+
+        if not data.get('success') or not data.get('data'):
+            return pd.DataFrame()
+
+        df = pd.DataFrame(data['data'])
+        if self.topic == 'user-interactions':
+            if 'metadata' in df.columns: # explode metadata col json
+                df = df.join(pd.json_normalize(df.pop("metadata"), sep=".").add_prefix("metadata_"))
+            df = df.dropna(subset=['eventName'])
+            # remape dateIndex
+            df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
+        return df
+
+
+class ExperimentJoiner(BaseEstimator, TransformerMixin):
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, df):
+        if df.empty or 'experimentId' not in df.columns:
+            return df
+
+        unique_exp_ids = df['experimentId'].dropna().unique()
+        if len(unique_exp_ids) == 0:
+            return df
+
+        resp = supabase.table('experiments').select(
+            'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, task:tasks(task_name, task_description, task_def_of_done)'
+        ).in_('id', unique_exp_ids.tolist()).execute()
+
+        if not resp.data:
+            return df
+
+        exp_df = pd.DataFrame(resp.data)
+
+        # flatten task nested object if present
+        if 'task' in exp_df.columns and exp_df['task'].notnull().any():
+            task_normalized = pd.json_normalize(exp_df['task'].dropna())
+            task_normalized.index = exp_df[exp_df['task'].notnull()].index
+            exp_df = exp_df.drop(columns=['task']).join(task_normalized, rsuffix='_task')
+
+        # rename experiment columns for clarity
+        exp_df = exp_df.rename(columns={
+            'id': 'experimentId',
+            'subject_name': 'exp_subject',
+            'xp_human_only': 'exp_human_only',
+            'xp_market_mode': 'exp_market_mode',
+            'xp_task_id': 'exp_task_id'
+        })
+
+        df = df.merge(exp_df, on='experimentId', how='left')
+        return df
+
+
+class EventTitleAugmenter(BaseEstimator, TransformerMixin):
+    def fit(self, X=None, y=None):
+        return self
+
+    def transform(self, df):
+        # from taking standard view_item_page in eventName to view_item_page_{metadata_schema}
+        # we want metadata schema to create product specific event names
+
+        # only create price buckets if we have enough unique prices
+        if df["metadata_price"].notnull().sum() > 0:
+            try:
+                price_buckets = pd.qcut(
+                    df["metadata_price"],
+                    q=N_PRICE_BUCKETS,
+                    labels=[f"PB_{i+1}" for i in range(N_PRICE_BUCKETS)],
+                    duplicates='drop'  # handle duplicate bin edges
+                )
+            except ValueError:
+                # fallback: if still not enough unique values, use cut with fixed ranges or just use raw price
+                price_buckets = df["metadata_price"].apply(lambda x: f"P_{int(x)}" if pd.notnull(x) else "")
+        else:
+            price_buckets = pd.Series([""] * len(df), index=df.index)
+
+        # metadata_schema: _product_id@price_bucket_{i} only if we have product metadata otherswise keep original event name
+        # TODO: make this adaptive, if we have hover_over_title we append the title, if its view_page we say which page
+        df["metadata_schema"] = np.where(
+            df["productId"].notnull() & df["metadata_price"].notnull(),
+            "_" + df["productId"].astype(str) + "@" + price_buckets.astype(str),
+            ""
+        )
+        df["eventName"] = df["eventName"] + df["metadata_schema"].astype(str)
+        return df
+
+
+def chunk_shared_data(interactions_df: pd.DataFrame,
+                      price_logs_df: pd.DataFrame,
+                      window_size: str = '30s',
+                      ts_col: str = 'ts') -> Tuple[List[Dict], List[Dict]]:
+    """
+    Chunk interaction and price data into aligned time windows.
+
+    Args:
+        interactions_df: interaction data with timestamp column
+        price_logs_df: price log data with timestamp column
+        window_size: pandas freq string ('30s', '1min', '1h', etc)
+        ts_col: name of timestamp column
+
+    Returns:
+        tuple of (interaction_chunks, price_chunks) where each is list of dicts:
+        {
+            'window_start': timestamp,
+            'window_end': timestamp,
+            'data': dataframe for this window
+        }
+    """
+    if interactions_df.empty and price_logs_df.empty:
+        return [], []
+
+    # convert timestamps to datetime
+    interactions_df = interactions_df.copy()
+    price_logs_df = price_logs_df.copy()
+
+    if not interactions_df.empty:
+        if not pd.api.types.is_datetime64_any_dtype(interactions_df[ts_col]):
+            interactions_df[ts_col] = pd.to_datetime(interactions_df[ts_col])
+
+    if not price_logs_df.empty:
+        if not pd.api.types.is_datetime64_any_dtype(price_logs_df[ts_col]):
+            price_logs_df[ts_col] = pd.to_datetime(price_logs_df[ts_col])
+
+    # find global time bounds
+    times = []
+    if not interactions_df.empty:
+        times.extend([interactions_df[ts_col].min(), interactions_df[ts_col].max()])
+    if not price_logs_df.empty:
+        times.extend([price_logs_df[ts_col].min(), price_logs_df[ts_col].max()])
+
+    if not times:
+        return [], []
+
+    earliest = min(times)
+    latest = max(times)
+
+    # create shared time windows
+    windows = pd.date_range(start=earliest, end=latest, freq=window_size)
+
+    if len(windows) < 2:
+        return [], []
+
+    # chunk both datasets
+    interaction_chunks = []
+    price_chunks = []
+
+    for i in range(len(windows) - 1):
+        window_start = windows[i]
+        window_end = windows[i + 1]
+
+        # filter interactions in this window
+        if not interactions_df.empty:
+            mask = (interactions_df[ts_col] >= window_start) & (interactions_df[ts_col] < window_end)
+            interaction_chunk = interactions_df[mask]
+        else:
+            interaction_chunk = pd.DataFrame()
+
+        interaction_chunks.append({
+            'window_start': window_start,
+            'window_end': window_end,
+            'data': interaction_chunk
+        })
+
+        # filter price logs in this window
+        if not price_logs_df.empty:
+            mask = (price_logs_df[ts_col] >= window_start) & (price_logs_df[ts_col] < window_end)
+            price_chunk = price_logs_df[mask]
+        else:
+            price_chunk = pd.DataFrame()
+
+        price_chunks.append({
+            'window_start': window_start,
+            'window_end': window_end,
+            'data': price_chunk
+        })
+
+    return interaction_chunks, price_chunks

experiments/procesing/mapping.py

@@ -0,0 +1,158 @@
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator, TransformerMixin
+
+def build_transition_prob_matrix(df: pd.DataFrame):
+    df = df.dropna(subset=['eventName'])
+    events = df['eventName'].tolist()
+    labels = pd.Index(events).unique().tolist()
+    idx = {e:i for i,e in enumerate(labels)}
+    M = np.zeros((len(labels), len(labels)), dtype=float)
+    for a, b in zip(events, events[1:]):
+        M[idx[a], idx[b]] += 1
+    row_sums = M.sum(axis=1, keepdims=True)
+    with np.errstate(divide='ignore', invalid='ignore'):
+        P = np.divide(M, row_sums, where=row_sums>0)  # row-normalized
+    return P, labels
+
+# https://medium.com/data-science/time-series-data-markov-transition-matrices-7060771e362b
+from graphviz import Digraph
+import numpy as np
+import pandas as pd
+
+def _as_prob_df(matrix, labels=None):
+    """Return a square DataFrame with index=columns=labels."""
+    if isinstance(matrix, pd.DataFrame):
+        # Ensure square and aligned
+        assert (matrix.index == matrix.columns).all(), "Index/columns must match."
+        return matrix
+    matrix = np.asarray(matrix, dtype=float)
+    assert matrix.shape[0] == matrix.shape[1], "Matrix must be square."
+    if labels is None:
+        raise ValueError("labels are required when matrix is not a DataFrame")
+    assert len(labels) == matrix.shape[0], "labels length must match matrix size."
+    return pd.DataFrame(matrix, index=list(labels), columns=list(labels))
+
+def _df_to_edgelist(P: pd.DataFrame, threshold=0.0, round_digits=2):
+    """Build weighted edges > threshold."""
+    edges = []
+    for src in P.index:
+        for dst in P.columns:
+            w = float(P.loc[src, dst])
+            if w > threshold:
+                edges.append((str(src), str(dst), f"{w:.{round_digits}f}"))
+    return edges
+
+def render_graph(fname, matrix, ls_index=None, threshold=0.0, fmt="svg", view=False):
+    """
+    fname: output file stem (no extension)
+    matrix: NumPy array or pandas DataFrame of transition PROBABILITIES
+    ls_index: ordered labels (required if matrix is not a DataFrame)
+    threshold: hide edges with weight <= threshold
+    fmt: 'svg'|'png'|'pdf' etc.
+    view: open after rendering
+    """
+    P = _as_prob_df(matrix, labels=ls_index)
+    edges = _df_to_edgelist(P, threshold=threshold)
+
+    g = Digraph(format=fmt)
+    g.attr(rankdir="LR", size="30")
+    g.attr("node", shape="circle")
+
+    # ensure isolated nodes appear
+    for node in P.index:
+        g.node(str(node), width="1", height="1")
+
+    for src, dst, label in edges:
+        g.edge(src, dst, label=label)
+
+    g.render(fname, view=view, cleanup=True)
+    return g
+
+
+class TransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0):
+        self.threshold = threshold
+        self.P_ = None
+        self.labels_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        P, labels = build_transition_prob_matrix(X)
+        self.P_ = P
+        self.labels_ = labels
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        return self.P_, self.labels_
+
+    def render(self, fname: str, fmt="svg", view=False):
+        if self.P_ is None or self.labels_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return render_graph(
+            fname,
+            self.P_,
+            ls_index=self.labels_,
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+
+
+class SessionTransitionProbMatrixTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, threshold=0.0, session_col='sessionId'):
+        self.threshold = threshold
+        self.session_col = session_col
+        self.session_matrices_ = None
+
+    def fit(self, X: pd.DataFrame, y=None):
+        if self.session_col not in X.columns:
+            raise ValueError(f"Column '{self.session_col}' not found in DataFrame")
+
+        session_matrices = {}
+        for session_id, grp in X.groupby(self.session_col):
+            if len(grp) > 1:  # need at least 2 events for transitions
+                P, labels = build_transition_prob_matrix(grp)
+                session_matrices[session_id] = {'matrix': P, 'labels': labels}
+
+        self.session_matrices_ = session_matrices
+        return self
+
+    def transform(self, X: pd.DataFrame = None):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        return pd.Series(self.session_matrices_)
+
+    def render_session(self, session_id: str, fname: str, fmt="svg", view=False):
+        if self.session_matrices_ is None:
+            raise ValueError("Transformer has not been fitted yet.")
+        if session_id not in self.session_matrices_:
+            raise ValueError(f"Session '{session_id}' not found in fitted data.")
+
+        sess_data = self.session_matrices_[session_id]
+        return render_graph(
+            fname,
+            sess_data['matrix'],
+            ls_index=sess_data['labels'],
+            threshold=self.threshold,
+            fmt=fmt,
+            view=view
+        )
+if __name__ == "__main__":
+    # Example usage
+    data = {
+        'eventName': [
+            'A', 'B', 'A', 'C', 'B', 'A', 'A', 'C', 'B', 'C',
+            'A', 'B', 'C', 'A', 'B', 'C', 'A', 'B', 'C', 'A'
+        ]
+    }
+    df = pd.DataFrame(data)
+
+    transformer = TransitionProbMatrixTransformer(threshold=0.1)
+    transformer.fit(df)
+    P, labels = transformer.transform(None)
+
+    print("Transition Probability Matrix:")
+    print(pd.DataFrame(P, index=labels, columns=labels))
+
+    # Render the graph
+    transformer.render("transition_graph", fmt="svg", view=False)

experiments/procesing/metrics.py

@@ -1,245 +0,0 @@
-"""
-Revenue and KPI benchmark framework for pricing strategies.
-
-Computes session-level and aggregate metrics to compare pricing functions:
-    - Revenue: R_T = Σ P_t^T · Q_t
-    - Conversion rate
-    - Average order value (AOV)
-    - Agent exploitation loss: L_agent = R_oracle - R_observed
-"""
-from typing import Dict, List, Any, Optional
-from dataclasses import dataclass, field, asdict
-import pandas as pd
-import numpy as np
-
-
-@dataclass
-class SessionMetrics:
-    """KPIs for single session."""
-    session_id: str
-    experiment_id: Optional[str] = None
-
-    # interaction metrics
-    total_interactions: int = 0
-    page_views: int = 0
-    item_views: int = 0
-    searches: int = 0
-    cart_adds: int = 0
-
-    # revenue metrics
-    items_purchased: int = 0
-    total_revenue: float = 0.0
-    avg_item_price: float = 0.0
-    conversion_rate: float = 0.0
-
-    # pricing signals
-    total_price_shown: float = 0.0  # sum of all prices displayed
-    avg_markup: float = 0.0  # avg (price / base_price)
-
-    # behavioral features (for agent detection)
-    interaction_velocity: float = 0.0  # interactions per minute
-    session_duration_sec: float = 0.0
-    unique_products_viewed: int = 0
-
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return asdict(self)
-
-
-@dataclass
-class AggregateMetrics:
-    """Aggregate KPIs across sessions/experiments."""
-    experiment_id: Optional[str] = None
-    n_sessions: int = 0
-
-    # revenue aggregates
-    total_revenue: float = 0.0
-    avg_revenue_per_session: float = 0.0
-    median_revenue_per_session: float = 0.0
-
-    # conversion aggregates
-    total_conversions: int = 0
-    conversion_rate: float = 0.0  # purchases / sessions
-
-    # pricing aggregates
-    avg_markup: float = 0.0
-    median_markup: float = 0.0
-
-    # agent exploitation metrics
-    estimated_agent_sessions: int = 0  # sessions flagged as agent-driven
-    agent_revenue: float = 0.0
-    human_revenue: float = 0.0
-    agent_loss: float = 0.0  # L_agent = R_oracle - R_observed (if available)
-
-    def to_dict(self) -> Dict[str, Any]:
-        return asdict(self)
-
-
-class MetricsComputer:
-    """Compute session and aggregate metrics from interaction/price logs."""
-
-    @staticmethod
-    def compute_session_metrics(
-        session_id: str,
-        interactions: pd.DataFrame,
-        price_logs: pd.DataFrame,
-        purchases: Optional[pd.DataFrame] = None,
-        experiment_id: Optional[str] = None
-    ) -> SessionMetrics:
-        """
-        Compute metrics for single session.
-
-        Args:
-            session_id: session identifier
-            interactions: user-interactions events for this session
-            price_logs: price-logs for this session
-            purchases: purchase events (if available)
-            experiment_id: experiment identifier
-        """
-        metrics = SessionMetrics(session_id=session_id, experiment_id=experiment_id)
-
-        if interactions.empty:
-            return metrics
-
-        # interaction counts
-        event_counts = interactions['eventName'].value_counts().to_dict()
-        metrics.total_interactions = len(interactions)
-        metrics.page_views = event_counts.get('page_view', 0) + event_counts.get('view_item_page', 0)
-        metrics.item_views = event_counts.get('view_item_page', 0)
-        metrics.searches = event_counts.get('search', 0)
-        metrics.cart_adds = event_counts.get('add_item_to_cart', 0)
-
-        # unique products viewed
-        metrics.unique_products_viewed = interactions['productId'].dropna().nunique()
-
-        # session duration
-        if 'ts' in interactions.columns:
-            timestamps = pd.to_datetime(interactions['ts'])
-            metrics.session_duration_sec = (timestamps.max() - timestamps.min()).total_seconds()
-            if metrics.session_duration_sec > 0:
-                metrics.interaction_velocity = (metrics.total_interactions / metrics.session_duration_sec) * 60
-
-        # revenue from purchases
-        if purchases is not None and not purchases.empty:
-            metrics.items_purchased = len(purchases)
-            metrics.total_revenue = purchases['price'].sum() if 'price' in purchases.columns else 0.0
-            metrics.avg_item_price = metrics.total_revenue / metrics.items_purchased if metrics.items_purchased > 0 else 0.0
-            metrics.conversion_rate = 1.0 if metrics.items_purchased > 0 else 0.0
-
-        # pricing metrics
-        if not price_logs.empty:
-            metrics.total_price_shown = price_logs['price'].sum()
-            # compute markup if base_price available in price logs or join with product catalog
-            if 'base_price' in price_logs.columns:
-                valid_markup = price_logs[price_logs['base_price'] > 0]
-                if not valid_markup.empty:
-                    metrics.avg_markup = (valid_markup['price'] / valid_markup['base_price']).mean()
-
-        return metrics
-
-    @staticmethod
-    def compute_aggregate_metrics(
-        session_metrics_list: List[SessionMetrics],
-        experiment_id: Optional[str] = None,
-        agent_detector_fn: Optional[callable] = None
-    ) -> AggregateMetrics:
-        """
-        Aggregate metrics across sessions.
-
-        Args:
-            session_metrics_list: list of SessionMetrics
-            experiment_id: experiment identifier
-            agent_detector_fn: optional function to classify session as agent (returns bool)
-        """
-        agg = AggregateMetrics(experiment_id=experiment_id)
-        agg.n_sessions = len(session_metrics_list)
-
-        if agg.n_sessions == 0:
-            return agg
-
-        df = pd.DataFrame([m.to_dict() for m in session_metrics_list])
-
-        # revenue aggregates
-        agg.total_revenue = df['total_revenue'].sum()
-        agg.avg_revenue_per_session = df['total_revenue'].mean()
-        agg.median_revenue_per_session = df['total_revenue'].median()
-
-        # conversion aggregates
-        agg.total_conversions = (df['items_purchased'] > 0).sum()
-        agg.conversion_rate = agg.total_conversions / agg.n_sessions
-
-        # pricing aggregates
-        valid_markups = df[df['avg_markup'] > 0]
-        if not valid_markups.empty:
-            agg.avg_markup = valid_markups['avg_markup'].mean()
-            agg.median_markup = valid_markups['avg_markup'].median()
-
-        # agent detection (if detector provided)
-        if agent_detector_fn is not None:
-            agent_flags = [agent_detector_fn(m) for m in session_metrics_list]
-            agg.estimated_agent_sessions = sum(agent_flags)
-
-            agent_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if is_agent)
-            human_revenue = sum(m.total_revenue for m, is_agent in zip(session_metrics_list, agent_flags) if not is_agent)
-
-            agg.agent_revenue = agent_revenue
-            agg.human_revenue = human_revenue
-
-        return agg
-
-    @staticmethod
-    def compare_pricing_strategies(
-        experiments: Dict[str, List[SessionMetrics]],
-        baseline_experiment_id: Optional[str] = None
-    ) -> pd.DataFrame:
-        """
-        Compare multiple pricing strategies/experiments.
-
-        Args:
-            experiments: dict mapping experiment_id -> list of SessionMetrics
-            baseline_experiment_id: experiment to use as baseline for comparison
-
-        Returns:
-            DataFrame with comparative metrics
-        """
-        results = []
-        baseline_agg = None
-
-        for exp_id, session_metrics in experiments.items():
-            agg = MetricsComputer.compute_aggregate_metrics(session_metrics, experiment_id=exp_id)
-            result = agg.to_dict()
-
-            if exp_id == baseline_experiment_id:
-                baseline_agg = agg
-
-            results.append(result)
-
-        df = pd.DataFrame(results)
-
-        # add relative metrics if baseline exists
-        if baseline_agg is not None:
-            df['revenue_lift_pct'] = ((df['total_revenue'] - baseline_agg.total_revenue) / baseline_agg.total_revenue * 100)
-            df['conversion_lift_pct'] = ((df['conversion_rate'] - baseline_agg.conversion_rate) / baseline_agg.conversion_rate * 100)
-
-        return df
-
-
-def simple_agent_detector(session_metrics: SessionMetrics, velocity_threshold: float = 5.0) -> bool:
-    """
-    Simple heuristic agent detector based on interaction velocity.
-
-    Args:
-        session_metrics: SessionMetrics instance
-        velocity_threshold: interactions per minute threshold (default: 5.0)
-
-    Returns:
-        True if session likely agent-driven
-    """
-    # agents tend to have higher interaction velocity and lower session duration
-    if session_metrics.interaction_velocity > velocity_threshold:
-        return True
-    # agents often view many products quickly without converting
-    if session_metrics.unique_products_viewed > 10 and session_metrics.conversion_rate == 0:
-        return True
-    return False

experiments/procesing/pipeline.py

@@ -0,0 +1,90 @@
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+import pandas as pd
+import logging
+log = logging.getLogger(__name__)
+
+from extract import KafkaDataFetcher, ExperimentJoiner, EventTitleAugmenter, chunk_shared_data
+from mapping import SessionTransitionProbMatrixTransformer, render_graph
+from demand import DemandEstimator, ChunkInteractionsIntoSteps
+from elasticity import TemporalElasticityEstimator, aggregate_price_logs
+
+
+
+# elasticity pipeline components (not sklearn compatible, manual orchestration)
+def elasticity_pipeline(interactions_df, price_logs_df, window_size='30s', store_mode='hotel'):
+    """
+    Compute price elasticity from interaction and price data.
+
+    Args:
+        interactions_df: raw interaction data from demand_data_pipeline
+        price_logs_df: price log data from price_data_pipeline
+        window_size: time window for chunking
+        store_mode: 'hotel' or 'airline'
+
+    Returns:
+        df with [productId, elasticity, std_error, n_obs]
+    """
+    # step 1: chunk interactions into time windows
+    chunker = ChunkInteractionsIntoSteps(window_size=window_size, return_metadata=True)
+    interaction_chunks = chunker.transform(interactions_df)
+    log.info(f"Chunked interactions into {len(interaction_chunks)} windows of size {window_size}")
+
+    if not interaction_chunks:
+        return None
+
+    # step 2: compute demand per window
+    demand_estimator = DemandEstimator(store_mode=store_mode)
+    demand_chunks = []
+    for chunk in interaction_chunks:
+        demand_vector = demand_estimator.transform(chunk['data'])
+        demand_chunks.append({
+            'window_start': chunk['window_start'],
+            'window_end': chunk['window_end'],
+            'demand_vector': demand_vector # each has a full list of all products, even if demand is 0
+        })
+    # [q_chunk1, q_chunk2, ...]
+
+    # step 3: aggregate price logs into windows
+    price_chunks = aggregate_price_logs(price_logs_df, window_size=window_size)
+
+    # step 4: compute elasticity
+    elasticity_estimator = TemporalElasticityEstimator(method='point', min_observations=2)
+    elasticity_df = elasticity_estimator.transform(demand_chunks, price_chunks, store_mode=store_mode)
+
+    return elasticity_df
+
+
+# exposable pipelines
+interaction_pipeline = Pipeline([
+    ('kafka_fetch', KafkaDataFetcher(topic='user-interactions')),
+    ('experiment_join', ExperimentJoiner()),
+    ('event_augment', EventTitleAugmenter()),
+])
+
+price_data_pipeline = Pipeline([
+    ('kafka_fetch', KafkaDataFetcher(topic='price-logs')),
+])
+
+# interaction_data + price_data -> elasticity (demand)
+# elasticity -> pricing
+
+pricing_pipeline = Pipeline([
+    ('demand_estimation', DemandEstimator()),
+])
+if __name__ == "__main__":
+    # fetch both datasets
+    interaction_data = interaction_pipeline.fit_transform(None)
+    pricing_data = price_data_pipeline.fit_transform(None)
+    if interaction_data.empty or pricing_data.empty:
+        print("Insufficient data for elasticity computation"); exit(0)
+    # compute elasticity via unified pipeline
+    window_size = "30s"
+    elasticity_results = elasticity_pipeline(interaction_data, pricing_data, window_size=window_size)
+    elasticity_value_array = elasticity_results['elasticity'].values if elasticity_results is not None else np.array([])
+    print(elasticity_value_array)
+
+    if elasticity_results is not None and not elasticity_results.empty:
+        print(elasticity_results.to_string(index=False))
+    else:
+        print("\nInsufficient data for elasticity computation")

experiments/procesing/pipelines.py

@@ -1,138 +0,0 @@
-from sklearn.pipeline import Pipeline
-import pandas as pd
-from procesing.context import PipelineContext
-from procesing.providers import SupabaseProvider, BackendAPIProvider
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-    JoinExperimentsStep,
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep,
-    ChunkByTimeWindowStep,
-    ComputeDemandForChunksStep,
-    AggregatePriceLogsStep,
-    # BuildStateSpaceStep,
-    FitPricingFunctionStep,
-    PredictPricesStep,
-    ComputeDemandStep,
-    JoinProductFeaturesStep
-)
-from procesing.pricers import SimpleSurgePricer
-
-def interaction_extraction_pipeline(context: PipelineContext):
-    """Pipeline for extracting and augmenting interaction data"""
-    return Pipeline([
-        ('fetch', FetchInteractionsStep(context)),
-        ('create_buckets', CreatePriceBucketsStep(context)),
-        ('augment_events', AugmentEventNamesStep(context)),
-    ])
-
-
-def price_extraction_pipeline(context: PipelineContext):
-    """Pipeline for extracting price logs"""
-    return Pipeline([
-        ('fetch', FetchPriceLogsStep(context)),
-    ])
-
-
-def product_features_pipeline(context: PipelineContext,
-                                    interactions_df: pd.DataFrame,
-                                    price_logs_df: pd.DataFrame):
-    demand_step = ComputeDemandStep(context)
-    price_step = AggregatePriceLogsStep(context)
-    join_step = JoinProductFeaturesStep(context)
-
-
-    demand_data = demand_step.transform(interactions_df)
-    price_data= price_step.transform(price_logs_df)
-    joined_data = join_step.transform((demand_data, price_data))
-
-    return joined_data
-
-
-
-def pricing_pipeline(context: "PipelineContext",
-                     data: pd.DataFrame,
-                     high_threshold: int = 10,
-                     low_threshold: int = 2,
-                     surge_multiplier: float = 1.2,
-                     discount_multiplier: float = 0.9) -> pd.DataFrame:
-
-    if data.empty or 'productId' not in data.columns:
-        return pd.DataFrame()
-
-    surge_pricer = SimpleSurgePricer()
-    surge_pricer.fit(data)
-    data['optimal_price'] = surge_pricer.predict()
-    return data
-
-
-def full_pipeline(context: PipelineContext,
-                  high_threshold: int = 10,
-                  low_threshold: int = 2,
-                  surge_multiplier: float = 1.2,
-                  discount_multiplier: float = 0.9):
-    """
-    Complete end-to-end pipeline: data extraction -> demand/price aggregation -> surge pricing
-
-    Args:
-        context: Pipeline context
-        high_threshold: Demand threshold for surge pricing
-        low_threshold: Demand threshold for discounts
-        surge_multiplier: Price multiplier for high demand
-        discount_multiplier: Price multiplier for low demand
-
-    Returns:
-        tuple: (product_features_df, optimal_prices_df)
-            - product_features_df: [productId, demand_score, price]
-            - optimal_prices_df: [productId, current_price, optimal_price, demand_score]
-    """
-    interaction_pipe = interaction_extraction_pipeline(context)
-    price_pipe = price_extraction_pipeline(context)
-
-    interactions_df = interaction_pipe.fit_transform(None)
-    price_logs_df = price_pipe.fit_transform(None)
-    product_features_df = product_features_pipeline(context, interactions_df, price_logs_df)
-    print(product_features_df.to_string())
-
-    # generate optimal prices using surge rules
-    optimal_prices_df = pricing_pipeline(context, product_features_df,
-                                          high_threshold=high_threshold,
-                                          low_threshold=low_threshold,
-                                          surge_multiplier=surge_multiplier,
-                                          discount_multiplier=discount_multiplier)
-
-    return product_features_df, optimal_prices_df
-
-
-
-
-if __name__ == '__main__':
-
-    class Provider(SupabaseProvider, BackendAPIProvider):
-        def __init__(self, backend_url: str):
-            SupabaseProvider.__init__(self)
-            BackendAPIProvider.__init__(self, backend_url=backend_url)
-
-
-    class HistoricalProvider(SupabaseProvider, BackendAPIProvider):
-        def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-            path = "/home/velocitatem/Documents/Projects/PHANTOM/experiments/collected_data/858c61ab-0a7f-4595-ae49-33f4365517b9/"
-            interactions_file = "messages(2).json"
-            prices_file = "messages(3).json"
-
-            data = pd.read_json(path + (interactions_file if topic == "user-interactions" else prices_file))
-            data = [r['payload'] for r in data['value'].to_list()]
-            data = pd.DataFrame(data)
-            return data
-
-
-    # example run
-    context = PipelineContext(
-        provider=HistoricalProvider(),
-        store_mode='hotel',
-    )
-
-    product_features, prices = full_pipeline(context)
-    print(prices.to_string())

experiments/procesing/pricers/__init__.py

@@ -1,14 +0,0 @@
-from procesing.pricers.base import PricingFunction
-from procesing.pricers.elasticity import ElasticityBasedPricer
-from procesing.pricers.simple import StaticPricer, RandomPricer, SimpleSurgePricer
-from procesing.pricers.session_aware import SessionAwarePricer, ProductSpecificSessionPricer
-
-__all__ = [
-    'PricingFunction',
-    'ElasticityBasedPricer',
-    'StaticPricer',
-    'RandomPricer',
-    'SimpleSurgePricer',
-    'SessionAwarePricer',
-    'ProductSpecificSessionPricer'
-]

experiments/procesing/pricers/base.py

@@ -1,70 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Optional, Dict, Any, List
-import numpy as np
-import pandas as pd
-
-
-class PricingFunction(ABC):
-    """
-    Abstract base for pricing functions.
-
-    Defines mapping: f(Q_t, P_t, S_t, H_t) -> P_{t+1}
-
-    Where:
-        Q_t ∈ R^n: demand vector at time t
-        P_t ∈ R^n: price vector at time t
-        S_t: session features (behavioral signals, interactions)
-        H_t = {Q_{t-k}, P_{t-k}, S_{t-k}}: historical state trajectory
-
-    Objective:
-        maximize E[R_T] = E[Σ P_t^T · Q_t]
-        subject to:
-            Q_t = g(P_t, S_t)  (demand response via elasticity)
-            P_t ≥ C  (cost floor)
-            minimize L_agent = R_oracle - R_observed
-    """
-
-    @abstractmethod
-    def fit(self, *kwargs):
-        """
-        Offline training on historical data.
-
-        Args:
-            historical_data: DataFrame with elasticity, prices, demand signals
-            **kwargs: additional training parameters
-        """
-        pass
-
-    @abstractmethod
-    def predict(self, *kwargs) -> np.ndarray:
-        """
-        Generate optimal prices given current state.
-
-        Args:
-            state_space: StateSpace object containing Q_t, P_t, S_t, H_t
-
-        Returns:
-            P_{t+1}: price vector in R^n
-        """
-        pass
-
-    def update(self, observation: Dict[str, Any]):
-        """
-        Online learning update (optional).
-
-        Args:
-            observation: dict with {state, action, reward, next_state}
-                - state: StateSpace before pricing decision
-                - action: prices shown (P_t)
-                - reward: revenue/conversion signal
-                - next_state: StateSpace after user interaction
-        """
-        pass  # default: no online learning
-
-    def get_params(self) -> Dict[str, Any]:
-        """Return pricing function parameters for serialization."""
-        return {}
-
-    def set_params(self, params: Dict[str, Any]):
-        """Load pricing function parameters from dict."""
-        pass

experiments/procesing/pricers/elasticity.py

@@ -1,59 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-
-
-class ElasticityBasedPricer(PricingFunction):
-    """
-    Pricing based on demand elasticity estimates.
-    f(Q, S) = base_price * (1 + alpha * elasticity * demand_deviation)
-    """
-
-    def __init__(self, alpha: float = 0.1, price_floor: float = 0.0, price_ceil: float = np.inf):
-        self.alpha = alpha
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-
-    def fit(self, historical_data: pd.DataFrame):
-        """
-        Calibrate from historical elasticity estimates.
-        Expects: [productId, elasticity, base_price, mean_demand]
-        """
-        if 'elasticity' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity' column")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """
-        Adjust prices based on demand deviation and elasticity.
-        Higher demand -> increase price (but less for elastic goods)
-        """
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        if len(demand) != len(self.elasticity):
-            raise ValueError(f"Demand vector size {len(demand)} != elasticity size {len(self.elasticity)}")
-
-        # compute demand deviation from mean
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-
-        # adjust price: if demand high and elastic, don't increase much
-        # if demand high and inelastic, increase more
-        price_multiplier = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
-        prices = self.base_prices * price_multiplier
-
-        # enforce bounds
-        prices = np.clip(prices, self.price_floor, self.price_ceil)
-        return prices

experiments/procesing/pricers/session_aware.py

@@ -1,172 +0,0 @@
-"""
-Session-aware pricing functions that leverage behavioral features S_t.
-These pricers aim to minimize L_agent = R_oracle - R_observed.
-"""
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-from procesing.pricers.elasticity import ElasticityBasedPricer
-
-
-class SessionAwarePricer(PricingFunction):
-    """
-    Extends elasticity-based pricing with session behavioral signals.
-
-    f(Q, P, S) = base_price * elasticity_factor * session_factor
-
-    Where session_factor adjusts for:
-        - interaction_velocity (agent detection proxy)
-        - product_view_depth (interest signal)
-        - cart_to_view_ratio (conversion intent)
-
-    Strategy: charge higher prices to suspected agents (high velocity)
-    to recover oracle revenue from reconnaissance sessions.
-    """
-
-    def __init__(self,
-                 alpha: float = 0.1,
-                 beta_velocity: float = 0.05,
-                 beta_attention: float = 0.03,
-                 agent_velocity_threshold: float = 5.0,
-                 agent_markup: float = 1.2,
-                 price_floor: float = 0.0,
-                 price_ceil: float = np.inf):
-        """
-        Args:
-            alpha: elasticity sensitivity
-            beta_velocity: interaction velocity weight
-            beta_attention: product attention weight
-            agent_velocity_threshold: velocity above which to apply agent markup
-            agent_markup: price multiplier for suspected agent sessions
-            price_floor, price_ceil: price bounds
-        """
-        self.alpha = alpha
-        self.beta_velocity = beta_velocity
-        self.beta_attention = beta_attention
-        self.agent_velocity_threshold = agent_velocity_threshold
-        self.agent_markup = agent_markup
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-
-        # fitted parameters
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-
-    def fit(self, historical_data: pd.DataFrame, **kwargs):
-        """Calibrate from historical elasticity data."""
-        if 'elasticity' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity'")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Generate prices with session awareness."""
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        n_products = len(demand)
-
-        # base elasticity-driven pricing
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-        elasticity_factor = 1 + self.alpha * np.abs(self.elasticity) * demand_dev
-
-        # session-aware adjustments
-        session_factor = np.ones(n_products)
-
-        if not state_space.session_features.empty:
-            sf = state_space.session_features.iloc[0]  # single session features
-
-            # agent detection via velocity
-            velocity = sf.get('interaction_velocity', 0.0)
-            if velocity > self.agent_velocity_threshold:
-                # suspected agent: apply markup to recover oracle revenue
-                session_factor *= self.agent_markup
-
-            # attention signal: higher view depth -> user interested -> can charge more
-            view_depth = sf.get('product_view_depth', 0)
-            if view_depth > 0:
-                attention_boost = 1 + self.beta_attention * np.log1p(view_depth)
-                session_factor *= attention_boost
-
-            # cart presence: if user has items in cart, slightly increase prices
-            cart_to_view = sf.get('cart_to_view_ratio', 0.0)
-            if cart_to_view > 0.1:
-                session_factor *= (1 + 0.02)  # small boost for conversion intent
-
-        prices = self.base_prices * elasticity_factor * session_factor
-        prices = np.clip(prices, self.price_floor, self.price_ceil)
-
-        return prices
-
-
-class ProductSpecificSessionPricer(PricingFunction):
-    """
-    Session-aware pricer with product-specific demand signals.
-
-    Uses S_t to extract per-product interaction counts and adjusts pricing
-    for products the user has already viewed/hovered.
-
-    Strategy: products viewed multiple times = high interest -> price up
-    """
-
-    def __init__(self,
-                 alpha: float = 0.1,
-                 view_boost: float = 0.02,
-                 max_view_boost: float = 0.15,
-                 price_floor: float = 0.0,
-                 price_ceil: float = np.inf):
-        self.alpha = alpha
-        self.view_boost = view_boost
-        self.max_view_boost = max_view_boost
-        self.price_floor = price_floor
-        self.price_ceil = price_ceil
-
-        self.elasticity = None
-        self.base_prices = None
-        self.mean_demand = None
-        self.product_ids = None
-
-    def fit(self, historical_data: pd.DataFrame, **kwargs):
-        if 'elasticity' not in historical_data.columns or 'productId' not in historical_data.columns:
-            raise ValueError("historical_data must contain 'elasticity' and 'productId'")
-
-        self.elasticity = historical_data['elasticity'].values
-        self.base_prices = (historical_data['base_price'].values
-                           if 'base_price' in historical_data.columns
-                           else np.ones(len(historical_data)) * 100)
-        self.mean_demand = (historical_data['mean_demand'].values
-                           if 'mean_demand' in historical_data.columns
-                           else np.ones(len(historical_data)) * 10)
-        self.product_ids = historical_data['productId'].values
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        if self.elasticity is None:
-            raise ValueError("Must call fit() before predict()")
-
-        demand = np.asarray(state_space.demand)
-        n_products = len(demand)
-
-        # base pricing
-        demand_dev = (demand - self.mean_demand) / (self.mean_demand + 1e-6)
-        base_prices = self.base_prices * (1 + self.alpha * np.abs(self.elasticity) * demand_dev)
-
-        # product-specific session adjustments
-        if not state_space.session_features.empty and state_space.product_ids is not None:
-            # extract product interaction counts from session metadata
-            # (this would require session features to include per-product signals)
-            # for now, use uniform boost as placeholder
-            # TODO: extend session feature extraction to include product-specific counts
-            pass
-
-        prices = np.clip(base_prices, self.price_floor, self.price_ceil)
-        return prices

experiments/procesing/pricers/simple.py

@@ -1,91 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.pricers.base import PricingFunction
-
-
-class StaticPricer(PricingFunction):
-    """Static pricing: always return fixed base prices"""
-
-    def __init__(self, base_prices: np.ndarray = None):
-        self.base_prices = base_prices
-
-    def fit(self, historical_data: pd.DataFrame):
-        """Extract base prices from historical data"""
-        if 'base_price' in historical_data.columns:
-            self.base_prices = historical_data['base_price'].values
-        elif 'price' in historical_data.columns:
-            self.base_prices = historical_data['price'].values
-        else:
-            raise ValueError("historical_data must contain 'base_price' or 'price' column")
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Return static base prices regardless of state"""
-        if self.base_prices is None:
-            raise ValueError("Must call fit() or provide base_prices in constructor")
-        return self.base_prices.copy()
-
-
-class RandomPricer(PricingFunction):
-    """Random pricing within bounds (for baseline comparison)"""
-
-    def __init__(self, price_min: float = 50.0, price_max: float = 500.0, seed: int = None):
-        self.price_min = price_min
-        self.price_max = price_max
-        self.seed = seed
-        self.n_products = None
-        self.rng = np.random.default_rng(seed)
-
-    def fit(self, historical_data: pd.DataFrame):
-        """Learn number of products"""
-        self.n_products = len(historical_data)
-        return self
-
-    def predict(self, state_space) -> np.ndarray:
-        """Generate random prices"""
-        if self.n_products is None:
-            self.n_products = len(state_space.demand)
-        return self.rng.uniform(self.price_min, self.price_max, size=self.n_products)
-
-
-class SimpleSurgePricer(PricingFunction):
-    """
-    Rule-based surge pricer adjusting prices via demand thresholds.
-    Logic: if demand > high_threshold -> surge, if demand < low_threshold -> discount.
-    Simpler and more controllable than curve fitting approaches.
-    """
-
-    def __init__(self,
-                 base_prices: np.ndarray = None,
-                 high_threshold: int = 10,
-                 low_threshold: int = 2,
-                 surge_multiplier: float = 1.2,
-                 discount_multiplier: float = 0.9):
-        self.base_prices = base_prices
-        self.high_threshold = high_threshold
-        self.low_threshold = low_threshold
-        self.surge_multiplier = surge_multiplier
-        self.discount_multiplier = discount_multiplier
-
-    def fit(self, market_data : pd.DataFrame):
-        """Extract base prices from product catalog or historical averages"""
-        self.base_prices = market_data['base_price'].to_numpy() if 'base_price' in market_data.columns else market_data['price'].values
-        self.demand_history = market_data['demand'].to_numpy() if 'demand' in market_data.columns else np.zeros_like(self.base_prices)
-
-    def predict(self) -> np.ndarray:
-        """
-        Adjust prices based on current demand using surge rules.
-        state_space.demand: demand counts per product
-        state_space.prices: current prices (fallback if base_prices not set)
-        """
-        current_prices = self.base_prices if self.base_prices is not None else np.ones_like(demand_vector) * 99.99
-        demand = self.demand_history if self.demand_history is not None else np.zeros_like(current_prices)
-        new_prices = current_prices.copy()
-
-        high_mask = demand >= self.high_threshold
-        new_prices[high_mask] *= self.surge_multiplier
-
-        low_mask = demand <= self.low_threshold
-        new_prices[low_mask] *= self.discount_multiplier
-
-        return new_prices

experiments/procesing/pricing.py

@@ -35,9 +35,8 @@ from sklearn.base import BaseEstimator, TransformerMixin
 import numpy as np
 import pandas as pd
 import os
-from dotenv import load_dotenv
-load_dotenv()
 from supabase import create_client, Client
+from pipeline import interaction_pipeline, price_data_pipeline, elasticity_pipeline
 
 SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL", "")
 SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY", "")
@@ -80,136 +79,18 @@ class PricingFunction(BaseEstimator, TransformerMixin, ABC):
 class SimpleLinearPricingFunction(PricingFunction):
     def __init__(self, price_sensitivity: float = -0.1):
         super().__init__()
-        self.price_sensitivity = price_sensitivity
+        self.price_sensitivity = price_sensitivity  # simple coefficient
 
     def fit(self, historical_data):
         return self
 
     def transform(self, state_space: StateSpace) -> np.ndarray:
-        new_prices = state_space.prices + self.price_sensitivity * state_space.demand
+        # Simple linear adjustment: P_{t+1} = P_t + sensitivity * Q_t
+        new_prices = state_space.prices + self.price_sensitivity * state_space.demand # this is not great
         return np.maximum(new_prices, 0)
 
-
-class ElasticityBasedPricingFunction(PricingFunction):
-    """
-    Revenue-maximizing pricing using elasticity estimates.
-
-    For each product, optimal price P* maximizes R = P * Q(P)
-    where Q(P) follows power law: Q(P) = Q_0 * (P/P_0)^ε
-
-    Taking derivative dR/dP = 0 gives optimal markup:
-    P* = P_0 * (1 + 1/ε) if ε < -1 (elastic)
-
-    For inelastic demand (|ε| < 1), we apply bounded markup.
-    """
-
-    def __init__(self,
-                 cost_floor: float = 0.5,
-                 max_markup: float = 2.0,
-                 min_markup: float = 1.0,
-                 inelastic_markup: float = 1.3):
-        super().__init__()
-        self.cost_floor = cost_floor  # prices as fraction of base
-        self.max_markup = max_markup  # max price = base * max_markup
-        self.min_markup = min_markup  # min price = base * min_markup
-        self.inelastic_markup = inelastic_markup  # default for |ε| < 1
-        self.elasticity_map = {}  # productId -> elasticity
-
-    def fit(self, elasticity_df: pd.DataFrame):
-        """
-        Args:
-            elasticity_df: df with [productId, elasticity, std_error, n_obs]
-        """
-        if elasticity_df is not None and not elasticity_df.empty:
-            self.elasticity_map = dict(zip(
-                elasticity_df['productId'],
-                elasticity_df['elasticity']
-            ))
-        return self
-
-    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
-        """
-        Args:
-            state_space: current state (prices = base prices)
-            product_ids: array of productIds aligned with state_space.prices
-
-        Returns:
-            optimized prices P_{t+1}
-        """
-        base_prices = state_space.prices
-
-        if product_ids is None:
-            # fallback: use positional index as productId (not ideal)
-            product_ids = np.arange(len(base_prices))
-
-        new_prices = np.zeros_like(base_prices)
-
-        for i, (base_p, pid) in enumerate(zip(base_prices, product_ids)):
-            elasticity = self.elasticity_map.get(pid, 0.0)
-
-            if elasticity < -1:  # elastic demand
-                # optimal markup: (1 + 1/ε)
-                markup = 1 + (1 / elasticity)
-                optimal_p = base_p * markup
-            elif elasticity > -1 and elasticity < 0:  # inelastic
-                # conservative markup
-                optimal_p = base_p * self.inelastic_markup
-            else:  # ε ≥ 0 (demand increases with price, or no data)
-                # no elasticity data or anomalous, keep base price
-                optimal_p = base_p
-
-            # apply bounds
-            optimal_p = np.clip(
-                optimal_p,
-                base_p * self.min_markup,
-                base_p * self.max_markup
-            )
-            optimal_p = max(optimal_p, self.cost_floor)
-
-            new_prices[i] = optimal_p
-
-        return new_prices
-
-
-class ContextualElasticityPricing(PricingFunction):
-    """
-    Revenue optimization with contextual adjustments based on session features.
-
-    Combines elasticity-based pricing with surge/demand-based multipliers.
-    """
-
-    def __init__(self,
-                 base_pricer: ElasticityBasedPricingFunction = None,
-                 demand_sensitivity: float = 0.1,
-                 surge_threshold: float = 0.7):
-        super().__init__()
-        self.base_pricer = base_pricer or ElasticityBasedPricingFunction()
-        self.demand_sensitivity = demand_sensitivity
-        self.surge_threshold = surge_threshold
-
-    def fit(self, elasticity_df: pd.DataFrame):
-        self.base_pricer.fit(elasticity_df)
-        return self
-
-    def transform(self, state_space: StateSpace, product_ids: np.ndarray = None) -> np.ndarray:
-        # get base optimal prices from elasticity
-        base_optimal = self.base_pricer.transform(state_space, product_ids)
-
-        # compute surge multiplier from demand
-        if len(state_space.demand) > 0:
-            demand_normalized = state_space.demand / (state_space.demand.max() + 1e-8)
-            surge_multiplier = 1 + self.demand_sensitivity * np.maximum(
-                demand_normalized - self.surge_threshold, 0
-            )
-        else:
-            surge_multiplier = np.ones_like(base_optimal)
-
-        return base_optimal * surge_multiplier
-
 # Example usage:
 if __name__ == "__main__":
-    from pipeline import interaction_pipeline, price_data_pipeline, elasticity_pipeline
-
     store_mode = 'hotel'
     interaction_data = interaction_pipeline.fit_transform(None)
     price_data = price_data_pipeline.fit_transform(None)

experiments/procesing/providers/__init__.py

@@ -1,5 +0,0 @@
-from procesing.providers.base import DataProvider
-from procesing.providers.supabase import SupabaseProvider
-from procesing.providers.backend import BackendAPIProvider
-
-__all__ = ['DataProvider', 'SupabaseProvider', 'BackendAPIProvider']

experiments/procesing/providers/backend.py

@@ -1,19 +0,0 @@
-import os
-import pandas as pd
-import requests
-from typing import List
-from procesing.providers.base import DataProvider
-
-class BackendAPIProvider(DataProvider):
-    """Concrete backend API implementation"""
-    def __init__(self, backend_url: str = None):
-        self.backend_url = backend_url or os.getenv("BACKEND_URL", "http://localhost:5000")
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        resp = requests.get(f"{self.backend_url}/api/kafka/dump?topic={topic}")
-        resp.raise_for_status()
-        data = resp.json()
-
-        if not data.get('success') or not data.get('data'):
-            return pd.DataFrame()
-
-        return pd.DataFrame(data['data'])

experiments/procesing/providers/base.py

@@ -1,21 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List
-import pandas as pd
-
-class DataProvider(ABC):
-    """Abstract interface for data access, enables DI and testing"""
-
-    @abstractmethod
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        """Fetch product catalog for given store mode"""
-        pass
-
-    @abstractmethod
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        """Fetch experiment metadata for given IDs"""
-        pass
-
-    @abstractmethod
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        """Fetch data from Kafka topic via backend API"""
-        pass

experiments/procesing/providers/supabase.py

@@ -1,35 +0,0 @@
-import os
-import pandas as pd
-import requests
-from typing import List
-from supabase import create_client, Client
-from procesing.providers.base import DataProvider
-from dotenv import load_dotenv
-
-class SupabaseProvider(DataProvider):
-    """Concrete Supabase + backend API implementation"""
-
-    def __init__(self,
-                 supabase_url: str = None,
-                 supabase_key: str = None,):
-        load_dotenv()
-        self.supabase_url = supabase_url or os.getenv("NEXT_PUBLIC_SUPABASE_URL")
-        self.supabase_key = supabase_key or os.getenv("NEXT_PUBLIC_SUPABASE_ANON_KEY")
-        self.supabase: Client = create_client(self.supabase_url, self.supabase_key)
-
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        resp = self.supabase.table(f'{store_mode}_products').select(
-            "id, room_type, date_index, metadata, availability"
-        ).execute()
-        return pd.DataFrame(resp.data) if resp.data else pd.DataFrame()
-
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        if not experiment_ids:
-            return pd.DataFrame()
-
-        resp = self.supabase.table('experiments').select(
-            'id, subject_name, xp_human_only, xp_market_mode, xp_task_id, '
-            'task:tasks(task_name, task_description, task_def_of_done)'
-        ).in_('id', experiment_ids).execute()
-
-        return pd.DataFrame(resp.data) if resp.data else pd.DataFrame()

experiments/procesing/steps/__init__.py

@@ -1,29 +0,0 @@
-from procesing.steps.base import BaseContextStep
-from procesing.steps.fetch import FetchInteractionsStep, FetchPriceLogsStep, FetchExperimentsStep
-from procesing.steps.join import JoinExperimentsStep, JoinProductFeaturesStep
-from procesing.steps.augment import CreatePriceBucketsStep, AugmentEventNamesStep, AugmentInteractionsStep
-from procesing.steps.chunk import ChunkByTimeWindowStep
-from procesing.steps.demand import ComputeDemandStep, ComputeDemandForChunksStep
-from procesing.steps.elasticity import AggregatePriceLogsStep
-from procesing.steps.pricing import FitPricingFunctionStep, PredictPricesStep
-from procesing.steps.session import ExtractSessionFeaturesStep, _extract_features_for_session
-
-__all__ = [
-    'BaseContextStep',
-    'FetchInteractionsStep',
-    'FetchPriceLogsStep',
-    'FetchExperimentsStep',
-    'JoinExperimentsStep',
-    'JoinProductFeaturesStep',
-    'CreatePriceBucketsStep',
-    'AugmentEventNamesStep',
-    'AugmentInteractionsStep',
-    'ChunkByTimeWindowStep',
-    'ComputeDemandStep',
-    'ComputeDemandForChunksStep',
-    'AggregatePriceLogsStep',
-    'FitPricingFunctionStep',
-    'PredictPricesStep',
-    'ExtractSessionFeaturesStep',
-    '_extract_features_for_session',
-]

experiments/procesing/steps/augment.py

@@ -1,140 +0,0 @@
-import numpy as np
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-
-class AugmentInteractionsStep(BaseContextStep):
-    """
-    Consolidated step: create price buckets, augment event names, join experiments.
-    Input: (interactions_df, price_logs_df)
-    Output: enriched interactions_df
-    """
-
-    def transform(self, data: tuple):
-        interactions_df, price_logs_df = data
-
-        if interactions_df.empty:
-            return interactions_df
-
-        # Step 1: Create price buckets
-        interactions_df = self._create_price_buckets(interactions_df)
-
-        # Step 2: Augment event names
-        interactions_df = self._augment_event_names(interactions_df)
-
-        # Step 3: Join experiments (optional)
-        if 'experimentId' in interactions_df.columns:
-            interactions_df = self._join_experiments(interactions_df)
-
-        return interactions_df
-
-    def _create_price_buckets(self, df: pd.DataFrame):
-        """Create price bucket labels from price data"""
-        if 'metadata_price' not in df.columns:
-            df['price_bucket'] = ""
-            return df
-
-        n_buckets = self.context.config.get('n_price_buckets', 5)
-
-        if df['metadata_price'].notnull().sum() > 0:
-            try:
-                price_buckets = pd.qcut(
-                    df['metadata_price'],
-                    q=n_buckets,
-                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
-                    duplicates='drop'
-                )
-            except ValueError:
-                # fallback for insufficient unique values
-                price_buckets = df['metadata_price'].apply(
-                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
-                )
-        else:
-            price_buckets = pd.Series([""] * len(df), index=df.index)
-
-        df['price_bucket'] = price_buckets
-        return df
-
-    def _augment_event_names(self, df: pd.DataFrame):
-        """Augment event names with product and price bucket schema"""
-        # Create schema: _productId@price_bucket
-        has_product = df.get('productId', pd.Series()).notnull()
-        has_bucket = df.get('price_bucket', pd.Series()).notnull()
-
-        df['metadata_schema'] = np.where(
-            has_product & has_bucket,
-            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
-            ""
-        )
-
-        df['eventName'] = df['eventName'] + df['metadata_schema']
-        return df
-
-    def _join_experiments(self, df: pd.DataFrame):
-        """Join experiment metadata if experimentId present"""
-        exp_ids = df['experimentId'].dropna().unique().tolist()
-        if not exp_ids:
-            return df
-
-        experiments_df = self.context.provider.fetch_experiments(exp_ids)
-        if experiments_df.empty:
-            return df
-
-        return df.merge(
-            experiments_df,
-            left_on='experimentId',
-            right_on='id',
-            how='left',
-            suffixes=('', '_exp')
-        )
-
-
-class CreatePriceBucketsStep(BaseContextStep):
-    """Create price bucket labels from price data"""
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty or 'metadata_price' not in df.columns:
-            df['price_bucket'] = ""
-            return df
-
-        n_buckets = self.context.config.get('n_price_buckets', 5)
-
-        if df['metadata_price'].notnull().sum() > 0:
-            try:
-                price_buckets = pd.qcut(
-                    df['metadata_price'],
-                    q=n_buckets,
-                    labels=[f"PB_{i+1}" for i in range(n_buckets)],
-                    duplicates='drop'
-                )
-            except ValueError:
-                # fallback for insufficient unique values
-                price_buckets = df['metadata_price'].apply(
-                    lambda x: f"P_{int(x)}" if pd.notnull(x) else ""
-                )
-        else:
-            price_buckets = pd.Series([""] * len(df), index=df.index)
-
-        df['price_bucket'] = price_buckets
-        return df
-
-
-class AugmentEventNamesStep(BaseContextStep):
-    """Augment event names with product and price bucket schema"""
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty:
-            return df
-
-        # Create schema: _productId@price_bucket
-        has_product = df.get('productId', pd.Series()).notnull()
-        has_bucket = df.get('price_bucket', pd.Series()).notnull()
-
-        df['metadata_schema'] = np.where(
-            has_product & has_bucket,
-            "_" + df['productId'].astype(str) + "@" + df['price_bucket'].astype(str),
-            ""
-        )
-
-        df['eventName'] = df['eventName'] + df['metadata_schema']
-        return df

experiments/procesing/steps/base.py

@@ -1,31 +0,0 @@
-from abc import ABC, abstractmethod
-from sklearn.base import BaseEstimator, TransformerMixin
-from procesing.context import PipelineContext
-
-class BaseContextStep(BaseEstimator, TransformerMixin, ABC):
-    """
-    Base for all pipeline steps.
-    Each step is stateless, context-driven, and performs ONE transformation.
-    """
-
-    def __init__(self, context: PipelineContext):
-        self.context = context
-
-    def fit(self, X=None, y=None):
-        """Most steps don't need training"""
-        return self
-
-    @abstractmethod
-    def transform(self, X):
-        """Transform input using context. Must be implemented by subclass."""
-        pass
-
-    def get_params(self, deep=True):
-        """sklearn compatibility"""
-        return {'context': self.context}
-
-    def set_params(self, **params):
-        """sklearn compatibility"""
-        if 'context' in params:
-            self.context = params['context']
-        return self

experiments/procesing/steps/chunk.py

@@ -1,34 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class ChunkByTimeWindowStep(BaseContextStep):
-    """
-    Chunk dataframe into time windows.
-    Returns list of dicts with window metadata.
-    """
-
-    def transform(self, df: pd.DataFrame):
-        if df.empty:
-            return []
-
-        df = df.copy()
-        ts_col = self.context.config.get('ts_col', 'ts')
-        window_size = self.context.window_size
-
-        # ensure datetime
-        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
-            df[ts_col] = pd.to_datetime(df[ts_col])
-
-        df = df.sort_values(ts_col)
-        df['_window'] = df[ts_col].dt.floor(window_size)
-
-        chunks = []
-        for idx, (window_start, group) in enumerate(df.groupby('_window')):
-            chunks.append({
-                'window_start': window_start,
-                'window_end': window_start + pd.Timedelta(window_size),
-                'window_idx': idx,
-                'data': group.drop(columns=['_window'])
-            })
-
-        return chunks

experiments/procesing/steps/demand.py

@@ -1,61 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class ComputeDemandStep(BaseContextStep):
-    """
-    Compute demand vector for a single time window or dataframe.
-    Input: single chunk dict OR raw dataframe
-    Output: demand dataframe with [productId, demand_score]
-    """
-
-    def transform(self, chunk):
-        # handle both chunk dict and raw dataframe
-        if isinstance(chunk, dict):
-            interactions = chunk['data']
-            window_meta = {k: v for k, v in chunk.items() if k != 'data'}
-        else:
-            interactions = chunk
-            window_meta = {}
-
-        products = self.context.products
-        unique_products = products['id'].unique()
-
-        # apply filters if configured
-        session_filter = self.context.config.get('session_filter')
-        experiment_filter = self.context.config.get('experiment_filter')
-
-        if session_filter and 'sessionId' in interactions.columns:
-            interactions = interactions[interactions['sessionId'] == session_filter]
-        if experiment_filter and 'experimentId' in interactions.columns:
-            interactions = interactions[interactions['experimentId'] == experiment_filter]
-
-        interactions_with_products = interactions.dropna(subset=['productId'])
-
-        if interactions_with_products.empty:
-            demand_df = pd.DataFrame({
-                'productId': unique_products,
-                'demand_score': 0
-            })
-        else:
-            # crosstab for simple demand count
-            demand_df = pd.crosstab(
-                interactions_with_products['productId'],
-                'count'
-            ).reindex(unique_products, fill_value=0).reset_index()
-            demand_df.columns = ['productId', 'demand_score']
-
-        # attach window metadata if present
-        if window_meta:
-            return {**window_meta, 'demand_vector': demand_df}
-        return demand_df
-
-
-class ComputeDemandForChunksStep(BaseContextStep):
-    """Apply ComputeDemandStep to list of chunks"""
-
-    def transform(self, chunks: list):
-        if not chunks:
-            return []
-
-        demand_step = ComputeDemandStep(self.context)
-        return [demand_step.transform(chunk) for chunk in chunks]

experiments/procesing/steps/elasticity.py

@@ -1,42 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import Dict, List
-from procesing.steps.base import BaseContextStep
-
-class AggregatePriceLogsStep(BaseContextStep):
-    """
-    Aggregate price logs into time windows using VECTORIZED operations.
-    Input: price_logs_df
-    Output: list of price chunks with [productId, price]
-    """
-
-    def transform(self, price_logs_df: pd.DataFrame):
-        if price_logs_df.empty:
-            return []
-
-        df = price_logs_df.copy()
-        ts_col = self.context.config.get('ts_col', 'ts')
-        #window_size = self.context.window_size WE ARE NOT USING CHUNKS ANYMORE
-
-        # ensure datetime
-        if not pd.api.types.is_datetime64_any_dtype(df[ts_col]):
-            df[ts_col] = pd.to_datetime(df[ts_col])
-
-        df = df.sort_values([ts_col, 'productId'])
-        products = self.context.products
-        # get base price from metadata if available 1) read the metadata col as json and get the base_price
-        products['base_price'] = products.apply(
-            lambda row: row['metadata'].get('base_price', 0) if isinstance(row['metadata'], dict) else 0,
-            axis=1
-        )
-
-        unique_products = products['id'].unique()
-
-        df_indexed = df.set_index(ts_col)
-        # we return a df of average price per product over the entire period
-        # TODO: maybe consider different opration to handle price aggregation over time
-        avg_prices = df_indexed.groupby('productId')['price'].mean().reindex(unique_products, fill_value=0).reset_index()
-        avg_prices.columns = ['productId', 'price']
-        # fill 0s with base_price from products
-        base_price_map = products.set_index('id')['base_price'].to_dict()
-        return avg_prices

experiments/procesing/steps/fetch.py

@@ -1,73 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class FetchInteractionsStep(BaseContextStep):
-    """Fetch raw interaction data from Kafka topic with optional time filtering"""
-
-    def __init__(self, context, lookback: str = None):
-        super().__init__(context)
-        self.lookback = lookback
-
-    def transform(self, X=None):
-        df = self.context.provider.fetch_kafka_topic('user-interactions')
-
-        if df.empty:
-            return df
-
-        # Explode metadata JSON column
-        if 'metadata' in df.columns:
-            df = df.join(
-                pd.json_normalize(df.pop('metadata'), sep='.').add_prefix('metadata_')
-            )
-
-        df = df.dropna(subset=['eventName'])
-        # drop all where page has /admin/
-        df = df[~df['page'].str.contains('/admin/', na=False)]
-
-        # Remap dateIndex if present
-        if 'metadata_dateIndex' in df.columns:
-            df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
-
-        # Apply time filtering if lookback specified
-        if self.lookback and 'ts' in df.columns:
-            df['ts'] = pd.to_datetime(df['ts'])
-            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
-            df = df[df['ts'] >= cutoff]
-
-        return df
-
-
-class FetchPriceLogsStep(BaseContextStep):
-    """Fetch price log data from Kafka topic with optional time filtering"""
-
-    def __init__(self, context, lookback: str = None):
-        super().__init__(context)
-        self.lookback = lookback
-
-    def transform(self, X=None):
-        df = self.context.provider.fetch_kafka_topic('price-logs')
-
-        if df.empty:
-            return df
-
-        # Apply time filtering if lookback specified
-        if self.lookback and 'ts' in df.columns:
-            df['ts'] = pd.to_datetime(df['ts'])
-            cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
-            df = df[df['ts'] >= cutoff]
-
-        return df
-
-
-class FetchExperimentsStep(BaseContextStep):
-    """Fetch experiment metadata for given interaction data"""
-
-    def transform(self, interactions_df: pd.DataFrame):
-        if interactions_df.empty or 'experimentId' not in interactions_df.columns:
-            return pd.DataFrame()
-
-        exp_ids = interactions_df['experimentId'].dropna().unique().tolist()
-        if not exp_ids:
-            return pd.DataFrame()
-
-        return self.context.provider.fetch_experiments(exp_ids)

experiments/procesing/steps/join.py

@@ -1,58 +0,0 @@
-import pandas as pd
-from procesing.steps.base import BaseContextStep
-
-class JoinExperimentsStep(BaseContextStep):
-    """Join experiment metadata to interactions"""
-
-    def transform(self, data: tuple):
-        """
-        Args:
-            data: (interactions_df, experiments_df)
-        Returns:
-            merged interactions dataframe
-        """
-        interactions_df, experiments_df = data
-
-        if experiments_df.empty:
-            return interactions_df
-
-        # Flatten nested task field if present
-        if 'task' in experiments_df.columns and experiments_df['task'].notnull().any():
-            task_norm = pd.json_normalize(experiments_df['task'].dropna())
-            task_norm.index = experiments_df[experiments_df['task'].notnull()].index
-            experiments_df = experiments_df.drop('task', axis=1).join(task_norm, rsuffix='_task')
-
-        # Rename for clarity
-        experiments_df = experiments_df.rename(columns={
-            'id': 'experimentId',
-            'subject_name': 'exp_subject',
-            'xp_human_only': 'exp_human_only',
-            'xp_market_mode': 'exp_market_mode',
-            'xp_task_id': 'exp_task_id'
-        })
-
-        return interactions_df.merge(experiments_df, on='experimentId', how='left')
-
-class JoinProductFeaturesStep(BaseContextStep):
-    """Join product features to interactions"""
-
-    def transform(self, data: tuple):
-        """
-        Args:
-            data: (interactions_df, products_df)
-        Returns:
-            merged interactions dataframe
-        """
-        demand_df, price_df = data
-
-        # get base prices from products if available
-        products = self.context.products
-        products['base_price'] = products.apply(
-            lambda row: float(row['metadata'].get('base_price', 0.0)) if isinstance(row['metadata'], dict) else 0,
-            axis=1
-        )
-        products = products[['id', 'base_price']].rename(columns={'id': 'productId'})
-
-        if price_df.empty:
-            return demand_df
-        return demand_df.merge(price_df, on='productId', how='left').merge(products, on='productId', how='left')

experiments/procesing/steps/pricing.py

@@ -1,55 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import Optional, List, Dict, Any
-from dataclasses import dataclass, field
-from procesing.pricers.simple import StaticPricer
-from procesing.steps.base import BaseContextStep
-from procesing.pricers import ElasticityBasedPricer
-
-class State:
-    def __init__(self,
-                 last_action : str,
-                 last_productId : str,
-                 last_price : float,
-                 session_features : np.ndarray
-                 ):
-        pass
-
-
-
-class FitPricingFunctionStep(BaseContextStep):
-    """
-    Fit pricing function using data.
-    Input: pricing_data
-    Output: fitted pricing function instance
-    """
-
-    def transform(self, pricing_data: pd.DataFrame):
-        pricing_class = self.context.config.get('pricing_function_class', StaticPricer)
-        pricing_params = self.context.config.get('pricing_function_params', {})
-
-        pricer = pricing_class(**pricing_params)
-        pricer.fit(pricing_data)
-
-        return pricer
-
-
-class PredictPricesStep(BaseContextStep):
-    """
-    Predict optimal prices using fitted pricing function.
-    Input: (pricer, state_space)
-    Output: prices_df [productId, predicted_price]
-    """
-
-    def transform(self, data: tuple):
-        pricer, state_space = data
-
-        products = self.context.products
-        product_ids = products['id'].values
-
-        predicted_prices = pricer.predict(state_space)
-
-        return pd.DataFrame({
-            'productId': product_ids,
-            'predicted_price': predicted_prices
-        })

experiments/procesing/steps/session.py

@@ -1,159 +0,0 @@
-"""
-Session feature extraction for S_t component of state space.
-Computes behavioral signals from interaction data already in pipeline.
-"""
-import pandas as pd
-import numpy as np
-from typing import Optional, Dict, Any
-from collections import Counter
-from procesing.steps.base import BaseContextStep
-
-def _extract_features_for_session(session_df: pd.DataFrame, session_timeout_sec: float = 900) -> Dict[str, Any]:
-    """Compute features for single session.
-
-    Args:
-        session_df: interaction events for this session
-        session_timeout_sec: max gap between events before resetting duration (default 900s = 15min)
-    """
-    features = {}
-
-    # basic counts
-    features['total_interactions'] = len(session_df)
-
-    event_counts = session_df['eventName'].value_counts().to_dict()
-    features['page_views'] = event_counts.get('page_view', 0) + event_counts.get('view_item_page', 0)
-    features['item_views'] = event_counts.get('view_item_page', 0)
-    features['searches'] = event_counts.get('search', 0)
-    features['cart_adds'] = event_counts.get('add_item_to_cart', 0)
-
-    # hover events
-    hover_events = ['hover_over_title', 'hover_over_paragraph', 'hover_over_link', 'hover_over_button']
-    features['hovers'] = sum(event_counts.get(ev, 0) for ev in hover_events)
-
-    # product-level signals
-    product_ids = session_df['productId'].dropna()
-    features['unique_products_viewed'] = product_ids.nunique()
-
-    if len(product_ids) > 0:
-        product_view_counts = Counter(product_ids)
-        features['product_view_depth'] = max(product_view_counts.values())
-    else:
-        features['product_view_depth'] = 0
-
-    # temporal features with session timeout logic
-    if 'ts' in session_df.columns:
-        timestamps = session_df['ts'].sort_values()
-
-        # compute active duration considering timeout gaps
-        if len(timestamps) > 1:
-            time_diffs = timestamps.diff().dropna().dt.total_seconds()
-            # only count gaps shorter than timeout towards active session duration
-            active_diffs = time_diffs[time_diffs <= session_timeout_sec]
-            features['session_duration_sec'] = active_diffs.sum() if len(active_diffs) > 0 else 0.0
-
-            features['avg_time_between_events'] = time_diffs.mean()
-            features['std_time_between_events'] = time_diffs.std()
-        else:
-            features['session_duration_sec'] = 0.0
-            features['avg_time_between_events'] = 0.0
-            features['std_time_between_events'] = 0.0
-
-        if features['session_duration_sec'] > 0:
-            features['interaction_velocity'] = (features['total_interactions'] / features['session_duration_sec']) * 60
-        else:
-            features['interaction_velocity'] = 0.0
-    else:
-        features['session_duration_sec'] = 0.0
-        features['interaction_velocity'] = 0.0
-        features['avg_time_between_events'] = 0.0
-        features['std_time_between_events'] = 0.0
-
-    # cart/conversion signals
-    features['cart_to_view_ratio'] = features['cart_adds'] / features['item_views'] if features['item_views'] > 0 else 0.0
-
-    return features
-
-
-def _apply_to_slice(df: pd.DataFrame) -> pd.DataFrame:
-    """Apply feature extraction to sliding window of interactions."""
-    # add columns of all features at each step
-    new_cols = ["total_interactions", "page_views", "item_views", "searches",
-                "cart_adds", "hovers", "unique_products_viewed", "product_view_depth",
-                "session_duration_sec", "interaction_velocity",
-                "avg_time_between_events", "std_time_between_events",
-                "cart_to_view_ratio"]
-    for col in new_cols: df[col] = np.nan
-    for idx in range(1, len(df) + 1):
-        features = _extract_features_for_session(df.iloc[:idx])
-        # fillna kinda meh
-        features = { k: (v if not pd.isna(v) else 0.0) for k, v in features.items() }
-        for col in new_cols:
-            df.at[df.index[idx - 1], col] = features[col]
-        #print(f"Processed {idx}/{len(df)} events for session {df['sessionId'].iloc[0]}")
-    return df
-
-class BuildStateSpaceStep(BaseContextStep):
-    """
-    Build state space representation S_t from session features.
-
-    Input: session_features DataFrame
-    Output: state_space_df DataFrame with S_t vectors
-    """
-
-    def transform(self, rich_dataset: pd.DataFrame) -> pd.DataFrame:
-        # check if features are present
-        required_cols = ["total_interactions", "page_views", "item_views", "searches",
-                         "cart_adds", "hovers", "unique_products_viewed", "product_view_depth",
-                         "session_duration_sec", "interaction_velocity",
-                         "avg_time_between_events", "std_time_between_events",
-                         "cart_to_view_ratio"]
-        if not all(col in rich_dataset.columns for col in required_cols):
-            raise ValueError("Missing required columns for feature extraction.")
-        if rich_dataset.empty:
-            return pd.DataFrame()
-
-
-        # For simplicity, we return as is
-        return rich_dataset.copy()
-
-
-
-
-class ExtractSessionFeaturesStep(BaseContextStep):
-    """
-    Extract session-level behavioral features from interaction logs.
-
-    Input: interactions_df (user-interactions from earlier pipeline step)
-    Output: interactions_df with added session feature columns
-    """
-
-    def transform(self, interactions_df: pd.DataFrame) -> pd.DataFrame:
-        if interactions_df.empty:
-            return pd.DataFrame()
-
-        # ensure timestamp column
-        if 'ts' in interactions_df.columns:
-            interactions_df = interactions_df.copy()
-            interactions_df['ts'] = pd.to_datetime(interactions_df['ts'])
-
-        # group by session and compute features
-        session_features = []
-        for session_id, session_df in interactions_df.groupby('sessionId'):
-            new_slice = _apply_to_slice(session_df.sort_values('ts'))
-            session_features.append(new_slice)
-
-        return pd.concat(session_features, ignore_index=True)
-
-
-
-class FilterSessionInteractionsStep(BaseContextStep):
-    """
-    Filter interactions DataFrame to specific session.
-
-    Input: (interactions_df, session_id)
-    Output: interactions_df filtered to session_id
-    """
-
-    def transform(self, data: tuple) -> pd.DataFrame:
-        interactions_df, session_id = data
-        return interactions_df[interactions_df['sessionId'] == session_id].copy()

experiments/procesing/tests/conftest.py

@@ -1,271 +0,0 @@
-import pytest
-import pandas as pd
-from typing import List
-from procesing.providers.base import DataProvider
-from procesing.context import PipelineContext
-
-
-class MockProvider(DataProvider):
-    """Mock provider for testing, holds in-memory fixtures"""
-
-    def __init__(self, products_df=None, experiments_df=None, kafka_data=None):
-        self._products = products_df if products_df is not None else pd.DataFrame()
-        self._experiments = experiments_df if experiments_df is not None else pd.DataFrame()
-        self._kafka_data = kafka_data if kafka_data is not None else {}
-
-    def fetch_products(self, store_mode: str) -> pd.DataFrame:
-        return self._products.copy()
-
-    def fetch_experiments(self, experiment_ids: List[str]) -> pd.DataFrame:
-        if self._experiments.empty:
-            return pd.DataFrame()
-        return self._experiments[
-            self._experiments['id'].isin(experiment_ids)
-        ].copy()
-
-    def fetch_kafka_topic(self, topic: str) -> pd.DataFrame:
-        return self._kafka_data.get(topic, pd.DataFrame()).copy()
-
-
-@pytest.fixture
-def mock_products():
-    """Standard product catalog fixture with realistic IDs from test data"""
-    return pd.DataFrame({
-        'id': [
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ],
-        'name': ['Junior Suite', 'Superior Room', 'Deluxe Room'],
-        'base_price': [200.0, 150.0, 180.0]
-    })
-
-
-@pytest.fixture
-def mock_interactions_raw_kafka():
-    """Raw Kafka message structure for interactions, matches production format"""
-    return [
-        {
-            'partitionID': 0, 'offset': 203, 'timestamp': 1764102082676,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'learn_more_about_item',
-                    'page': '/hotel/products/d018efc1-25e9-4284-b276-80386e048b25',
-                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
-                    'metadata': {'type': 'hotel', 'dateIndex': 1, 'roomType': 'Junior Suite'},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:22.674Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 204, 'timestamp': 1764102086982,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'page_view',
-                    'page': '/hotel/products',
-                    'productId': None,
-                    'metadata': {'referrer': ''},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:26.947Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 205, 'timestamp': 1764102091825,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'eventName': 'hover_over_title',
-                    'page': '/hotel/products',
-                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-                    'metadata': {'elementText': 'Superior Room', 'dateIndex': 1, 'dwellTime': 1200},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:31.823Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 206, 'timestamp': 1764102094193,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
-                    'eventName': 'hover_over_paragraph',
-                    'page': '/hotel/products',
-                    'productId': '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1307},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:34.191Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 207, 'timestamp': 1764102101970,
-            'value': {
-                'payload': {
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': 'bbbbcccc-dddd-eeee-ffff-000011112222',
-                    'eventName': 'hover_over_paragraph',
-                    'page': '/hotel/products',
-                    'productId': 'd018efc1-25e9-4284-b276-80386e048b25',
-                    'metadata': {'elementText': 'price', 'dateIndex': 1, 'dwellTime': 1201},
-                    'storeMode': 'hotel',
-                    'ts': '2025-11-25T20:21:41.967Z'
-                }
-            }
-        }
-    ]
-
-
-@pytest.fixture
-def mock_interactions(mock_interactions_raw_kafka):
-    """Processed interaction DataFrame (what provider.fetch_kafka_topic returns)"""
-    records = [msg['value']['payload'] for msg in mock_interactions_raw_kafka]
-    df = pd.DataFrame(records)
-    df['timestamp'] = pd.to_datetime(df['ts'])
-    return df
-
-
-@pytest.fixture
-def mock_price_logs_raw_kafka():
-    """Raw Kafka message structure for price logs, matches production format"""
-    return [
-        {
-            'partitionID': 0, 'offset': 32, 'timestamp': 1764104757969,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 162.47,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'shop',
-                    'ts': '2025-11-25T21:05:57.967Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 33, 'timestamp': 1764104757995,
-            'value': {
-                'payload': {
-                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
-                    'price': 743.49,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'shop',
-                    'ts': '2025-11-25T21:05:57.993Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 34, 'timestamp': 1764104758011,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 163.87,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'shop',
-                    'ts': '2025-11-25T21:05:58.009Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 35, 'timestamp': 1764104758050,
-            'value': {
-                'payload': {
-                    'productId': '2ddabbfc-4127-48fc-86dc-ebc4c677efa2',
-                    'price': 397.46,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'shop',
-                    'ts': '2025-11-25T21:05:58.049Z'
-                }
-            }
-        },
-        {
-            'partitionID': 0, 'offset': 36, 'timestamp': 1764104768865,
-            'value': {
-                'payload': {
-                    'productId': '2cd7f756-fc65-4ba0-ab01-74521c1fff43',
-                    'price': 401.66,
-                    'sessionId': 'd423ce8a-77aa-4c9a-94d4-d1adddcc3472',
-                    'experimentId': '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35',
-                    'storeMode': 'shop',
-                    'ts': '2025-11-25T21:06:08.864Z'
-                }
-            }
-        }
-    ]
-
-
-@pytest.fixture
-def mock_price_logs(mock_price_logs_raw_kafka):
-    """Processed price logs DataFrame (what provider.fetch_kafka_topic returns)"""
-    # extract payloads and flatten
-    records = [msg['value']['payload'] for msg in mock_price_logs_raw_kafka]
-    df = pd.DataFrame(records)
-    df['timestamp'] = pd.to_datetime(df['ts'])
-    return df
-
-
-@pytest.fixture
-def mock_experiments():
-    """Standard experiment metadata fixture matching Supabase schema"""
-    return pd.DataFrame({
-        'id': ['53aefd07-f66a-4d7f-ba8b-7ea1fc562d35', 'bbbbcccc-dddd-eeee-ffff-000011112222'],
-        'created_at': pd.to_datetime(['2025-11-25T20:00:00Z', '2025-11-26T10:00:00Z']),
-        'subject_name': ['Session A', 'Session B'],
-        'xp_human_only': [True, False],
-        'xp_market_mode': ['hotel', 'shop'],
-        'xp_task_id': [None, None]
-    })
-
-
-@pytest.fixture
-def mock_provider(mock_products, mock_experiments, mock_interactions, mock_price_logs):
-    """Fully configured mock provider"""
-    return MockProvider(
-        products_df=mock_products,
-        experiments_df=mock_experiments,
-        kafka_data={
-            'user-interactions': mock_interactions,
-            'price-logs': mock_price_logs
-        }
-    )
-
-
-@pytest.fixture
-def pipeline_context(mock_provider):
-    """Standard pipeline context for testing"""
-    return PipelineContext(
-        provider=mock_provider,
-        store_mode='hotel',
-        window_size='30s',
-        n_price_buckets=3
-    )
-
-
-@pytest.fixture
-def empty_provider():
-    """Provider with no data, for edge case testing"""
-    return MockProvider(
-        products_df=pd.DataFrame(columns=['id', 'name', 'base_price']),
-        experiments_df=pd.DataFrame(columns=['id', 'created_at', 'subject_name', 'xp_human_only', 'xp_market_mode', 'xp_task_id']),
-        kafka_data={'user-interactions': pd.DataFrame(), 'price-logs': pd.DataFrame()}
-    )
-
-
-@pytest.fixture
-def empty_context(empty_provider):
-    """Context with empty provider"""
-    return PipelineContext(
-        provider=empty_provider,
-        store_mode='hotel',
-        window_size='30s'
-    )

experiments/procesing/tests/test_augement.py

@@ -1,45 +0,0 @@
-import pytest
-import random
-import pandas as pd
-from procesing.steps import (
-    CreatePriceBucketsStep,
-    AugmentEventNamesStep
-)
-
-def test_bucketing(pipeline_context):
-    step = CreatePriceBucketsStep(context=pipeline_context)
-
-    # Test with normal price data
-    df = pd.DataFrame({
-        'metadata_price': random.sample(range(10, 1000), 100)
-    })
-    result = step.transform(df)
-    assert 'price_bucket' in result.columns
-    # test if is categorical
-    assert isinstance(result['price_bucket'].dtype, pd.CategoricalDtype)
-    assert result['price_bucket'].nunique() == 3 # as per context config
-    # distribution check
-    counts = result['price_bucket'].value_counts()
-    assert all(counts > 0)
-    assert counts.max() - counts.min() <= 10  # roughly equal distribution for 100 samples
-    # Test with empty DataFrame
-    df = pd.DataFrame()
-    result = step.transform(df)
-    assert 'price_bucket' in result.columns
-    assert result.empty
-
-
-def test_augment_names(pipeline_context):
-    df = pd.DataFrame({
-        'eventName': ['click', 'view', 'purchase'],
-        'productId': ['prod_1', 'prod_2', None],
-        'price_bucket': ['PB_1', None, 'PB_3']
-    })
-    step = AugmentEventNamesStep(context=pipeline_context)
-    result = step.transform(df)
-    expected_event_names = [
-        'click_prod_1@PB_1',
-        'view',
-        'purchase'
-    ]
-    assert result['eventName'].tolist() == expected_event_names

experiments/procesing/tests/test_demand.py

@@ -1,49 +0,0 @@
-import pytest
-import random
-import pandas as pd
-from procesing.steps import (
-    ComputeDemandStep
-)
-
-def test_compute_demand(pipeline_context):
-    step = ComputeDemandStep(context=pipeline_context)
-
-    # Test with normal interaction data
-    df = pd.DataFrame({
-        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
-        'productId': random.choices([
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ], k=100),
-        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
-    })
-    result = step.transform(df)
-    assert type(result) == pd.DataFrame
-    assert not result.empty
-    assert set(result['productId']) == set(pipeline_context.products['id'])
-    assert all(result['demand_score'] > 100/3 -10)
-
-
-def test_compute_demand_skewed(pipeline_context):
-    step = ComputeDemandStep(context=pipeline_context)
-
-    # Test with normal interaction data
-    df = pd.DataFrame({
-        'ts': pd.date_range(start='2023-01-01', periods=100, freq='h'),
-        'productId': random.choices([
-            'd018efc1-25e9-4284-b276-80386e048b25',
-            '51266ddb-5b07-47b7-89ee-5b5cae94bb11',
-            '2cd7f756-fc65-4ba0-ab01-74521c1fff43'
-        ], weights=[0.7, 0.2, 0.1], k=100),
-        'eventName': random.choices(['view', 'click', 'purchase'], k=100)
-    })
-    result = step.transform(df)
-    assert type(result) == pd.DataFrame
-    assert not result.empty
-    assert set(result['productId']) == set(pipeline_context.products['id'])
-    # test for skewness
-    scores = result.set_index('productId')['demand_score'].to_dict()
-    assert scores['d018efc1-25e9-4284-b276-80386e048b25'] > \
-           scores['51266ddb-5b07-47b7-89ee-5b5cae94bb11'] > \
-           scores['2cd7f756-fc65-4ba0-ab01-74521c1fff43']

experiments/procesing/tests/test_fetch.py

@@ -1,51 +0,0 @@
-import pytest
-import pandas as pd
-from procesing.steps import (
-    FetchInteractionsStep,
-    FetchPriceLogsStep,
-    FetchExperimentsStep,
-)
-
-
-def test_fetch_interactions_data(pipeline_context):
-    step = FetchInteractionsStep(pipeline_context)
-    data = step.transform(None)
-    assert data is not None
-    assert isinstance(data, pd.DataFrame)
-    expected_cols = [
-        "eventName",
-        "dateIndex",
-        "experimentId",
-        "storeMode",
-        "metadata_elementText"
-    ]
-    for expected in expected_cols:
-        assert expected in data.columns
-
-def test_fetch_price_logs(pipeline_context):
-    step = FetchPriceLogsStep(pipeline_context)
-    data = step.transform(None)
-    assert data is not None
-    assert isinstance(data, pd.DataFrame)
-    expected_cols = [
-        "price",
-        "productId"
-    ]
-    for expected in expected_cols:
-        assert expected in data.columns
-    prices = data['price'].to_list()
-    assert min(prices) >= 0
-    assert max(prices) <= 9999
-
-
-def test_experiments_fetching(pipeline_context):
-    interactions = FetchInteractionsStep(pipeline_context).transform(None)
-    assert interactions is not None
-    experiments = FetchExperimentsStep(pipeline_context)
-    experiment_data = experiments.transform(interactions)
-    assert experiment_data is not None
-    assert isinstance(experiment_data, pd.DataFrame)
-    assert not experiment_data.empty
-    assert 'id' in experiment_data.columns
-    assert len(experiment_data) == 2
-    assert '53aefd07-f66a-4d7f-ba8b-7ea1fc562d35' in experiment_data['id'].values

experiments/procesing/tests/test_pricing.py

@@ -1,87 +0,0 @@
-import pytest
-import pandas as pd
-
-from procesing.pricers import (
-    StaticPricer,
-    RandomPricer,
-    ElasticityBasedPricer
-)
-
-
-def test_static_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'product_id': [1, 2, 3],
-        'base_price': [100.0, 150.0, 200.0]
-    })
-
-    # Initialize and fit StaticPricer
-    pricer = StaticPricer()
-    pricer.fit(historical_data)
-
-    # Predict prices
-    predicted_prices = pricer.predict(None)
-
-    # Assert that predicted prices match base prices
-    expected_prices = historical_data['base_price'].values
-    assert all(predicted_prices == expected_prices), "Predicted prices do not match base prices"
-
-
-def test_random_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'product_id': [1, 2, 3],
-        'base_price': [100.0, 150.0, 200.0]
-    })
-
-    # Initialize and fit RandomPricer
-    pricer = RandomPricer(price_min=50.0, price_max=250.0, seed=42)
-    pricer.fit(historical_data)
-
-    # Predict prices
-    predicted_prices = pricer.predict(None)
-
-    # Assert that predicted prices are within bounds
-    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
-    assert predicted_prices.max() <= 250.0, "Predicted prices are above maximum bound"
-    # distribution check (not so strict)
-    assert len(set(predicted_prices)) > 1, "Predicted prices are not varied enough"
-    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
-
-def test_elasticity_based_pricer_fit_and_predict():
-    # Sample historical data
-    historical_data = pd.DataFrame({
-        'productId': [1, 2, 3],
-        'elasticity': [-1.5, -0.5, -2.0],
-        'base_price': [100.0, 150.0, 200.0],
-        'mean_demand': [10, 20, 15]
-    })
-
-    # Initialize and fit ElasticityBasedPricer
-    pricer = ElasticityBasedPricer(alpha=0.1, price_floor=50.0, price_ceil=300.0)
-    pricer.fit(historical_data)
-
-    # Create a mock state space with demand deviations
-    class MockStateSpace:
-        def __init__(self, demand):
-            self.demand = demand
-
-    # Simulate demand higher than mean for all products
-    state_space = MockStateSpace(demand=[15, 25, 20])
-
-    # Predict prices
-    predicted_prices = pricer.predict(state_space)
-
-    # Assert that predicted prices are within bounds
-    assert predicted_prices.min() >= 50.0, "Predicted prices are below minimum bound"
-    assert predicted_prices.max() <= 300.0, "Predicted prices are above maximum bound"
-    assert len(predicted_prices) == len(historical_data), "Number of predicted prices does not match number of products"
-
-    # now we gotta check semantic validity
-    # since demand is higher than mean, prices should generally increase
-    for i, row in historical_data.iterrows():
-        base_price = row['base_price']
-        elasticity = row['elasticity']
-        expected_increase = base_price * (1 + 0.1 * abs(elasticity) * ((state_space.demand[i] - row['mean_demand']) / row['mean_demand']))
-        assert predicted_prices[i] >= base_price, f"Predicted price for product {row['productId']} did not increase as expected"
-        assert abs(predicted_prices[i] - expected_increase) < 1e-5, f"Predicted price for product {row['productId']} does not match expected calculation within 1e-5 tolerance"

experiments/pytest.ini

@@ -1,8 +0,0 @@
-[pytest]
-pythonpath = .
-testpaths = procesing/tests agents
-python_files = test*.py
-python_classes = Test*
-python_functions = test_*
-asyncio_mode = auto
-asyncio_default_fixture_loop_scope = function

lib/model_registry.py

@@ -1,180 +0,0 @@
-import redis
-import pickle
-import json
-import pandas as pd
-from typing import Optional, Dict, Any
-import os
-import logging
-log = logging.getLogger(__name__)
-
-class ModelRegistry:
-    """
-    Lightweight model registry using Redis for storing pricing models and elasticity data.
-    Models are serialized using pickle, metadata stored as JSON.
-    """
-
-    def __init__(self, redis_host: str = None, redis_port: int = None):
-        host = redis_host or os.getenv('REDIS_HOST', 'localhost')
-        port = redis_port or int(os.getenv('REDIS_PORT', '6378'))
-
-        self.redis_client = redis.Redis(
-            host=host,
-            port=port,
-            db=0,
-            decode_responses=False
-        )
-        self.metadata_prefix = "model:meta:"
-        self.data_prefix = "model:data:"
-        self.elasticity_prefix = "elasticity:"
-        self.prices_prefix = "prices:"
-
-    def publish_elasticity(self,
-                          elasticity_df: pd.DataFrame,
-                          model_name: str = 'latest',
-                          metadata: Optional[Dict[str, Any]] = None):
-        """
-        Store elasticity estimates in registry.
-
-        Args:
-            elasticity_df: df with [productId, elasticity, std_error, n_obs]
-            model_name: identifier for this elasticity snapshot
-            metadata: additional info (timestamp, window_size, etc)
-        """
-        key = f"{self.elasticity_prefix}{model_name}"
-
-        # serialize dataframe as JSON
-        data_json = elasticity_df.to_json(orient='records')
-
-        # store data
-        self.redis_client.set(key, data_json)
-
-        # store metadata
-        meta = metadata or {}
-        meta.update({
-            'n_products': len(elasticity_df),
-            'mean_elasticity': float(elasticity_df['elasticity'].mean()),
-            'model_type': 'elasticity_snapshot'
-        })
-
-        meta_key = f"{self.metadata_prefix}{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published elasticity model '{model_name}' with {len(elasticity_df)} products")
-
-    def get_elasticity(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
-        """Retrieve elasticity estimates from registry."""
-        key = f"{self.elasticity_prefix}{model_name}"
-        data_json = self.redis_client.get(key)
-
-        if data_json is None:
-            return None
-
-        # decode bytes to string if needed
-        if isinstance(data_json, bytes):
-            data_json = data_json.decode('utf-8')
-
-        return pd.read_json(data_json, orient='records')
-
-    def publish_pricing_model(self,
-                              pricing_function,
-                              model_name: str = 'latest',
-                              metadata: Optional[Dict[str, Any]] = None):
-        """
-        Store a fitted pricing function object.
-
-        Args:
-            pricing_function: fitted PricingFunction instance
-            model_name: identifier
-            metadata: additional info
-        """
-        key = f"{self.data_prefix}{model_name}"
-
-        # serialize object
-        model_bytes = pickle.dumps(pricing_function)
-        self.redis_client.set(key, model_bytes)
-
-        # store metadata
-        meta = metadata or {}
-        meta.update({
-            'model_class': pricing_function.__class__.__name__,
-            'model_type': 'pricing_function'
-        })
-
-        meta_key = f"{self.metadata_prefix}{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published pricing model '{model_name}' ({meta['model_class']})")
-
-    def get_pricing_model(self, model_name: str = 'latest'):
-        """Retrieve a pricing function from registry."""
-        key = f"{self.data_prefix}{model_name}"
-        model_bytes = self.redis_client.get(key)
-
-        if model_bytes is None:
-            return None
-
-        return pickle.loads(model_bytes)
-
-    def list_models(self) -> Dict[str, Any]:
-        """List all registered models with metadata."""
-        models = {}
-
-        for key in self.redis_client.scan_iter(f"{self.metadata_prefix}*"):
-            key_str = key.decode('utf-8') if isinstance(key, bytes) else key
-            model_name = key_str.replace(self.metadata_prefix, '')
-            meta_json = self.redis_client.get(key)
-
-            if meta_json:
-                if isinstance(meta_json, bytes):
-                    meta_json = meta_json.decode('utf-8')
-                models[model_name] = json.loads(meta_json)
-
-        return models
-
-    def publish_prices(self,
-                      prices_df: pd.DataFrame,
-                      model_name: str = 'latest',
-                      metadata: Optional[Dict[str, Any]] = None):
-        """Store predicted prices in registry.
-
-        Args:
-            prices_df: df with [productId, predicted_price, ...]
-            model_name: identifier for this price snapshot
-            metadata: additional info
-        """
-        key = f"{self.prices_prefix}{model_name}"
-        data_json = prices_df.to_json(orient='records')
-
-        self.redis_client.set(key, data_json)
-
-        meta = metadata or {}
-        meta.update({
-            'n_products': len(prices_df),
-            'model_type': 'predicted_prices'
-        })
-
-        meta_key = f"{self.metadata_prefix}prices_{model_name}"
-        self.redis_client.set(meta_key, json.dumps(meta))
-
-        log.info(f"Published prices '{model_name}' for {len(prices_df)} products")
-
-    def get_prices(self, model_name: str = 'latest') -> Optional[pd.DataFrame]:
-        """Retrieve predicted prices from registry."""
-        key = f"{self.prices_prefix}{model_name}"
-        data_json = self.redis_client.get(key)
-
-        if data_json is None:
-            return None
-
-        if isinstance(data_json, bytes):
-            data_json = data_json.decode('utf-8')
-
-        return pd.read_json(data_json, orient='records')
-
-    def health_check(self) -> bool:
-        """Check if Redis connection is alive."""
-        try:
-            self.redis_client.ping()
-            return True
-        except:
-            return False

pytest.ini

@@ -1,5 +1,4 @@
 [pytest]
-pythonpath = experiments
 testpaths = experiments
 python_files = test*.py
 python_classes = Test*

requirements.txt

@@ -11,4 +11,3 @@ pytest-asyncio
 uv
 scikit-learn
 supabase
-pymc

web/package-lock.json

@@ -10,7 +10,7 @@
       "dependencies": {
         "@supabase/ssr": "^0.7.0",
         "@supabase/supabase-js": "^2.81.1",
-        "next": "^16.0.0",
+        "next": "16.0.0",
         "react": "19.2.0",
         "react-dom": "19.2.0",
         "zod": "^4.1.12"
@@ -526,15 +526,15 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.0.tgz",
-      "integrity": "sha512-gpaNgUh5nftFKRkRQGnVi5dpcYSKGcZZkQffZ172OrG/XkrnS7UBTQ648YY+8ME92cC4IojpI2LqTC8sTDhAaw==",
+      "integrity": "sha512-s5j2iFGp38QsG1LWRQaE2iUY3h1jc014/melHFfLdrsMJPqxqDQwWNwyQTcNoUSGZlCVZuM7t7JDMmSyRilsnA==",
       "license": "MIT"
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.0.0.tgz",
-      "integrity": "sha512-LlDtCYOEj/rfSnEn/Idi+j1QKHxY9BJFmxx7108A6D8K0SB+bNgfYQATPk/4LqOl4C0Wo3LACg2ie6s7xqMpJg==",
+      "integrity": "sha512-/CntqDCnk5w2qIwMiF0a9r6+9qunZzFmU0cBX4T82LOflE72zzH6gnOjCwUXYKOBlQi8OpP/rMj8cBIr18x4TA==",
       "cpu": [
         "arm64"
       ],
@@ -548,9 +548,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.0.0.tgz",
-      "integrity": "sha512-rtZ7BhnVvO1ICf3QzfW9H3aPz7GhBrnSIMZyr4Qy6boXF0b5E3QLs+cvJmg3PsTCG2M1PBoC+DANUi4wCOKXpA==",
+      "integrity": "sha512-hB4GZnJGKa8m4efvTGNyii6qs76vTNl+3dKHTCAUaksN6KjYy4iEO3Q5ira405NW2PKb3EcqWiRaL9DrYJfMHg==",
       "cpu": [
         "x64"
       ],
@@ -564,9 +564,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.0.0.tgz",
-      "integrity": "sha512-mloD5WcPIeIeeZqAIP5c2kdaTa6StwP4/2EGy1mUw8HiexSHGK/jcM7lFuS3u3i2zn+xH9+wXJs6njO7VrAqww==",
+      "integrity": "sha512-E2IHMdE+C1k+nUgndM13/BY/iJY9KGCphCftMh7SXWcaQqExq/pJU/1Hgn8n/tFwSoLoYC/yUghOv97tAsIxqg==",
       "cpu": [
         "arm64"
       ],
@@ -580,9 +580,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.0.0.tgz",
-      "integrity": "sha512-+ksWNrZrthisXuo9gd1XnjHRowCbMtl/YgMpbRvFeDEqEBd523YHPWpBuDjomod88U8Xliw5DHhekBC3EOOd9g==",
+      "integrity": "sha512-xzgl7c7BVk4+7PDWldU+On2nlwnGgFqJ1siWp3/8S0KBBLCjonB6zwJYPtl4MUY7YZJrzzumdUpUoquu5zk8vg==",
       "cpu": [
         "arm64"
       ],
@@ -596,9 +596,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.0.0.tgz",
-      "integrity": "sha512-4WtJU5cRDxpEE44Ana2Xro1284hnyVpBb62lIpU5k85D8xXxatT+rXxBgPkc7C1XwkZMWpK5rXLXTh9PFipWsA==",
+      "integrity": "sha512-sdyOg4cbiCw7YUr0F/7ya42oiVBXLD21EYkSwN+PhE4csJH4MSXUsYyslliiiBwkM+KsuQH/y9wuxVz6s7Nstg==",
       "cpu": [
         "x64"
       ],
@@ -612,9 +612,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.0.0.tgz",
-      "integrity": "sha512-HYlhqIP6kBPXalW2dbMTSuB4+8fe+j9juyxwfMwCe9kQPPeiyFn7NMjNfoFOfJ2eXkeQsoUGXg+O2SE3m4Qg2w==",
+      "integrity": "sha512-IAXv3OBYqVaNOgyd3kxR4L3msuhmSy1bcchPHxDOjypG33i2yDWvGBwFD94OuuTjjTt/7cuIKtAmoOOml6kfbg==",
       "cpu": [
         "x64"
       ],
@@ -628,9 +628,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.0.0.tgz",
-      "integrity": "sha512-EviG+43iOoBRZg9deGauXExjRphhuYmIOJ12b9sAPy0eQ6iwcPxfED2asb/s2/yiLYOdm37kPaiZu8uXSYPs0Q==",
+      "integrity": "sha512-bmo3ncIJKUS9PWK1JD9pEVv0yuvp1KPuOsyJTHXTv8KDrEmgV/K+U0C75rl9rhIaODcS7JEb6/7eJhdwXI0XmA==",
       "cpu": [
         "arm64"
       ],
@@ -644,9 +644,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.0.0.tgz",
-      "integrity": "sha512-gniPjy55zp5Eg0896qSrf3yB1dw4F/3s8VK1ephdsZZ129j2n6e1WqCbE2YgcKhW9hPB9TVZENugquWJD5x0ug==",
+      "integrity": "sha512-O1cJbT+lZp+cTjYyZGiDwsOjO3UHHzSqobkPNipdlnnuPb1swfcuY6r3p8dsKU4hAIEO4cO67ZCfVVH/M1ETXA==",
       "cpu": [
         "x64"
       ],
@@ -1447,12 +1447,12 @@
       }
     },
     "node_modules/next": {
-      "version": "16.0.7",
+      "version": "16.0.0",
-      "resolved": "https://registry.npmjs.org/next/-/next-16.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/next/-/next-16.0.0.tgz",
-      "integrity": "sha512-3mBRJyPxT4LOxAJI6IsXeFtKfiJUbjCLgvXO02fV8Wy/lIhPvP94Fe7dGhUgHXcQy4sSuYwQNcOLhIfOm0rL0A==",
+      "integrity": "sha512-nYohiNdxGu4OmBzggxy9rczmjIGI+TpR5vbKTsE1HqYwNm1B+YSiugSrFguX6omMOKnDHAmBPY4+8TNJk0Idyg==",
       "license": "MIT",
       "dependencies": {
-        "@next/env": "16.0.7",
+        "@next/env": "16.0.0",
         "@swc/helpers": "0.5.15",
         "caniuse-lite": "^1.0.30001579",
         "postcss": "8.4.31",
@@ -1465,14 +1465,14 @@
         "node": ">=20.9.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "16.0.7",
+        "@next/swc-darwin-arm64": "16.0.0",
-        "@next/swc-darwin-x64": "16.0.7",
+        "@next/swc-darwin-x64": "16.0.0",
-        "@next/swc-linux-arm64-gnu": "16.0.7",
+        "@next/swc-linux-arm64-gnu": "16.0.0",
-        "@next/swc-linux-arm64-musl": "16.0.7",
+        "@next/swc-linux-arm64-musl": "16.0.0",
-        "@next/swc-linux-x64-gnu": "16.0.7",
+        "@next/swc-linux-x64-gnu": "16.0.0",
-        "@next/swc-linux-x64-musl": "16.0.7",
+        "@next/swc-linux-x64-musl": "16.0.0",
-        "@next/swc-win32-arm64-msvc": "16.0.7",
+        "@next/swc-win32-arm64-msvc": "16.0.0",
-        "@next/swc-win32-x64-msvc": "16.0.7",
+        "@next/swc-win32-x64-msvc": "16.0.0",
         "sharp": "^0.34.4"
       },
       "peerDependencies": {

web/package.json

@@ -10,7 +10,7 @@
   "dependencies": {
     "@supabase/ssr": "^0.7.0",
     "@supabase/supabase-js": "^2.81.1",
-    "next": "^16.0.0",
+    "next": "16.0.0",
     "react": "19.2.0",
     "react-dom": "19.2.0",
     "zod": "^4.1.12"

web/src/app/airline/checkout/page.tsx

@@ -1,11 +0,0 @@
-export default function AirlineCheckout() {
-  return (
-    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-sky-50 to-blue-50">
-      <div className="text-center p-8">
-        <h1 className="text-4xl font-light text-gray-800 mb-4">
-          Thank you for flying with us
-        </h1>
-      </div>
-    </div>
-  );
-}

web/src/app/api/pricing/route.ts

@@ -20,40 +20,10 @@ export async function GET(req: NextRequest) {
         );
     }
 
+    // stub: call external pricing provider (random for now)
+    const basePrice = 100 + Math.random() * 900; // 100-1000 range
+    const price = Math.round(basePrice * 100) / 100;
     const timestamp = new Date().toISOString();
-    let price: number;
-    let basePrice: number | undefined;
-    let markup: number | undefined;
-    let elasticity: number | undefined;
-
-    // call real pricing provider
-    const providerUrl = process.env.PRICING_PROVIDER_URL || 'http://localhost:5001';
-    try {
-        const queryParams = new URLSearchParams();
-        if (sessionId) queryParams.append('sessionId', sessionId);
-        if (experimentId) queryParams.append('experimentId', experimentId);
-
-        const providerResponse = await fetch(
-            `${providerUrl}/api/${storeMode}/price/${productId}?${queryParams.toString()}`,
-            { headers: { 'Accept': 'application/json' }, cache: 'no-store' }
-        );
-
-        if (!providerResponse.ok) {
-            throw new Error(`Provider returned ${providerResponse.status}`);
-        }
-
-        const providerData = await providerResponse.json();
-        price = providerData.price;
-        basePrice = providerData.base_price;
-        markup = providerData.markup;
-        elasticity = providerData.elasticity;
-
-    } catch (err) {
-        console.error('[pricing-provider-error]', err);
-        // fallback to random pricing if provider unavailable
-        const randomBase = 100 + Math.random() * 900;
-        price = Math.round(randomBase * 100) / 100;
-    }
 
     // log price to kafka for elasticity computation
     if (sessionId) {
@@ -73,13 +43,19 @@ export async function GET(req: NextRequest) {
             });
         } catch (err) {
             console.error('[price-log-error]', err);
+            // don't fail the pricing request if logging fails
         }
     }
 
+    // log in dev
     if (process.env.NODE_ENV === 'development') {
         console.log('[pricing-api]', {
-            productId, sessionId, experimentId, storeMode,
+            productId,
-            price, basePrice, markup, elasticity, timestamp,
+            sessionId,
+            experimentId,
+            storeMode,
+            price,
+            timestamp,
         });
     }
 

web/src/app/cart/page.tsx

@@ -96,10 +96,7 @@ export default function CartPage() {
                                 <span className="text-3xl font-bold">${total.toFixed(2)}</span>
                             </div>
                             <button
-                                onClick={() => {
+                                onClick={() => dispatchInteraction('checkout_start', undefined, { total, itemCount })}
-                                    dispatchInteraction('checkout_start', undefined, { total, itemCount });
-                                    window.location.href = '/checkout';
-                                }}
                                 className="w-full py-3 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors"
                             >
                                 Proceed to Checkout

web/src/app/hotel/checkout/page.tsx

@@ -1,11 +0,0 @@
-export default function HotelCheckout() {
-  return (
-    <div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-blue-50 to-indigo-50">
-      <div className="text-center p-8">
-        <h1 className="text-4xl font-light text-gray-800 mb-4">
-          Thank you for staying with us
-        </h1>
-      </div>
-    </div>
-  );
-}

web/src/components/feats/hotel/HotelCard.tsx

@@ -21,7 +21,7 @@ const AmenityIcon = ({ name }: { name: string }) => {
         breakfast: 'Breakfast',
         spa: 'Spa',
     };
-    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name.replaceAll("_", " ")}</span>;
+    return <span className="feature-tag">{iconMap[name.toLowerCase()] || name}</span>;
 };
 
 export default function HotelCard({ hotel }: { hotel: Hotel }) {
@@ -47,31 +47,18 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
         window.location.href = `/hotel/products/${hotel.id}`;
     };
 
-    const imageUrl = `https://images.unsplash.com/photo-1551882547-ff40c63fe5fa?w=400&h=300&fit=crop`;
-
     return (
         <div
             className="hotel-card cursor-pointer"
             onClick={handleCardClick}
         >
-            <div className="hotel-image relative overflow-hidden">
+            <div className="hotel-image bg-gray-200 flex items-center justify-center">
-                <img
-                    src={imageUrl}
-                    alt={hotel.name}
-                    className="w-full h-full object-cover"
-                    onError={(e) => {
-                        e.currentTarget.style.display = 'none';
-                        const fallback = e.currentTarget.nextElementSibling as HTMLElement;
-                        if (fallback) fallback.style.display = 'flex';
-                    }}
-                />
-                <div className="absolute inset-0 bg-gray-200 flex items-center justify-center" style={{ display: 'none' }}>
                 <span className="text-gray-400 text-sm">Image</span>
             </div>
-            </div>
 
             <div className="hotel-info">
                 <h3 ref={titleRef} className="hotel-name">{hotel.name}</h3>
+                <div className="hotel-location text-sm mb-2">{hotel.roomType}</div>
                 <div className="text-sm text-[var(--text-secondary)] mb-2">
                     {hotel.checkIn} - {hotel.checkOut}
                 </div>
@@ -80,6 +67,9 @@ export default function HotelCard({ hotel }: { hotel: Hotel }) {
                         <AmenityIcon key={a} name={a} />
                     ))}
                 </div>
+                {hotel.refundable && (
+                    <div className="free-cancellation mt-2">Free cancellation</div>
+                )}
             </div>
 
             <div className="hotel-pricing">

web/src/components/feats/hotel/HotelDetails.tsx

@@ -1,8 +1,6 @@
 'use client';
 
-import { useState, useEffect } from 'react';
 import type { Hotel } from '@/lib/hotel-utils';
-import PriceDisplay from '@/components/ui/PriceDisplay';
 
 interface HotelDetailsProps {
   product: Hotel;
@@ -10,63 +8,19 @@ interface HotelDetailsProps {
   addedToCart: boolean;
 }
 
-const PriceTotalDisplay = ({ productId, nights }: { productId: string; nights: number }) => {
-  const [price, setPrice] = useState<number | null>(null);
-
-  useEffect(() => {
-    const fetchPrice = async () => {
-      try {
-        const sessionRes = await fetch('/api/session');
-        const sessionData = await sessionRes.json();
-        const params = new URLSearchParams({
-          productId,
-          sessionId: sessionData.sessionId || '',
-          experimentId: sessionData.experimentId || '',
-        });
-        const res = await fetch(`/api/pricing?${params.toString()}`);
-        const data = await res.json();
-        setPrice(data.price);
-      } catch (err) {
-        console.error('failed to fetch price for total:', err);
-      }
-    };
-    fetchPrice();
-  }, [productId]);
-
-  if (!price) return <span className="text-4xl font-bold text-gray-900">Loading...</span>;
-
-  return (
-    <span className="text-4xl font-bold text-gray-900">
-      ${(price * nights).toFixed(2)}
-    </span>
-  );
-};
-
 export default function HotelDetails({ product, onAddToCart, addedToCart }: HotelDetailsProps) {
-  const imageUrl = `https://images.unsplash.com/photo-1566073771259-6a8506099945?w=800&h=600&fit=crop`;
-
   return (
     <div className="w-full flex flex-col lg:flex-row gap-12 py-8">
-      <div className="w-full lg:w-1/2 rounded-lg aspect-[4/3] overflow-hidden shrink-0">
+      {/* Image Section - Larger and cleaner */}
-        <img
+      <div className="w-full lg:w-1/2 bg-gray-100 rounded-lg aspect-[4/3] flex items-center justify-center shrink-0">
-          src={imageUrl}
-          alt={product.name}
-          className="w-full h-full object-cover"
-          onError={(e) => {
-            e.currentTarget.style.display = 'none';
-            if (e.currentTarget.nextElementSibling) {
-              (e.currentTarget.nextElementSibling as HTMLElement).style.display = 'flex';
-            }
-          }}
-        />
-        <div className="w-full h-full bg-gray-100 rounded-lg flex items-center justify-center" style={{ display: 'none' }}>
         <span className="text-gray-400 text-lg font-medium">Hotel Image</span>
       </div>
-      </div>
 
+      {/* Details Section - Full height/width usage */}
       <div className="flex-1 flex flex-col">
         <div className="border-b pb-6 mb-6">
           <h1 className="text-4xl font-bold text-gray-900 mb-2">{product.name}</h1>
+          <p className="text-xl text-gray-500">{product.roomType}</p>
         </div>
 
         <div className="grid grid-cols-2 gap-8 mb-8">
@@ -85,17 +39,24 @@ export default function HotelDetails({ product, onAddToCart, addedToCart }: Hote
           <div className="flex flex-wrap gap-3">
             {product.amenities.map(a => (
               <span key={a} className="px-3 py-1.5 bg-gray-100 text-gray-700 rounded-md text-sm font-medium">
-                  {a.replaceAll('_', ' ')}
+                {a}
               </span>
             ))}
           </div>
         </div>
 
+        {product.refundable && (
+          <div className="mb-8 p-4 bg-green-50 text-green-800 rounded-md inline-block">
+            <span className="font-medium">Free cancellation available</span>
+          </div>
+        )}
+
         <div className="mt-auto pt-6 border-t flex items-center justify-between">
           <div>
-            <p className="text-sm text-gray-500 mb-1">Price per night</p>
+            <p className="text-sm text-gray-500 mb-1">Total for {product.nights} night{product.nights > 1 ? 's' : ''}</p>
-            <div className="mb-3">
+            <div className="flex items-baseline gap-2">
-              <PriceDisplay productId={product.id} className="!text-2xl" />
+              <span className="text-4xl font-bold text-gray-900">${product.pricePerNight * product.nights}</span>
+              <span className="text-gray-500">/ {product.nights} nights</span>
             </div>
           </div>
 

web/src/components/ui/DateInput.tsx

@@ -1,29 +1,7 @@
-import { InputHTMLAttributes, useMemo } from 'react';
+import { InputHTMLAttributes } from 'react';
 
 interface DateInpProps extends Omit<InputHTMLAttributes<HTMLInputElement>, 'type'> {}
 
 export default function DateInput({ className = '', ...props }: DateInpProps) {
-  const { minDate, maxDate } = useMemo(() => {
+  return <input type="date" className={`input-field ${className}`.trim()} {...props} />;
-    const today = new Date();
-    const tomorrow = new Date(today);
-    tomorrow.setDate(today.getDate() + 1);
-
-    const tenDaysOut = new Date(tomorrow);
-    tenDaysOut.setDate(tomorrow.getDate() + 9); // tomorrow + 9 = 10 days total
-
-    return {
-      minDate: tomorrow.toISOString().split('T')[0],
-      maxDate: tenDaysOut.toISOString().split('T')[0]
-    };
-  }, []);
-
-  return (
-    <input
-      type="date"
-      className={`input-field ${className}`.trim()}
-      min={minDate}
-      max={maxDate}
-      {...props}
-    />
-  );
 }

web/src/components/ui/Navigation.tsx

@@ -20,7 +20,7 @@ const NavLink = ({ href, children }: { href: string; children: React.ReactNode }
       href={href}
       className={`px-4 py-2 rounded-md transition-colors ${
         isActive
-          ? 'bg-[var(--accent-primary)] font-semibold'
+          ? 'bg-[var(--accent-primary)] text-white font-semibold'
           : 'hover:bg-[var(--accent-primary-light)] text-[var(--text-primary)]'
       }`}
     >
@@ -37,7 +37,9 @@ export default function Navigation() {
           <div className="flex items-center space-x-1">
             <NavLink href="/">Home</NavLink>
             <NavLink href="/products">Products</NavLink>
+            <NavLink href="/search">Search</NavLink>
             <NavLink href="/cart">Cart</NavLink>
+            <NavLink href="/checkout">Checkout</NavLink>
           </div>
         </div>
       </div>

web/src/lib/hotel-utils.ts

@@ -21,6 +21,7 @@ export interface Hotel {
   checkOut: string;
   dateIndex: number;
   amenities: string[];
+  refundable: boolean;
   pricePerNight: number;
   nights: number;
 }
@@ -29,37 +30,19 @@ const EPOCH = new Date(0);
 
 export const transformProduct = (p: HotelProduct): Hotel => {
   const { id, room_type, date_index, metadata } = p;
-
+  const checkIn = new Date(EPOCH.getTime() + date_index * 86400000);
-  // DB stores date_index as days since epoch
-  // but if value is small (<1000), treat as days from today for backward compat
-  let checkIn: Date;
-  if (date_index < 1000) {
-    // legacy: treat as offset from today
-    const today = new Date();
-    today.setHours(0, 0, 0, 0);
-    checkIn = new Date(today.getTime() + date_index * 86400000);
-  } else {
-    // proper: days since epoch
-    checkIn = new Date(EPOCH.getTime() + date_index * 86400000);
-  }
-
   const nights = 1;
   const checkOut = new Date(checkIn.getTime() + nights * 86400000);
 
-  const formatOpts: Intl.DateTimeFormatOptions = {
-    month: 'short',
-    day: 'numeric',
-    year: checkIn.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined
-  };
-
   return {
     id,
     name: metadata?.name || room_type,
     roomType: room_type,
-    checkIn: checkIn.toLocaleDateString('en-US', formatOpts),
+    checkIn: checkIn.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
-    checkOut: checkOut.toLocaleDateString('en-US', formatOpts),
+    checkOut: checkOut.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
     dateIndex: date_index,
     amenities: metadata?.amenities || [],
+    refundable: metadata?.refundable || false,
     pricePerNight: metadata?.base_price || 100,
     nights,
   };