Add dynamic pricing E2E test suite with Playwright

Implement comprehensive E2E tests to validate the surge pricing pipeline:
- Test SimpleSurgePricer with configurable thresholds (high=3, surge=1.5x)
- Verify discount pricing when demand is below low_threshold
- Test multi-product differential pricing based on demand signals
- Validate price propagation from pipeline through Redis to API

Test infrastructure:
- Playwright configuration with custom fixtures
- Python pipeline worker for direct test execution (bypasses Airflow)
- API client for event ingestion and price verification
- Event generator for creating realistic interaction sequences
- docker-compose.e2e.yml with minimal services for testing

e2e/lib/api-client.ts

@@ -0,0 +1,191 @@
+import { testConfig } from '../playwright.config';
+
+/**
+ * Event payload structure matching the backend API
+ */
+export interface EventPayload {
+  sessionId: string;
+  experimentId?: string;
+  eventName: string;
+  page: string;
+  productId?: string;
+  metadata?: Record<string, unknown>;
+  storeMode: 'hotel' | 'airline';
+  userAgent?: string;
+  ts?: string;
+}
+
+/**
+ * Price log payload structure
+ */
+export interface PriceLogPayload {
+  productId: string;
+  price: number;
+  sessionId: string;
+  experimentId?: string;
+  storeMode: 'hotel' | 'airline';
+  ts?: string;
+}
+
+/**
+ * Price response from the pricing provider
+ */
+export interface PriceResponse {
+  productId: string;
+  price: number;
+  base_price: number;
+  markup: number;
+  elasticity: number | null;
+  model_version: string;
+}
+
+/**
+ * API client for interacting with PHANTOM services
+ */
+export class PhantomApiClient {
+  private backendUrl: string;
+  private providerUrl: string;
+
+  constructor(
+    backendUrl: string = testConfig.backendUrl,
+    providerUrl: string = testConfig.providerUrl
+  ) {
+    this.backendUrl = backendUrl;
+    this.providerUrl = providerUrl;
+  }
+
+  /**
+   * Send a user interaction event to the ingestion API
+   */
+  async ingestEvent(event: EventPayload): Promise<{ success: boolean }> {
+    const payload: EventPayload = {
+      ...event,
+      ts: event.ts || new Date().toISOString(),
+    };
+
+    const response = await fetch(`${this.backendUrl}/api/kafka/ingest`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to ingest event: ${response.status} ${await response.text()}`);
+    }
+
+    return response.json();
+  }
+
+  /**
+   * Send multiple events in rapid succession
+   */
+  async ingestEvents(events: EventPayload[], delayMs: number = 100): Promise<void> {
+    for (const event of events) {
+      await this.ingestEvent(event);
+      if (delayMs > 0) {
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+      }
+    }
+  }
+
+  /**
+   * Log a price observation
+   */
+  async logPrice(priceLog: PriceLogPayload): Promise<{ success: boolean }> {
+    const payload: PriceLogPayload = {
+      ...priceLog,
+      ts: priceLog.ts || new Date().toISOString(),
+    };
+
+    const response = await fetch(`${this.backendUrl}/api/kafka/price-log`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to log price: ${response.status} ${await response.text()}`);
+    }
+
+    return response.json();
+  }
+
+  /**
+   * Get the current price for a product from the pricing provider
+   */
+  async getPrice(
+    mode: 'hotel' | 'airline',
+    productId: string,
+    sessionId?: string
+  ): Promise<PriceResponse> {
+    const params = new URLSearchParams();
+    if (sessionId) {
+      params.set('sessionId', sessionId);
+    }
+
+    const url = `${this.providerUrl}/api/${mode}/price/${productId}${params.toString() ? '?' + params.toString() : ''}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      throw new Error(`Failed to get price: ${response.status} ${await response.text()}`);
+    }
+
+    return response.json();
+  }
+
+  /**
+   * Dump events from Kafka topic for debugging
+   */
+  async dumpKafkaEvents(
+    topic: 'user-interactions' | 'price-logs' = 'user-interactions',
+    lastN?: number
+  ): Promise<{ success: boolean; count: number; data: unknown[] }> {
+    const params = new URLSearchParams({ topic });
+    if (lastN) {
+      params.set('last_n', String(lastN));
+    }
+
+    const response = await fetch(`${this.backendUrl}/api/kafka/dump?${params.toString()}`);
+
+    if (!response.ok) {
+      throw new Error(`Failed to dump Kafka events: ${response.status}`);
+    }
+
+    return response.json();
+  }
+
+  /**
+   * Check health of backend service
+   */
+  async checkBackendHealth(): Promise<{ status: string; kafka: string }> {
+    const response = await fetch(`${this.backendUrl}/health`);
+    return response.json();
+  }
+
+  /**
+   * Check health of pricing provider
+   */
+  async checkProviderHealth(): Promise<{ status: string; redis: boolean }> {
+    const response = await fetch(`${this.providerUrl}/health`);
+    return response.json();
+  }
+
+  /**
+   * List registered models in the pricing provider
+   */
+  async listModels(): Promise<Record<string, unknown>> {
+    const response = await fetch(`${this.providerUrl}/models`);
+    return response.json();
+  }
+
+  /**
+   * Reload models in the pricing provider
+   */
+  async reloadModels(): Promise<{ elasticity_loaded: boolean; pricing_model_loaded: boolean }> {
+    const response = await fetch(`${this.providerUrl}/models/reload`, { method: 'POST' });
+    return response.json();
+  }
+}
+
+// Singleton instance for convenience
+export const apiClient = new PhantomApiClient();

e2e/lib/event-generator.ts

@@ -0,0 +1,249 @@
+import { EventPayload, PriceLogPayload } from './api-client';
+import { v4 as uuidv4 } from 'uuid';
+
+/**
+ * Canonical event names matching the frontend
+ */
+export const EventNames = {
+  // Navigation events
+  PAGE_VIEW: 'page_view',
+  VIEW_ITEM_PAGE: 'view_item_page',
+  LEARN_MORE: 'learn_more_about_item',
+
+  // Cart events
+  ADD_TO_CART: 'add_item_to_cart',
+  REMOVE_FROM_CART: 'remove_item',
+  CHECKOUT_START: 'checkout_start',
+  PURCHASE_COMPLETE: 'purchase_complete',
+
+  // Search/Filter events
+  SEARCH: 'search',
+  FILTER_DATE: 'filter_for_date',
+  FILTER_AMENITIES: 'filter_for_amenities',
+  FILTER_PRICE: 'filter_for_price',
+  SORT_CHANGE: 'sort_change',
+
+  // Dwell signals (engagement)
+  HOVER_TITLE: 'hover_over_title',
+  HOVER_PARAGRAPH: 'hover_over_paragraph',
+  HOVER_LINK: 'hover_over_link',
+  HOVER_BUTTON: 'hover_over_button',
+
+  // Session
+  SESSION_START: 'session_start',
+} as const;
+
+export type EventName = typeof EventNames[keyof typeof EventNames];
+
+/**
+ * Test product configuration
+ */
+export interface TestProduct {
+  id: string;
+  basePrice: number;
+  storeMode: 'hotel' | 'airline';
+  name?: string;
+}
+
+/**
+ * Generates test events for dynamic pricing E2E tests
+ */
+export class EventGenerator {
+  private sessionId: string;
+  private experimentId: string;
+  private storeMode: 'hotel' | 'airline';
+
+  constructor(options?: {
+    sessionId?: string;
+    experimentId?: string;
+    storeMode?: 'hotel' | 'airline';
+  }) {
+    this.sessionId = options?.sessionId || uuidv4();
+    this.experimentId = options?.experimentId || uuidv4();
+    this.storeMode = options?.storeMode || 'hotel';
+  }
+
+  get session(): string {
+    return this.sessionId;
+  }
+
+  get experiment(): string {
+    return this.experimentId;
+  }
+
+  /**
+   * Create a new session for isolation between test scenarios
+   */
+  newSession(): string {
+    this.sessionId = uuidv4();
+    return this.sessionId;
+  }
+
+  /**
+   * Generate a single event
+   */
+  createEvent(
+    eventName: EventName,
+    productId: string,
+    metadata?: Record<string, unknown>
+  ): EventPayload {
+    return {
+      sessionId: this.sessionId,
+      experimentId: this.experimentId,
+      eventName,
+      page: `/${this.storeMode}/products/${productId}`,
+      productId,
+      metadata: metadata || {},
+      storeMode: this.storeMode,
+      userAgent: 'PHANTOM-E2E-Test/1.0',
+      ts: new Date().toISOString(),
+    };
+  }
+
+  /**
+   * Generate a product view event
+   */
+  viewProduct(productId: string): EventPayload {
+    return this.createEvent(EventNames.VIEW_ITEM_PAGE, productId, {
+      referrer: `/${this.storeMode}/products`,
+      viewport: { width: 1920, height: 1080 },
+    });
+  }
+
+  /**
+   * Generate a "learn more" event (high intent signal)
+   */
+  learnMore(productId: string): EventPayload {
+    return this.createEvent(EventNames.LEARN_MORE, productId, {
+      section: 'details',
+    });
+  }
+
+  /**
+   * Generate a hover event (engagement signal)
+   */
+  hover(productId: string, element: 'title' | 'paragraph' | 'button' = 'title'): EventPayload {
+    const eventMap = {
+      title: EventNames.HOVER_TITLE,
+      paragraph: EventNames.HOVER_PARAGRAPH,
+      button: EventNames.HOVER_BUTTON,
+    };
+    return this.createEvent(eventMap[element], productId, {
+      duration_ms: Math.floor(Math.random() * 2000) + 500,
+    });
+  }
+
+  /**
+   * Generate an add-to-cart event
+   */
+  addToCart(productId: string, quantity: number = 1): EventPayload {
+    return this.createEvent(EventNames.ADD_TO_CART, productId, {
+      quantity,
+      cart_size: quantity,
+    });
+  }
+
+  /**
+   * Generate a sequence of high-velocity events for surge pricing trigger
+   * This simulates rapid user interest in a product
+   */
+  generateSurgeSequence(productId: string, count: number): EventPayload[] {
+    const events: EventPayload[] = [];
+
+    for (let i = 0; i < count; i++) {
+      // Mix of different event types to simulate realistic behavior
+      events.push(this.viewProduct(productId));
+
+      if (i % 2 === 0) {
+        events.push(this.learnMore(productId));
+      }
+
+      if (i % 3 === 0) {
+        events.push(this.hover(productId, 'title'));
+      }
+    }
+
+    return events;
+  }
+
+  /**
+   * Generate a normal browsing session (not triggering surge)
+   */
+  generateNormalSession(productId: string): EventPayload[] {
+    return [
+      this.viewProduct(productId),
+      this.hover(productId, 'title'),
+    ];
+  }
+
+  /**
+   * Generate high-velocity agent-like behavior
+   * This should trigger SessionAwarePricer's agent detection
+   */
+  generateAgentBehavior(productIds: string[]): EventPayload[] {
+    const events: EventPayload[] = [];
+
+    // Rapid-fire product views across multiple products
+    for (const productId of productIds) {
+      events.push(this.viewProduct(productId));
+      // Very quick succession - agent-like behavior
+    }
+
+    return events;
+  }
+
+  /**
+   * Generate a price log entry
+   */
+  createPriceLog(productId: string, price: number): PriceLogPayload {
+    return {
+      productId,
+      price,
+      sessionId: this.sessionId,
+      experimentId: this.experimentId,
+      storeMode: this.storeMode,
+      ts: new Date().toISOString(),
+    };
+  }
+}
+
+/**
+ * Pre-configured test products for E2E tests
+ * These should match products in your test database
+ */
+export const TestProducts = {
+  // Hotel products with known base prices
+  hotel1: {
+    id: 'e2e-test-hotel-001',
+    basePrice: 150.00,
+    storeMode: 'hotel' as const,
+    name: 'E2E Test Hotel 1',
+  },
+  hotel2: {
+    id: 'e2e-test-hotel-002',
+    basePrice: 200.00,
+    storeMode: 'hotel' as const,
+    name: 'E2E Test Hotel 2',
+  },
+  hotel3: {
+    id: 'e2e-test-hotel-003',
+    basePrice: 100.00,
+    storeMode: 'hotel' as const,
+    name: 'E2E Test Hotel 3',
+  },
+
+  // Airline products
+  airline1: {
+    id: 'e2e-test-airline-001',
+    basePrice: 350.00,
+    storeMode: 'airline' as const,
+    name: 'E2E Test Flight 1',
+  },
+};
+
+/**
+ * Generate a unique test product ID for isolation
+ */
+export function generateTestProductId(prefix: string = 'e2e-test'): string {
+  return `${prefix}-${uuidv4().slice(0, 8)}`;
+}

e2e/lib/fixtures.ts

@@ -0,0 +1,143 @@
+import { test as base, expect } from '@playwright/test';
+import { PhantomApiClient, apiClient } from './api-client';
+import { EventGenerator, TestProducts } from './event-generator';
+import { runPricingPipeline, waitForPriceUpdate, PipelineResult } from './pipeline-runner';
+import { testConfig } from '../playwright.config';
+
+/**
+ * Extended test fixtures for PHANTOM E2E tests
+ */
+export interface PhantomTestFixtures {
+  /** API client for interacting with PHANTOM services */
+  api: PhantomApiClient;
+
+  /** Event generator for creating test events */
+  events: EventGenerator;
+
+  /** Run the pricing pipeline and wait for updates */
+  triggerPriceUpdate: (options?: {
+    storeMode?: 'hotel' | 'airline';
+    highThreshold?: number;
+    lowThreshold?: number;
+    surgeMultiplier?: number;
+    discountMultiplier?: number;
+  }) => Promise<PipelineResult>;
+
+  /** Wait for a specific price condition */
+  waitForPrice: (
+    productId: string,
+    condition: (price: number, basePrice: number) => boolean,
+    storeMode?: 'hotel' | 'airline'
+  ) => Promise<{ price: number; basePrice: number; markup: number }>;
+
+  /** Test configuration */
+  config: typeof testConfig;
+}
+
+/**
+ * Custom test with PHANTOM fixtures
+ */
+export const test = base.extend<PhantomTestFixtures>({
+  api: async ({}, use) => {
+    await use(apiClient);
+  },
+
+  events: async ({}, use) => {
+    // Create a new event generator with a fresh session for each test
+    const generator = new EventGenerator({
+      storeMode: 'hotel',
+    });
+    await use(generator);
+  },
+
+  triggerPriceUpdate: async ({}, use) => {
+    const trigger = async (options = {}) => {
+      const result = await runPricingPipeline({
+        storeMode: 'hotel',
+        highThreshold: testConfig.pricing.highThreshold,
+        lowThreshold: testConfig.pricing.lowThreshold,
+        surgeMultiplier: testConfig.pricing.surgeMultiplier,
+        discountMultiplier: testConfig.pricing.discountMultiplier,
+        ...options,
+      });
+
+      // Wait a moment for Redis to be fully updated
+      await new Promise(resolve => setTimeout(resolve, 500));
+
+      return result;
+    };
+
+    await use(trigger);
+  },
+
+  waitForPrice: async ({ api }, use) => {
+    const waiter = async (
+      productId: string,
+      condition: (price: number, basePrice: number) => boolean,
+      storeMode: 'hotel' | 'airline' = 'hotel'
+    ) => {
+      let lastPrice = 0;
+      let lastBasePrice = 0;
+
+      const updated = await waitForPriceUpdate(async () => {
+        const priceResponse = await api.getPrice(storeMode, productId);
+        lastPrice = priceResponse.price;
+        lastBasePrice = priceResponse.base_price;
+        return condition(priceResponse.price, priceResponse.base_price);
+      });
+
+      if (!updated) {
+        throw new Error(
+          `Price condition not met within timeout. Last price: ${lastPrice}, base: ${lastBasePrice}`
+        );
+      }
+
+      return {
+        price: lastPrice,
+        basePrice: lastBasePrice,
+        markup: lastPrice / lastBasePrice,
+      };
+    };
+
+    await use(waiter);
+  },
+
+  config: async ({}, use) => {
+    await use(testConfig);
+  },
+});
+
+export { expect };
+
+/**
+ * Helper assertions for pricing tests
+ */
+export const PricingAssertions = {
+  /**
+   * Assert that a price has surge markup applied
+   */
+  isSurged: (price: number, basePrice: number, expectedMultiplier: number, tolerance = 0.01) => {
+    const actualMarkup = price / basePrice;
+    const minExpected = expectedMultiplier * (1 - tolerance);
+    const maxExpected = expectedMultiplier * (1 + tolerance);
+    return actualMarkup >= minExpected && actualMarkup <= maxExpected;
+  },
+
+  /**
+   * Assert that a price has discount applied
+   */
+  isDiscounted: (price: number, basePrice: number, expectedMultiplier: number, tolerance = 0.01) => {
+    const actualMarkup = price / basePrice;
+    const minExpected = expectedMultiplier * (1 - tolerance);
+    const maxExpected = expectedMultiplier * (1 + tolerance);
+    return actualMarkup >= minExpected && actualMarkup <= maxExpected;
+  },
+
+  /**
+   * Assert that a price is at base (no surge/discount)
+   */
+  isBase: (price: number, basePrice: number, tolerance = 0.01) => {
+    const actualMarkup = price / basePrice;
+    return actualMarkup >= (1 - tolerance) && actualMarkup <= (1 + tolerance);
+  },
+};

e2e/lib/index.ts

@@ -0,0 +1,6 @@
+// Re-export all test utilities
+
+export * from './api-client';
+export * from './event-generator';
+export * from './pipeline-runner';
+export * from './fixtures';

e2e/lib/pipeline-runner.ts

@@ -0,0 +1,152 @@
+import { spawn } from 'child_process';
+import path from 'path';
+import { testConfig } from '../playwright.config';
+
+/**
+ * Pipeline execution result
+ */
+export interface PipelineResult {
+  success: boolean;
+  interactions_count: number;
+  products_count: number;
+  prices_published: boolean;
+  prices?: Array<{
+    productId: string;
+    current_price: number;
+    base_price: number;
+    optimal_price: number;
+    demand_score: number;
+  }>;
+  timestamp?: string;
+  message?: string;
+  error?: string;
+}
+
+/**
+ * Pipeline configuration options
+ */
+export interface PipelineOptions {
+  storeMode?: 'hotel' | 'airline';
+  highThreshold?: number;
+  lowThreshold?: number;
+  surgeMultiplier?: number;
+  discountMultiplier?: number;
+  dryRun?: boolean;
+}
+
+/**
+ * Execute the pricing pipeline to update prices based on current events
+ */
+export async function runPricingPipeline(options: PipelineOptions = {}): Promise<PipelineResult> {
+  const {
+    storeMode = 'hotel',
+    highThreshold = testConfig.pricing.highThreshold,
+    lowThreshold = testConfig.pricing.lowThreshold,
+    surgeMultiplier = testConfig.pricing.surgeMultiplier,
+    discountMultiplier = testConfig.pricing.discountMultiplier,
+    dryRun = false,
+  } = options;
+
+  const workerPath = path.join(__dirname, 'pipeline-worker.py');
+
+  const args = [
+    workerPath,
+    '--store-mode', storeMode,
+    '--high-threshold', String(highThreshold),
+    '--low-threshold', String(lowThreshold),
+    '--surge-multiplier', String(surgeMultiplier),
+    '--discount-multiplier', String(discountMultiplier),
+    '--json-output',
+  ];
+
+  if (dryRun) {
+    args.push('--dry-run');
+  }
+
+  return new Promise((resolve, reject) => {
+    const python = spawn('python3', args, {
+      env: {
+        ...process.env,
+        BACKEND_URL: testConfig.backendUrl,
+        REDIS_HOST: testConfig.redisHost,
+        REDIS_PORT: String(testConfig.redisPort),
+        KAFKA_HOST: testConfig.kafkaHost,
+        KAFKA_PORT: String(testConfig.kafkaPort),
+      },
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    python.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    python.stderr.on('data', (data) => {
+      stderr += data.toString();
+      // Log pipeline output for debugging
+      console.log('[Pipeline]', data.toString().trim());
+    });
+
+    python.on('close', (code) => {
+      if (code === 0) {
+        try {
+          // Find JSON output in stdout (last JSON object)
+          const jsonMatch = stdout.match(/\{[\s\S]*\}$/);
+          if (jsonMatch) {
+            const result = JSON.parse(jsonMatch[0]);
+            resolve(result);
+          } else {
+            resolve({
+              success: true,
+              interactions_count: 0,
+              products_count: 0,
+              prices_published: false,
+              message: 'Pipeline completed but no JSON output',
+            });
+          }
+        } catch (parseError) {
+          resolve({
+            success: true,
+            interactions_count: 0,
+            products_count: 0,
+            prices_published: false,
+            message: 'Pipeline completed but output not parseable',
+          });
+        }
+      } else {
+        reject(new Error(`Pipeline exited with code ${code}: ${stderr}`));
+      }
+    });
+
+    python.on('error', (error) => {
+      reject(new Error(`Failed to start pipeline: ${error.message}`));
+    });
+  });
+}
+
+/**
+ * Wait for prices to be updated in Redis and available via the pricing API
+ */
+export async function waitForPriceUpdate(
+  checkFn: () => Promise<boolean>,
+  maxWaitMs: number = testConfig.timing.maxPriceWait,
+  intervalMs: number = testConfig.timing.priceCheckInterval
+): Promise<boolean> {
+  const startTime = Date.now();
+
+  while (Date.now() - startTime < maxWaitMs) {
+    try {
+      const updated = await checkFn();
+      if (updated) {
+        return true;
+      }
+    } catch (error) {
+      // Ignore errors during polling
+    }
+
+    await new Promise(resolve => setTimeout(resolve, intervalMs));
+  }
+
+  return false;
+}

e2e/lib/pipeline-worker.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+E2E Test Pipeline Worker
+
+A lightweight worker that runs the surge pricing pipeline for E2E tests.
+This bypasses Airflow for faster, more reliable test execution.
+
+Usage:
+    python pipeline-worker.py --store-mode hotel --high-threshold 3 --surge-multiplier 1.5
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+from typing import Optional
+from datetime import datetime
+
+# Add project paths
+project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, project_root)
+sys.path.insert(0, os.path.join(project_root, 'experiments'))
+sys.path.insert(0, os.path.join(project_root, 'lib'))
+
+from procesing.context import PipelineContext
+from procesing.providers import BackendAPIProvider
+from procesing.steps import (
+    FetchInteractionsStep,
+    FetchPriceLogsStep,
+    ComputeDemandStep,
+    AggregatePriceLogsStep,
+    JoinProductFeaturesStep,
+)
+from procesing.pricers.simple import SimpleSurgePricer
+from lib.model_registry import ModelRegistry
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(message)s'
+)
+log = logging.getLogger(__name__)
+
+
+class E2ETestProvider(BackendAPIProvider):
+    """Provider configured for E2E test environment"""
+
+    def __init__(self, backend_url: str = None):
+        self.backend_url = backend_url or os.getenv('BACKEND_URL', 'http://localhost:5000')
+        super().__init__()
+
+
+def run_pricing_pipeline(
+    store_mode: str = 'hotel',
+    high_threshold: int = 3,
+    low_threshold: int = 1,
+    surge_multiplier: float = 1.5,
+    discount_multiplier: float = 0.9,
+    dry_run: bool = False
+) -> dict:
+    """
+    Execute the surge pricing pipeline and publish results to Redis.
+
+    Args:
+        store_mode: 'hotel' or 'airline'
+        high_threshold: Demand threshold for surge pricing
+        low_threshold: Demand threshold for discount pricing
+        surge_multiplier: Price multiplier for high demand
+        discount_multiplier: Price multiplier for low demand
+        dry_run: If True, don't publish to Redis
+
+    Returns:
+        dict with pipeline results and statistics
+    """
+    log.info(f"Starting E2E pricing pipeline: mode={store_mode}, "
+             f"high_threshold={high_threshold}, surge_multiplier={surge_multiplier}")
+
+    # Initialize provider and context
+    provider = E2ETestProvider()
+    context = PipelineContext(provider=provider, store_mode=store_mode)
+
+    # Step 1: Fetch interactions from Kafka
+    log.info("Fetching interactions from Kafka...")
+    fetch_interactions = FetchInteractionsStep(context)
+    interactions_df = fetch_interactions.transform(None)
+    log.info(f"Fetched {len(interactions_df)} interaction records")
+
+    if interactions_df.empty:
+        log.warning("No interactions found. Pipeline will produce no price updates.")
+        return {
+            'success': True,
+            'interactions_count': 0,
+            'products_count': 0,
+            'prices_published': False,
+            'message': 'No interactions to process'
+        }
+
+    # Step 2: Fetch price logs from Kafka
+    log.info("Fetching price logs from Kafka...")
+    fetch_prices = FetchPriceLogsStep(context)
+    price_logs_df = fetch_prices.transform(None)
+    log.info(f"Fetched {len(price_logs_df)} price log records")
+
+    # Step 3: Compute demand scores
+    log.info("Computing demand scores...")
+    compute_demand = ComputeDemandStep(context)
+    demand_df = compute_demand.transform(interactions_df)
+    log.info(f"Computed demand for {len(demand_df)} products")
+
+    if demand_df.empty:
+        log.warning("No demand data computed.")
+        return {
+            'success': True,
+            'interactions_count': len(interactions_df),
+            'products_count': 0,
+            'prices_published': False,
+            'message': 'No demand data to process'
+        }
+
+    # Step 4: Aggregate price logs
+    log.info("Aggregating price logs...")
+    aggregate_prices = AggregatePriceLogsStep(context)
+    price_agg_df = aggregate_prices.transform(price_logs_df)
+    log.info(f"Aggregated prices for {len(price_agg_df)} products")
+
+    # Step 5: Join product features
+    log.info("Joining product features...")
+    join_features = JoinProductFeaturesStep(context)
+    features_df = join_features.transform((demand_df, price_agg_df))
+    log.info(f"Joined features for {len(features_df)} products")
+
+    if features_df.empty:
+        log.warning("No product features after join.")
+        return {
+            'success': True,
+            'interactions_count': len(interactions_df),
+            'products_count': 0,
+            'prices_published': False,
+            'message': 'No product features to price'
+        }
+
+    # Step 6: Apply surge pricing
+    log.info(f"Applying surge pricing (high={high_threshold}, surge={surge_multiplier}x)...")
+
+    # Rename columns for pricer compatibility
+    data = features_df.rename(columns={'demand_score': 'demand'})
+
+    surge_pricer = SimpleSurgePricer(
+        high_threshold=high_threshold,
+        low_threshold=low_threshold,
+        surge_multiplier=surge_multiplier,
+        discount_multiplier=discount_multiplier
+    )
+    surge_pricer.fit(data)
+    data['optimal_price'] = surge_pricer.predict()
+
+    # Prepare output DataFrame
+    prices_df = data[['productId', 'price', 'base_price', 'optimal_price', 'demand']].rename(columns={
+        'price': 'current_price',
+        'demand': 'demand_score'
+    })
+
+    log.info(f"Generated optimal prices for {len(prices_df)} products")
+
+    # Log pricing decisions
+    for _, row in prices_df.iterrows():
+        markup = row['optimal_price'] / row['base_price'] if row['base_price'] > 0 else 1.0
+        log.info(f"  {row['productId'][:8]}...: base=${row['base_price']:.2f} "
+                 f"-> optimal=${row['optimal_price']:.2f} (demand={row['demand_score']:.0f}, markup={markup:.2f}x)")
+
+    # Step 7: Publish to Redis
+    if not dry_run:
+        log.info("Publishing prices to Redis registry...")
+        registry = ModelRegistry()
+
+        metadata = {
+            'timestamp': datetime.utcnow().isoformat(),
+            'store_mode': store_mode,
+            'pipeline': 'e2e_test_worker',
+            'high_threshold': high_threshold,
+            'low_threshold': low_threshold,
+            'surge_multiplier': surge_multiplier,
+            'discount_multiplier': discount_multiplier,
+        }
+
+        registry.publish_prices(prices_df, model_name='latest', metadata=metadata)
+        log.info(f"✅ Published {len(prices_df)} prices to Redis")
+    else:
+        log.info("Dry run - skipping Redis publish")
+
+    return {
+        'success': True,
+        'interactions_count': len(interactions_df),
+        'products_count': len(prices_df),
+        'prices_published': not dry_run,
+        'prices': prices_df.to_dict(orient='records'),
+        'timestamp': datetime.utcnow().isoformat()
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description='E2E Test Pipeline Worker')
+    parser.add_argument('--store-mode', choices=['hotel', 'airline'], default='hotel',
+                        help='Store mode (hotel or airline)')
+    parser.add_argument('--high-threshold', type=int, default=3,
+                        help='Demand threshold for surge pricing')
+    parser.add_argument('--low-threshold', type=int, default=1,
+                        help='Demand threshold for discount pricing')
+    parser.add_argument('--surge-multiplier', type=float, default=1.5,
+                        help='Price multiplier for high demand')
+    parser.add_argument('--discount-multiplier', type=float, default=0.9,
+                        help='Price multiplier for low demand')
+    parser.add_argument('--dry-run', action='store_true',
+                        help='Run without publishing to Redis')
+    parser.add_argument('--json-output', action='store_true',
+                        help='Output results as JSON')
+
+    args = parser.parse_args()
+
+    try:
+        result = run_pricing_pipeline(
+            store_mode=args.store_mode,
+            high_threshold=args.high_threshold,
+            low_threshold=args.low_threshold,
+            surge_multiplier=args.surge_multiplier,
+            discount_multiplier=args.discount_multiplier,
+            dry_run=args.dry_run
+        )
+
+        if args.json_output:
+            print(json.dumps(result, indent=2))
+        else:
+            log.info(f"Pipeline completed: {result['products_count']} products priced")
+
+        sys.exit(0 if result['success'] else 1)
+
+    except Exception as e:
+        log.error(f"Pipeline failed: {e}")
+        if args.json_output:
+            print(json.dumps({'success': False, 'error': str(e)}))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()