chore: refactoring the providers docker config and requirements

backend/provider/requirements.txt

@@ -0,0 +1,26 @@
+# Web Framework & API
+fastapi>=0.104.0
+uvicorn[standard]>=0.24.0
+pydantic>=2.0.0
+
+# Core ML/Data Science
+numpy>=1.24.0
+pandas>=2.0.0
+scikit-learn>=1.3.0
+
+# Data Storage & Messaging
+redis>=5.0.0
+supabase>=2.0.0
+confluent-kafka>=2.3.0
+kafka-python>=2.0.2
+
+# Visualization & Graphing
+graphviz>=0.20.1
+
+# Utilities
+python-dotenv>=1.0.0
+requests>=2.31.0
+typing-extensions>=4.8.0
+
+# Serialization
+pickle5>=0.0.11; python_version < '3.8'

docker-compose.yml

@@ -100,11 +100,14 @@ services:
       - _AIRFLOW_WWW_USER_CREATE=true
       - _AIRFLOW_WWW_USER_USERNAME=admin
       - _AIRFLOW_WWW_USER_PASSWORD=admin
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
     volumes:
       - ./experiments/airflow/dags:/opt/airflow/dags
       - ./experiments/airflow/logs:/opt/airflow/logs
       - ./experiments/airflow/plugins:/opt/airflow/plugins
       - ./experiments/procesing:/opt/airflow/procesing
+      - ./lib:/opt/airflow/lib
     command: version
     restart: "no"
 
@@ -116,6 +119,7 @@ services:
     depends_on:
       - postgres
       - airflow-init
+      - redis
     environment:
       - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
       - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
@@ -128,6 +132,8 @@ services:
       - BACKEND_URL=http://backend:5000
       - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
       - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
     ports:
       - "${AIRFLOW_WEBSERVER_PORT:-8085}:8080"
     volumes:
@@ -135,6 +141,7 @@ services:
       - ./experiments/airflow/logs:/opt/airflow/logs
       - ./experiments/airflow/plugins:/opt/airflow/plugins:ro
       - ./experiments/procesing:/opt/airflow/procesing:ro
+      - ./lib:/opt/airflow/lib:ro
     command: webserver
     restart: unless-stopped
     healthcheck:
@@ -152,6 +159,8 @@ services:
     depends_on:
       airflow-webserver:
         condition: service_healthy
+      redis:
+        condition: service_started
     environment:
       - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
       - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
@@ -163,11 +172,14 @@ services:
       - BACKEND_URL=http://backend:5000
       - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
       - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
     volumes:
       - ./experiments/airflow/dags:/opt/airflow/dags:ro
       - ./experiments/airflow/logs:/opt/airflow/logs
       - ./experiments/airflow/plugins:/opt/airflow/plugins:ro
       - ./experiments/procesing:/opt/airflow/procesing:ro
+      - ./lib:/opt/airflow/lib:ro
     command: scheduler
     restart: unless-stopped
     healthcheck:
@@ -177,6 +189,31 @@ services:
       retries: 5
       start_period: 30s
 
+  pricing-provider:
+    container_name: "PHANTOM-pricing-provider"
+    build:
+      context: .
+      dockerfile: docker/Provider.dockerfile
+    depends_on:
+      - redis
+      - kafka
+    environment:
+      - PROVIDER_PORT=5001
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - KAFKA_HOST=kafka
+      - KAFKA_PORT=29092
+      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
+      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
+    ports:
+      - "${PROVIDER_PORT:-5001}:5001"
+    volumes:
+      - ./lib:/app/lib:ro
+      - ./experiments/procesing:/app/procesing:ro
+      - ./backend/provider:/app/provider:ro
+    command: python -m uvicorn provider.app:app --host 0.0.0.0 --port 5001
+    restart: unless-stopped
+
 volumes:
   phantom_kafka_data:
   phantom_redis_data:

docker/Provider.dockerfile

@@ -2,26 +2,19 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
+# Install system dependencies including graphviz
 RUN apt-get update && apt-get install -y \
     gcc \
     g++ \
+    graphviz \
+    libgraphviz-dev \
     && rm -rf /var/lib/apt/lists/*
 
-COPY pyproject.toml /app/
-RUN pip install --no-cache-dir \
-    fastapi \
-    uvicorn \
-    pydantic \
-    numpy \
-    pandas \
-    scikit-learn \
-    redis \
-    supabase \
-    kafka \
-    confluent-kafka
+# Copy and install Python dependencies
+COPY backend/provider/requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
 
-
-# structure will be mounted via volumes:
+# Structure will be mounted via volumes:
 # /app/lib -> lib/
 # /app/procesing -> experiments/procesing/
 # /app/provider -> backend/provider/