velocitatem/PHANTOM
1
0
Fork 0
mirror of https://github.com/velocitatem/PHANTOM.git synced 2026-05-31 16:43:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improving interface after experiment01 (#30)

Browse Source 
* fix: fixes of backwords

* fixing hotel information with image placeholders

* chore: clean up product display in hotel and cleaner interfacing

* adding loader with historical data loading

* feature: cleaning up pipeline

* chore: simple surge pricer

* created new pricing pipeline

* adding a checkout page to both sites

* fix: fixing stale pacakge

* test: we wont be using elasticity anymore so its okay

* chore: cleaning elasticity references

* chore: store sting

* feature: e2e intro pipline surge pricing

* fix: CVE vulnerability patching
This commit is contained in:
Daniel Alves Rösel
2025-12-06 17:47:14 +01:00
committed by GitHub
parent 59d4fb7891
commit 8751583764
27 changed files with 709 additions and 1096 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
experiments/procesing/steps/fetch.py
View File

@@ -2,7 +2,11 @@ import pandas as pd
from procesing.steps.base import BaseContextStep
class FetchInteractionsStep(BaseContextStep):
"""Fetch raw interaction data from Kafka topic"""
"""Fetch raw interaction data from Kafka topic with optional time filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
self.lookback = lookback
def transform(self, X=None):
df = self.context.provider.fetch_kafka_topic('user-interactions')
@@ -24,14 +28,35 @@ class FetchInteractionsStep(BaseContextStep):
if 'metadata_dateIndex' in df.columns:
df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
# Apply time filtering if lookback specified
if self.lookback and 'ts' in df.columns:
df['ts'] = pd.to_datetime(df['ts'])
cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
df = df[df['ts'] >= cutoff]
return df
class FetchPriceLogsStep(BaseContextStep):
"""Fetch price log data from Kafka topic"""
"""Fetch price log data from Kafka topic with optional time filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
self.lookback = lookback
def transform(self, X=None):
return self.context.provider.fetch_kafka_topic('price-logs')
df = self.context.provider.fetch_kafka_topic('price-logs')
if df.empty:
return df
# Apply time filtering if lookback specified
if self.lookback and 'ts' in df.columns:
df['ts'] = pd.to_datetime(df['ts'])
cutoff = pd.Timestamp.now() - pd.Timedelta(self.lookback)
df = df[df['ts'] >= cutoff]
return df
class FetchExperimentsStep(BaseContextStep):