velocitatem/PHANTOM
1
0
Fork 0
mirror of https://github.com/velocitatem/PHANTOM.git synced 2026-05-31 16:43:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update provider and pricing snitch with agnostic system

Browse Source
This commit is contained in:
Daniel Rosel
2025-12-06 19:06:40 +01:00
parent d45b344264
commit 4426b0ff74
3 changed files with 22 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
experiments/procesing/steps/fetch.py
View File

@@ -2,7 +2,7 @@ import pandas as pd
from procesing.steps.base import BaseContextStep
class FetchInteractionsStep(BaseContextStep):
"""Fetch raw interaction data from Kafka topic with optional time filtering"""
"""Fetch raw interaction data from Kafka topic with optional time and store_mode filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
@@ -24,6 +24,10 @@ class FetchInteractionsStep(BaseContextStep):
# drop all where page has /admin/
df = df[~df['page'].str.contains('/admin/', na=False)]
# filter by store_mode from context
if 'storeMode' in df.columns:
df = df[df['storeMode'] == self.context.store_mode]
# Remap dateIndex if present
if 'metadata_dateIndex' in df.columns:
df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
@@ -38,7 +42,7 @@ class FetchInteractionsStep(BaseContextStep):
class FetchPriceLogsStep(BaseContextStep):
"""Fetch price log data from Kafka topic with optional time filtering"""
"""Fetch price log data from Kafka topic with optional time and store_mode filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
@@ -50,6 +54,10 @@ class FetchPriceLogsStep(BaseContextStep):
if df.empty:
return df
# filter by store_mode from context
if 'storeMode' in df.columns:
df = df[df['storeMode'] == self.context.store_mode]
# Apply time filtering if lookback specified
if self.lookback and 'ts' in df.columns:
df['ts'] = pd.to_datetime(df['ts'])