velocitatem/PHANTOM
1
0
Fork 0
mirror of https://github.com/velocitatem/PHANTOM.git synced 2026-05-31 16:43:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Catchup airline (#31)

Browse Source 
* chore: update provider and pricing snitch with agnostic system

* cloning pipelines per mode instance

* updating airline hero section

* fix: must keep airflow secretkey

* fix: fixture update to hotel not shop

* chore: refactored to factory design pattern of pipelines

* chore: clean up definition of composite class of providers
This commit is contained in:
Daniel Alves Rösel
2025-12-11 21:56:12 +01:00
committed by GitHub
parent d45b344264
commit ef98141ca8
10 changed files with 384 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
experiments/procesing/steps/fetch.py
View File

@@ -2,7 +2,7 @@ import pandas as pd
from procesing.steps.base import BaseContextStep
class FetchInteractionsStep(BaseContextStep):
"""Fetch raw interaction data from Kafka topic with optional time filtering"""
"""Fetch raw interaction data from Kafka topic with optional time and store_mode filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
@@ -24,6 +24,10 @@ class FetchInteractionsStep(BaseContextStep):
# drop all where page has /admin/
df = df[~df['page'].str.contains('/admin/', na=False)]
# filter by store_mode from context
if 'storeMode' in df.columns:
df = df[df['storeMode'] == self.context.store_mode]
# Remap dateIndex if present
if 'metadata_dateIndex' in df.columns:
df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
@@ -38,7 +42,7 @@ class FetchInteractionsStep(BaseContextStep):
class FetchPriceLogsStep(BaseContextStep):
"""Fetch price log data from Kafka topic with optional time filtering"""
"""Fetch price log data from Kafka topic with optional time and store_mode filtering"""
def __init__(self, context, lookback: str = None):
super().__init__(context)
@@ -50,6 +54,10 @@ class FetchPriceLogsStep(BaseContextStep):
if df.empty:
return df
# filter by store_mode from context
if 'storeMode' in df.columns:
df = df[df['storeMode'] == self.context.store_mode]
# Apply time filtering if lookback specified
if self.lookback and 'ts' in df.columns:
df['ts'] = pd.to_datetime(df['ts'])