mirror of
https://github.com/velocitatem/PHANTOM.git
synced 2026-05-31 08:33:36 +00:00
ad9423bf59
* introducing airflow to run pipeline * chore: updating dag with upload to registry * introducing complete provider (non refactored and noisy) * chore: removing old shit * generic pricing baselines * feature: super simple model registry (to be updated maybe third party OS software) * chore: refactoring the providers docker config and requirements * chore: refactored and broke down components (braking * exporting all * local pipeline excution working * fix: fixing import structures from nonrelativistic * chore: enables cross comm pickling with fully e2e pipeline compilation * docs: what the pipeline is like now * pipelines local running and pipeline high level definition * cleaning old pipeline and vectorization * leaked but fixing, not so important * test: started with pipeline step testing * chore: cleaning up provider of prices * test: extra tests wit hsemantic meaning checks * migrating pricers * feature: introducing pricing predictors (pricers) * chore: e2e is done with new pipeline * extra session feature extraction * feature: experiemntal sessin pricer and metrics(vibe) * chore: redefined and connected pricers (#29)
47 lines
1.4 KiB
Python
Executable File
47 lines
1.4 KiB
Python
Executable File
import pandas as pd
|
|
from procesing.steps.base import BaseContextStep
|
|
|
|
class FetchInteractionsStep(BaseContextStep):
|
|
"""Fetch raw interaction data from Kafka topic"""
|
|
|
|
def transform(self, X=None):
|
|
df = self.context.provider.fetch_kafka_topic('user-interactions')
|
|
|
|
if df.empty:
|
|
return df
|
|
|
|
# Explode metadata JSON column
|
|
if 'metadata' in df.columns:
|
|
df = df.join(
|
|
pd.json_normalize(df.pop('metadata'), sep='.').add_prefix('metadata_')
|
|
)
|
|
|
|
df = df.dropna(subset=['eventName'])
|
|
|
|
# Remap dateIndex if present
|
|
if 'metadata_dateIndex' in df.columns:
|
|
df['dateIndex'] = df['metadata_dateIndex'].astype('Int64')
|
|
|
|
return df
|
|
|
|
|
|
class FetchPriceLogsStep(BaseContextStep):
|
|
"""Fetch price log data from Kafka topic"""
|
|
|
|
def transform(self, X=None):
|
|
return self.context.provider.fetch_kafka_topic('price-logs')
|
|
|
|
|
|
class FetchExperimentsStep(BaseContextStep):
|
|
"""Fetch experiment metadata for given interaction data"""
|
|
|
|
def transform(self, interactions_df: pd.DataFrame):
|
|
if interactions_df.empty or 'experimentId' not in interactions_df.columns:
|
|
return pd.DataFrame()
|
|
|
|
exp_ids = interactions_df['experimentId'].dropna().unique().tolist()
|
|
if not exp_ids:
|
|
return pd.DataFrame()
|
|
|
|
return self.context.provider.fetch_experiments(exp_ids)
|