diff --git a/paper/src/chapters/figures/results/generated/final/final_focus_alpha_deltas.csv b/paper/src/chapters/figures/results/generated/final/final_focus_alpha_deltas.csv new file mode 100644 index 0000000..32bbd73 --- /dev/null +++ b/paper/src/chapters/figures/results/generated/final/final_focus_alpha_deltas.csv @@ -0,0 +1,12 @@ +alpha,revenue_delta,revenue_delta_pct,reward_delta,reward_delta_pct,volatility_delta,supra_delta,coi_leakage_delta +0.0,-17982.383542886935,-5.11072862876989,-17145.799161982606,-5.235033672101227,0.001232973729699119,0.0,-0.0030412479577408003 +0.1,-14962.041501283413,-4.410637208586118,-14303.760282736213,-4.531344436782669,0.0011858665298920962,0.0,-0.004133727080174038 +0.2,-16153.416666167905,-4.826514761457546,-15398.621298776357,-4.9418165571901715,0.00200624274016295,0.0,-0.0033201883450373615 +0.3,-17294.9275360335,-5.382423616385397,-16544.91845114401,-5.533399709364953,-0.0011022484400295268,0.0,-0.0029151149203366505 +0.4,-19661.294346174283,-6.250307313590199,-18728.35578200908,-6.3953153560217535,3.582812967113658e-05,0.0,-0.0038123361988749577 +0.5,-16411.03168918495,-5.3630681206030015,-15638.77510066732,-5.4888928630525315,0.00015428950526953644,0.0,-0.00439661338956944 +0.6,-14729.668247641937,-5.069964928178309,-13912.22417824401,-5.148827377884945,-0.002735776807082743,0.0,-0.004310129386364658 +0.7,-21160.81910514756,-7.351404104505076,-20171.762105623755,-7.525169314210056,-0.0008903632602569461,0.0,-0.0026198461183787186 +0.8,-16404.76825612632,-5.9342582959227075,-15645.025250480074,-6.078699946285722,0.0010338614665691137,0.0,-0.002542765270289696 +0.9,-8674.090655496111,-3.2592966246269577,-8371.30734891587,-3.378943339994106,-0.0005579187914590139,0.0,-0.0013720835439427759 +1.0,768.8099906174757,0.2991618705853567,399.7394696234842,0.16706914330070038,0.0014659834822295797,0.0,-0.0007600066499474645 diff --git a/paper/src/chapters/figures/results/generated/final/final_focus_alpha_mode_summary.csv b/paper/src/chapters/figures/results/generated/final/final_focus_alpha_mode_summary.csv new file mode 100644 index 0000000..50051aa --- /dev/null +++ b/paper/src/chapters/figures/results/generated/final/final_focus_alpha_mode_summary.csv @@ -0,0 +1,23 @@ +alpha,mode,runs,revenue_mean,reward_mean,supra_mean,volatility_mean,coi_leakage_mean,coi_level_mean +0.0,baseline,36,351855.57381502265,327520.32242613373,0.0,0.06922494093544151,0.11931704468268205,136.80105514058158 +0.0,defended,35,333873.1902721357,310374.5232641511,0.0,0.07045791466514063,0.11627579672494125,136.81832905386602 +0.1,baseline,32,339226.3020897988,315662.6136522988,0.0,0.06952778671756812,0.11924519238669087,136.47864859317326 +0.1,defended,33,324264.2605885154,301358.8533695626,0.0,0.07071365324746022,0.11511146530651684,136.7200845824852 +0.2,baseline,31,334680.76789409376,311598.399506997,0.0,0.06848006194428993,0.11597869134898402,136.83684469591932 +0.2,defended,35,318527.35122792586,296199.77820822067,0.0,0.07048630468445288,0.11265850300394666,137.2758153292305 +0.3,baseline,30,321322.30327214615,299000.9636054795,0.0,0.07085669473747759,0.11527347603412934,136.4452630715689 +0.3,defended,44,304027.37573611265,282456.0451543355,0.0,0.06975444629744806,0.11235836111379269,136.4704115371568 +0.4,baseline,33,314565.2423109539,292844.914432166,0.0,0.07031811881503117,0.11300307992768284,136.72547178046122 +0.4,defended,38,294903.9479647796,274116.55865015695,0.0,0.0703539469447023,0.10919074372880788,136.75671002806396 +0.5,baseline,33,306000.80625751516,284916.7489847879,0.0,0.06938663916591635,0.11118137138243217,136.9528780620641 +0.5,defended,35,289589.7745683302,269277.9738841206,0.0,0.06954092867118589,0.10678475799286273,136.65018588845163 +0.6,baseline,28,290528.0106727377,270201.7985298805,0.0,0.07139577980623227,0.11081647254398667,135.258395468266 +0.6,defended,41,275798.3424250958,256289.57435163652,0.0,0.06866000299914952,0.10650634315762202,136.3194947785247 +0.7,baseline,40,287847.3119465684,268057.25244656845,0.0,0.07132313199532896,0.10746267580456732,137.0170522633547 +0.7,defended,40,266686.49284142087,247885.4903409447,0.0,0.07043276873507201,0.1048428296861886,136.56834095392904 +0.8,baseline,26,276441.76303208206,257374.52726285128,0.0,0.06945655282263205,0.1063246766773884,136.66765260798618 +0.8,defended,39,260036.99477595574,241729.5020123712,0.0,0.07049041428920116,0.1037819114070987,136.61222667078658 +0.9,baseline,35,266133.8213268301,247749.2667554015,0.0,0.0709569180547784,0.10455882265976374,136.5370653814206 +0.9,defended,39,257459.73067133396,239377.95940648564,0.0,0.07039899926331938,0.10318673911582096,136.7368893225831 +1.0,baseline,35,256987.96076959255,239265.888198164,0.0,0.06888231148034313,0.10369761394735275,136.68691718467974 +1.0,defended,30,257756.77076021003,239665.62766778748,0.0,0.07034829496257271,0.10293760729740528,136.65287739235566 diff --git a/paper/src/chapters/figures/results/generated/final/final_focus_headline_summary.json b/paper/src/chapters/figures/results/generated/final/final_focus_headline_summary.json new file mode 100644 index 0000000..e257560 --- /dev/null +++ b/paper/src/chapters/figures/results/generated/final/final_focus_headline_summary.json @@ -0,0 +1,27 @@ +{ + "bundle": "engine/studies/results/wandb_sweep_bundles/bundle_20260317_093826", + "focus_cohort": "max_alpha_coverage", + "alpha_cells": 11, + "alpha_min": 0.0, + "alpha_max": 1.0, + "mean_revenue_delta_pct": -4.787221975639986, + "mean_reward_delta_pct": -4.91730667541704, + "zone_summary": [ + { + "zone": "high_alpha_0_7_plus", + "alpha_cells": 4, + "revenue_delta_pct_mean": -4.0614492886173466, + "reward_delta_pct_mean": -4.2039358642972955, + "coi_leakage_delta_mean": -0.0018236753956396637, + "volatility_delta_mean": 0.00026289072427068336 + }, + { + "zone": "low_alpha_below_0_7", + "alpha_cells": 7, + "revenue_delta_pct_mean": -5.201949225367208, + "reward_delta_pct_mean": -5.324947138914036, + "coi_leakage_delta_mean": -0.0037041938968711296, + "volatility_delta_mean": 0.00011102505536893832 + } + ] +} diff --git a/paper/src/chapters/figures/results/generated/final/final_focus_zone_summary.csv b/paper/src/chapters/figures/results/generated/final/final_focus_zone_summary.csv new file mode 100644 index 0000000..224a022 --- /dev/null +++ b/paper/src/chapters/figures/results/generated/final/final_focus_zone_summary.csv @@ -0,0 +1,3 @@ +zone,alpha_cells,revenue_delta_pct_mean,reward_delta_pct_mean,coi_leakage_delta_mean,volatility_delta_mean +high_alpha_0_7_plus,4,-4.0614492886173466,-4.2039358642972955,-0.0018236753956396637,0.00026289072427068336 +low_alpha_below_0_7,7,-5.201949225367208,-5.324947138914036,-0.0037041938968711296,0.00011102505536893832 diff --git a/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_by_alpha.pdf b/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_by_alpha.pdf new file mode 100644 index 0000000..343539e Binary files /dev/null and b/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_by_alpha.pdf differ diff --git a/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_delta.pdf b/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_delta.pdf new file mode 100644 index 0000000..157fc7a Binary files /dev/null and b/paper/src/chapters/figures/results/generated/final/plots/final_focus_revenue_delta.pdf differ diff --git a/paper/src/chapters/figures/results/generated/final/plots/final_focus_risk_deltas.pdf b/paper/src/chapters/figures/results/generated/final/plots/final_focus_risk_deltas.pdf new file mode 100644 index 0000000..9bbbf7a Binary files /dev/null and b/paper/src/chapters/figures/results/generated/final/plots/final_focus_risk_deltas.pdf differ diff --git a/paper/src/chapters/figures/results/includes/final/final_focus_revenue_by_alpha.tex b/paper/src/chapters/figures/results/includes/final/final_focus_revenue_by_alpha.tex new file mode 100644 index 0000000..f2fa4a2 --- /dev/null +++ b/paper/src/chapters/figures/results/includes/final/final_focus_revenue_by_alpha.tex @@ -0,0 +1 @@ +\includegraphics[width=0.98\linewidth]{chapters/figures/results/generated/final/plots/final_focus_revenue_by_alpha.pdf} diff --git a/paper/src/chapters/figures/results/includes/final/final_focus_revenue_delta.tex b/paper/src/chapters/figures/results/includes/final/final_focus_revenue_delta.tex new file mode 100644 index 0000000..0a13ca4 --- /dev/null +++ b/paper/src/chapters/figures/results/includes/final/final_focus_revenue_delta.tex @@ -0,0 +1 @@ +\includegraphics[width=0.95\linewidth]{chapters/figures/results/generated/final/plots/final_focus_revenue_delta.pdf} diff --git a/paper/src/chapters/figures/results/includes/final/final_focus_risk_deltas.tex b/paper/src/chapters/figures/results/includes/final/final_focus_risk_deltas.tex new file mode 100644 index 0000000..c694faf --- /dev/null +++ b/paper/src/chapters/figures/results/includes/final/final_focus_risk_deltas.tex @@ -0,0 +1 @@ +\includegraphics[width=0.95\linewidth]{chapters/figures/results/generated/final/plots/final_focus_risk_deltas.pdf} diff --git a/paper/src/chapters/figures/results/process_final_sweeps.py b/paper/src/chapters/figures/results/process_final_sweeps.py new file mode 100644 index 0000000..f874e2b --- /dev/null +++ b/paper/src/chapters/figures/results/process_final_sweeps.py @@ -0,0 +1,409 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +def _project_root() -> Path: + return Path(__file__).resolve().parents[5] + + +def _default_bundle_dir() -> Path: + base = _project_root() / "engine" / "studies" / "results" / "wandb_sweep_bundles" + bundles = sorted( + [path for path in base.glob("bundle_*") if path.is_dir()], + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + if not bundles: + raise FileNotFoundError(f"No sweep bundle directories found in {base}") + return bundles[0] + + +def _default_output_dir() -> Path: + return Path(__file__).resolve().parent / "generated" / "final" + + +def _default_plot_dir(output_dir: Path) -> Path: + return output_dir / "plots" + + +def _truthy(value: Any) -> bool: + if isinstance(value, bool): + return value + if value is None: + return False + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _mode_of(row: pd.Series) -> str: + mode_hint = str(row.get("study_mode", "")).strip().lower() + if mode_hint in {"baseline", "no_robust"}: + return "baseline" + if mode_hint in {"defended", "robust"}: + return "defended" + if _truthy(row.get("baseline_mode")) or _truthy(row.get("no_robust")): + return "baseline" + return "defended" + + +def _coerce_numeric(frame: pd.DataFrame, columns: list[str]) -> None: + for column in columns: + if column in frame.columns: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + + +def _configure_style() -> None: + plt.rcParams.update( + { + "font.family": "serif", + "font.size": 10, + "axes.titlesize": 10, + "axes.labelsize": 9, + "legend.fontsize": 8, + "xtick.labelsize": 8, + "ytick.labelsize": 8, + "figure.dpi": 220, + "savefig.dpi": 320, + "axes.spines.top": False, + "axes.spines.right": False, + "axes.grid": True, + "grid.alpha": 0.22, + } + ) + + +def _load_runs(bundle_dir: Path) -> pd.DataFrame: + path = bundle_dir / "runs_finished.csv" + if not path.exists(): + raise FileNotFoundError(f"Missing required file: {path}") + frame = pd.read_csv(path) + frame["mode"] = frame.apply(_mode_of, axis=1) + _coerce_numeric( + frame, + [ + "alpha", + "n_products", + "eval_revenue_mean", + "eval_reward_mean", + "eval_supra_share_mean", + "eval_volatility_mean", + "eval_coi_level_mean", + "eval_coi_leakage_mean", + "objective_score", + ], + ) + return frame + + +def _focus_sweep(runs: pd.DataFrame) -> str: + coverage = ( + runs.groupby("sweep_id", as_index=False) + .agg( + n_alpha=("alpha", lambda s: int(pd.Series(s).dropna().nunique())), + max_alpha=("alpha", "max"), + run_count=("run_id", "size"), + ) + .sort_values( + ["n_alpha", "max_alpha", "run_count"], ascending=[False, False, False] + ) + ) + if coverage.empty: + raise ValueError("No sweep rows available in runs_finished.csv") + return str(coverage.iloc[0]["sweep_id"]) + + +def _alpha_mode_summary(runs: pd.DataFrame) -> pd.DataFrame: + return ( + runs.groupby(["alpha", "mode"], as_index=False) + .agg( + runs=("run_id", "size"), + revenue_mean=("eval_revenue_mean", "mean"), + reward_mean=("eval_reward_mean", "mean"), + supra_mean=("eval_supra_share_mean", "mean"), + volatility_mean=("eval_volatility_mean", "mean"), + coi_leakage_mean=("eval_coi_leakage_mean", "mean"), + coi_level_mean=("eval_coi_level_mean", "mean"), + ) + .sort_values(["alpha", "mode"]) + .reset_index(drop=True) + ) + + +def _alpha_deltas(alpha_mode: pd.DataFrame) -> pd.DataFrame: + rows: list[dict[str, float]] = [] + for alpha, group in alpha_mode.groupby("alpha", sort=True): + defended = group[group["mode"] == "defended"] + baseline = group[group["mode"] == "baseline"] + if defended.empty or baseline.empty: + continue + d_rev = float(defended["revenue_mean"].iloc[0]) + b_rev = float(baseline["revenue_mean"].iloc[0]) + d_reward = float(defended["reward_mean"].iloc[0]) + b_reward = float(baseline["reward_mean"].iloc[0]) + d_vol = float(defended["volatility_mean"].iloc[0]) + b_vol = float(baseline["volatility_mean"].iloc[0]) + d_supra = float(defended["supra_mean"].iloc[0]) + b_supra = float(baseline["supra_mean"].iloc[0]) + d_coi_leak = float(defended["coi_leakage_mean"].iloc[0]) + b_coi_leak = float(baseline["coi_leakage_mean"].iloc[0]) + rows.append( + { + "alpha": float(alpha), + "revenue_delta": d_rev - b_rev, + "revenue_delta_pct": 0.0 + if b_rev == 0.0 + else 100.0 * (d_rev - b_rev) / b_rev, + "reward_delta": d_reward - b_reward, + "reward_delta_pct": 0.0 + if b_reward == 0.0 + else 100.0 * (d_reward - b_reward) / b_reward, + "volatility_delta": d_vol - b_vol, + "supra_delta": d_supra - b_supra, + "coi_leakage_delta": d_coi_leak - b_coi_leak, + } + ) + return pd.DataFrame(rows).sort_values("alpha").reset_index(drop=True) + + +def _zone_summary(alpha_deltas: pd.DataFrame) -> pd.DataFrame: + if alpha_deltas.empty: + return pd.DataFrame() + data = alpha_deltas.copy() + data["zone"] = np.where( + data["alpha"] >= 0.7, "high_alpha_0_7_plus", "low_alpha_below_0_7" + ) + return ( + data.groupby("zone", as_index=False) + .agg( + alpha_cells=("alpha", "size"), + revenue_delta_pct_mean=("revenue_delta_pct", "mean"), + reward_delta_pct_mean=("reward_delta_pct", "mean"), + coi_leakage_delta_mean=("coi_leakage_delta", "mean"), + volatility_delta_mean=("volatility_delta", "mean"), + ) + .sort_values("zone") + ) + + +def _save_plot(fig: plt.Figure, path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(path, bbox_inches="tight") + plt.close(fig) + return path + + +def _plot_focus_revenue_by_alpha(alpha_mode: pd.DataFrame, out_path: Path) -> Path: + fig, ax = plt.subplots(figsize=(7.8, 4.8), constrained_layout=True) + for mode, color, label in ( + ("baseline", "#4C72B0", "Baseline"), + ("defended", "#C44E52", "Defended"), + ): + sub = alpha_mode[alpha_mode["mode"] == mode].sort_values("alpha") + if sub.empty: + continue + ax.plot( + sub["alpha"], + sub["revenue_mean"], + marker="o", + linewidth=1.9, + markersize=4, + color=color, + label=label, + ) + ax.axvline(0.7, color="#666666", linewidth=1.0, linestyle="--") + ax.set_xlabel(r"Contamination $\alpha$") + ax.set_ylabel("Mean episode revenue") + ax.set_title("Final Cohort Revenue Curves") + ax.legend(loc="lower left") + return _save_plot(fig, out_path) + + +def _plot_focus_revenue_delta(alpha_deltas: pd.DataFrame, out_path: Path) -> Path: + fig, ax = plt.subplots(figsize=(7.8, 4.8), constrained_layout=True) + x = alpha_deltas["alpha"].to_numpy(dtype=float) + y = alpha_deltas["revenue_delta_pct"].to_numpy(dtype=float) + ax.plot(x, y, marker="o", linewidth=2.0, markersize=4, color="#C44E52") + ax.fill_between(x, y, 0.0, color="#C44E52", alpha=0.12) + ax.axhline(0.0, color="#444444", linewidth=1.0, linestyle="--") + ax.axvline(0.7, color="#666666", linewidth=1.0, linestyle="--") + high = alpha_deltas[alpha_deltas["alpha"] >= 0.7] + if not high.empty: + best = high.reindex( + high["revenue_delta_pct"].abs().sort_values(ascending=False).index + ).iloc[0] + ax.scatter( + [best["alpha"]], + [best["revenue_delta_pct"]], + color="#1f77b4", + s=45, + zorder=3, + ) + ax.annotate( + f"high-alpha peak {best['revenue_delta_pct']:.2f}%", + (float(best["alpha"]), float(best["revenue_delta_pct"])), + textcoords="offset points", + xytext=(6, 6), + fontsize=8, + ) + ax.set_xlabel(r"Contamination $\alpha$") + ax.set_ylabel("Defended minus baseline revenue (%)") + ax.set_title("Revenue Delta by Contamination (Final Cohort)") + return _save_plot(fig, out_path) + + +def _plot_focus_risk_deltas(alpha_deltas: pd.DataFrame, out_path: Path) -> Path: + fig, ax = plt.subplots(figsize=(7.8, 4.8), constrained_layout=True) + x = alpha_deltas["alpha"].to_numpy(dtype=float) + ax.plot( + x, + alpha_deltas["coi_leakage_delta"].to_numpy(dtype=float), + marker="o", + linewidth=1.8, + markersize=4, + color="#55A868", + label="COI leakage delta", + ) + ax.plot( + x, + alpha_deltas["volatility_delta"].to_numpy(dtype=float), + marker="s", + linewidth=1.8, + markersize=3.8, + color="#8172B3", + label="Volatility delta", + ) + ax.axhline(0.0, color="#444444", linewidth=1.0, linestyle="--") + ax.axvline(0.7, color="#666666", linewidth=1.0, linestyle="--") + ax.set_xlabel(r"Contamination $\alpha$") + ax.set_ylabel("Defended minus baseline") + ax.set_title("Leakage and Stability Deltas (Final Cohort)") + ax.legend(loc="lower left") + return _save_plot(fig, out_path) + + +def _write_include(path: Path, figure_rel_path: str, width: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"\\includegraphics[width={width}]{{{figure_rel_path}}}\n") + return path + + +def run(bundle_dir: Path, output_dir: Path, plot_dir: Path) -> list[Path]: + all_runs = _load_runs(bundle_dir) + focus_id = _focus_sweep(all_runs) + focus_runs = all_runs[all_runs["sweep_id"] == focus_id].copy() + alpha_mode = _alpha_mode_summary(focus_runs) + deltas = _alpha_deltas(alpha_mode) + zones = _zone_summary(deltas) + + output_dir.mkdir(parents=True, exist_ok=True) + plot_dir.mkdir(parents=True, exist_ok=True) + + written: list[Path] = [] + alpha_mode_path = output_dir / "final_focus_alpha_mode_summary.csv" + alpha_mode.to_csv(alpha_mode_path, index=False) + written.append(alpha_mode_path) + + delta_path = output_dir / "final_focus_alpha_deltas.csv" + deltas.to_csv(delta_path, index=False) + written.append(delta_path) + + zone_path = output_dir / "final_focus_zone_summary.csv" + zones.to_csv(zone_path, index=False) + written.append(zone_path) + + headline = { + "bundle": str(bundle_dir), + "focus_cohort": "max_alpha_coverage", + "alpha_cells": int(deltas["alpha"].nunique()) if not deltas.empty else 0, + "alpha_min": float(deltas["alpha"].min()) if not deltas.empty else None, + "alpha_max": float(deltas["alpha"].max()) if not deltas.empty else None, + "mean_revenue_delta_pct": float(deltas["revenue_delta_pct"].mean()) + if not deltas.empty + else None, + "mean_reward_delta_pct": float(deltas["reward_delta_pct"].mean()) + if not deltas.empty + else None, + "zone_summary": zones.to_dict(orient="records"), + } + headline_path = output_dir / "final_focus_headline_summary.json" + headline_path.write_text(json.dumps(headline, indent=2) + "\n") + written.append(headline_path) + + written.append( + _plot_focus_revenue_by_alpha( + alpha_mode, + plot_dir / "final_focus_revenue_by_alpha.pdf", + ) + ) + written.append( + _plot_focus_revenue_delta( + deltas, + plot_dir / "final_focus_revenue_delta.pdf", + ) + ) + written.append( + _plot_focus_risk_deltas( + deltas, + plot_dir / "final_focus_risk_deltas.pdf", + ) + ) + + include_dir = Path(__file__).resolve().parent / "includes" / "final" + written.append( + _write_include( + include_dir / "final_focus_revenue_by_alpha.tex", + "chapters/figures/results/generated/final/plots/final_focus_revenue_by_alpha.pdf", + "0.98\\linewidth", + ) + ) + written.append( + _write_include( + include_dir / "final_focus_revenue_delta.tex", + "chapters/figures/results/generated/final/plots/final_focus_revenue_delta.pdf", + "0.95\\linewidth", + ) + ) + written.append( + _write_include( + include_dir / "final_focus_risk_deltas.tex", + "chapters/figures/results/generated/final/plots/final_focus_risk_deltas.pdf", + "0.95\\linewidth", + ) + ) + return written + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate final paper figures/tables from the final sweep cohort" + ) + parser.add_argument("--bundle-dir", type=Path, default=_default_bundle_dir()) + parser.add_argument("--output-dir", type=Path, default=_default_output_dir()) + parser.add_argument("--plot-dir", type=Path, default=None) + args = parser.parse_args() + + _configure_style() + plot_dir = ( + args.plot_dir + if args.plot_dir is not None + else _default_plot_dir(args.output_dir) + ) + outputs = run( + bundle_dir=args.bundle_dir, output_dir=args.output_dir, plot_dir=plot_dir + ) + for path in outputs: + print(path) + + +if __name__ == "__main__": + main()