added pct_improvement metrics and updated space

app.py

@@ -10,6 +10,8 @@ import seaborn as sns
 from io import BytesIO
 import base64
 
+from utils import scores_to_battles, compute_bt_elo, compute_pct_improvement_over_baseline
+
 # TabBench currently supports the following models, with new additions that keep coming:
 # - **NICL (Neuralk In-Context-Learning)**: Our in-house tabular foundation model based on an in-context learning architecture (proprietary).
 # - **TabICL**: A transformer-based model that performs feature compression before doing in-context learning on tabular data by conditioning on labeled support examples to predict unseen queries without task-specific training.
@@ -182,14 +184,56 @@ def highlight_max(df: pd.DataFrame):
     )
 
 
+def compute_elo_for_subset(df: pd.DataFrame, metric: str = "Accuracy") -> pd.DataFrame:
+    """
+    Recompute Elo scores for a filtered subset of results.
+
+    Takes a DataFrame with per-dataset model scores and computes Elo ratings
+    based only on the battles within that subset.
+
+    Args:
+        df: DataFrame with columns 'model', 'dataset_name', and the metric column
+        metric: The metric to use for determining battle winners (default: "Accuracy")
+
+    Returns:
+        DataFrame with updated 'Elo_score' column
+    """
+    if df.empty:
+        return df
+
+    # Convert scores to pairwise battles
+    battles = scores_to_battles(df, metric)
+
+    if not battles:
+        return df
+
+    # Compute Elo scores from battles
+    elo_dict = compute_bt_elo(battles)
+
+    # Map Elo scores back to each row
+    df = df.copy()
+    df["Elo_score"] = df["model"].map(elo_dict)
+
+    return df
+
+
 with open("complete_results_acad.json", 'r') as f:
     public_per_dataset = pd.json_normalize(json.load(f))
 
+# Recompute Elo scores for the full public dataset
+public_per_dataset = compute_elo_for_subset(public_per_dataset, metric="Accuracy")
+# Compute percentage improvement over XGBoost
+public_per_dataset = compute_pct_improvement_over_baseline(public_per_dataset, baseline_model="xgboost_ensemble", metric="Accuracy")
 public_enter_per_dataset = public_per_dataset
 
 with open("complete_results_indus.json", "r") as f:
     private_per_dataset = pd.json_normalize(json.load(f))
 
+# Recompute Elo scores for the full private dataset
+private_per_dataset = compute_elo_for_subset(private_per_dataset, metric="Accuracy")
+# Compute percentage improvement over XGBoost
+private_per_dataset = compute_pct_improvement_over_baseline(private_per_dataset, baseline_model="xgboost_ensemble", metric="Accuracy")
+
 with open("qrt.json", "r") as f:
     qrt_scores = json.load(f)
 
@@ -298,6 +342,16 @@ css = """
     white-space: normal !important;
 }
 
+.dataframe {
+    height: auto !important;
+    max-height: none !important;
+}
+
+.dataframe .table-wrap {
+    height: auto !important;
+    max-height: none !important;
+}
+
 .dataframe td, .dataframe th {
     max-width: 200px; /* adjust column width as needed */
     overflow-wrap: break-word;
@@ -395,14 +449,14 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                 gr.Markdown('  ')
                 with gr.Row(equal_height=False):
                     with gr.Column(scale=8):
-                        public_model_agg = public_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score']].mean().reset_index()
-                        public_enter_model_agg = public_enter_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score']].mean().reset_index()
+                        public_model_agg = public_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score', 'Pct_Improvement_over_XGBoost']].mean().reset_index()
+                        public_enter_model_agg = public_enter_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score', 'Pct_Improvement_over_XGBoost']].mean().reset_index()
                         gr.Markdown("## 🏆 Overview of the Leaderboard *(Evaluation on publicly available Enterprise Datasets)*")
                         gr.Markdown('  ')
 
                         with gr.Row():
                             metric_selector = gr.Dropdown(
-                                choices=["Accuracy", "Precision", "Recall", "F1_score", "AUC", "Elo_score"],
+                                choices=["Accuracy", "Precision", "Recall", "F1_score", "AUC", "Elo_score", "Pct_Improvement_over_XGBoost"],
                                 value="Accuracy",
                                 label="📊 Metric to display",
                                 elem_classes=["compact-dropdown"]
@@ -489,7 +543,7 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
             gr.Markdown("## 🏆 Overview ")
 
             # Compute per-model averages from public_per_dataset
-            public_model_agg = public_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score']].mean().reset_index()
+            public_model_agg = public_per_dataset.groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score', 'Pct_Improvement_over_XGBoost']].mean().reset_index()
             public_model_agg = public_model_agg.round(3)
             # Add model_type from model_info.json
             with open("model_info.json", "r") as f:
@@ -497,7 +551,7 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
 
             public_model_agg = public_model_agg.merge(model_info[["model_name", "model_type"]], left_on="model", right_on="model_name", how="left").drop(columns=["model_name"])
             # Only keep the required columns, with model_type second
-            public_model_agg = public_model_agg[['model', 'model_type', 'Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score']]
+            public_model_agg = public_model_agg[['model', 'model_type', 'Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score', 'Pct_Improvement_over_XGBoost']]
             public_model_agg = public_model_agg.sort_values(by=["model"], key=lambda x: x != "NICL")
             model_df = gr.DataFrame(
                 value=highlight_max(public_model_agg),
@@ -576,6 +630,7 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                         "F1_score",
                         "AUC",
                         "Elo_score",
+                        "Pct_Improvement_over_XGBoost",
                     ]
 
                     # --- Filtering & aggregation ---
@@ -592,6 +647,11 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                         if filtered.empty:
                             return pd.DataFrame(columns=dataset_perf_cols)
 
+                        # Recompute Elo scores for this filtered subset
+                        filtered = compute_elo_for_subset(filtered, metric="Accuracy")
+                        # Recompute percentage improvement over XGBoost
+                        filtered = compute_pct_improvement_over_baseline(filtered, baseline_model="xgboost_ensemble", metric="Accuracy")
+
                         per_model_avg = (
                             filtered
                             .groupby("model")[dataset_perf_cols[1:]]
@@ -638,10 +698,14 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                 )
                 domain_per_dataset = domain_per_dataset[domain_per_dataset['dataset_industry'] == industry]
 
+                # Recompute Elo scores for this industry subset
+                domain_per_dataset = compute_elo_for_subset(domain_per_dataset, metric="Accuracy")
+                # Recompute percentage improvement over XGBoost for this industry subset
+                domain_per_dataset = compute_pct_improvement_over_baseline(domain_per_dataset, baseline_model="xgboost_ensemble", metric="Accuracy")
 
                 model_agg = (
                     domain_per_dataset
-                    .groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score']]
+                    .groupby('model')[['Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC', 'Elo_score', 'Pct_Improvement_over_XGBoost']]
                     .mean()
                     .round(3)
                     .reset_index()
@@ -668,6 +732,7 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                     interactive=False,
                     wrap=True,
                     type="pandas",
+                    row_count=(len(model_agg), "fixed"),
                 )
 
                 gr.Markdown("## 🔍 Explore performance by dataset")
@@ -697,6 +762,7 @@ with gr.Blocks(css=css, theme=gr.themes.Default()) as demo:
                     "F1_score",
                     "AUC",
                     "Elo_score",
+                    "Pct_Improvement_over_XGBoost",
                 ]   
                 
                 # Available datasets

requirements.txt

@@ -14,3 +14,5 @@ scikit-learn
 requests
 
 pyarrow
+
+scipy

utils.py

@@ -0,0 +1,154 @@
+import math
+from collections import defaultdict
+from typing import Dict, Iterable
+import pandas as pd
+
+import pandas as pd
+from itertools import combinations
+import scipy
+
+
+def compute_pct_improvement_over_baseline(
+    df: pd.DataFrame,
+    baseline_model: str = "xgboost_ensemble",
+    metric: str = "Accuracy"
+) -> pd.DataFrame:
+    """
+    Compute the percentage improvement of each model over a baseline model.
+
+    For each dataset, computes: ((model_metric - baseline_metric) / baseline_metric) * 100
+
+    Args:
+        df: DataFrame with columns 'model', 'dataset_name', and the metric column
+        baseline_model: The model to use as baseline (default: "xgboost_ensemble")
+        metric: The metric to compute improvement on (default: "Accuracy")
+
+    Returns:
+        DataFrame with a new 'Pct_Improvement_over_XGBoost' column
+    """
+    if df.empty:
+        return df
+
+    df = df.copy()
+
+    # Get baseline scores per dataset
+    baseline_scores = (
+        df[df["model"] == baseline_model]
+        .set_index("dataset_name")[metric]
+        .to_dict()
+    )
+
+    # Compute percentage improvement for each row
+    def calc_pct_improvement(row):
+        baseline = baseline_scores.get(row["dataset_name"])
+        if baseline is None or baseline == 0:
+            return None
+        return ((row[metric] - baseline) / baseline) * 100
+
+    df["Pct_Improvement_over_XGBoost"] = df.apply(calc_pct_improvement, axis=1)
+
+    return df
+
+
+def scores_to_battles(df: pd.DataFrame, metric: str = "Accuracy") -> pd.DataFrame:
+    battles = []
+    
+    for dataset, group in df.groupby("dataset_name"):
+        # Sort classifiers in descending order of metric
+        group_sorted = group.sort_values(by=metric, ascending=False)
+        for (i1, row1), (i2, row2) in combinations(group_sorted.iterrows(), 2):
+            if row1[metric] == row2[metric]:
+                winner = "tie"
+            elif row1[metric] > row2[metric]:
+                winner = "model_a"
+            else:
+                winner = "model_b"
+            battles.append({
+                "model_a": row1["model"],
+                "model_b": row2["model"],
+                "winner": winner,
+                "dataset": dataset,
+            })
+    
+    return battles
+
+
+def _sigmoid(x: float, eps: float = 1e-7) -> float:
+    """Stable sigmoid with clipped output."""
+    val = 0.5 * (1 + math.tanh(0.5 * x))
+    return max(eps, min(1.0 - eps, val))
+
+
+def compute_bt_elo(
+    battles: Iterable[Dict[str, str]],
+    SCALE: float = 400.0,
+    BASE: float = 10.0,
+    INIT_RATING: float = 1000.0,
+    lr: float = 0.05,
+    n_iter: int = 1000,
+    use_scipy: bool = True,
+) -> Dict[str, float]:
+    """Fit a Bradley--Terry model.
+
+    ``BASE`` controls the link function scale.  If ``BASE=10`` (the default),
+    the win probability follows the usual Elo form
+
+    ``P(win) = 1 / (1 + BASE ** ((rating_b - rating_a) / SCALE))``.
+
+    The function will use :mod:`scipy.optimize` if available for a fast
+    optimisation of the negative log-likelihood.  If SciPy is not installed,
+    it falls back to the simple gradient-descent routine previously used.
+    """
+
+    models = sorted({b["model_a"] for b in battles} | {b["model_b"] for b in battles})
+    battles_list = list(battles)
+
+    if use_scipy:
+        try:
+            import numpy as np
+            from scipy.optimize import minimize
+        except Exception:  # pragma: no cover - SciPy not available
+            use_scipy = False
+
+    if use_scipy:
+        idx = {m: k for k, m in enumerate(models)}
+
+        def nll(theta_vec: "np.ndarray") -> float:
+            loss = 0.0
+            for row in battles_list:
+                i = idx[row["model_a"]]
+                j = idx[row["model_b"]]
+                s = math.log(BASE) * (theta_vec[i] - theta_vec[j])
+                p = _sigmoid(s)
+                y = 1.0 if row["winner"] == "model_a" else 0.0
+                if str(row["winner"]).startswith("tie"):
+                    y = 0.5
+                # Binary cross entropy with y in [0, 1]
+                loss -= y * math.log(p) + (1 - y) * math.log(1 - p)
+            return loss
+
+        theta0 = [0.0] * len(models)
+        res = minimize(nll, theta0, method="BFGS")
+        theta_opt = res.x - sum(res.x) / len(res.x)
+        theta = {m: theta_opt[idx[m]] for m in models}
+    else:
+        theta = {m: 0.0 for m in models}
+        for _ in range(n_iter):
+            grad = {m: 0.0 for m in models}
+            for row in battles_list:
+                i = row["model_a"]
+                j = row["model_b"]
+                w = row["winner"]
+                y = 1.0 if w == "model_a" else 0.0
+                if str(w).startswith("tie"):
+                    y = 0.5
+                s = math.log(BASE) * (theta[i] - theta[j])
+                p = _sigmoid(s)
+                diff = (p - y) * math.log(BASE)
+                grad[i] += diff
+                grad[j] -= diff
+            for m in models:
+                theta[m] -= lr * grad[m] / len(battles_list)
+
+    return {m: SCALE * theta[m] + INIT_RATING for m in sorted(models, key=lambda x: -theta[x])}
+