Add GIFT-Eval forecasting benchmark

.gitignore

@@ -221,5 +221,8 @@ scoringbench_out/
 evals/
 datasets.json
 
+# Time Series benchmarks
+gift_eval_out/
+
 # Checkpoint files
 *.pt

benchmark/README.md

@@ -9,3 +9,5 @@ This folder contains benchmark integrations for `structured-data-models`:
   - [**`talent/`**](tabular/talent/): [TALENT](https://github.com/LAMDA-Tabular/TALENT)
 - [**`relational/`**](relational/): Relational benchmarks
   - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena)
+- [**`timeseries/`**](timeseries/): Time series benchmarks
+  - [**`gift_eval/`**](timeseries/gift_eval/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval)

benchmark/relational/README.md

@@ -0,0 +1,3 @@
+# Relational Benchmarks
+
+This directory contains relational benchmarks of `structured-data-models` on [RelArena-α](relarena/).

benchmark/relational/relarena/README.md

@@ -1,4 +1,4 @@
-# Relational Models on RelArena-α
+# Relational Foundation Models on RelArena-α
 
 This benchmark evaluates `structured-data-models` on the [RelArena-α](https://github.com/PriorLabs/relarena) benchmark.
 

benchmark/timeseries/README.md

@@ -0,0 +1,3 @@
+# Time Series Benchmarks
+
+This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/).

benchmark/timeseries/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

benchmark/timeseries/gift_eval/README.md

@@ -0,0 +1,32 @@
+# Time Series Forecasting Foundation Models on GIFT-Eval
+
+This benchmark evaluates `structured-data-models` on the [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) benchmark.
+
+## Setup
+
+Run the commands below from the repository root:
+
+```bash
+pip install . pandas gluonts datasets python-dotenv
+pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d"
+```
+
+## Run
+
+> [!NOTE]
+> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE).
+> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively.
+
+```bash
+python -m benchmark.timeseries.gift_eval.main --model timesfm3
+```
+
+Results will be written to `benchmark/timeseries/gift_eval_out/<model>/`, and already completed configurations are skipped.
+
+## Evaluate
+
+Evaluate all available model results with:
+
+```bash
+python -m benchmark.timeseries.gift_eval.evaluate
+```

benchmark/timeseries/gift_eval/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+GIFT_EVAL_URL = (
+    "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/"
+    "577a65cfa3acb812405c00d989230bf59f7f8a3d"
+)

benchmark/timeseries/gift_eval/datasets.py

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+DATASETS = [
+    "m4_yearly",
+    "m4_quarterly",
+    "m4_monthly",
+    "m4_weekly",
+    "m4_daily",
+    "m4_hourly",
+    "electricity/15T",
+    "electricity/H",
+    "electricity/D",
+    "electricity/W",
+    "solar/10T",
+    "solar/H",
+    "solar/D",
+    "solar/W",
+    "hospital",
+    "covid_deaths",
+    "us_births/D",
+    "us_births/M",
+    "us_births/W",
+    "saugeenday/D",
+    "saugeenday/M",
+    "saugeenday/W",
+    "temperature_rain_with_missing",
+    "kdd_cup_2018_with_missing/H",
+    "kdd_cup_2018_with_missing/D",
+    "car_parts_with_missing",
+    "restaurant",
+    "hierarchical_sales/D",
+    "hierarchical_sales/W",
+    "LOOP_SEATTLE/5T",
+    "LOOP_SEATTLE/H",
+    "LOOP_SEATTLE/D",
+    "SZ_TAXI/15T",
+    "SZ_TAXI/H",
+    "M_DENSE/H",
+    "M_DENSE/D",
+    "ett1/15T",
+    "ett1/H",
+    "ett1/D",
+    "ett1/W",
+    "ett2/15T",
+    "ett2/H",
+    "ett2/D",
+    "ett2/W",
+    "jena_weather/10T",
+    "jena_weather/H",
+    "jena_weather/D",
+    "bitbrains_fast_storage/5T",
+    "bitbrains_fast_storage/H",
+    "bitbrains_rnd/5T",
+    "bitbrains_rnd/H",
+    "bizitobs_application",
+    "bizitobs_service",
+    "bizitobs_l2c/5T",
+    "bizitobs_l2c/H",
+]
+
+MED_LONG_DATASETS = [
+    "electricity/15T",
+    "electricity/H",
+    "solar/10T",
+    "solar/H",
+    "kdd_cup_2018_with_missing/H",
+    "LOOP_SEATTLE/5T",
+    "LOOP_SEATTLE/H",
+    "SZ_TAXI/15T",
+    "M_DENSE/H",
+    "ett1/15T",
+    "ett1/H",
+    "ett2/15T",
+    "ett2/H",
+    "jena_weather/10T",
+    "jena_weather/H",
+    "bitbrains_fast_storage/5T",
+    "bitbrains_rnd/5T",
+    "bizitobs_application",
+    "bizitobs_service",
+    "bizitobs_l2c/5T",
+    "bizitobs_l2c/H",
+]
+
+PRETTY_NAMES = {
+    "saugeenday": "saugeen",
+    "temperature_rain_with_missing": "temperature_rain",
+    "kdd_cup_2018_with_missing": "kdd_cup_2018",
+    "car_parts_with_missing": "car_parts",
+}

benchmark/timeseries/gift_eval/evaluate.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Evaluate SDM time series forecasting results on GIFT-Eval."""
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from benchmark.timeseries.gift_eval import GIFT_EVAL_URL
+from benchmark.timeseries.model import MODEL_CONFIGS
+
+BASELINE = "Seasonal_Naive"
+REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE)
+METRICS = {
+    "eval_metrics/MASE[0.5]": "MASE",
+    "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
+}
+
+root = Path(__file__).parent.parent / "gift_eval_out"
+
+dfs = []
+for config in MODEL_CONFIGS.values():
+    path = root / config.name / "all_results.csv"
+    if path.exists():
+        dfs.append(pd.read_csv(path))
+datasets = set.intersection(*(set(df["dataset"]) for df in dfs))
+
+for name in REFERENCES:
+    url = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv"
+    dfs.append(pd.read_csv(url))
+
+df = pd.concat(dfs)
+df = df[df["dataset"].isin(datasets)].set_index(["model", "dataset"])
+df = df[list(METRICS)].rename(columns=METRICS)
+
+terms = df.index.get_level_values("dataset").str.split("/").str[-1]
+scores = np.log(df.div(df.loc[BASELINE], level="dataset"))
+for i, (group, score) in enumerate([("all", scores), *scores.groupby(terms)]):
+    if i > 0:
+        print()
+    summary = np.exp(score.groupby("model").mean())
+    summary = summary.sort_values("CRPS")
+    print(f"GIFT-Eval leaderboard ({group}):")
+    print(summary.reset_index().to_string(index=False, float_format="%.4f"))

benchmark/timeseries/gift_eval/main.py

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Run an SDM time series forecasting model on GIFT-Eval."""
+
+import argparse
+import logging
+import os
+import warnings
+from functools import partial
+from pathlib import Path
+from unittest.mock import patch
+
+import pandas as pd
+import torch
+from gift_eval.data import Dataset
+from gluonts.ev import metrics
+from gluonts.model import evaluate_model
+from gluonts.time_feature import get_seasonality
+from huggingface_hub import snapshot_download
+from tqdm.auto import tqdm
+
+from benchmark.timeseries.gift_eval import GIFT_EVAL_URL
+from benchmark.timeseries.gift_eval.datasets import (
+    DATASETS,
+    MED_LONG_DATASETS,
+    PRETTY_NAMES,
+)
+from benchmark.timeseries.gift_eval.predictor import SDMPredictor
+from benchmark.timeseries.model import MODEL_CONFIGS
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--model",
+    choices=tuple(MODEL_CONFIGS),
+    default="timesfm3",
+    help="SDM model to benchmark.",
+)
+parser.add_argument(
+    "--dataset",
+    action="append",
+    help="Run only the selected GIFT-Eval dataset; repeat to combine.",
+)
+args = parser.parse_args()
+
+warnings.filterwarnings("ignore", message=r"'.*' is deprecated.*")
+logging.getLogger("gluonts.model.forecast_generator").addFilter(
+    lambda r: not r.getMessage().startswith("The mean prediction")
+)
+
+METRICS = [
+    metrics.MSE(forecast_type="mean"),
+    metrics.MSE(forecast_type="0.5"),
+    metrics.MAE(),
+    metrics.MASE(),
+    metrics.MAPE(),
+    metrics.SMAPE(),
+    metrics.MSIS(),
+    metrics.RMSE(),
+    metrics.NRMSE(),
+    metrics.ND(),
+    metrics.MeanWeightedSumQuantileLoss(
+        quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
+    ),
+]
+
+config = MODEL_CONFIGS[args.model]
+path = Path(__file__).parent.parent
+path = path / "gift_eval_out" / config.name / "all_results.csv"
+path.parent.mkdir(parents=True, exist_ok=True)
+
+completed = set(pd.read_csv(path)["dataset"]) if path.exists() else set()
+properties = pd.read_json(f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json")
+
+names = args.dataset or DATASETS
+if not os.getenv("GIFT_EVAL"):
+    os.environ["GIFT_EVAL"] = snapshot_download(
+        repo_id="Salesforce/GiftEval",
+        repo_type="dataset",
+        allow_patterns=[f"{name}/*" for name in names],
+    )
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = config.factory(device)
+
+for name in names:
+    key, _, freq = name.partition("/")
+    key = PRETTY_NAMES.get(key.lower(), key.lower())
+    freq = freq or properties[key]["frequency"]
+    for term in ("short", "medium", "long"):
+        if term != "short" and name not in MED_LONG_DATASETS:
+            continue
+        dataset_id = f"{key}/{freq}/{term}"
+        if dataset_id in completed:
+            continue
+
+        dataset = Dataset(name=name, term=term, to_univariate=False)
+        predictor = SDMPredictor(model, config, dataset.prediction_length)
+        with patch(
+            "gluonts.model.evaluation.tqdm",
+            partial(tqdm, total=len(dataset.test_data), desc=dataset_id),
+        ):
+            result = evaluate_model(
+                predictor,
+                test_data=dataset.test_data,
+                metrics=METRICS,
+                seasonality=get_seasonality(dataset.freq),
+            )
+        print(
+            f"{dataset_id}: "
+            f"MASE={result.iloc[0]['MASE[0.5]']:.4f}, "
+            f"CRPS={result.iloc[0]['mean_weighted_sum_quantile_loss']:.4f}"
+        )
+        row = pd.DataFrame(
+            [
+                {
+                    "dataset": dataset_id,
+                    "model": f"[SDM] {config.name}",
+                    **{
+                        f"eval_metrics/{k}": v
+                        for k, v in result.iloc[0].items()
+                    },
+                    "domain": properties[key]["domain"],
+                    "num_variates": properties[key]["num_variates"],
+                }
+            ]
+        )
+        row.to_csv(path, mode="a", header=not path.exists(), index=False)
+        completed.add(dataset_id)