Benchmark

Benchmark

A time series forecasting benchmark consisting of multiple Task objects.

Attributes:

Name	Type	Description
tasks	list[Task]	Collection of tasks in the benchmark.

Source code in src/fev/benchmark.py

class Benchmark:
    """A time series forecasting benchmark consisting of multiple [`Task`][fev.Task] objects.

    Attributes
    ----------
    tasks : list[Task]
        Collection of tasks in the benchmark.
    """

    def __init__(self, tasks: list[Task]):
        for t in tasks:
            if not isinstance(t, Task):
                raise ValueError(f"`tasks` must be a list of `Task` objects (got {type(t)})")
        self.tasks: list[Task] = tasks  # declare type explicitly to correctly show up in the docs

    @classmethod
    def from_yaml(cls, file_path: str | Path) -> "Benchmark":
        """Load benchmark definition from a YAML file.

        The YAML file should contain the key `'tasks'` with a list of dictionaries with task definitions.

            tasks:
            - dataset_path: autogluon/chronos_datasets
              dataset_config: m4_hourly
              horizon: 24
              num_windows: 2
            - dataset_path: autogluon/chronos_datasets
              dataset_config: monash_cif_2016
              horizon: 12

        Parameters
        ----------
        file_path : str | Path
            URL or path of a YAML file containing the task definitions.
        """
        try:
            if str(file_path).startswith(("http://", "https://")):
                response = requests.get(file_path)
                response.raise_for_status()
                config = yaml.safe_load(response.text)
            else:
                with open(file_path) as file:
                    config = yaml.safe_load(file)
        except Exception:
            raise ValueError("Failed to load the file")

        return cls.from_list(config["tasks"])

    @classmethod
    def from_list(cls, task_configs: list[dict]) -> "Benchmark":
        """Load benchmark definition from a list of dictionaries.

        Each dictionary must follow the schema compatible with a `fev.task.Task`.
        """
        return cls(tasks=[Task(**conf) for conf in task_configs])

Attributes

tasks: list[Task] = tasks instance-attribute

Functions

from_yaml(file_path: str | Path) -> Benchmark classmethod

Load benchmark definition from a YAML file.

The YAML file should contain the key 'tasks' with a list of dictionaries with task definitions.

tasks:
- dataset_path: autogluon/chronos_datasets
  dataset_config: m4_hourly
  horizon: 24
  num_windows: 2
- dataset_path: autogluon/chronos_datasets
  dataset_config: monash_cif_2016
  horizon: 12

Parameters:

Name	Type	Description	Default
file_path	str | Path	URL or path of a YAML file containing the task definitions.	required

Source code in src/fev/benchmark.py

@classmethod
def from_yaml(cls, file_path: str | Path) -> "Benchmark":
    """Load benchmark definition from a YAML file.

    The YAML file should contain the key `'tasks'` with a list of dictionaries with task definitions.

        tasks:
        - dataset_path: autogluon/chronos_datasets
          dataset_config: m4_hourly
          horizon: 24
          num_windows: 2
        - dataset_path: autogluon/chronos_datasets
          dataset_config: monash_cif_2016
          horizon: 12

    Parameters
    ----------
    file_path : str | Path
        URL or path of a YAML file containing the task definitions.
    """
    try:
        if str(file_path).startswith(("http://", "https://")):
            response = requests.get(file_path)
            response.raise_for_status()
            config = yaml.safe_load(response.text)
        else:
            with open(file_path) as file:
                config = yaml.safe_load(file)
    except Exception:
        raise ValueError("Failed to load the file")

    return cls.from_list(config["tasks"])

from_list(task_configs: list[dict]) -> Benchmark classmethod

Load benchmark definition from a list of dictionaries.

Each dictionary must follow the schema compatible with a fev.task.Task.

Source code in src/fev/benchmark.py

@classmethod
def from_list(cls, task_configs: list[dict]) -> "Benchmark":
    """Load benchmark definition from a list of dictionaries.

    Each dictionary must follow the schema compatible with a `fev.task.Task`.
    """
    return cls(tasks=[Task(**conf) for conf in task_configs])