Data Dataset Io

dataset_io

FUNCTION	DESCRIPTION
save_dataset	A utility function to save [DatasetWeighted][spectre.core.DatasetWeighted]
load_dataset	Load DatasetWeighted from text file.

Classes

Functions

save_dataset(filename: str, dataset: DatasetWeighted, fmt: str = '%.6f', header: bool = True, header_prefix: str | None = None, delimiter: str = ' ', validate_path: bool = True) -> None

A utility function to save [DatasetWeighted][spectre.core.DatasetWeighted] to text file.

The file will contain data columns, followed by weight columns, and optionally target columns. Column headers can be included.

PARAMETER	DESCRIPTION
filename	Output file path. TYPE: str
dataset	Dataset to save. TYPE: DatasetWeighted
fmt	Numerical format string. TYPE: str, optional, by default "%.6f" DEFAULT: '%.6f'
header	Include column headers. TYPE: bool, optional, by default True DEFAULT: True
header_prefix	Header prefix string. TYPE: str | None, optional, by default None DEFAULT: None
delimiter	Column delimiter. TYPE: str, optional, by default " " DEFAULT: ' '
validate_path	Validate output path before writing. TYPE: bool, optional, by default True DEFAULT: True

Examples:

Basic usage:

>>> dataset = DatasetWeighted(data, weights)
>>> save_dataset("output.txt", dataset, header=True)

Examples with target:

>>> dataset_with_target = DatasetWeighted(data, weights, target)
>>> save_dataset("output_with_target.txt", dataset_with_target, header=True)

Source code in spectre/data/dataset_io.py

def save_dataset(
    filename: str,
    dataset: DatasetWeighted,
    fmt: str = "%.6f",
    header: bool = True,
    header_prefix: str | None = None,
    delimiter: str = " ",
    validate_path: bool = True,
) -> None:
    """
    A utility function to save [DatasetWeighted][spectre.core.DatasetWeighted]
    to text file.

    The file will contain data columns, followed by weight columns, and optionally
    target columns. Column headers can be included.

    Parameters
    ----------
    filename : str
        Output file path.

    dataset : DatasetWeighted
        Dataset to save.

    fmt : str, optional, by default "%.6f"
        Numerical format string.

    header : bool, optional, by default True
        Include column headers.

    header_prefix : str | None, optional, by default None
        Header prefix string.

    delimiter : str, optional, by default " "
        Column delimiter.

    validate_path : bool, optional, by default True
        Validate output path before writing.

    Examples
    --------
    Basic usage:

    >>> dataset = DatasetWeighted(data, weights)
    >>> save_dataset("output.txt", dataset, header=True)

    Examples with target:

    >>> dataset_with_target = DatasetWeighted(data, weights, target)
    >>> save_dataset("output_with_target.txt", dataset_with_target, header=True)
    """
    _validate_save_inputs(filename, dataset, header, header_prefix, validate_path)

    data_tensor = _prepare_data_tensor(dataset)

    if header:
        header_str = _generate_header_string(dataset, header, header_prefix, delimiter)
    else:
        header_str = None

    _write_tensor_to_file(filename, data_tensor, fmt, header_str, delimiter)

load_dataset(filename: str, n_data_cols: int, n_weight_cols: int = 1, n_target_cols: int = 0, delimiter: str | None = None, skip_header: bool = False, dtype: torch.dtype = torch.float32) -> DatasetWeighted

Load DatasetWeighted from text file.

The file should contain data columns, followed by weight columns, and optionally target columns. The first row can be skipped as a header.

PARAMETER	DESCRIPTION
filename	Input file path. TYPE: str
n_data_cols	Number of data columns. TYPE: int
n_weight_cols	Number of weight columns. TYPE: int, optional, by default 1 DEFAULT: 1
n_target_cols	Number of target columns. TYPE: int, optional, by default 0 DEFAULT: 0
delimiter	Column delimiter, if None, any whitespace is used. TYPE: str | None, optional, by default None DEFAULT: None
skip_header	Skip first row as header. TYPE: bool, optional, by default False DEFAULT: False
dtype	Data type for tensors. TYPE: torch.dtype, optional, by default torch.float32 DEFAULT: float32

RETURNS	DESCRIPTION
DatasetWeighted	Loaded dataset.

RAISES	DESCRIPTION
OSError	If file cannot be read.
ValueError	If file format is invalid or column counts do not match.

Examples:

1D weights and optional target:

>>> # Assuming the number of samples is 100 in all examples...
>>> dataset = load_dataset("data.txt", n_data_cols=3, n_weight_cols=1)
>>> print(dataset.data.shape, dataset.weights.shape)
(100, 3) (100,)

Examples with target columns:

>>> dataset = load_dataset(
...     "data_with_target.txt",
...     n_data_cols=3,
...     n_weight_cols=1,
...     n_target_cols=1,
... )
>>> print(
...     dataset.data.shape,
...     dataset.weights.shape,
...     dataset.target.shape,
... )
(100, 3) (100,) (100,)

Multi-dimensional weights and targets:

>>>  dataset = load_dataset(
...     "data_with_multi_weights.txt", n_data_cols=3, n_weight_cols=2
... )
>>> print(dataset.data.shape, dataset.weights.shape)
(100, 3) (100, 2)
>>> dataset = load_dataset(
...     "data_multi_target.txt",
...     n_data_cols=3,
...     n_weight_cols=1,
...     n_target_cols=2,
... )
>>> print(
...     dataset.data.shape,
...     dataset.weights.shape,
...     dataset.target.shape,
... )
(100, 3) (100,) (100, 2)

Source code in spectre/data/dataset_io.py

def load_dataset(
    filename: str,
    n_data_cols: int,
    n_weight_cols: int = 1,
    n_target_cols: int = 0,
    delimiter: str | None = None,
    skip_header: bool = False,
    dtype: torch.dtype = torch.float32,
) -> DatasetWeighted:
    """
    Load DatasetWeighted from text file.

    The file should contain data columns, followed by weight columns,
    and optionally target columns. The first row can be skipped as a header.

    Parameters
    ----------
    filename : str
        Input file path.

    n_data_cols : int
        Number of data columns.

    n_weight_cols : int, optional, by default 1
        Number of weight columns.

    n_target_cols : int, optional, by default 0
        Number of target columns.

    delimiter : str | None, optional, by default None
        Column delimiter, if None, any whitespace is used.

    skip_header : bool, optional, by default False
        Skip first row as header.

    dtype : torch.dtype, optional, by default torch.float32
        Data type for tensors.

    Returns
    -------
    DatasetWeighted
        Loaded dataset.

    Raises
    ------
    OSError
        If file cannot be read.

    ValueError
        If file format is invalid or column counts do not match.

    Examples
    --------
    1D weights and optional target:

    >>> # Assuming the number of samples is 100 in all examples...
    >>> dataset = load_dataset("data.txt", n_data_cols=3, n_weight_cols=1)
    >>> print(dataset.data.shape, dataset.weights.shape)
    (100, 3) (100,)

    Examples with target columns:

    >>> dataset = load_dataset(
    ...     "data_with_target.txt",
    ...     n_data_cols=3,
    ...     n_weight_cols=1,
    ...     n_target_cols=1,
    ... )
    >>> print(
    ...     dataset.data.shape,
    ...     dataset.weights.shape,
    ...     dataset.target.shape,
    ... )
    (100, 3) (100,) (100,)

    Multi-dimensional weights and targets:

    >>>  dataset = load_dataset(
    ...     "data_with_multi_weights.txt", n_data_cols=3, n_weight_cols=2
    ... )
    >>> print(dataset.data.shape, dataset.weights.shape)
    (100, 3) (100, 2)
    >>> dataset = load_dataset(
    ...     "data_multi_target.txt",
    ...     n_data_cols=3,
    ...     n_weight_cols=1,
    ...     n_target_cols=2,
    ... )
    >>> print(
    ...     dataset.data.shape,
    ...     dataset.weights.shape,
    ...     dataset.target.shape,
    ... )
    (100, 3) (100,) (100, 2)
    """
    try:
        # Load data from file
        data_numpy = np.loadtxt(
            filename,
            delimiter=delimiter,
            skiprows=1 if skip_header else 0,
            ndmin=2,
        )

    except (OSError, ValueError) as e:
        raise OSError(f"Failed to load file '{filename}': {e}") from e

    # Validate column count
    expected_cols = n_data_cols + n_weight_cols + n_target_cols
    if data_numpy.shape[1] != expected_cols:
        raise ValueError(
            f"Expected {expected_cols} columns but found {data_numpy.shape[1]} "
            f"in file '{filename}'."
        )

    # Convert to tensors and split columns
    data_tensor = torch.from_numpy(data_numpy).to(dtype)

    # Extract data, weights, and targets
    col_idx = 0

    # Data columns
    data = data_tensor[:, col_idx : col_idx + n_data_cols]
    col_idx += n_data_cols

    # Weight columns
    weights = data_tensor[:, col_idx : col_idx + n_weight_cols]
    if n_weight_cols == 1:
        weights = weights.squeeze(1)
    col_idx += n_weight_cols

    # Target columns (if present)
    target = None
    if n_target_cols > 0:
        target = data_tensor[:, col_idx : col_idx + n_target_cols]
        if n_target_cols == 1:
            target = target.squeeze(1)

    return DatasetWeighted(X=data, weights=weights, target=target)

get_file_info(filename: str) -> dict[str, Any]

Get information about a data file.

PARAMETER	DESCRIPTION
filename	File path to analyze. TYPE: str

RETURNS	DESCRIPTION
dict[str, Any]	File information including shape, size, and sample content.

Examples:

>>> info = get_file_info("data.txt")
>>> print(f"Shape: {info['shape']}, Size: {info['size_mb']:.2f} MB")

Source code in spectre/data/dataset_io.py

def get_file_info(filename: str) -> dict[str, Any]:
    """
    Get information about a data file.

    Parameters
    ----------
    filename : str
        File path to analyze.

    Returns
    -------
    dict[str, Any]
        File information including shape, size, and sample content.

    Examples
    --------
    >>> info = get_file_info("data.txt")
    >>> print(f"Shape: {info['shape']}, Size: {info['size_mb']:.2f} MB")
    """
    try:
        path = Path(filename)
        if not path.exists():
            raise FileNotFoundError(f"File '{filename}' not found.")

        # Get file size
        size_bytes = path.stat().st_size
        size_mb = size_bytes / (1024 * 1024)

        # Try to load first few rows to get shape info
        try:
            sample_data = np.loadtxt(filename, max_rows=5, ndmin=2)
            n_cols = sample_data.shape[1]

            # Get total number of rows (excluding potential header)
            with open(filename, "r") as f:
                n_rows = sum(1 for _ in f)

            # Check if first row might be header
            first_line = next(open(filename, "r")).strip()
            has_header = not _is_numeric_line(first_line)

            if has_header:
                n_rows -= 1

        except (ValueError, UnicodeDecodeError):
            n_rows, n_cols = None, None
            has_header = None

        return {
            "filename": str(path.absolute()),
            "size_bytes": size_bytes,
            "size_mb": size_mb,
            "shape": (n_rows, n_cols) if n_rows and n_cols else None,
            "has_header": has_header,
            "exists": True,
        }

    except Exception as e:
        return {
            "filename": filename,
            "error": str(e),
            "exists": False,
        }