Source code for quantdata.utils.pd

"""
Panda Utilities
===============
"""

from __future__ import annotations

import pandas as pd
from collections.abc import Mapping
from typing import Any
import numpy as np
from cdxcore.err import verify_inp
from cdxcore.util import fmt_list


[docs] def create_empty_df( schema ): """ Creates an empty :class:`pandas.DataFrame` from a schema. A schema is a dictionary of the form:: { "col_a": np.dtype("float32"), "col_b": np.dtype("int64"), } """ return pd.DataFrame({c: pd.Series(dtype=dtype) for c,dtype in schema.items()})
def _as_pandas_dtype(dtype: Any, *, col: str) -> Any: try: return pd.api.types.pandas_dtype(dtype) except (TypeError, ValueError) as exc: verify_inp(False, f"Invalid dtype for column {col!r}: {dtype!r}. Error: {exc}")
[docs] def verify_schema( df: pd.DataFrame, schema: Mapping[str, Any], header : str|callable[[],str]| None = None, *, allow_extra_columns: bool = False, ) -> pd.DataFrame: """ Validate a :class:`pandas.DataFrame` against a schema of the form:: { "col_a": np.dtype("float32"), "col_b": np.dtype("int64"), } Checks: - all schema columns exist - no extra columns, unless allow_extra_columns=True - exact dtype match Returns df unchanged, so it can be used inline. Raises :class:`ValueError` if validation fails. The error message will be preeded by ``header`` which can either be a string or a callable returning a string (to avoid expensive message construction if validation passes). Parameters ---------- df : :class:`pandas.DataFrame` Data frame to validate. schema : Mapping[str, Any] Mapping of column name to expected dtype. header : str|callable[[],str]| None, optional Header for error messages. Can be a string or a callable returning a string. Using a callable delays string composition to avoid unnecessary overhead when validation passes. allow_extra_columns : bool, default False Whether to allow columns in the DataFrame that are not in the schema. """ def real_header(): if header is None: return "" elif isinstance(header, str): return header + ": " else: return header() + ": " assert isinstance(df, pd.DataFrame), f"{real_header()} must be a pandas DataFrame" assert isinstance(schema, Mapping), f"{real_header()}schema must be a mapping of column name -> dtype" expected_cols = set(schema.keys()) actual_cols = set(df.columns) missing = sorted(expected_cols - actual_cols) extra = sorted(actual_cols - expected_cols) if len(missing) > 0: if len(extra) > 0: raise ValueError( f"{real_header()}is missing columns: {fmt_list(missing)} and has extra columns: {fmt_list(extra)}.") raise ValueError( f"{real_header()}is missing columns {fmt_list(missing)}.") if len(extra) > 0 and not allow_extra_columns: raise ValueError( f"{real_header()}has extra columns {fmt_list(extra)}.") dtype_errors: list[str] = [] for col, expected_dtype_raw in schema.items(): if col not in df.columns: continue expected_dtype = _as_pandas_dtype(expected_dtype_raw, col=col) actual_dtype = _as_pandas_dtype(df[col].dtype, col=col) if actual_dtype != expected_dtype: dtype_errors.append( f"{col!r}: expected {expected_dtype!r}, got {actual_dtype!r}" ) if dtype_errors: raise ValueError( f"{real_header()} has schema mismatches: {fmt_list(dtype_errors)}.") return df