Source code for quantdata.utils.pl

"""
Polars Utilities
================

"""

from __future__ import annotations

from collections.abc import Mapping
from typing import Any

import polars as pl

from cdxcore.err import verify_inp
from cdxcore.util import fmt_list

[docs] def create_empty_df(schema: Mapping[str, Any]) -> pl.DataFrame: """ Creates an empty :class:`polars.DataFrame` from a schema. A schema is a dictionary of the form:: { "col_a": pl.Float32, "col_b": pl.Int64, } """ assert isinstance(schema, Mapping), "schema must be a mapping of column name -> dtype" return pl.DataFrame(schema=dict(schema))
def _as_polars_dtype(dtype: Any, *, col: str) -> pl.DataType: try: return pl.Series(col, [], dtype=dtype).dtype except (TypeError, ValueError) as exc: verify_inp(False, f"Invalid dtype for column {col!r}: {dtype!r}. Error: {exc}")
[docs] def verify_schema( df: pl.DataFrame, schema: Mapping[str, Any], header: str | callable[[], str] | None = None, *, allow_extra_columns: bool = False, ) -> pl.DataFrame: """ Validate a :class:`polars.DataFrame` against a schema of the form:: { "col_a": pl.Float32, "col_b": pl.Int64, } Checks: - all schema columns exist - no extra columns, unless allow_extra_columns=True - exact dtype match Returns df unchanged, so it can be used inline. Raises :class:`ValueError` if validation fails. The error message will be preceeded by ``header`` which can either be a string or a callable returning a string (to avoid expensive message construction if validation passes). Parameters ---------- df : :class:`polars.DataFrame` Data frame to validate. schema : Mapping[str, Any] Mapping of column name to expected dtype. header : str|callable[[],str]| None, optional Header for error messages. Can be a string or a callable returning a string. Using a callable delays string composition to avoid unnecessary overhead when validation passes. allow_extra_columns : bool, default False Whether to allow columns in the DataFrame that are not in the schema. """ def real_header(): if header is None: return "" elif isinstance(header, str): return header + ": " else: return header() + ": " assert isinstance(df, pl.DataFrame), f"{real_header()} must be a polars DataFrame" assert isinstance(schema, Mapping), f"{real_header()}schema must be a mapping of column name -> dtype" expected_cols = set(schema.keys()) actual_cols = set(df.columns) missing = sorted(expected_cols - actual_cols) extra = sorted(actual_cols - expected_cols) if len(missing) > 0: if len(extra) > 0: raise ValueError(f"{real_header()}is missing columns: {fmt_list(missing)} and has extra columns: {fmt_list(extra)}.") raise ValueError(f"{real_header()}is missing columns {fmt_list(missing)}.") if len(extra) > 0 and not allow_extra_columns: raise ValueError(f"{real_header()}has extra columns {fmt_list(extra)}.") dtype_errors: list[str] = [] for col, expected_dtype_raw in schema.items(): if col not in df.columns: continue expected_dtype = _as_polars_dtype(expected_dtype_raw, col=col) actual_dtype = df.schema[col] if actual_dtype != expected_dtype: dtype_errors.append( f"{col!r}: expected {expected_dtype!r}, got {actual_dtype!r}" ) if dtype_errors: raise ValueError(f"{real_header()} has schema mismatches: {fmt_list(dtype_errors)}.") return df