Source code for quantdata.utils.pl
"""
Polars Utilities
================
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
import polars as pl
from cdxcore.err import verify_inp
from cdxcore.util import fmt_list
[docs]
def create_empty_df(schema: Mapping[str, Any]) -> pl.DataFrame:
"""
Creates an empty :class:`polars.DataFrame` from a schema.
A schema is a dictionary of the form::
{
"col_a": pl.Float32,
"col_b": pl.Int64,
}
"""
assert isinstance(schema, Mapping), "schema must be a mapping of column name -> dtype"
return pl.DataFrame(schema=dict(schema))
def _as_polars_dtype(dtype: Any, *, col: str) -> pl.DataType:
try:
return pl.Series(col, [], dtype=dtype).dtype
except (TypeError, ValueError) as exc:
verify_inp(False, f"Invalid dtype for column {col!r}: {dtype!r}. Error: {exc}")
[docs]
def verify_schema(
df: pl.DataFrame,
schema: Mapping[str, Any],
header: str | callable[[], str] | None = None,
*,
allow_extra_columns: bool = False,
) -> pl.DataFrame:
"""
Validate a :class:`polars.DataFrame` against a schema of the form::
{
"col_a": pl.Float32,
"col_b": pl.Int64,
}
Checks:
- all schema columns exist
- no extra columns, unless allow_extra_columns=True
- exact dtype match
Returns df unchanged, so it can be used inline.
Raises :class:`ValueError` if validation fails. The error message will be
preceeded by ``header`` which can either be a string or a callable returning
a string (to avoid expensive message construction if validation passes).
Parameters
----------
df : :class:`polars.DataFrame`
Data frame to validate.
schema : Mapping[str, Any]
Mapping of column name to expected dtype.
header : str|callable[[],str]| None, optional
Header for error messages. Can be a string or a callable returning a string. Using a callable delays string composition to avoid unnecessary overhead when validation passes.
allow_extra_columns : bool, default False
Whether to allow columns in the DataFrame that are not in the schema.
"""
def real_header():
if header is None:
return ""
elif isinstance(header, str):
return header + ": "
else:
return header() + ": "
assert isinstance(df, pl.DataFrame), f"{real_header()} must be a polars DataFrame"
assert isinstance(schema, Mapping), f"{real_header()}schema must be a mapping of column name -> dtype"
expected_cols = set(schema.keys())
actual_cols = set(df.columns)
missing = sorted(expected_cols - actual_cols)
extra = sorted(actual_cols - expected_cols)
if len(missing) > 0:
if len(extra) > 0:
raise ValueError(f"{real_header()}is missing columns: {fmt_list(missing)} and has extra columns: {fmt_list(extra)}.")
raise ValueError(f"{real_header()}is missing columns {fmt_list(missing)}.")
if len(extra) > 0 and not allow_extra_columns:
raise ValueError(f"{real_header()}has extra columns {fmt_list(extra)}.")
dtype_errors: list[str] = []
for col, expected_dtype_raw in schema.items():
if col not in df.columns:
continue
expected_dtype = _as_polars_dtype(expected_dtype_raw, col=col)
actual_dtype = df.schema[col]
if actual_dtype != expected_dtype:
dtype_errors.append(
f"{col!r}: expected {expected_dtype!r}, got {actual_dtype!r}"
)
if dtype_errors:
raise ValueError(f"{real_header()} has schema mismatches: {fmt_list(dtype_errors)}.")
return df