"""Validated centered PCA of log implied-volatility surface levels."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
import numpy as np
from cdxcore.err import verify_inp
from cdxcore.version import version
[docs]
@dataclass(frozen=True, kw_only=True)
class ValidatedPCAMarket:
"""Arrays required to fit and simulate the direct PCA benchmark."""
dates: np.ndarray
time_y: np.ndarray
mid_vols: np.ndarray
nms: np.ndarray
expiries: np.ndarray
xstrikes: np.ndarray
spot_ref: np.ndarray
spot_ret: np.ndarray
@property
def shape(self) -> tuple[int, int, int]:
return self.mid_vols.shape
def _floating_array(value: Any, *, name: str) -> np.ndarray:
output = np.asarray(value)
verify_inp(
np.issubdtype(output.dtype, np.floating),
lambda: f"market field {name!r} must have floating dtype, got {output.dtype}",
)
verify_inp(
np.all(np.isfinite(output)),
lambda: f"market field {name!r} contains non-finite values",
)
return output.astype(np.float64, copy=False)
[docs]
@version("0.1")
def validate_pca_market(market: Mapping[str, Any]) -> ValidatedPCAMarket:
"""Validate the exact historical-grid inputs used by the PCA benchmark."""
required = {
"dates",
"time_y",
"mid_vols",
"nms",
"expiries",
"xstrikes",
"spot_ref",
"spot_ret",
}
missing = required.difference(market)
verify_inp(not missing, lambda: f"PCA market is missing fields {sorted(missing)}")
dates = np.asarray(market["dates"])
time_y = _floating_array(market["time_y"], name="time_y")
mid_vols = _floating_array(market["mid_vols"], name="mid_vols")
nms = _floating_array(market["nms"], name="nms")
expiries = _floating_array(market["expiries"], name="expiries")
xstrikes = _floating_array(market["xstrikes"], name="xstrikes")
spot_ref = _floating_array(market["spot_ref"], name="spot_ref")
spot_ret = _floating_array(market["spot_ret"], name="spot_ret")
verify_inp(
mid_vols.ndim == 3 and min(mid_vols.shape) > 0,
lambda: f"'mid_vols' must have shape (D,T,N), got {mid_vols.shape}",
)
D, T, N = mid_vols.shape
expected = {
"dates": (D,),
"time_y": (D,),
"nms": (D, T, N),
"expiries": (D, T),
"xstrikes": (D, T, N + 2),
"spot_ref": (D,),
"spot_ret": (D,),
}
actual = {
"dates": dates.shape,
"time_y": time_y.shape,
"nms": nms.shape,
"expiries": expiries.shape,
"xstrikes": xstrikes.shape,
"spot_ref": spot_ref.shape,
"spot_ret": spot_ret.shape,
}
for name, shape in expected.items():
verify_inp(
actual[name] == shape,
lambda name=name, shape=shape: (
f"market field {name!r} must have shape {shape}, got {actual[name]}"
),
)
verify_inp(
np.all(mid_vols > 0.0),
lambda: (
"PCA input 'mid_vols' must be strictly positive; minimum is "
f"{float(np.min(mid_vols)):g}"
),
)
verify_inp(
np.all(spot_ref > 0.0),
lambda: f"cash spots must be strictly positive, minimum {np.min(spot_ref):g}",
)
verify_inp(
np.all(np.diff(time_y) > 0.0),
lambda: f"'time_y' must increase strictly, got {time_y}",
)
verify_inp(
np.all(expiries > 0.0) and np.all(np.diff(expiries, axis=1) > 0.0),
"expiries must be strictly positive and increasing within every date",
)
verify_inp(
np.all(np.diff(nms, axis=2) > 0.0),
"normalized-moneyness coordinates must increase within every expiry",
)
verify_inp(
np.all(np.diff(xstrikes, axis=2) > 0.0) and np.all(xstrikes > 0.0),
"extended relative-strike grids must be positive and increasing",
)
# The decoder has one fixed (T,z) grid. The data route should produce this
# exactly, but tolerate insignificant serialization roundoff.
grid_tolerance = 64.0 * np.finfo(np.float64).eps
verify_inp(
np.allclose(nms, nms[:1], rtol=grid_tolerance, atol=grid_tolerance),
"the PCA benchmark requires one common NMS grid across dates",
)
verify_inp(
np.allclose(
expiries,
expiries[:1],
rtol=grid_tolerance,
atol=grid_tolerance,
),
"the PCA benchmark requires one common expiry grid across dates",
)
return ValidatedPCAMarket(
dates=dates,
time_y=time_y,
mid_vols=mid_vols,
nms=nms,
expiries=expiries,
xstrikes=xstrikes,
spot_ref=spot_ref,
spot_ret=spot_ret,
)
[docs]
@version("0.1")
@dataclass(frozen=True, kw_only=True)
class LogIVPCA:
"""Centered, unweighted PCA representation of log-IV surface levels."""
mean_log_iv: np.ndarray
components: np.ndarray
explained_variance: np.ndarray
explained_variance_ratio: np.ndarray
cumulative_explained_variance_ratio: np.ndarray
expiries: np.ndarray
nms: np.ndarray
def __post_init__(self) -> None:
verify_inp(
self.mean_log_iv.ndim == 1,
lambda: f"mean log-IV must be one-dimensional, got {self.mean_log_iv.shape}",
)
H, P = self.components.shape
verify_inp(
self.mean_log_iv.shape == (P,),
lambda: (
f"component width {P} differs from mean width "
f"{self.mean_log_iv.shape}"
),
)
verify_inp(
self.expiries.ndim == 1
and self.nms.ndim == 2
and self.nms.shape[0] == self.expiries.size
and self.nms.size == P,
lambda: (
"expiry/NMS grid does not match PCA width: "
f"{self.expiries.shape}/{self.nms.shape}/{P}"
),
)
verify_inp(
self.explained_variance.shape
== self.explained_variance_ratio.shape
== self.cumulative_explained_variance_ratio.shape
== (H,),
"PCA explained-variance arrays must contain one value per component",
)
for name, value in (
("mean_log_iv", self.mean_log_iv),
("components", self.components),
("explained_variance", self.explained_variance),
("explained_variance_ratio", self.explained_variance_ratio),
(
"cumulative_explained_variance_ratio",
self.cumulative_explained_variance_ratio,
),
("expiries", self.expiries),
("nms", self.nms),
):
verify_inp(
np.issubdtype(value.dtype, np.floating)
and np.all(np.isfinite(value)),
lambda name=name: f"PCA field {name!r} must be finite floating data",
)
verify_inp(
np.all(self.expiries > 0.0) and np.all(np.diff(self.expiries) > 0.0),
"PCA expiries must be positive and increasing",
)
verify_inp(
np.all(np.diff(self.nms, axis=1) > 0.0),
"PCA NMS coordinates must increase within every expiry",
)
identity = self.components @ self.components.T
verify_inp(
np.allclose(identity, np.eye(H), atol=1.0e-10, rtol=1.0e-10),
lambda: (
"PCA component rows are not orthonormal; maximum error "
f"{float(np.max(np.abs(identity - np.eye(H)))):g}"
),
)
@property
def num_components(self) -> int:
return self.components.shape[0]
@property
def surface_shape(self) -> tuple[int, int]:
return self.nms.shape
[docs]
def encode(self, implied_vols: np.ndarray) -> np.ndarray:
"""Encode positive IV surfaces whose trailing axes are ``(T,N)``."""
implied_vols = np.asarray(implied_vols)
verify_inp(
implied_vols.shape[-2:] == self.surface_shape,
lambda: (
f"IV surfaces must end in {self.surface_shape}, got "
f"{implied_vols.shape}"
),
)
verify_inp(
np.issubdtype(implied_vols.dtype, np.floating)
and np.all(np.isfinite(implied_vols))
and np.all(implied_vols > 0.0),
"encoded implied volatilities must be finite, floating, and positive",
)
flat = np.log(implied_vols).reshape(*implied_vols.shape[:-2], -1)
return (flat - self.mean_log_iv) @ self.components.T
[docs]
def decode_log_iv(self, scores: np.ndarray) -> np.ndarray:
"""Decode PCA scores to log-IV surfaces on the training grid."""
scores = np.asarray(scores)
verify_inp(
scores.ndim >= 1 and scores.shape[-1] == self.num_components,
lambda: (
f"scores must end in {self.num_components}, got {scores.shape}"
),
)
verify_inp(
np.issubdtype(scores.dtype, np.floating) and np.all(np.isfinite(scores)),
"PCA scores must be finite floating data",
)
output = self.mean_log_iv + scores @ self.components
return output.reshape(*scores.shape[:-1], *self.surface_shape)
[docs]
def decode(self, scores: np.ndarray) -> np.ndarray:
"""Decode PCA scores to strictly positive IV grid surfaces."""
output = np.exp(self.decode_log_iv(scores))
if not np.all(np.isfinite(output)) or not np.all(output > 0.0):
raise FloatingPointError("decoded PCA implied volatility is non-finite")
return output
[docs]
def to_state_dict(self) -> dict[str, np.ndarray]:
"""Return a cache-safe array tree."""
return {
"mean_log_iv": self.mean_log_iv,
"components": self.components,
"explained_variance": self.explained_variance,
"explained_variance_ratio": self.explained_variance_ratio,
"cumulative_explained_variance_ratio": (
self.cumulative_explained_variance_ratio
),
"expiries": self.expiries,
"nms": self.nms,
}
[docs]
@classmethod
def from_state_dict(cls, state: Mapping[str, np.ndarray]) -> LogIVPCA:
return cls(**{name: np.asarray(value) for name, value in state.items()})
[docs]
@dataclass(frozen=True, kw_only=True)
class PCAFitDiagnostics:
"""Reconstruction and eigensurface diagnostics for one PCA fit."""
scores: np.ndarray
reconstruction: dict[str, Any]
errors_by_expiry: list[dict[str, float]]
errors_by_nms_region: dict[str, dict[str, float | int]]
first_five_iv_eigensurfaces: np.ndarray
[docs]
def report(self, model: LogIVPCA) -> dict[str, Any]:
return {
"explained_variance": model.explained_variance.tolist(),
"explained_variance_ratio": model.explained_variance_ratio.tolist(),
"cumulative_explained_variance_ratio": (
model.cumulative_explained_variance_ratio.tolist()
),
"reconstruction": self.reconstruction,
"errors_by_expiry": self.errors_by_expiry,
"errors_by_nms_region": self.errors_by_nms_region,
"first_five_iv_eigensurfaces": (
self.first_five_iv_eigensurfaces.tolist()
),
"eigensurface_definition": (
"one-score-standard-deviation change from the mean log-IV surface"
),
}
def _deterministic_component_signs(
components: np.ndarray,
scores: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
"""Make the largest-magnitude loading of every component positive."""
components = components.copy()
scores = scores.copy()
pivots = np.argmax(np.abs(components), axis=1)
signs = np.sign(components[np.arange(components.shape[0]), pivots])
signs[signs == 0.0] = 1.0
components *= signs[:, None]
scores *= signs[None, :]
return components, scores
def _error_summary(error: np.ndarray) -> dict[str, float]:
return {
"rmse": float(np.sqrt(np.mean(np.square(error)))),
"maximum_absolute_error": float(np.max(np.abs(error))),
}
[docs]
@version("0.1", dependencies=[LogIVPCA])
def fit_log_iv_pca(
implied_vols: np.ndarray,
*,
expiries: np.ndarray,
nms: np.ndarray,
num_components: int = 20,
) -> tuple[LogIVPCA, PCAFitDiagnostics]:
"""Fit ordinary centered, unweighted PCA to log-IV surface levels."""
implied_vols = np.asarray(implied_vols)
expiries = np.asarray(expiries)
nms = np.asarray(nms)
verify_inp(
implied_vols.ndim == 3 and min(implied_vols.shape) > 0,
lambda: f"implied volatilities must have shape (D,T,N), got {implied_vols.shape}",
)
D, T, N = implied_vols.shape
verify_inp(
expiries.shape == (T,) and nms.shape == (T, N),
lambda: (
f"expiry/NMS geometry must be {(T,)}/{(T, N)}, got "
f"{expiries.shape}/{nms.shape}"
),
)
verify_inp(
np.issubdtype(implied_vols.dtype, np.floating)
and np.all(np.isfinite(implied_vols))
and np.all(implied_vols > 0.0),
"PCA input implied volatilities must be finite, floating, and positive",
)
maximum_components = min(D, T * N)
verify_inp(
isinstance(num_components, int) and 0 < num_components <= maximum_components,
lambda: (
f"'num_components' must be in [1,{maximum_components}], got "
f"{num_components!r}"
),
)
log_iv = np.log(implied_vols.astype(np.float64, copy=False)).reshape(D, T * N)
mean_log_iv = np.mean(log_iv, axis=0)
centered = log_iv - mean_log_iv
left, singular_values, components = np.linalg.svd(centered, full_matrices=False)
scores = left[:, :num_components] * singular_values[:num_components]
components = components[:num_components]
components, scores = _deterministic_component_signs(components, scores)
denominator = max(D - 1, 1)
all_explained_variance = np.square(singular_values) / denominator
total_variance = float(np.sum(all_explained_variance))
verify_inp(total_variance > 0.0, "log-IV surface levels have zero total variance")
explained_variance = all_explained_variance[:num_components]
explained_ratio = explained_variance / total_variance
model = LogIVPCA(
mean_log_iv=mean_log_iv,
components=components,
explained_variance=explained_variance,
explained_variance_ratio=explained_ratio,
cumulative_explained_variance_ratio=np.cumsum(explained_ratio),
expiries=np.asarray(expiries, dtype=np.float64),
nms=np.asarray(nms, dtype=np.float64),
)
reconstructed_log = model.decode_log_iv(scores)
reconstructed_iv = np.exp(reconstructed_log)
log_error = reconstructed_log - log_iv.reshape(D, T, N)
iv_error = reconstructed_iv - implied_vols
errors_by_expiry = [
{
"expiry": float(expiries[index]),
**{f"iv_{name}": value for name, value in _error_summary(iv_error[:, index]).items()},
**{
f"log_iv_{name}": value
for name, value in _error_summary(log_error[:, index]).items()
},
}
for index in range(T)
]
region_masks = {
"left_wing_z_lt_minus_0.5": nms < -0.5,
"center_abs_z_le_0.5": np.abs(nms) <= 0.5,
"right_wing_z_gt_0.5": nms > 0.5,
}
region_errors: dict[str, dict[str, float | int]] = {}
for name, mask in region_masks.items():
if not np.any(mask):
region_errors[name] = {"grid_points": 0, "iv_rmse": float("nan"), "log_iv_rmse": float("nan")}
continue
region_errors[name] = {
"grid_points": int(np.count_nonzero(mask)),
"iv_rmse": float(np.sqrt(np.mean(np.square(iv_error[:, mask])))),
"iv_maximum_absolute_error": float(np.max(np.abs(iv_error[:, mask]))),
"log_iv_rmse": float(np.sqrt(np.mean(np.square(log_error[:, mask])))),
"log_iv_maximum_absolute_error": float(np.max(np.abs(log_error[:, mask]))),
}
component_count = min(5, num_components)
mean_iv = np.exp(mean_log_iv.reshape(T, N))
shifted_log = (
mean_log_iv[None, :]
+ np.sqrt(explained_variance[:component_count])[:, None]
* components[:component_count]
)
eigensurfaces = np.exp(shifted_log).reshape(component_count, T, N) - mean_iv
diagnostics = PCAFitDiagnostics(
scores=scores,
reconstruction={
**{f"iv_{name}": value for name, value in _error_summary(iv_error).items()},
**{
f"log_iv_{name}": value
for name, value in _error_summary(log_error).items()
},
},
errors_by_expiry=errors_by_expiry,
errors_by_nms_region=region_errors,
first_five_iv_eigensurfaces=eigensurfaces,
)
return model, diagnostics
__all__ = [
"LogIVPCA",
"PCAFitDiagnostics",
"ValidatedPCAMarket",
"fit_log_iv_pca",
"validate_pca_market",
]