import math
import warnings
from dataclasses import dataclass
import numpy as np
import torch
from cdxcore.err import verify_inp
from cdxcore.pretty import PrettyObject
from cdxcore.uniquehash import unique_hash8
from cdxcore.verbose import Context
from cdxcore.version import version
from quantdata.biz_calendar import DAYS_PER_YEAR
from quantdata.dysanos.grid.torch_model import SanosModel
from quantdata.dysanos.grid.torch_pricer import SanosPricer
[docs]
@version("0.3")
@dataclass(frozen=True)
class ARSamplerConfig:
num_alpha : int|float|None = 0.95 # number of PCA factors as integer or float for % vriance cut off
# Retaining fewer residual factors is a low-rank diffusion approximation.
# Do not inflate those factors to restore discarded variance: that changes
# the fitted covariance and makes factor-count comparisons inconsistent.
rescale_var : bool = False
seed : int = 113123231
use_matrix_exp : bool = True # if True, use exact affine matrix-exponential AR transition
surface_first : bool = True # fit stationary h before adding log-spot s
def __str__(self) -> str:
return (
f"nalp={self.num_alpha},{'rescl' if self.rescale_var else 'nat'},"
f"{'mxp' if self.use_matrix_exp else 'eul'},"
f"{'sf' if self.surface_first else 'joint'},seed={self.seed}"
)
@property
def rng_current_state_uid(self) -> np.random.Generator:
""" Utility to mimic the same function in ARSampler, but for the config object. """
return unique_hash8( np.random.default_rng(seed=self.seed).bit_generator.state )
[docs]
@version("0.1")
@dataclass(frozen=True, kw_only=True)
class GaussianSpotImportanceConfig:
"""Path-level proposal which broadens selected future log-spot marginals.
A positive ``base_probability`` component retains the original Gaussian
path law. Every other mixture component changes one future log-spot ``s``
marginal to the same mean and a wider standard deviation. In the
surface-first model the complete surface path is retained and the scalar
conditional spot residual is bridged; the legacy joint model uses a full
Gaussian state bridge.
"""
spot_std_multiplier: float = 8.0
base_probability: float = 0.1
target_action_count: int = 3
def __post_init__(self) -> None:
verify_inp(
math.isfinite(self.spot_std_multiplier)
and self.spot_std_multiplier > 1.0,
lambda: (
"'spot_std_multiplier' must be finite and greater than one, "
f"got {self.spot_std_multiplier}"
),
)
verify_inp(
math.isfinite(self.base_probability)
and 0.0 < self.base_probability < 1.0,
lambda: (
"'base_probability' must lie strictly between zero and one, "
f"got {self.base_probability}"
),
)
verify_inp(
isinstance(self.target_action_count, int)
and self.target_action_count > 0,
lambda: (
"'target_action_count' must be a positive integer, got "
f"{self.target_action_count!r}"
),
)
[docs]
@dataclass(frozen=True, kw_only=True)
class GaussianSpotImportanceSample:
"""One exact Gaussian importance sample and direct-P control moments."""
paths: np.ndarray
log_weights: np.ndarray
proposal_components: np.ndarray
target_steps: tuple[int, ...]
target_log_spot_std: np.ndarray
reference_log_spot_sum: np.ndarray
reference_log_spot_square_sum: np.ndarray
reference_realized_variance_sum: float
reference_realized_variance_square_sum: float
[docs]
@dataclass(frozen=True)
class AR1FitQuality:
"""One-step deterministic-drift fit diagnostics."""
r2 : float
r2_by_state : np.ndarray
rmse : float
rmse_by_state : np.ndarray
sample_mean : np.ndarray
sample_std : np.ndarray
fitted_mean : np.ndarray
mean_error : np.ndarray
[docs]
def summary_statistics(self) -> dict:
"""Return scale-independent mean-fit statistics and log-spot details."""
mean_error_in_std = np.divide(
self.mean_error,
self.sample_std,
out=np.full_like(self.mean_error, np.nan, dtype=float),
where=self.sample_std > 0.,
)
finite = np.isfinite(mean_error_in_std)
return {
"mean_error_in_std": mean_error_in_std,
"mean_error_rms_in_std": (
float(np.sqrt(np.mean(mean_error_in_std[finite] ** 2)))
if np.any(finite) else math.nan
),
"mean_error_max_abs_in_std": (
float(np.max(np.abs(mean_error_in_std[finite])))
if np.any(finite) else math.nan
),
"log_spot_sample_mean": float(self.sample_mean[0]),
"log_spot_fitted_mean": float(self.fitted_mean[0]),
"log_spot_sample_std": float(self.sample_std[0]),
"log_spot_mean_error_in_std": float(mean_error_in_std[0]),
}
def __str__(self) -> str:
stats = self.summary_statistics()
if math.isfinite(stats["log_spot_fitted_mean"]):
spot_fit = (
f"fitted={stats['log_spot_fitted_mean']:.4g}, "
f"error={stats['log_spot_mean_error_in_std']:.3g} std"
)
else:
spot_fit = "fitted mean=N/A (unit root)"
return (
f"one-step R²={self.r2:.1%}, RMSE={self.rmse:.4g}; "
f"mean fit RMS={stats['mean_error_rms_in_std']:.3g} std, "
f"max={stats['mean_error_max_abs_in_std']:.3g} std; "
f"log-spot mean: sample={stats['log_spot_sample_mean']:.4g}, "
f"sample std={stats['log_spot_sample_std']:.4g}, "
f"{spot_fit}"
)
[docs]
@dataclass(frozen=True)
class AR1PCA:
r"""
dh_t = (U0 - k h_t)dt + \alpha' dU \sqrt{dt}
"""
U0 : np.ndarray
alpha : np.ndarray
dU : np.ndarray
k : np.ndarray
ndt : np.ndarray # dt/mean(dt)
mean_dt: float # mean(dt)
state_center: np.ndarray|None # coordinate center used for stable simulation
state_scale : np.ndarray|None # coordinate scale used for stable simulation
# explained
vars_explained : np.ndarray
var_explained : float
fit_quality : AR1FitQuality|None = None
surface_first : bool = False
num_surface_alpha: int|None = None
surface_var_explained: float|None = None
spot_surface_r2: float|None = None
spot_residual_std: float|None = None
[docs]
@version(
"1.1",
dependencies=[
ARSamplerConfig,
GaussianSpotImportanceConfig,
SanosPricer,
SanosModel,
],
)
class ARSampler(object):
Config : ARSamplerConfig
def __init__(self, market,
model,
torch_data=None, *,
surface_states: np.ndarray | None = None,
config : ARSamplerConfig = ARSamplerConfig(),
verbose : Context = Context.all ):
self.market = market
self.torch_data = torch_data
self.model = model
self.config = config
verify_inp(
(torch_data is None) != (surface_states is None),
"provide exactly one of 'torch_data' and 'surface_states'",
)
if surface_states is None:
torch_output = model(torch_data, training=False)
self.pricer = SanosPricer.create(
smoothness=model.smoothness,
model_output=torch_output,
debug=model.debug,
)
self.model_output = {
name: value.detach().cpu().numpy()
for name, value in torch_output.items()
}
else:
state = np.asarray(surface_states)
verify_inp(
state.shape == (len(market.dates), model.config.num_state)
and np.issubdtype(state.dtype, np.floating)
and np.all(np.isfinite(state)),
lambda: (
"'surface_states' must be a finite floating array with shape "
f"{(len(market.dates), model.config.num_state)}, got "
f"{state.shape}/{state.dtype}"
),
)
# Path generation needs only the fitted surface coordinates. The
# full training output/pricer would retain multi-GB objective
# tensors which are unrelated to the AR fit.
self.pricer = None
self.model_output = {"state": state}
self.rng = np.random.default_rng(seed=config.seed)
spot_ref = self.market['spot_ref']
self.log_nspot = np.log( spot_ref / spot_ref[0] )
self.xstate = np.concatenate( [ self.log_nspot[:,None], self.model_output['state'] ], axis=1 ) # (T, num_state+1)
self.expiries = self.market['expiries']
self.time_y = self.market['time_y']
self.spot_ref = self.xstate[:,0]
T = len(market.dates)
assert self.xstate.shape == (T, self.model.config.num_state+1), f"Internal error {self.xstate.shape} != {(T, self.model.config.num_state+1)}"
assert self.spot_ref.shape == (T,), f"Internal error {self.spot_ref.shape} != {(T,)}"
# PCA XAR1
# ---------
# dX_t = ( U0 - k X_t ) dt + ( alpha + noise ) @ Y sqrt(nt)
num_xstate = self.model.config.num_state+1
fit_function = (
fit_surface_first_xar1_pca
if config.surface_first
else fit_xar1_pca
)
self.pcaxar1: AR1PCA = fit_function(
self.xstate,
self.time_y,
num_alpha=config.num_alpha,
rescale_var=config.rescale_var,
use_matrix_exp=config.use_matrix_exp,
)
num_alpha = self.pcaxar1.alpha.shape[1]
assert self.pcaxar1.U0.shape == (1,num_xstate), f"Internal error u0 {self.pcaxar1.U0.shape} != {(1,num_xstate)}"
assert self.pcaxar1.dU.shape == (num_alpha,num_xstate), f"Internal error du {self.pcaxar1.dU.shape} != {(num_alpha,num_xstate)}"
assert self.pcaxar1.k.shape == (num_xstate,num_xstate), f"Internal error k {self.pcaxar1.k.shape} != {(num_xstate,num_xstate)}"
assert self.pcaxar1.alpha.shape == (T-1,num_alpha), f"Internal error alpha {self.pcaxar1.alpha.shape} != {(T-1,num_alpha)}"
assert self.pcaxar1.ndt.shape == (T-1,), f"Internal error ndt {self.pcaxar1.ndt.shape} != {(T-1,)}"
assert self.pcaxar1.vars_explained.shape == (num_xstate,), f"Internal error vars_explained s{self.pcaxar1.vars_explained.shape} != {(num_xstate,)}"
eig_k = np.linalg.eigvals(self.pcaxar1.k)
if self.pcaxar1.surface_first:
eig_surface = np.linalg.eigvals(self.pcaxar1.k[1:, 1:])
tolerance = 100.0 * np.finfo(float).eps
assert (
np.all(np.isfinite(eig_k))
and np.all(eig_surface.real > 0.0)
and np.min(np.abs(eig_k)) <= tolerance
), (
"Unstable surface-first generator: expected one zero spot "
f"eigenvalue and a stable surface block, found {eig_k}"
)
else:
assert np.all(np.isfinite(eig_k)) and np.all(eig_k.real > 0.), \
f"Unstable fitted k: expected all eigenvalues to have positive real parts, found {eig_k}"
factor_description = (
f"{self.pcaxar1.num_surface_alpha} surface factors plus one "
"conditional spot factor"
if self.pcaxar1.surface_first
else f"{num_alpha} joint factors"
)
verbose.write(
lambda: (
f"Model {model.qualified_label}: using {factor_description} "
f"for {num_xstate} states over {T} dates. The retained factors "
f"explain {self.pcaxar1.var_explained:.1%} of joint innovation "
f"variance; {self.pcaxar1.fit_quality}."
)
)
# Prepare simulation
# ------------------
alpha_hist = self.pcaxar1.alpha
self.alpha_mean = np.mean(alpha_hist, axis=0)
alpha_cov = np.cov(alpha_hist, rowvar=False, bias=True)
self.cholesky = np.linalg.cholesky( alpha_cov + 1e-12 * np.eye(self.num_alpha) )
@property
def num_alpha(self) -> int:
return self.pcaxar1.alpha.shape[1]
@property
def num_xstate(self) -> int:
return self.model.config.num_state+1
[docs]
def reset_rng(self):
""" Reset the random number generator to the original seed. """
self.rng = np.random.default_rng(seed=self.config.seed)
[docs]
def invariant_surface_distribution(
self,
) -> tuple[np.ndarray, np.ndarray]:
"""Return mean and covariance of the stationary ML-SANOS state block.
This distribution exists when the fitted surface generator has all
eigenvalues in the open right half-plane. Log-spot is deliberately
excluded because it has a unit root in the surface-first model.
"""
from scipy.linalg import solve_continuous_lyapunov
verify_inp(
self.pcaxar1.surface_first,
"the invariant surface distribution requires surface_first=True",
)
surface_k = self.pcaxar1.k[1:, 1:]
eig_k = np.linalg.eigvals(surface_k)
verify_inp(
np.all(np.isfinite(eig_k)) and np.all(eig_k.real > 0.0),
lambda: (
"the surface generator is not stationary; eigenvalues are "
f"{eig_k}"
),
)
alpha_cov = np.atleast_2d(
np.cov(self.pcaxar1.alpha, rowvar=False, bias=True)
)
state_cov = self.pcaxar1.dU.T @ alpha_cov @ self.pcaxar1.dU
surface_cov_rate = state_cov[1:, 1:]
covariance = solve_continuous_lyapunov(
surface_k.T,
surface_cov_rate,
)
covariance = 0.5 * (covariance + covariance.T)
values = np.linalg.eigvalsh(covariance)
tolerance = 1.0e3 * np.finfo(values.dtype).eps * max(
1.0, float(np.max(np.abs(values)))
)
verify_inp(
values[0] >= -tolerance,
lambda: (
"invariant surface covariance is not positive semidefinite; "
f"minimum eigenvalue {values[0]:g}"
),
)
mean = np.linalg.solve(
surface_k.T,
self.pcaxar1.U0[0, 1:],
)
return mean, covariance
[docs]
def sample_invariant_surface_states(self, num_samples: int) -> np.ndarray:
"""Draw Gaussian states from the invariant surface distribution."""
verify_inp(
isinstance(num_samples, int) and num_samples > 0,
lambda: f"'num_samples' must be a positive integer, got {num_samples!r}",
)
mean, covariance = self.invariant_surface_distribution()
values, vectors = np.linalg.eigh(covariance)
values = np.maximum(values, 0.0)
factor = vectors * np.sqrt(values)[None, :]
samples = mean[None, :] + self.rng.normal(
size=(num_samples, mean.size)
) @ factor.T
if not np.all(np.isfinite(samples)):
raise FloatingPointError("invariant surface sampling produced non-finite states")
return samples
@property
def rng_current_state_uid(self) -> np.random.Generator:
""" Identifies current 'seed'. Used for caching samples """
return unique_hash8( self.rng.bit_generator.state )
[docs]
def report_system(self):
eig_k = np.linalg.eigvals(self.pcaxar1.k)
rho_euler = np.max(np.abs(1. - self.pcaxar1.ndt[:,None] * eig_k[None,:]))
rho_exp = np.max(np.abs(np.exp(-self.pcaxar1.ndt[:,None] * eig_k[None,:])))
""" Print some statistics about the fitted PCA-AR1 model. """
print("alpha mean:", self.pcaxar1.alpha.mean(axis=0))
print("min real eigenvalue(k):", eig_k.real.min())
print("max Euler transition spectral radius:", rho_euler)
print("max matrix-exp transition spectral radius:", rho_exp, flush=True)
print("u0", self.pcaxar1.U0)
[docs]
def hist_sample_xstates(self,
index : int,
num_steps : int, *,
num_samples : int = 0,
bandwidth : float = 0.,
replay : bool = True,
) -> np.ndarray:
"""
Generate ``num_samples`` of ``num_steps`` days of extended state scenarios starting in the state of ``index``
and using the PCA factors from the historical data. The PCA factors can be replayed or sampled, and noise can be added to the PCA factors.
If ``replay=True`` and ``bandwidth=0`` this replays historic factor loadings.
Parameters
----------
index : int
The index of the starting state in the time series.
num_steps : int
The number of steps to simulate. This includes ``index``; so if ``num_steps=1``, the returned state is just the
state at ``index``.
num_samples : int, default 0
The number of independent samples to generate.
Set to zero to generate one sample, and return a matrix of shape (num_steps, num_xstate) instead of (num_samples, num_steps, num_xstate).
bandwidth : float, default 0.
The bandwidth for the PCA noise. A value of 0 means no noise, while a value of 1 means full noise. Must be in [0,1].
replay : bool, default True
If True, the PCA factors are replayed from the historical data, instead of being sampled.
That means with bandwidth=0, the historical path is generated.
Returns
-------
Result : :class:`np.ndarray`
Returns a matrix of form ``(num_samples, num_steps, num_xstate)`` if ``num_samples>0`` else ``(num_steps, num_xstate)``.
"""
verify_inp( num_steps > 0, lambda: f"num_steps {num_steps} must be positive")
verify_inp( num_samples >= 0, lambda: f"num_samples {num_samples} must be non-negative")
verify_inp( 0 <= index < self.xstate.shape[0], lambda: f"index {index} must be in [0,{self.xstate.shape[0]-1}]")
verify_inp( index+num_steps <= self.xstate.shape[0], lambda: f"Index {index} with num_steps {num_steps} must not exceed {self.xstate.shape[0]}")
verify_inp( bandwidth >= 0. and bandwidth <= 1. , lambda: f"bandwidth {bandwidth} must be in [0,1]")
squeeze_sample = num_samples == 0
num_samples = 1 if squeeze_sample else num_samples
num_alpha = self.num_alpha
num_xstate = self.num_xstate
num_steps = num_steps-1
if num_steps == 0:
new_xstates = np.repeat(self.xstate[index,:][None,None,:], num_samples, axis=0)
return new_xstates[0] if squeeze_sample else new_xstates
ndt = self.pcaxar1.ndt[index:index+num_steps]
Tm1 = self.pcaxar1.alpha.shape[0]
if not replay:
idx = self.rng.integers(0, Tm1, size=(num_samples*num_steps,))
alpha = self.pcaxar1.alpha[idx,:].reshape((num_samples, num_steps, num_alpha))
else:
alpha = self.pcaxar1.alpha[index:index+num_steps,:][None,:,:]
if num_samples > 1:
alpha = np.repeat(alpha, num_samples, axis=0)
if bandwidth > 0.:
z = self.rng.normal(size=(num_samples, num_steps, num_alpha)) @ self.cholesky.T
alpha = self.alpha_mean + math.sqrt(max(0., 1. - bandwidth**2)) * (alpha - self.alpha_mean) + bandwidth * z
noise = (alpha @ self.pcaxar1.dU) * np.sqrt(ndt[None, :, None])
assert noise.shape == (num_samples, num_steps, num_xstate), "Internal error"
k = self.pcaxar1.k[:,:]
U0 = self.pcaxar1.U0
X0 = self.xstate[index,:][None,:]
assert X0.shape == ( 1, num_xstate), f"Internal error {X0.shape} != {( 1, num_xstate)}"
assert k.shape == (num_xstate, num_xstate), f"Internal error {k.shape} != {(num_xstate, num_xstate)}"
assert U0.shape == (1, num_xstate), f"Internal error {U0.shape} != {(1, num_xstate)}"
new_xstates = np.zeros((num_samples, num_steps+1, num_xstate), dtype=X0.dtype )
new_xstates[:,0,:] = X0
Xt = np.repeat(X0, num_samples, axis=0)
transition_cache = {}
for j in range(num_steps):
if self.config.use_matrix_exp:
h = float(ndt[j])
A, b = transition_cache.get(h, (None, None))
if A is None:
A, b = _xar1_matrix_exp_terms(U0=U0, k=k, h=h)
transition_cache[h] = (A, b)
Xt = Xt @ A + b + noise[:,j,:]
else:
Xt = Xt + ( U0 - Xt @ k ) * ndt[j] + noise[:,j,:]
assert Xt.shape == (num_samples, num_xstate), f"Internal error {Xt.shape} != {(num_samples, num_xstate)}"
new_xstates[:,j+1,:] = Xt
return new_xstates[0,...] if squeeze_sample else new_xstates
[docs]
def hist_sample_log_nspot_states(self,
index : int,
num_steps : int, *,
num_samples : int = 0,
bandwidth : float = 0.,
replay : bool = True,
) -> tuple[np.ndarray, np.ndarray]:
"""
Generate ``num_samples`` of ``num_steps``-day log normalized spot/state scenarios starting in the state of ``index``.
Parameters
----------
index : int
The index of the starting state in the time series.
num_steps : int
The number of steps to simulate. This includes ``index``; so if ``num_steps=1``, the returned state is just the
state at ``index``.
num_samples : int, default 0
The number of independent samples to generate.
Set to zero to generate one sample, and return a matrix of shape (num_steps, num_xstate) instead of (num_samples, num_steps, num_xstate).
bandwidth : float, default 0.
The bandwidth for the PCA noise. A value of 0 means no noise, while a value of 1 means full noise. Must be in [0,1].
replay : bool, default True
If True, the PCA factors are replayed from the historical data, instead of being sampled.
That means with bandwidth=0, the historical path is generated.
Returns
-------
State : tuple[ np.ndarray, np.ndarray ]
log-spot and states
"""
xstate = self.hist_sample_xstates(index=index, num_steps=num_steps, num_samples=num_samples, bandwidth=bandwidth, replay=replay)
return ( xstate[:,:,0], xstate[:,:,1:] ) if num_samples > 0 else ( xstate[:,0], xstate[:,1:] )
[docs]
def free_sample_xstates(self,
num_steps : int,
num_samples : int = 1,
dt : float = 1./DAYS_PER_YEAR, *,
index : int = None,
bandwidth : float = 0.,
random_start : bool = True,
max_internal_dt : float|None = None,
initial_state : np.ndarray|None = None,
) -> np.ndarray:
"""
Generate ``num_samples`` paths on a grid with spacing ``dt``.
The PCA innovation law is calibrated at approximately ``pcaxar1.mean_dt``.
To simulate a larger output step robustly, each interval is therefore split
into internal substeps no larger than ``max_internal_dt``. Every internal
substep uses an independently sampled PCA loading and the exact affine
matrix-exponential drift. Only the states on the requested output grid are
returned.
Parameters
----------
num_steps : int
Number of returned states. The initial state is included, so
``num_steps=1`` returns only the starting state.
num_samples : int, default 1
Number of independent paths. Set to zero to generate one path and
remove the leading sample dimension from the result.
dt : float, default ``1/DAYS_PER_YEAR``
Time between returned states, in the same units as ``time_y``.
index : int or None
Starting-state cut-off. If ``random_start=False``, start exactly at
``xstate[index]``. If ``random_start=True``, sample starting states
from ``xstate[:index+1]``. With ``index=None``, use the full history.
bandwidth : float, default 0
Gaussian smoothing of the empirical PCA loadings. Zero uses pure
historical resampling; one uses a Gaussian with the same covariance.
random_start : bool, default True
Whether to sample starting states up to ``index``.
max_internal_dt : float or None, default None
Maximum internal simulation step used when ``bandwidth < 1``.
``None`` uses the mean historical step ``pcaxar1.mean_dt``. In full
normal mode (``bandwidth == 1``), the exact Gaussian OU transition is
sampled directly over ``dt`` and this argument is ignored.
initial_state : numpy.ndarray or None, default None
Exact raw-coordinate state from which every path starts. This is
useful for conditional Monte Carlo. It cannot be combined with
``index``; ``random_start`` has no effect when it is supplied.
Returns
-------
Result : :class:`np.ndarray`
Shape ``(num_samples, num_steps, num_xstate)`` when
``num_samples>0``; otherwise ``(num_steps, num_xstate)``.
"""
verify_inp( num_steps > 0, lambda: f"num_steps {num_steps} must be positive")
verify_inp( num_samples >= 0, lambda: f"num_samples {num_samples} must be non-negative")
verify_inp( bandwidth >= 0. and bandwidth <= 1., lambda: f"bandwidth {bandwidth} must be in [0,1]")
verify_inp( np.isfinite(dt) and dt > 0., lambda: f"dt {dt} must be finite and positive")
verify_inp(
initial_state is not None or index is not None or random_start,
"'random_start' cannot be False if neither 'index' nor "
"'initial_state' is supplied",
)
verify_inp(
initial_state is None or index is None,
"'initial_state' cannot be combined with 'index'",
)
Tm1 = self.pcaxar1.alpha.shape[0]
verify_inp(
index is None or (0 <= index < Tm1),
lambda: f"index {index} must be in [0,{Tm1-1}]",
)
if bandwidth < 1.:
if max_internal_dt is None:
max_internal_dt = float(self.pcaxar1.mean_dt)*10.
verify_inp(
np.isfinite(max_internal_dt) and max_internal_dt > 0.,
lambda: f"max_internal_dt {max_internal_dt} must be finite and positive",
)
squeeze_sample = num_samples == 0
num_samples = 1 if squeeze_sample else num_samples
num_alpha = self.num_alpha
num_xstate = self.num_xstate
# Work in the normalized time used to fit the XAR(1) model.
ndt = float(dt / self.pcaxar1.mean_dt)
internal_ndt = None if bandwidth == 1. else _split_xar1_step( ndt, float(max_internal_dt / self.pcaxar1.mean_dt) )
if initial_state is not None:
initial_state = np.asarray(initial_state)
verify_inp(
initial_state.shape == (num_xstate,)
and np.issubdtype(initial_state.dtype, np.floating)
and np.all(np.isfinite(initial_state)),
lambda: (
"'initial_state' must be a finite floating vector with "
f"shape {(num_xstate,)}, got "
f"{initial_state.shape}/{initial_state.dtype}"
),
)
X0 = np.repeat(initial_state[None, :], num_samples, axis=0)
alpha_pool_size = Tm1
elif index is None:
X0 = self.xstate[self.rng.integers(0, Tm1, size=num_samples), :]
alpha_pool_size = Tm1
else:
X0 = (
self.xstate[index, :][None, :]
if not random_start
else self.xstate[self.rng.integers(0, index + 1, size=num_samples), :]
)
alpha_pool_size = index + 1
# Simulate in standardized coordinates. The raw decoder states differ
# greatly in scale, making matrix exponentials and Lyapunov covariance
# calculations poorly conditioned when performed after mapping back.
center = self.pcaxar1.state_center
scale = self.pcaxar1.state_scale
k_raw = self.pcaxar1.k
U0_raw = self.pcaxar1.U0
k = k_raw * scale[:, None] / scale[None, :] if self.config.use_matrix_exp else k_raw
U0 = (U0_raw - center[None, :] @ k_raw) / scale[None, :] if self.config.use_matrix_exp else U0_raw
dU = self.pcaxar1.dU / scale[None, :] if self.config.use_matrix_exp else self.pcaxar1.dU
X0 = (X0 - center[None, :]) / scale[None, :] if self.config.use_matrix_exp else X0
alpha_hist = self.pcaxar1.alpha
assert X0.shape == (1, num_xstate) or X0.shape == (num_samples, num_xstate), f"Internal X0 error {X0.shape}"
assert k.shape == (num_xstate, num_xstate), f"Internal k error {k.shape} != {(num_xstate, num_xstate)}"
assert U0.shape == (1, num_xstate), f"Internal U0 error {U0.shape} != {(1, num_xstate)}"
assert dU.shape == (num_alpha, num_xstate), f"Internal dU error {dU.shape} != {(num_alpha, num_xstate)}"
new_xstates = np.zeros((num_samples, num_steps, num_xstate), dtype=X0.dtype)
new_xstates[:, 0] = X0
Xt = new_xstates[:, 0, :]
if not self.config.use_matrix_exp:
# Original vectorized Euler path. Generate the entire innovation
# tensor at once and take exactly one transition per output step.
# This deliberately bypasses exact-OU covariance construction and
# empirical internal substeps.
num_transitions = num_steps - 1
gaussian_weight = bandwidth
historical_weight = math.sqrt(max(0., 1. - bandwidth**2))
if bandwidth < 1.:
idx = self.rng.integers( 0, alpha_pool_size, size=(num_samples, num_transitions) )
alpha = alpha_hist[idx, :]
if bandwidth > 0.:
z = self.rng.normal( size=(num_samples, num_transitions, num_alpha) ) @ self.cholesky.T
alpha = ( self.alpha_mean + historical_weight * (alpha - self.alpha_mean) + gaussian_weight * z )
else:
z = self.rng.normal( size=(num_samples, num_transitions, num_alpha) )
alpha = self.alpha_mean + z @ self.cholesky.T
noise = (alpha @ dU) * math.sqrt(ndt)
assert noise.shape == (num_samples, num_transitions, num_xstate), \
f"Internal Euler noise error {noise.shape}"
A = np.eye(num_xstate, dtype=k.dtype) - ndt * k
b = ndt * U0
for j in range(num_transitions):
Xt = Xt @ A + b + noise[:, j, :]
assert Xt.shape == (num_samples, num_xstate), \
f"Internal Xt error {Xt.shape} != {(num_samples, num_xstate)}"
new_xstates[:, j + 1, :] = Xt
new_xstates = center[None, None, :] + new_xstates * scale[None, None, :]
return new_xstates[0, ...] if squeeze_sample else new_xstates
if bandwidth == 1.:
# Exact Gaussian OU transition and its integrated covariance.
A, b = _xar1_matrix_exp_terms(U0=U0, k=k, h=ndt)
verify_inp(
np.all(np.isfinite(A)) and np.all(np.isfinite(b)),
lambda: "Non-finite matrix-exponential transition; "
"the fitted k may be unstable for this time step",
)
alpha_cov = self.cholesky @ self.cholesky.T
state_cov = dU.T @ alpha_cov @ dU
Lh = _xar1_gaussian_step_factor(k=k, C=state_cov, h=ndt, A=A)
Q = Lh @ Lh.T
noise = _sample_surface_then_spot_gaussian_noise(
covariance=Q,
size=(num_samples, num_steps - 1),
rng=self.rng,
)
assert noise.shape == (num_samples, num_steps - 1, num_xstate), \
f"Internal Gaussian noise error {noise.shape}"
for j in range(num_steps - 1):
Xt = Xt @ A + b + noise[:, j, :]
assert Xt.shape == (num_samples, num_xstate), \
f"Internal Xt error {Xt.shape} != {(num_samples, num_xstate)}"
new_xstates[:, j + 1, :] = Xt
new_xstates = center[None, None, :] + new_xstates * scale[None, None, :]
return new_xstates[0, ...] if squeeze_sample else new_xstates
transition_cache = {}
gaussian_weight = bandwidth
historical_weight = math.sqrt(max(0., 1. - bandwidth**2))
# With any empirical component left, preserve the PCA innovation law at
# its fitted time scale. Sequential application automatically damps early
# shocks by the remaining matrix-exponential transitions.
for j in range(num_steps - 1):
for h in internal_ndt:
h = float(h)
A, b = transition_cache.get(h, (None, None))
if A is None:
A, b = _xar1_matrix_exp_terms(U0=U0, k=k, h=h)
verify_inp(
np.all(np.isfinite(A)) and np.all(np.isfinite(b)),
lambda: "Non-finite state transition; the fitted k may "
"be unstable for this time step",
)
transition_cache[h] = (A, b)
idx = self.rng.integers(0, alpha_pool_size, size=num_samples)
alpha = alpha_hist[idx, :]
if bandwidth > 0.:
z = self.rng.normal(size=(num_samples, num_alpha)) @ self.cholesky.T
alpha = (
self.alpha_mean
+ historical_weight * (alpha - self.alpha_mean)
+ gaussian_weight * z
)
noise = (alpha @ dU) * math.sqrt(h)
assert noise.shape == (num_samples, num_xstate), \
f"Internal noise error {noise.shape} != {(num_samples, num_xstate)}"
Xt = Xt @ A + b + noise
assert Xt.shape == (num_samples, num_xstate), \
f"Internal Xt error {Xt.shape} != {(num_samples, num_xstate)}"
new_xstates[:, j + 1, :] = Xt
new_xstates = center[None, None, :] + new_xstates * scale[None, None, :]
return new_xstates[0, ...] if squeeze_sample else new_xstates
[docs]
def sample_conditional_xstates_at_offsets(
self,
*,
initial_state: np.ndarray,
step_offsets: np.ndarray,
num_samples: int,
dt: float = 1.0 / DAYS_PER_YEAR,
) -> np.ndarray:
"""Draw exact Gaussian states at selected offsets from one state.
This is the sparse-time conditional counterpart of
:meth:`free_sample_xstates`. It avoids constructing every intervening
daily state when only option-expiry states are required.
"""
verify_inp(
self.config.use_matrix_exp and self.pcaxar1.surface_first,
"sparse conditional sampling requires the surface-first exact "
"Gaussian model",
)
verify_inp(
isinstance(num_samples, int) and num_samples > 0,
lambda: f"'num_samples' must be positive, got {num_samples!r}",
)
verify_inp(
np.isfinite(dt) and dt > 0.0,
lambda: f"'dt' must be finite-positive, got {dt!r}",
)
initial_state = np.asarray(initial_state)
offsets = np.asarray(step_offsets)
states = self.num_xstate
verify_inp(
initial_state.shape == (states,)
and np.issubdtype(initial_state.dtype, np.floating)
and np.all(np.isfinite(initial_state)),
lambda: (
"'initial_state' must be a finite floating vector with shape "
f"{(states,)}, got {initial_state.shape}/{initial_state.dtype}"
),
)
verify_inp(
offsets.ndim == 1
and offsets.size >= 2
and np.issubdtype(offsets.dtype, np.integer)
and offsets[0] == 0
and np.all(np.diff(offsets) > 0),
lambda: (
"'step_offsets' must be an increasing integer vector starting "
f"at zero, got {offsets}"
),
)
center = np.asarray(self.pcaxar1.state_center, dtype=np.float64)
scale = np.asarray(self.pcaxar1.state_scale, dtype=np.float64)
verify_inp(
center.shape == scale.shape == (states,)
and np.all(np.isfinite(center))
and np.all(np.isfinite(scale))
and np.all(scale > 0.0),
"invalid state normalization for conditional sampling",
)
raw_k = np.asarray(self.pcaxar1.k, dtype=np.float64)
k = raw_k * scale[:, None] / scale[None, :]
U0 = (
np.asarray(self.pcaxar1.U0, dtype=np.float64)
- center[None, :] @ raw_k
) / scale[None, :]
dU = np.asarray(self.pcaxar1.dU, dtype=np.float64) / scale[None, :]
alpha_covariance = self.cholesky @ self.cholesky.T
covariance_rate = dU.T @ alpha_covariance @ dU
one_step_h = float(dt / self.pcaxar1.mean_dt)
one_step_A, one_step_b = _xar1_matrix_exp_terms(
U0=U0, k=k, h=one_step_h
)
one_step_factor = _xar1_gaussian_step_factor(
k=k,
C=covariance_rate,
h=one_step_h,
A=one_step_A,
)
one_step_covariance = one_step_factor @ one_step_factor.T
result = np.empty(
(num_samples, offsets.size, states), dtype=np.float64
)
Xt = np.repeat(
((initial_state.astype(np.float64) - center) / scale)[None, :],
num_samples,
axis=0,
)
result[:, 0] = Xt
for output_index, step_count in enumerate(np.diff(offsets), start=1):
# Compose the exact one-day transition instead of evaluating the
# generic Van Loan block exponential over the entire interval.
# The surface-first model contains a spot unit root, for which the
# long-horizon block exponential is unnecessarily ill-conditioned.
A = np.eye(states, dtype=np.float64)
b = np.zeros((1, states), dtype=np.float64)
covariance = np.zeros((states, states), dtype=np.float64)
for _ in range(int(step_count)):
covariance = (
one_step_A.T @ covariance @ one_step_A
+ one_step_covariance
)
b = b @ one_step_A + one_step_b
A = A @ one_step_A
covariance = 0.5 * (covariance + covariance.T)
noise = _sample_surface_then_spot_gaussian_noise(
covariance=covariance,
size=(num_samples,),
rng=self.rng,
)
Xt = Xt @ A + b + noise
result[:, output_index] = Xt
result *= scale[None, None, :]
result += center[None, None, :]
if not np.all(np.isfinite(result)):
raise FloatingPointError("conditional state sample is non-finite")
return result
[docs]
def importance_sample_xstates(
self,
*,
num_steps: int,
num_samples: int,
dt: float,
target_steps: tuple[int, ...],
config: GaussianSpotImportanceConfig,
proposal_rng: np.random.Generator,
) -> GaussianSpotImportanceSample:
"""Sample exact Gaussian paths under a log-spot tail proposal.
For the surface-first model, the original surface path is retained and
only the spot residual conditional on that complete path is widened.
For the legacy joint fit, the original draw supplies the Gaussian state
residual conditional on the selected future log spot. In both cases
the returned weight is the likelihood ratio of the original path law
to the *full* proposal mixture, not merely to the selected component.
"""
verify_inp(
self.config.use_matrix_exp,
"Gaussian path importance sampling requires use_matrix_exp=True",
)
verify_inp(
self.pcaxar1.surface_first
or self.config.num_alpha is None
or self.num_alpha == self.num_xstate,
lambda: (
"Gaussian path importance sampling requires the full state "
f"innovation dimension, got {self.num_alpha}/{self.num_xstate}"
),
)
verify_inp(
isinstance(proposal_rng, np.random.Generator),
lambda: f"'proposal_rng' must be a NumPy Generator, got {type(proposal_rng)}",
)
verify_inp(
bool(target_steps)
and tuple(sorted(set(target_steps))) == target_steps
and target_steps[0] > 0
and target_steps[-1] < num_steps,
lambda: (
"target steps must be unique, increasing, positive, and below "
f"{num_steps}; got {target_steps}"
),
)
reference = self.free_sample_xstates(
num_steps=num_steps,
num_samples=num_samples,
dt=dt,
bandwidth=1.0,
)
reference_log_spot = reference[..., 0]
reference_log_spot_sum = reference_log_spot.sum(axis=0)
reference_log_spot_square_sum = np.square(reference_log_spot).sum(axis=0)
reference_rv = np.square(np.diff(reference_log_spot, axis=1)).sum(axis=1)
reference_rv_sum = float(reference_rv.sum())
reference_rv_square_sum = float(np.square(reference_rv).sum())
center = self.pcaxar1.state_center
scale = self.pcaxar1.state_scale
verify_inp(
center is not None and scale is not None,
"matrix-exponential importance sampling requires state normalization",
)
# Standardize the direct-P draw in place. At million-path scale an
# additional full (sample, time, state) temporary costs tens of GB.
# The direct-P control moments above are the only raw-coordinate
# quantities needed after this point.
standardized_reference = reference
standardized_reference -= center[None, None, :]
standardized_reference /= scale[None, None, :]
ndt = float(dt / self.pcaxar1.mean_dt)
k = self.pcaxar1.k * scale[:, None] / scale[None, :]
U0 = (
self.pcaxar1.U0 - center[None, :] @ self.pcaxar1.k
) / scale[None, :]
dU = self.pcaxar1.dU / scale[None, :]
A, b = _xar1_matrix_exp_terms(U0=U0, k=k, h=ndt)
alpha_cov = self.cholesky @ self.cholesky.T
state_cov = dU.T @ alpha_cov @ dU
noise_factor = _xar1_gaussian_step_factor(
k=k,
C=state_cov,
h=ndt,
A=A,
)
Q = noise_factor @ noise_factor.T
importance_function = (
_importance_sample_spot_given_surface_paths
if self.pcaxar1.surface_first
else _importance_sample_gaussian_spot_paths
)
standardized, log_weights, components, target_std = (
importance_function(
reference_paths=standardized_reference,
A=A,
b=b,
Q=Q,
target_steps=target_steps,
config=config,
rng=proposal_rng,
)
)
standardized *= scale[None, None, :]
standardized += center[None, None, :]
paths = standardized
if not np.all(np.isfinite(paths)) or not np.all(np.isfinite(log_weights)):
raise FloatingPointError("Gaussian importance sample is non-finite")
return GaussianSpotImportanceSample(
paths=paths.astype(reference.dtype, copy=False),
log_weights=log_weights,
proposal_components=components,
target_steps=target_steps,
target_log_spot_std=target_std * float(scale[0]),
reference_log_spot_sum=reference_log_spot_sum,
reference_log_spot_square_sum=reference_log_spot_square_sum,
reference_realized_variance_sum=reference_rv_sum,
reference_realized_variance_square_sum=reference_rv_square_sum,
)
def _sample_surface_then_spot_gaussian_noise(
*,
covariance: np.ndarray,
size: tuple[int, ...],
rng: np.random.Generator,
) -> np.ndarray:
"""Sample joint Gaussian noise by drawing surface, then conditional spot.
The extended state is ordered as ``(s, h)``, where ``s`` is log-spot and
``h`` contains the autonomous ML-SANOS implied-volatility parameters. The
construction is valid for singular surface covariance matrices and makes
the sampling order explicit without changing the joint Gaussian law.
"""
from scipy.linalg import eigh
verify_inp(
covariance.ndim == 2
and covariance.shape[0] == covariance.shape[1]
and covariance.shape[0] >= 2,
lambda: f"'covariance' must be square with at least two states, got {covariance.shape}",
)
verify_inp(
isinstance(size, tuple)
and all(isinstance(value, int) and value >= 0 for value in size),
lambda: f"'size' must be a tuple of non-negative integers, got {size!r}",
)
Q = 0.5 * (covariance + covariance.T)
verify_inp(np.all(np.isfinite(Q)), "'covariance' contains non-finite values")
surface_covariance = Q[1:, 1:]
values, vectors = eigh(surface_covariance)
scale = max(1.0, float(np.max(np.abs(values))))
tolerance = 1.0e3 * np.finfo(float).eps * scale
verify_inp(
values[0] >= -tolerance,
lambda: f"surface covariance is not positive semidefinite: {values[0]:g}",
)
keep = values > tolerance
surface_factor = vectors[:, keep] * np.sqrt(values[keep])[None, :]
surface_noise = (
rng.normal(size=(*size, int(np.sum(keep)))) @ surface_factor.T
)
surface_inverse = np.linalg.pinv(
surface_covariance,
rcond=1.0e3 * np.finfo(float).eps,
hermitian=True,
)
spot_given_surface = surface_inverse @ Q[1:, 0]
conditional_variance = float(
Q[0, 0] - Q[0, 1:] @ spot_given_surface
)
variance_tolerance = 1.0e3 * np.finfo(float).eps * max(
1.0, abs(float(Q[0, 0]))
)
verify_inp(
conditional_variance >= -variance_tolerance,
lambda: (
"conditional spot variance is negative: "
f"{conditional_variance:g}"
),
)
conditional_std = math.sqrt(max(conditional_variance, 0.0))
spot_noise = surface_noise @ spot_given_surface
if conditional_std > 0.0:
spot_noise += conditional_std * rng.normal(size=size)
result = np.empty((*size, Q.shape[0]), dtype=Q.dtype)
result[..., 0] = spot_noise
result[..., 1:] = surface_noise
if not np.all(np.isfinite(result)):
raise FloatingPointError("surface-first Gaussian sampling produced non-finite noise")
return result
def _xar1_gaussian_step_factor(
k: np.ndarray,
C: np.ndarray,
h: float,
A: np.ndarray|None = None,
) -> np.ndarray:
r"""
Exact Gaussian noise factor for the row-vector OU transition.
For
dX_t = (U0 - X_t @ k) dt + dW_t @ B,
C = B.T @ B,
return ``L`` such that ``L @ L.T = Q_h``, where
Q_h = integral_0^h exp(-s k.T) C exp(-s k) ds.
A stable Lyapunov formulation is used whenever ``k`` is mean reverting;
Van Loan's block exponential is retained as a finite-horizon fallback.
An eigendecomposition, rather than Cholesky, permits singular low-rank PCA
covariance matrices.
"""
from scipy.linalg import eigh, eigvals, expm, solve_continuous_lyapunov
verify_inp(np.isfinite(h) and h >= 0., lambda: f"h must be finite and non-negative; found {h}")
verify_inp(k.ndim == 2 and k.shape[0] == k.shape[1], lambda: f"k must be square; found {k.shape}")
n = k.shape[0]
verify_inp(C.shape == (n, n), lambda: f"C must have shape {(n,n)}; found {C.shape}")
verify_inp(np.all(np.isfinite(k)) and np.all(np.isfinite(C)), lambda: "k and C must contain only finite values")
dtype = np.result_type(k, C, float)
k = np.asarray(k, dtype=dtype)
C = np.asarray(C, dtype=dtype)
C = 0.5 * (C + C.T)
if h == 0.:
return np.zeros((n, 0), dtype=dtype)
if A is None:
A = expm(-h * k)
else:
A = np.asarray(A, dtype=dtype)
verify_inp(A.shape == (n, n), lambda: f"A must have shape {(n,n)}; found {A.shape}")
eig_k = eigvals(k)
stability_tol = 100. * np.finfo(float).eps * max(1., np.linalg.norm(k, ord=2))
if np.min(eig_k.real) > stability_tol:
# Stationary covariance P solves k.T P + P k = C, and
# Q_h = P - A.T P A. This avoids the exponentially growing block
# present in the Van Loan construction and is robust for large h.
P = solve_continuous_lyapunov(k.T, C)
Q = P - A.T @ P @ A
else:
# Finite-horizon fallback valid even when no stationary covariance exists.
# For F=-k.T, Van Loan gives Q_h = M12 exp(F.T h) = M12 A.
G = np.zeros((2*n, 2*n), dtype=dtype)
G[:n, :n] = -k.T
G[:n, n:] = C
G[n:, n:] = k
M = expm(h * G)
Q = M[:n, n:] @ A
Q = 0.5 * (Q + Q.T)
verify_inp(np.all(np.isfinite(Q)), lambda: "Non-finite Gaussian transition covariance")
values, vectors = eigh(Q)
scale = max(1., float(np.max(np.abs(values))))
tolerance = 1000. * np.finfo(float).eps * scale
verify_inp(
values[0] >= -tolerance,
lambda: f"Gaussian transition covariance is not PSD; minimum eigenvalue {values[0]}",
)
values = np.maximum(values, 0.)
keep = values > tolerance
return vectors[:, keep] * np.sqrt(values[keep])[None, :]
def _importance_sample_spot_given_surface_paths(
*,
reference_paths: np.ndarray,
A: np.ndarray,
b: np.ndarray,
Q: np.ndarray,
target_steps: tuple[int, ...],
config: GaussianSpotImportanceConfig,
rng: np.random.Generator,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Broaden conditional log-spot tails while preserving every surface path.
The surface transition is autonomous. At each step its innovation is
sampled under the original law, and the simultaneous spot innovation is
decomposed into its conditional mean given that surface innovation plus an
independent scalar Gaussian residual. Importance sampling changes only
those scalar residuals. Consequently the simulated surface marginal and
every realized surface path are exactly unchanged.
"""
from scipy.special import logsumexp
verify_inp(
reference_paths.ndim == 3 and reference_paths.shape[-1] > 1,
lambda: (
"'reference_paths' must have shape (S, steps, states>1), got "
f"{reference_paths.shape}"
),
)
samples, steps, states = reference_paths.shape
verify_inp(
A.shape == Q.shape == (states, states) and b.shape == (1, states),
lambda: f"invalid affine Gaussian geometry {A.shape}/{b.shape}/{Q.shape}",
)
verify_inp(
tuple(sorted(set(target_steps))) == target_steps
and target_steps[0] > 0
and target_steps[-1] < steps,
lambda: f"invalid target steps {target_steps} for path length {steps}",
)
if not all(np.all(np.isfinite(value)) for value in (reference_paths, A, b, Q)):
raise ValueError("conditional Gaussian importance inputs contain non-finite values")
autonomy_tolerance = 1.0e3 * np.finfo(float).eps * max(
1.0, float(np.linalg.norm(A, ord=2))
)
verify_inp(
np.max(np.abs(A[0, 1:])) <= autonomy_tolerance,
lambda: (
"surface transition is not autonomous: current spot loads into "
f"the next surface by {A[0, 1:]}"
),
)
surface_covariance = 0.5 * (Q[1:, 1:] + Q[1:, 1:].T)
surface_inverse = np.linalg.pinv(
surface_covariance,
rcond=1.0e3 * np.finfo(float).eps,
hermitian=True,
)
spot_given_surface = surface_inverse @ Q[1:, 0]
residual_variance = float(
Q[0, 0] - Q[0, 1:] @ spot_given_surface
)
variance_tolerance = 1.0e3 * np.finfo(float).eps * max(
1.0, abs(float(Q[0, 0]))
)
verify_inp(
residual_variance > variance_tolerance,
lambda: (
"conditional spot residual variance must be positive for tail "
f"importance sampling, got {residual_variance:g}"
),
)
residual_std = math.sqrt(residual_variance)
innovations = (
reference_paths[:, 1:, :]
- reference_paths[:, :-1, :] @ A
- b[None, :, :]
)
surface_innovations = innovations[..., 1:]
spot_conditional_mean = surface_innovations @ spot_given_surface
residual_z = (
innovations[..., 0] - spot_conditional_mean
) / residual_std
# Conditional on the initial state and complete surface path, only the
# scalar residual shocks remain random. Propagate their zero-residual
# conditional mean and their target-step standard deviations.
conditional_mean = np.empty((samples, steps), dtype=np.float64)
conditional_mean[:, 0] = reference_paths[:, 0, 0]
for step in range(steps - 1):
conditional_mean[:, step + 1] = (
conditional_mean[:, step] * A[0, 0]
+ reference_paths[:, step, 1:] @ A[1:, 0]
+ b[0, 0]
+ spot_conditional_mean[:, step]
)
target_count = len(target_steps)
bridge_coefficients = np.zeros((target_count, steps - 1), dtype=np.float64)
target_variance = np.empty(target_count, dtype=np.float64)
for target_index, target_step in enumerate(target_steps):
powers = np.arange(target_step - 1, -1, -1)
coefficients = residual_std * np.power(A[0, 0], powers)
variance = float(coefficients @ coefficients)
verify_inp(
np.isfinite(variance) and variance > variance_tolerance,
lambda: f"invalid conditional spot variance at step {target_step}: {variance}",
)
target_variance[target_index] = variance
bridge_coefficients[target_index, :target_step] = coefficients / math.sqrt(variance)
target_std = np.sqrt(target_variance)
reference_target_z = (
reference_paths[:, target_steps, 0]
- conditional_mean[:, target_steps]
) / target_std[None, :]
reconstructed_target_z = residual_z @ bridge_coefficients.T
assert np.allclose(reference_target_z, reconstructed_target_z, atol=1.0e-8)
base_probability = config.base_probability
component_probability = (1.0 - base_probability) / target_count
probabilities = np.full(target_count + 1, component_probability)
probabilities[0] = base_probability
components = rng.choice(
target_count + 1,
size=samples,
replace=True,
p=probabilities,
).astype(np.int16, copy=False)
proposal_residual_z = residual_z.astype(np.float64, copy=True)
multiplier = config.spot_std_multiplier
for component in range(1, target_count + 1):
selected = np.flatnonzero(components == component)
if selected.size == 0:
continue
target_index = component - 1
proposed_target_z = multiplier * rng.standard_normal(selected.size)
difference = (
proposed_target_z
- reference_target_z[selected, target_index]
)
proposal_residual_z[selected] += (
difference[:, None]
* bridge_coefficients[target_index][None, :]
)
proposal = reference_paths.astype(np.float64, copy=True)
for step in range(steps - 1):
proposal[:, step + 1, 0] = (
proposal[:, step] @ A[:, 0]
+ b[0, 0]
+ spot_conditional_mean[:, step]
+ residual_std * proposal_residual_z[:, step]
)
assert np.array_equal(proposal[..., 1:], reference_paths[..., 1:])
proposal_target_z = (
proposal[:, target_steps, 0]
- conditional_mean[:, target_steps]
) / target_std[None, :]
log_component_ratio = (
-math.log(multiplier)
+ 0.5
* np.square(proposal_target_z)
* (1.0 - 1.0 / multiplier**2)
)
log_terms = np.concatenate(
(
np.full((samples, 1), math.log(base_probability)),
math.log(component_probability) + log_component_ratio,
),
axis=1,
)
log_weights = -logsumexp(log_terms, axis=1)
if not np.all(np.isfinite(proposal)) or not np.all(np.isfinite(log_weights)):
raise FloatingPointError("conditional spot proposal or likelihood is non-finite")
return proposal, log_weights, components, target_std
def _importance_sample_gaussian_spot_paths(
*,
reference_paths: np.ndarray,
A: np.ndarray,
b: np.ndarray,
Q: np.ndarray,
target_steps: tuple[int, ...],
config: GaussianSpotImportanceConfig,
rng: np.random.Generator,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Apply an exact future-log-spot Gaussian-mixture proposal.
``reference_paths`` must already be a draw from the affine Gaussian path
law conditional on its first state. For each target coordinate ``Y``, the
identity
``X | Y=y = X_P + Cov(X,Y)/Var(Y) * (y-Y_P)``
turns that draw into an exact Gaussian bridge. The proposal is a mixture
of the original path law and one widened marginal for every target step.
"""
from scipy.special import logsumexp
verify_inp(
reference_paths.ndim == 3 and reference_paths.shape[-1] > 1,
lambda: (
"'reference_paths' must have shape (S, steps, states>1), got "
f"{reference_paths.shape}"
),
)
samples, steps, states = reference_paths.shape
verify_inp(
A.shape == Q.shape == (states, states) and b.shape == (1, states),
lambda: f"invalid affine Gaussian geometry {A.shape}/{b.shape}/{Q.shape}",
)
verify_inp(
tuple(sorted(set(target_steps))) == target_steps
and target_steps[0] > 0
and target_steps[-1] < steps,
lambda: f"invalid target steps {target_steps} for path length {steps}",
)
if not all(np.all(np.isfinite(value)) for value in (reference_paths, A, b, Q)):
raise ValueError("Gaussian importance inputs contain non-finite values")
powers = np.empty((steps, states, states), dtype=np.float64)
variances = np.empty_like(powers)
powers[0] = np.eye(states)
variances[0] = 0.0
for step in range(1, steps):
powers[step] = powers[step - 1] @ A
variances[step] = A.T @ variances[step - 1] @ A + Q
target_count = len(target_steps)
target_variance = np.asarray(
[variances[step, 0, 0] for step in target_steps], dtype=np.float64
)
variance_floor = np.finfo(np.float64).eps * max(
1.0, float(np.max(np.abs(variances)))
)
verify_inp(
np.all(target_variance > variance_floor),
lambda: (
"future log-spot marginal variance is zero or numerically singular: "
f"{target_variance}"
),
)
betas = np.empty((target_count, steps, states), dtype=np.float64)
for target_index, target_step in enumerate(target_steps):
target_covariance = variances[target_step]
for step in range(steps):
if step <= target_step:
covariance = variances[step] @ powers[target_step - step]
betas[target_index, step] = covariance[:, 0]
else:
covariance = powers[step - target_step].T @ target_covariance
betas[target_index, step] = covariance[:, 0]
betas[target_index] /= target_variance[target_index]
assert np.allclose(betas[target_index, target_step, 0], 1.0)
target_lookup = {step: index for index, step in enumerate(target_steps)}
deterministic = reference_paths[:, 0].astype(np.float64, copy=True)
target_mean = np.empty((samples, target_count), dtype=np.float64)
for step in range(1, steps):
deterministic = deterministic @ A + b
target_index = target_lookup.get(step)
if target_index is not None:
target_mean[:, target_index] = deterministic[:, 0]
base_probability = config.base_probability
component_probability = (1.0 - base_probability) / target_count
probabilities = np.full(target_count + 1, component_probability)
probabilities[0] = base_probability
components = rng.choice(
target_count + 1,
size=samples,
replace=True,
p=probabilities,
).astype(np.int16, copy=False)
proposal = reference_paths.astype(np.float64, copy=True)
target_std = np.sqrt(target_variance)
multiplier = config.spot_std_multiplier
for component in range(1, target_count + 1):
selected = np.flatnonzero(components == component)
if selected.size == 0:
continue
target_index = component - 1
target_step = target_steps[target_index]
proposed_z = multiplier * rng.standard_normal(selected.size)
proposed_spot = (
target_mean[selected, target_index]
+ target_std[target_index] * proposed_z
)
difference = (
proposed_spot - reference_paths[selected, target_step, 0]
)
proposal[selected] += (
difference[:, None, None] * betas[target_index][None, :, :]
)
target_value = proposal[:, target_steps, 0]
standardized_target = (target_value - target_mean) / target_std[None, :]
log_component_ratio = (
-math.log(multiplier)
+ 0.5
* np.square(standardized_target)
* (1.0 - 1.0 / multiplier**2)
)
log_terms = np.concatenate(
(
np.full((samples, 1), math.log(base_probability)),
math.log(component_probability) + log_component_ratio,
),
axis=1,
)
log_weights = -logsumexp(log_terms, axis=1)
if not np.all(np.isfinite(proposal)) or not np.all(np.isfinite(log_weights)):
raise FloatingPointError("Gaussian bridge proposal or likelihood is non-finite")
verify_inp(
float(np.max(log_weights))
<= -math.log(base_probability) + 1.0e-10,
"importance likelihood exceeds its base-mixture bound",
)
return proposal, log_weights, components, target_std
def _split_xar1_step(h: float, max_h: float) -> np.ndarray:
"""Split one normalized XAR(1) step into full substeps plus a remainder."""
verify_inp(np.isfinite(h) and h > 0., lambda: f"h must be finite and positive; found {h}")
verify_inp(np.isfinite(max_h) and max_h > 0., lambda: f"max_h must be finite and positive; found {max_h}")
# Keeping full native-size substeps is preferable to equal subdivision:
# the empirical alpha distribution was calibrated at that native scale.
tol = 32. * np.finfo(float).eps * max(1., abs(h), abs(max_h))
n_full = int(math.floor((h + tol) / max_h))
remainder = h - n_full * max_h
if abs(remainder) <= tol:
remainder = 0.
elif remainder < 0.:
# Can only occur through floating-point round-off near an exact multiple.
n_full -= 1
remainder = h - n_full * max_h
parts = [max_h] * n_full
if remainder > 0.:
parts.append(remainder)
if not parts:
parts.append(h)
out = np.asarray(parts, dtype=float)
assert np.all(out > 0.)
assert abs(float(np.sum(out)) - h) <= 4. * tol
return out
def _xar1_matrix_exp_terms(U0: np.ndarray, k: np.ndarray, h: float) -> tuple[np.ndarray, np.ndarray]:
"""
Exact affine transition terms for dX/dt = U0 - X @ k using row-vector convention.
Returns ``A`` and ``b`` such that ``X_next = X @ A + b`` over normalized
time step ``h``. The augmented exponential avoids inverting ``k`` and is
therefore safe when ``k`` is singular or nearly singular.
"""
from scipy.linalg import expm
verify_inp( h >= 0., lambda: f"h must be non-negative; found {h}")
verify_inp( k.ndim == 2 and k.shape[0] == k.shape[1], lambda: f"k must be square; found {k.shape}")
n = k.shape[0]
verify_inp( U0.shape == (1,n) or U0.shape == (n,), lambda: f"U0 must have shape {(1,n)} or {(n,)}; found {U0.shape}")
dtype = np.result_type(U0, k, float)
G = np.zeros((n+1, n+1), dtype=dtype)
G[:n,:n] = -k
G[n,:n] = U0.reshape(n)
M = expm(h * G)
A = M[:n,:n]
b = M[n:n+1,:n]
return A, b
def _xar1_matrix_exp_drift(X: np.ndarray, U0: np.ndarray, k: np.ndarray, ndt: np.ndarray) -> np.ndarray:
"""
Exact deterministic AR drift for an array of starting states and time steps.
"""
verify_inp( X.ndim == 2, lambda: f"X must be two-dimensional; found {X.shape}")
verify_inp( ndt.shape == (X.shape[0],), lambda: f"ndt must have shape {(X.shape[0],)}; found {ndt.shape}")
out = np.empty_like(X)
cache = {}
for t, h in enumerate(ndt):
h = float(h)
A, b = cache.get(h, (None, None))
if A is None:
A, b = _xar1_matrix_exp_terms(U0=U0, k=k, h=h)
cache[h] = (A, b)
out[t:t+1,:] = X[t:t+1,:] @ A + b
return out
def _fit_daily_ou_drift(
X0: np.ndarray,
X1: np.ndarray,
ndt: np.ndarray,
initial_k: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
"""Fit a continuous OU generator from the dominant daily transition.
Most observations have one normalized daily interval. An affine least-
squares transition at that interval followed by one matrix logarithm is
both much faster and more robust than optimizing a matrix exponential at
every calibration iteration. If its logarithm is not a real stable
generator, retain the Euler regression and shift it minimally into the
stable half-plane.
"""
from scipy.linalg import logm
verify_inp(
X0.ndim == 2 and X1.shape == X0.shape,
lambda: f"Invalid OU observations {X0.shape}, {X1.shape}",
)
verify_inp(
ndt.shape == (X0.shape[0],),
lambda: f"ndt must have shape {(X0.shape[0],)}, found {ndt.shape}",
)
n = X0.shape[1]
verify_inp(
initial_k.shape == (n, n),
lambda: f"initial_k must have shape {(n, n)}, found {initial_k.shape}",
)
unique_h, h_counts = np.unique(ndt, return_counts=True)
common_h = float(unique_h[np.argmax(h_counts)])
common = ndt == common_h
if np.count_nonzero(common) >= n + 2:
design = np.concatenate(
(np.ones((np.count_nonzero(common), 1)), X0[common]), axis=1
)
if np.linalg.matrix_rank(design) == n + 1:
affine = np.linalg.lstsq(design, X1[common], rcond=None)[0]
candidate_complex = -logm(affine[1:, :]) / common_h
imaginary_scale = float(np.max(np.abs(candidate_complex.imag)))
real_scale = max(1., float(np.max(np.abs(candidate_complex.real))))
candidate_k = candidate_complex.real
try:
candidate_mu = np.linalg.solve(
(np.eye(n) - affine[1:, :]).T, affine[0, :]
)
except np.linalg.LinAlgError:
candidate_mu = np.full(n, np.nan)
eig_k = np.linalg.eigvals(candidate_k)
if (
np.all(np.isfinite(candidate_complex))
and np.all(np.isfinite(candidate_mu))
and imaginary_scale <= 1e-8 * real_scale
and np.all(eig_k.real > 0.)
):
return candidate_mu[None, :] @ candidate_k, candidate_k
dXdt = (X1 - X0) / ndt[:, None]
Xbar = np.mean(X0, axis=0, keepdims=True)
dXbar = np.mean(dXdt, axis=0, keepdims=True)
centered_k = -np.linalg.lstsq(X0 - Xbar, dXdt - dXbar, rcond=None)[0]
candidate_k = centered_k if np.all(np.isfinite(centered_k)) else initial_k.copy()
stability_floor = math.sqrt(np.finfo(candidate_k.dtype).eps)
minimum_real_eigenvalue = float(np.min(np.linalg.eigvals(candidate_k).real))
if minimum_real_eigenvalue <= stability_floor:
candidate_k = candidate_k + (
stability_floor - minimum_real_eigenvalue
) * np.eye(n, dtype=candidate_k.dtype)
return dXbar + Xbar @ candidate_k, candidate_k
def _fit_continuous_ou_drift(
X0: np.ndarray,
X1: np.ndarray,
ndt: np.ndarray,
initial_k: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
"""Fit an exact, stable continuous-time OU drift to irregular observations.
The generator is parameterized as ``k = L L.T + S``, with ``S``
skew-symmetric. Its symmetric part is therefore positive definite, which
makes ``exp(-h k)`` contractive for every positive horizon ``h``. The
equilibrium ``mu`` is fitted directly and ``U0 = mu @ k``.
"""
from scipy.optimize import minimize
verify_inp(X0.ndim == 2 and X1.shape == X0.shape, lambda: f"Invalid OU observations {X0.shape}, {X1.shape}")
verify_inp(ndt.shape == (X0.shape[0],), lambda: f"ndt must have shape {(X0.shape[0],)}, found {ndt.shape}")
n = X0.shape[1]
verify_inp(initial_k.shape == (n, n), lambda: f"initial_k must have shape {(n,n)}, found {initial_k.shape}")
# Obtain an exact discrete-to-continuous initial estimate from the most
# common observation interval. For X1 = c + X0 B, the corresponding OU
# generator is k = -log(B)/h. This puts the nonlinear irregular-time fit
# close to its optimum and avoids hundreds of expensive refinement steps.
unique_h, h_counts = np.unique(ndt, return_counts=True)
common_h = unique_h[np.argmax(h_counts)]
common = ndt == common_h
initial_mu = None
if np.count_nonzero(common) >= n + 2:
from scipy.linalg import logm
design = np.concatenate((np.ones((np.count_nonzero(common), 1)), X0[common]), axis=1)
affine = np.linalg.lstsq(design, X1[common], rcond=None)[0]
discrete_k = -logm(affine[1:, :]) / float(common_h)
imaginary_scale = float(np.max(np.abs(discrete_k.imag)))
real_scale = max(1., float(np.max(np.abs(discrete_k.real))))
if np.all(np.isfinite(discrete_k)) and imaginary_scale <= 1e-8 * real_scale:
initial_k = discrete_k.real
try:
candidate_mu = np.linalg.solve((np.eye(n) - affine[1:, :]).T, affine[0, :])
if np.all(np.isfinite(candidate_mu)):
initial_mu = candidate_mu
except np.linalg.LinAlgError:
pass
# Project the initial estimate onto the interior of the stable
# parameterization while retaining its rotational component.
sym = 0.5 * (initial_k + initial_k.T)
values, vectors = np.linalg.eigh(sym)
floor = 1e-6 * max(1., float(np.linalg.norm(initial_k, ord=2)))
sym = (vectors * np.maximum(values, floor)[None, :]) @ vectors.T
L0 = np.linalg.cholesky(sym)
S0 = 0.5 * (initial_k - initial_k.T)
mu0 = np.mean(np.concatenate((X0, X1[-1:, :]), axis=0), axis=0) if initial_mu is None else initial_mu
lower = np.tril_indices(n)
upper = np.triu_indices(n, 1)
diag_positions = np.flatnonzero(lower[0] == lower[1])
def pack(mu: np.ndarray, L: np.ndarray, S: np.ndarray) -> np.ndarray:
lp = L[lower].copy()
lp[diag_positions] = np.log(lp[diag_positions])
return np.concatenate((mu, lp, S[upper]))
def unpack(p: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
mu = p[:n]
lp = p[n:n + len(lower[0])].copy()
lp[diag_positions] = np.exp(lp[diag_positions])
L = np.zeros((n, n), dtype=p.dtype)
L[lower] = lp
sp = p[n + len(lower[0]):]
S = np.zeros((n, n), dtype=p.dtype)
S[upper] = sp
S[(upper[1], upper[0])] = -sp
return mu, L @ L.T + S
unique_h, inverse_h = np.unique(ndt, return_inverse=True)
tdtype = torch.float64
tx0 = torch.as_tensor(X0, dtype=tdtype)
tx1 = torch.as_tensor(X1, dtype=tdtype)
th = torch.as_tensor(unique_h, dtype=tdtype)
thinverse = torch.as_tensor(inverse_h, dtype=torch.long)
tsqrt_h = torch.as_tensor(np.sqrt(ndt)[:, None], dtype=tdtype)
tlower = (torch.as_tensor(lower[0]), torch.as_tensor(lower[1]))
tupper = (torch.as_tensor(upper[0]), torch.as_tensor(upper[1]))
tdiag_mask = torch.as_tensor(lower[0] == lower[1])
def objective_and_gradient(p: np.ndarray) -> tuple[float, np.ndarray]:
"""One batched matrix-exponential evaluation and its exact gradient."""
tp = torch.tensor(p, dtype=tdtype, requires_grad=True)
mu = tp[:n]
lp = tp[n:n + len(lower[0])]
lp = torch.where(tdiag_mask, torch.exp(lp), lp)
L = torch.zeros((n, n), dtype=tdtype)
L[tlower] = lp
sp = tp[n + len(lower[0]):]
S = torch.zeros((n, n), dtype=tdtype)
S[tupper] = sp
S[(tupper[1], tupper[0])] = -sp
k = L @ L.T + S
transitions = torch.matrix_exp(-th[:, None, None] * k[None, :, :])
centered = tx0 - mu
prediction = mu + torch.bmm(centered[:, None, :], transitions[thinverse]).squeeze(1)
standardized_error = (tx1 - prediction) / tsqrt_h
objective = 0.5 * torch.mean(standardized_error.square())
objective.backward()
return float(objective.detach()), tp.grad.detach().numpy()
p0 = pack(mu0, L0, S0)
diagonal_parameters = n + diag_positions
bounds = [(None, None)] * len(p0)
for ix in diagonal_parameters:
bounds[ix] = (-20., 10.)
result = minimize(
objective_and_gradient,
p0,
method="L-BFGS-B",
jac=True,
bounds=bounds,
options={"maxiter": 100, "ftol": 1e-10, "gtol": 1e-7, "maxls": 20},
)
verify_inp(
np.all(np.isfinite(result.x)) and np.isfinite(result.fun),
lambda: f"Continuous-time OU calibration produced a non-finite result: {result.message}",
)
if not result.success:
warnings.warn(
"Continuous-time OU calibration stopped before the optimizer's "
f"convergence criterion ({result.message}); using the best stable "
f"iterate after {result.nit} iterations with objective {result.fun:.6g}."
)
mu, k = unpack(result.x)
eig_k = np.linalg.eigvals(k)
verify_inp(
np.all(np.isfinite(eig_k)) and np.all(eig_k.real > 0.),
lambda: f"Continuous-time OU calibration returned an unstable generator with eigenvalues {eig_k}",
)
return mu[None, :] @ k, k
[docs]
def fit_xar1_pca(X, time_y, *, num_alpha=0.9, rescale_var=False, use_matrix_exp=False):
r"""
Fits
$$
dX_t = ( U0 - k X_t) dt + \sqrt{dt} * Y_t
$$
and
$$
Y_t = \alpha_t @ dU
$$
"""
T, N = X.shape
verify_inp(time_y.shape == (T,), lambda: f"time_y must have shape {(T,)}, found {time_y.shape}")
verify_inp(isinstance(use_matrix_exp, bool), lambda: f"use_matrix_exp must be a bool; found {type(use_matrix_exp)}")
# dX_t = (U0 - X_t k ) * ndt + \sqrt{ndt} * N_t
# dX_t/dt = (U0 - X_t k ) + N_t / \sqrt{ndt}
dX = np.diff(X, axis=0)
dt = np.diff(time_y)
verify_inp(np.all(dt > 0.), lambda: f"dt must be positive. Found min dt {np.min(dt)}")
dtbar =np.mean(dt)
ndt = dt / dtbar
dXdt = dX / ndt[:, None]
X0 = X[:-1,:]
Xbar = np.mean(X0, axis=0, keepdims=True)
dXbar = np.mean(dXdt, axis=0, keepdims=True)
Xc = X0 - Xbar
dXc = dXdt - dXbar
slope = np.linalg.lstsq(Xc, dXc, rcond=None)[0]
k = - slope
assert k.shape == (N, N), f"Internal error: expected k to have shape {(N, N)}, but found {k.shape}"
state_center = np.zeros(N, dtype=X.dtype)
state_scale = np.ones(N, dtype=X.dtype)
if use_matrix_exp:
# Decoder state coordinates can have vastly different scales. Fit the
# continuous-time OU in standardized coordinates, then map its generator
# back with an exact similarity transform. This preserves transitions
# and eigenvalues without forcing the optimizer to handle raw magnitudes.
state_center = np.mean(X, axis=0)
state_scale = np.std(X, axis=0)
scale_floor = math.sqrt(np.finfo(X.dtype).eps) * max(1., float(np.max(state_scale)))
state_scale = np.maximum(state_scale, scale_floor)
Z = (X - state_center[None, :]) / state_scale[None, :]
k_z_initial = k * state_scale[:, None] / state_scale[None, :]
# Estimate the dominant daily affine transition and take one matrix
# logarithm rather than solving a dense nonlinear problem repeatedly.
U0_z, k_z = _fit_daily_ou_drift(
Z[:-1, :], Z[1:, :], ndt, initial_k=k_z_initial
)
k = k_z * state_scale[None, :] / state_scale[:, None]
U0 = U0_z * state_scale[None, :] + state_center[None, :] @ k
X1_drift = _xar1_matrix_exp_drift(X0, U0=U0, k=k, ndt=ndt)
res = (X[1:,:] - X1_drift) / np.sqrt(ndt)[:, None]
else:
U0 = dXbar + Xbar @ k
hatZ = U0 - X0 @ k
X1_drift = X0 + ndt[:, None] * hatZ
res = (X[1:,:] - X1_drift) / np.sqrt(ndt)[:, None]
# Report fit quality in the original state coordinates. R² measures the
# deterministic one-step drift fit; PCA explained variance below measures
# how much of the remaining innovation variance is retained for simulation.
fit_error = X[1:,:] - X1_drift
squared_error = np.sum(fit_error**2, axis=0)
centered_target = X[1:,:] - np.mean(X[1:,:], axis=0, keepdims=True)
total_squares = np.sum(centered_target**2, axis=0)
r2_by_state = np.full(N, np.nan, dtype=float)
nonconstant = total_squares > 0.
r2_by_state[nonconstant] = 1. - squared_error[nonconstant] / total_squares[nonconstant]
total_squares_all = float(np.sum(total_squares))
sample_mean = np.mean(X, axis=0)
sample_std = np.std(X, axis=0)
fitted_mean = np.linalg.lstsq(k.T, U0.reshape(N), rcond=None)[0]
fit_quality = AR1FitQuality(
r2=(1. - float(np.sum(squared_error)) / total_squares_all)
if total_squares_all > 0. else math.nan,
r2_by_state=r2_by_state,
rmse=float(np.sqrt(np.mean(fit_error**2))),
rmse_by_state=np.sqrt(np.mean(fit_error**2, axis=0)),
sample_mean=sample_mean,
sample_std=sample_std,
fitted_mean=fitted_mean,
mean_error=fitted_mean - sample_mean,
)
# PCA describes innovations around the fitted deterministic OU drift. Any
# finite-sample residual mean must not be resampled as an additional drift.
res = res - np.mean(res, axis=0, keepdims=True)
U, SD, V = np.linalg.svd(res, full_matrices=False)
ev = SD**2
vars_explained = np.cumsum(ev) / np.sum(ev)
if num_alpha is None or num_alpha == -1:
num_alpha = N
elif isinstance(num_alpha, int):
verify_inp(0 < num_alpha <= N, lambda: f"'num_alpha' must be in [1,{N}], found {num_alpha}")
else:
verify_inp(isinstance(num_alpha, float), lambda: f"'num_alpha' must be a float or int, found {type(num_alpha)}")
verify_inp(0. < num_alpha <= 1., lambda: f"'num_alpha' as float must be in (0,1], found {num_alpha}")
num_alpha = min(np.searchsorted(vars_explained, num_alpha) + 1, N)
var_explained = vars_explained[num_alpha - 1]
scale = 1. if not rescale_var else 1. / math.sqrt(var_explained)
alpha = U[:, :num_alpha] * SD[:num_alpha] * scale
dU = V[:num_alpha,:]
assert alpha.shape == (T-1, num_alpha), f"Internal error: expected alpha to have shape {(T-1, num_alpha)}, but found {alpha.shape}"
assert dU.shape == (num_alpha, N), f"Internal error: expected dU to have shape {(num_alpha, N)}, but found {dU.shape}"
return AR1PCA(
U0=U0,
dU=dU,
k=k,
alpha=alpha,
ndt=ndt,
mean_dt=dtbar,
state_center=state_center,
state_scale=state_scale,
vars_explained=vars_explained,
var_explained=var_explained,
fit_quality=fit_quality)
[docs]
def fit_surface_first_xar1_pca(
X: np.ndarray,
time_y: np.ndarray,
*,
num_alpha: int | float | None = 0.9,
rescale_var: bool = False,
use_matrix_exp: bool = False,
) -> AR1PCA:
r"""Fit stationary surface dynamics before adding the log-spot driver.
The extended state is ``X=(s,h)``: the first coordinate ``s`` is log-spot
and the remaining coordinates ``h`` are ML-SANOS implied-volatility
parameters. The surface block is fitted without spot.
Log-spot has a constant drift and no mean reversion. Its innovation is
regressed on the retained surface PCA scores; one orthogonal residual spot
factor completes the driver. Thus ``num_alpha`` counts surface factors,
while the returned joint model has one additional spot factor.
"""
verify_inp(
X.ndim == 2 and X.shape[1] >= 2,
lambda: (
"surface-first fitting requires X=(s,h)=(log_spot,surface_state) with "
f"at least two columns, got {X.shape}"
),
)
T, N = X.shape
surface = X[:, 1:]
surface_fit = fit_xar1_pca(
surface,
time_y,
num_alpha=num_alpha,
rescale_var=rescale_var,
use_matrix_exp=use_matrix_exp,
)
ndt = surface_fit.ndt
sqrt_ndt = np.sqrt(ndt)
# A constant log-spot drift makes its standardized innovation mean zero.
# The surface generator is autonomous, so only the surface block can have
# an invariant distribution.
dlog_spot = np.diff(X[:, 0])
spot_drift = float(np.sum(dlog_spot) / np.sum(ndt))
U0 = np.zeros((1, N), dtype=X.dtype)
U0[0, 0] = spot_drift
U0[0, 1:] = surface_fit.U0
k = np.zeros((N, N), dtype=X.dtype)
k[1:, 1:] = surface_fit.k
if use_matrix_exp:
X1_drift = _xar1_matrix_exp_drift(X[:-1], U0=U0, k=k, ndt=ndt)
else:
X1_drift = X[:-1] + ndt[:, None] * (U0 - X[:-1] @ k)
residual = (X[1:] - X1_drift) / sqrt_ndt[:, None]
residual -= np.mean(residual, axis=0, keepdims=True)
surface_alpha = surface_fit.alpha
spot_residual = residual[:, 0]
spot_loading = np.linalg.lstsq(
surface_alpha,
spot_residual,
rcond=None,
)[0]
idiosyncratic_spot = spot_residual - surface_alpha @ spot_loading
idiosyncratic_spot -= np.mean(idiosyncratic_spot)
surface_factors = surface_alpha.shape[1]
alpha = np.concatenate(
(surface_alpha, idiosyncratic_spot[:, None]),
axis=1,
)
dU = np.zeros((surface_factors + 1, N), dtype=X.dtype)
dU[:surface_factors, 0] = spot_loading
dU[:surface_factors, 1:] = surface_fit.dU
dU[-1, 0] = 1.0
spot_total = float(np.sum(np.square(spot_residual)))
surface_total = float(np.sum(np.square(residual[:, 1:])))
total = spot_total + surface_total
surface_cumulative = surface_fit.vars_explained * surface_total
vars_explained = np.concatenate(
(
np.asarray([spot_total / total]),
(spot_total + surface_cumulative) / total,
)
)
var_explained = float(vars_explained[surface_factors])
spot_surface_r2 = (
1.0 - float(np.sum(np.square(idiosyncratic_spot))) / spot_total
if spot_total > 0.0
else math.nan
)
fit_error = X[1:] - X1_drift
squared_error = np.sum(np.square(fit_error), axis=0)
centered_target = X[1:] - np.mean(X[1:], axis=0, keepdims=True)
total_squares = np.sum(np.square(centered_target), axis=0)
r2_by_state = np.full(N, np.nan, dtype=float)
nonconstant = total_squares > 0.0
r2_by_state[nonconstant] = (
1.0 - squared_error[nonconstant] / total_squares[nonconstant]
)
total_squares_all = float(np.sum(total_squares))
sample_mean = np.mean(X, axis=0)
sample_std = np.std(X, axis=0)
assert surface_fit.fit_quality is not None
fitted_mean = np.concatenate(
(np.asarray([math.nan]), surface_fit.fit_quality.fitted_mean)
)
mean_error = fitted_mean - sample_mean
fit_quality = AR1FitQuality(
r2=(
1.0 - float(np.sum(squared_error)) / total_squares_all
if total_squares_all > 0.0
else math.nan
),
r2_by_state=r2_by_state,
rmse=float(np.sqrt(np.mean(np.square(fit_error)))),
rmse_by_state=np.sqrt(np.mean(np.square(fit_error), axis=0)),
sample_mean=sample_mean,
sample_std=sample_std,
fitted_mean=fitted_mean,
mean_error=mean_error,
)
assert surface_fit.state_center is not None
assert surface_fit.state_scale is not None
state_center = np.concatenate(
(np.asarray([np.mean(X[:, 0])]), surface_fit.state_center)
)
state_scale = np.concatenate(
(np.asarray([np.std(X[:, 0])]), surface_fit.state_scale)
)
scale_floor = math.sqrt(np.finfo(X.dtype).eps) * max(
1.0, float(np.max(state_scale))
)
state_scale = np.maximum(state_scale, scale_floor)
assert alpha.shape == (T - 1, surface_factors + 1)
assert dU.shape == (surface_factors + 1, N)
assert vars_explained.shape == (N,)
assert np.max(np.abs(np.mean(alpha, axis=0))) < 1.0e-8
return AR1PCA(
U0=U0,
alpha=alpha,
dU=dU,
k=k,
ndt=ndt,
mean_dt=surface_fit.mean_dt,
state_center=state_center,
state_scale=state_scale,
vars_explained=vars_explained,
var_explained=var_explained,
fit_quality=fit_quality,
surface_first=True,
num_surface_alpha=surface_factors,
surface_var_explained=surface_fit.var_explained,
spot_surface_r2=spot_surface_r2,
spot_residual_std=float(np.std(idiosyncratic_spot)),
)
[docs]
def plot_sampler(sampler):
from cdxcore.dynaplot import colors_tableau, figure, m_o_m
from IPython.display import Markdown, display
from matplotlib.ticker import MaxNLocator
market = sampler.market
model = sampler.model
dates = market.dates
num_alpha = sampler.num_alpha
with figure(row_size=4, col_size=6, columns=6 ) as fig:
display(Markdown(f"## {model.qualified_label}: using {sampler.num_alpha} Alphas for {sampler.num_xstate} Extended States"))
display(Markdown(f"### Real and simulated states, discrete to normal randomness"))
r = sampler.pcaxar1
xstate = sampler.xstate
ndt = r.ndt[:,None]
dX = ( r.U0 - sampler.xstate[:-1,:] @ r.k ) * ndt + r.alpha @ r.dU * np.sqrt(ndt)
recX = np.concatenate( [ sampler.xstate[:1,:], sampler.xstate[:1,:] + np.cumsum( dX, axis=0 ) ], axis=0 )
assert recX.shape == xstate.shape, f"{recX.shape} != {xstate.shape}"
rec = np.zeros((len(dates), xstate.shape[1], 5))
rec[...,0] = xstate
rec[...,1] = sampler.hist_sample_xstates( index=0, num_steps=len(dates), num_samples=0, bandwidth=0., replay=True )
rec[...,2] = sampler.hist_sample_xstates( index=0, num_steps=len(dates), num_samples=0, bandwidth=0.1, replay=False )
rec[...,3] = sampler.hist_sample_xstates( index=0, num_steps=len(dates), num_samples=0, bandwidth=0.5, replay=False )
rec[...,4] = sampler.hist_sample_xstates( index=0, num_steps=len(dates), num_samples=0, bandwidth=1., replay=False )
labels = ["Market", "0%", "10%", "50%", "100%"]
ax_ = None
ax = fig.add_subplot(f"State 0 (log-spot)")
for j in range(rec.shape[-1]):
ax.plot( dates, rec[:,0,j], "o" if j==0 else "-", label=labels[j])
ax.tick_params(axis="x", labelrotation=90)
ax.legend(loc="upper right")
for i in range(1,xstate.shape[1]):
ax = fig.add_subplot(f"State {i}")
for j, label in enumerate(labels):
ax.plot( dates, rec[:,i,j], "o" if j==0 else "-", )
ax.tick_params(axis="x", labelrotation=90)
with figure(row_size=6, col_size=4, columns=6 ) as fig:
display(Markdown(f"### {model.qualified_label} factors"))
U0 = sampler.pcaxar1.U0
dU = sampler.pcaxar1.dU
colors = colors_tableau()
x = np.arange(U0.shape[1])
ax0 = fig.add_subplot("U0")
ax0.bar( x, U0[0,:], width=0.2, color="black")
for i in range( dU.shape[0] ):
ax = fig.add_subplot(f"dU[{i}]")
ax.bar( x, dU[i,:], width=0.2, color=next(colors))
ax.sharex(ax0)
ax0.set_ylim( m_o_m( U0, dU ) )
with figure(row_size=5, col_size=5, columns=None ) as fig:
display(Markdown(f"### {model.qualified_label} weights"))
mev = sampler.pcaxar1.vars_explained
ax = fig.add_subplot("Explained")
fig.render()
ax.plot( np.arange(0,len(mev)), mev*100., ":", color="orange", label="All")
ax.plot( np.arange(0,num_alpha), mev[:num_alpha]*100., "-", color="red",label=f"First {num_alpha}")
ax.set_ylabel("% of total variance explained")
ax.set_xlabel("Number of Eigenvalues")
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
with figure(row_size=6, col_size=4, columns=6 ) as fig:
display(Markdown(f"### {model.qualified_label} weights"))
ax_ = None
colors = colors_tableau()
alpha = sampler.pcaxar1.alpha
for i in range(alpha.shape[1]):
ax = fig.add_subplot(f"Alpha {i}")
ax.plot( dates[:-1], alpha[:,i], "*:", label="Model", color=next(colors))
ax.tick_params(axis="x", labelrotation=90)
if not ax_ is None:
ax.sharey(ax_)
else:
ax_ = ax
ax_.set_ylim( m_o_m(alpha) )
display(Markdown(f"### {model.qualified_label} fit quality"))
display(sampler.pcaxar1.fit_quality)