"""Blockwise coherent spot/option universes for direct PCA IV surfaces."""
from __future__ import annotations
import math
from collections.abc import Iterator
from dataclasses import dataclass, field
import numpy as np
from cdxcore.err import verify_inp
from cdxcore.verbose import Context
from quantdata.dysanos.grid.gendysanos.terminal_universe import (
TerminalPathDiagnostics,
build_terminal_path_universe,
expiry_step_offsets,
)
from quantdata.dysanos.grid.gendysanos.torch_prep import (
_importance_control_diagnostics,
_importance_target_steps,
)
from .decoder import PCATotalVarianceDecoder, WingHitDiagnostics
from .experiment import ImpliedPCAExperiment
from .simulator import ImpliedPCASampler
from .static_arbitrage import StaticArbitrageAccumulator
[docs]
@dataclass(frozen=True, kw_only=True)
class PCAUniverseBlock:
"""One independently simulated path block and its global row interval."""
start: int
stop: int
universe: dict[str, np.ndarray]
[docs]
@dataclass(frozen=True, kw_only=True)
class PCAMatchedUniverseBlock:
"""Expiry and incremental markets derived from one simulated path block."""
start: int
stop: int
expiry: dict[str, np.ndarray]
incremental: dict[str, np.ndarray]
[docs]
@dataclass
class PCAUniverseStreamDiagnostics:
"""Mutable online diagnostics finalized after a universe stream is consumed."""
wing_hits: WingHitDiagnostics = WingHitDiagnostics(
queries=0, left_hits=0, right_hits=0
)
minimum_log_spot: float = math.inf
maximum_log_spot: float = -math.inf
importance_accumulator: dict[str, np.ndarray | float] | None = None
importance_component_counts: np.ndarray | None = None
minimum_log_weight: float = math.inf
maximum_log_weight: float = -math.inf
_path_geometry: TerminalPathDiagnostics | None = field(
default=None, init=False, repr=False
)
[docs]
def add_wing_hits(self, value: WingHitDiagnostics) -> None:
self.wing_hits = self.wing_hits + value
[docs]
def traded_nms_indices(grid_strikes: int, num_traded_nms: int) -> np.ndarray:
"""Return the deterministic evenly spaced subset of the training grid."""
verify_inp(
isinstance(grid_strikes, int)
and isinstance(num_traded_nms, int)
and 0 < num_traded_nms <= grid_strikes,
lambda: f"invalid grid/traded strike counts {grid_strikes}/{num_traded_nms}",
)
indices = np.rint(
np.linspace(0, grid_strikes - 1, num_traded_nms)
).astype(np.int64)
verify_inp(
np.unique(indices).size == num_traded_nms,
lambda: f"deterministic traded-grid selection is not unique: {indices}",
)
return indices
[docs]
def default_static_strike_mesh(points: int = 201) -> np.ndarray:
"""Dense fixed relative-strike mesh containing ATM exactly."""
verify_inp(
isinstance(points, int) and points >= 5 and points % 2 == 1,
lambda: f"static strike mesh requires an odd integer >=5, got {points!r}",
)
half = points // 2
return np.concatenate(
(
np.linspace(0.5, 1.0, half, endpoint=False),
np.linspace(1.0, 1.5, half + 1),
)
)
def _update_importance_accumulator(
diagnostics: PCAUniverseStreamDiagnostics,
*,
importance_sample,
) -> None:
log_weight = importance_sample.log_weights
weight = np.exp(log_weight)
log_spot = importance_sample.paths[..., 0].astype(np.float64, copy=False)
realized_variance = np.square(np.diff(log_spot, axis=1)).sum(axis=1)
block: dict[str, np.ndarray | float] = {
"weight_sum": float(weight.sum()),
"weight_square_sum": float(np.square(weight).sum()),
"reference_log_spot_sum": importance_sample.reference_log_spot_sum,
"reference_log_spot_square_sum": (
importance_sample.reference_log_spot_square_sum
),
"weighted_log_spot_sum": (weight[:, None] * log_spot).sum(axis=0),
"weighted_log_spot_square_sum": (
weight[:, None] * np.square(log_spot)
).sum(axis=0),
"reference_rv_sum": importance_sample.reference_realized_variance_sum,
"reference_rv_square_sum": (
importance_sample.reference_realized_variance_square_sum
),
"weighted_rv_sum": float(np.dot(weight, realized_variance)),
"weighted_rv_square_sum": float(
np.dot(weight, np.square(realized_variance))
),
}
if diagnostics.importance_accumulator is None:
diagnostics.importance_accumulator = block
else:
for name, value in block.items():
diagnostics.importance_accumulator[name] += value
assert diagnostics.importance_component_counts is not None
diagnostics.importance_component_counts += np.bincount(
importance_sample.proposal_components,
minlength=diagnostics.importance_component_counts.size,
)
diagnostics.minimum_log_weight = min(
diagnostics.minimum_log_weight, float(np.min(log_weight))
)
diagnostics.maximum_log_weight = max(
diagnostics.maximum_log_weight, float(np.max(log_weight))
)
[docs]
def iter_pca_universe_blocks(
experiment: ImpliedPCAExperiment,
*,
decoder: PCATotalVarianceDecoder,
sampler: ImpliedPCASampler,
diagnostics: PCAUniverseStreamDiagnostics,
static_accumulator: StaticArbitrageAccumulator | None = None,
static_strikes: np.ndarray | None = None,
verbose: Context = Context.all,
) -> Iterator[PCAUniverseBlock]:
"""Yield coherent path blocks without materializing a full second copy."""
verify_inp(
decoder.pca is sampler.pca,
"decoder and sampler must refer to the same fitted PCA representation",
)
expiries = decoder.pca.expiries
T, grid_strikes = decoder.pca.surface_shape
verify_inp(
grid_strikes == experiment.grid_strikes
and decoder.pca.num_components == experiment.num_surface_state,
lambda: (
"PCA geometry differs from experiment: "
f"grid {grid_strikes}/{experiment.grid_strikes}, states "
f"{decoder.pca.num_components}/{experiment.num_surface_state}"
),
)
expiry_offsets, maximum_expiry_time_error = expiry_step_offsets(
expiries, dt=experiment.dt
)
trading_times = experiment.trading_times
stride = experiment.trading_step_stride
option_return_offset = (
int(expiry_offsets[-1])
if experiment.return_horizon == "expiry"
else stride
)
last_action = (trading_times - 1) * stride
simulated_steps = last_action + max(option_return_offset, stride) + 1
traded_indices = traded_nms_indices(
grid_strikes, experiment.num_traded_nms
)
traded_nms = decoder.pca.nms[:, traded_indices]
trading_indices = np.arange(trading_times, dtype=np.int64) * stride
importance_config = experiment.importance_sampling
target_steps = (
_importance_target_steps(
trading_times=trading_times,
trading_step_stride=stride,
return_offsets=(
expiry_offsets
if experiment.return_horizon == "expiry"
else np.asarray([option_return_offset], dtype=np.int64)
),
action_count=importance_config.target_action_count,
)
if importance_config is not None
else ()
)
importance_rng = np.random.default_rng(experiment.seed + 8_000_003)
if importance_config is not None:
diagnostics.importance_component_counts = np.zeros(
len(target_steps) + 1, dtype=np.int64
)
if static_accumulator is not None:
verify_inp(
static_strikes is not None,
"a static strike mesh is required with a static accumulator",
)
static_strikes = np.asarray(static_strikes, dtype=np.float64)
verify_inp(
np.array_equal(static_accumulator.strikes, static_strikes)
and np.array_equal(static_accumulator.expiries, expiries),
"static accumulator geometry differs from the PCA decoder",
)
for start in range(0, experiment.num_samples, experiment.block_size):
stop = min(start + experiment.block_size, experiment.num_samples)
block_samples = stop - start
if importance_config is None:
importance_sample = None
xstate_path = sampler.free_sample_xstates(
num_steps=simulated_steps,
num_samples=block_samples,
dt=experiment.dt,
bandwidth=1.0,
)
else:
importance_sample = sampler.importance_sample_xstates(
num_steps=simulated_steps,
num_samples=block_samples,
dt=experiment.dt,
target_steps=target_steps,
config=importance_config,
proposal_rng=importance_rng,
)
xstate_path = importance_sample.paths
_update_importance_accumulator(
diagnostics, importance_sample=importance_sample
)
if not np.all(np.isfinite(xstate_path)):
raise FloatingPointError("PCA state sampler produced non-finite paths")
diagnostics.minimum_log_spot = min(
diagnostics.minimum_log_spot,
float(np.min(xstate_path[..., 0])),
)
diagnostics.maximum_log_spot = max(
diagnostics.maximum_log_spot,
float(np.max(xstate_path[..., 0])),
)
action_scores = xstate_path[:, trading_indices, 1:]
action_log_spot = xstate_path[:, trading_indices, 0]
action_spot = np.exp(action_log_spot)
maturity_query = expiries[:, None]
nms_query = traded_nms
relative_strikes = decoder.relative_strikes_from_nms(
action_scores, maturity_query, nms_query
)
normalized_calls, call_hits = decoder.normalized_call_prices(
action_scores, maturity_query, relative_strikes
)
cash_strikes = relative_strikes * action_spot[:, :, None, None]
deltas, delta_hits = decoder.cash_deltas(
action_scores,
action_spot[:, :, None, None],
maturity_query,
cash_strikes,
)
diagnostics.add_wing_hits(call_hits + delta_hits)
calls_tp1 = None
if experiment.return_horizon == "incremental":
next_indices = trading_indices + option_return_offset
next_scores = xstate_path[:, next_indices, 1:]
next_spot = np.exp(xstate_path[:, next_indices, 0])
remaining_maturity = (
expiries - option_return_offset * experiment.dt
)
alive = expiry_offsets > option_return_offset
calls_tp1 = np.zeros_like(normalized_calls)
if np.any(alive):
relative_strikes_tp1 = (
cash_strikes[:, :, alive, :]
/ next_spot[:, :, None, None]
)
alive_calls, repricing_hits = decoder.normalized_call_prices(
next_scores,
remaining_maturity[alive][None, None, :, None],
relative_strikes_tp1,
)
calls_tp1[:, :, alive, :] = alive_calls
diagnostics.add_wing_hits(repricing_hits)
if static_accumulator is not None:
assert static_strikes is not None
static_prices, static_hits = decoder.normalized_call_prices(
action_scores,
expiries[:, None],
static_strikes[None, :],
)
static_nms = decoder.nms_from_relative_strikes(
action_scores,
expiries[:, None],
static_strikes[None, :],
)
diagnostics.add_wing_hits(static_hits)
static_accumulator.update(
static_prices,
normalized_moneyness=static_nms,
log_path_weights=(
None
if importance_sample is None
else importance_sample.log_weights
),
)
universe = build_terminal_path_universe(
xstate_path=xstate_path,
trading_times=trading_times,
trading_step_stride=stride,
expiries=expiries,
expiry_offsets=expiry_offsets,
normalized_strikes=relative_strikes,
normalized_calls=normalized_calls,
normalized_deltas=deltas,
normalized_calls_tp1=calls_tp1,
option_return_step_offset=option_return_offset,
require_bounded_deltas=False,
)
if importance_sample is not None:
universe["log_path_weight"] = importance_sample.log_weights
universe["importance_component"] = (
importance_sample.proposal_components
)
for name, value in universe.items():
if not np.all(np.isfinite(value)):
raise FloatingPointError(
f"generated PCA universe field {name!r} is non-finite"
)
verbose.write(
lambda start=start, stop=stop: (
f"Generated PCA {experiment.mode}/{experiment.option_return_horizon} "
f"paths {start:,}:{stop:,}/{experiment.num_samples:,}"
)
)
yield PCAUniverseBlock(start=start, stop=stop, universe=universe)
diagnostics._path_geometry = TerminalPathDiagnostics(
mode=experiment.mode,
option_return_horizon=experiment.option_return_horizon,
trading_times=trading_times,
trading_step_stride=stride,
simulated_state_steps=simulated_steps,
expiry_step_offsets=tuple(int(value) for value in expiry_offsets),
dt=experiment.dt,
maximum_expiry_time_error=maximum_expiry_time_error,
minimum_log_spot=diagnostics.minimum_log_spot,
maximum_log_spot=diagnostics.maximum_log_spot,
importance_sampling=(
None
if importance_config is None
else _importance_control_diagnostics(
diagnostics.importance_accumulator,
samples=experiment.num_samples,
config=importance_config,
target_steps=target_steps,
component_counts=diagnostics.importance_component_counts,
minimum_log_weight=diagnostics.minimum_log_weight,
maximum_log_weight=diagnostics.maximum_log_weight,
)
),
)
[docs]
def iter_pca_matched_universe_blocks(
experiment: ImpliedPCAExperiment,
*,
decoder: PCATotalVarianceDecoder,
sampler: ImpliedPCASampler,
diagnostics: PCAUniverseStreamDiagnostics,
static_accumulator: StaticArbitrageAccumulator | None = None,
static_strikes: np.ndarray | None = None,
verbose: Context = Context.all,
) -> Iterator[PCAMatchedUniverseBlock]:
"""Yield both return horizons from one sufficiently long coherent path.
The experiment's ``return_horizon`` is deliberately ignored here. It is
not a simulation input: the same ten decoded trading surfaces and daily
spot path determine the ten expiry cohorts and nine incremental intervals.
"""
verify_inp(
experiment.num_trading_times == 10,
"matched reported markets require ten trading timestamps",
)
verify_inp(
decoder.pca is sampler.pca,
"decoder and sampler must refer to the same fitted PCA representation",
)
expiries = decoder.pca.expiries
T, grid_strikes = decoder.pca.surface_shape
verify_inp(
grid_strikes == experiment.grid_strikes
and decoder.pca.num_components == experiment.num_surface_state,
"PCA geometry differs from the versioned experiment",
)
expiry_offsets, maximum_expiry_time_error = expiry_step_offsets(
expiries, dt=experiment.dt
)
action_times = experiment.num_trading_times
stride = experiment.trading_step_stride
trading_indices = np.arange(action_times, dtype=np.int64) * stride
simulated_steps = int(trading_indices[-1] + expiry_offsets[-1] + 1)
traded_indices = traded_nms_indices(
grid_strikes, experiment.num_traded_nms
)
traded_nms = decoder.pca.nms[:, traded_indices]
importance_config = experiment.importance_sampling
target_steps = (
_importance_target_steps(
trading_times=action_times,
trading_step_stride=stride,
return_offsets=np.unique(
np.concatenate((expiry_offsets, np.asarray([stride])))
),
action_count=importance_config.target_action_count,
)
if importance_config is not None
else ()
)
importance_rng = np.random.default_rng(experiment.seed + 8_000_003)
if importance_config is not None:
diagnostics.importance_component_counts = np.zeros(
len(target_steps) + 1, dtype=np.int64
)
if static_accumulator is not None:
verify_inp(
static_strikes is not None,
"a static strike mesh is required with a static accumulator",
)
static_strikes = np.asarray(static_strikes, dtype=np.float64)
verify_inp(
np.array_equal(static_accumulator.strikes, static_strikes)
and np.array_equal(static_accumulator.expiries, expiries),
"static accumulator geometry differs from the PCA decoder",
)
for start in range(0, experiment.num_samples, experiment.block_size):
stop = min(start + experiment.block_size, experiment.num_samples)
block_samples = stop - start
if importance_config is None:
importance_sample = None
xstate_path = sampler.free_sample_xstates(
num_steps=simulated_steps,
num_samples=block_samples,
dt=experiment.dt,
bandwidth=1.0,
)
else:
importance_sample = sampler.importance_sample_xstates(
num_steps=simulated_steps,
num_samples=block_samples,
dt=experiment.dt,
target_steps=target_steps,
config=importance_config,
proposal_rng=importance_rng,
)
xstate_path = importance_sample.paths
_update_importance_accumulator(
diagnostics, importance_sample=importance_sample
)
if not np.all(np.isfinite(xstate_path)):
raise FloatingPointError("PCA state sampler produced non-finite paths")
diagnostics.minimum_log_spot = min(
diagnostics.minimum_log_spot, float(np.min(xstate_path[..., 0]))
)
diagnostics.maximum_log_spot = max(
diagnostics.maximum_log_spot, float(np.max(xstate_path[..., 0]))
)
action_scores = xstate_path[:, trading_indices, 1:]
action_spot = np.exp(xstate_path[:, trading_indices, 0])
relative_strikes = decoder.relative_strikes_from_nms(
action_scores, expiries[:, None], traded_nms
)
normalized_calls, call_hits = decoder.normalized_call_prices(
action_scores, expiries[:, None], relative_strikes
)
cash_strikes = relative_strikes * action_spot[:, :, None, None]
deltas, delta_hits = decoder.cash_deltas(
action_scores,
action_spot[:, :, None, None],
expiries[:, None],
cash_strikes,
)
diagnostics.add_wing_hits(call_hits + delta_hits)
incremental_calls = np.zeros_like(normalized_calls[:, :-1])
alive = expiry_offsets > stride
if np.any(alive):
next_scores = action_scores[:, 1:]
next_spot = action_spot[:, 1:]
next_relative_strikes = (
cash_strikes[:, :-1, alive]
/ next_spot[:, :, None, None]
)
alive_calls, repricing_hits = decoder.normalized_call_prices(
next_scores,
(expiries[alive] - stride * experiment.dt)[None, None, :, None],
next_relative_strikes,
)
incremental_calls[:, :, alive] = alive_calls
diagnostics.add_wing_hits(repricing_hits)
if static_accumulator is not None:
assert static_strikes is not None
static_prices, static_hits = decoder.normalized_call_prices(
action_scores,
expiries[:, None],
static_strikes[None, :],
)
static_nms = decoder.nms_from_relative_strikes(
action_scores,
expiries[:, None],
static_strikes[None, :],
)
diagnostics.add_wing_hits(static_hits)
static_accumulator.update(
static_prices,
normalized_moneyness=static_nms,
log_path_weights=(
None
if importance_sample is None
else importance_sample.log_weights
),
)
expiry = build_terminal_path_universe(
xstate_path=xstate_path,
trading_times=action_times,
trading_step_stride=stride,
expiries=expiries,
expiry_offsets=expiry_offsets,
normalized_strikes=relative_strikes,
normalized_calls=normalized_calls,
normalized_deltas=deltas,
require_bounded_deltas=False,
)
incremental = build_terminal_path_universe(
xstate_path=xstate_path,
trading_times=action_times - 1,
trading_step_stride=stride,
expiries=expiries,
expiry_offsets=expiry_offsets,
normalized_strikes=relative_strikes[:, :-1],
normalized_calls=normalized_calls[:, :-1],
normalized_deltas=deltas[:, :-1],
normalized_calls_tp1=incremental_calls,
option_return_step_offset=stride,
require_bounded_deltas=False,
)
if importance_sample is not None:
for universe in (expiry, incremental):
universe["log_path_weight"] = importance_sample.log_weights
universe["importance_component"] = (
importance_sample.proposal_components
)
for horizon, universe in (
("expiry", expiry),
("incremental", incremental),
):
for name, value in universe.items():
if not np.all(np.isfinite(value)):
raise FloatingPointError(
f"generated PCA {horizon} field {name!r} is non-finite"
)
verbose.write(
lambda start=start, stop=stop: (
f"Generated matched PCA {experiment.trading_frequency} paths "
f"{start:,}:{stop:,}/{experiment.num_samples:,}"
)
)
yield PCAMatchedUniverseBlock(
start=start,
stop=stop,
expiry=expiry,
incremental=incremental,
)
diagnostics._path_geometry = TerminalPathDiagnostics(
mode=f"matched-{experiment.trading_frequency}",
option_return_horizon="expiry+incremental",
trading_times=action_times,
trading_step_stride=stride,
simulated_state_steps=simulated_steps,
expiry_step_offsets=tuple(int(value) for value in expiry_offsets),
dt=experiment.dt,
maximum_expiry_time_error=maximum_expiry_time_error,
minimum_log_spot=diagnostics.minimum_log_spot,
maximum_log_spot=diagnostics.maximum_log_spot,
importance_sampling=(
None
if importance_config is None
else _importance_control_diagnostics(
diagnostics.importance_accumulator,
samples=experiment.num_samples,
config=importance_config,
target_steps=target_steps,
component_counts=diagnostics.importance_component_counts,
minimum_log_weight=diagnostics.minimum_log_weight,
maximum_log_weight=diagnostics.maximum_log_weight,
)
),
)
[docs]
def finalize_stream_diagnostics(
diagnostics: PCAUniverseStreamDiagnostics,
) -> TerminalPathDiagnostics:
"""Return path diagnostics after fully consuming the block iterator."""
verify_inp(
diagnostics._path_geometry is not None,
"PCA universe stream must be fully consumed before finalization",
)
assert diagnostics._path_geometry is not None
return diagnostics._path_geometry
__all__ = [
"PCAMatchedUniverseBlock",
"PCAUniverseBlock",
"PCAUniverseStreamDiagnostics",
"default_static_strike_mesh",
"finalize_stream_diagnostics",
"iter_pca_matched_universe_blocks",
"iter_pca_universe_blocks",
"traded_nms_indices",
]