"""Exact adaptive-neighbour diagnostics for conditional option-return tails.
The estimator uses a uniform ``k``-nearest-neighbour kernel. Its weights are
positive, so estimates of positive and negative return parts remain
nonnegative. Training valuation is leave-one-out; validation queries always
use the training split as their reference sample.
"""
from __future__ import annotations
import math
import time
from dataclasses import dataclass
from typing import Any, Callable, Literal, Mapping
import numpy as np
import torch
from cdxcore.err import verify_inp
from cdxcore.version import version
from torch import Tensor
from .torch_drift_change import UniverseGeometry, validate_universe
from .unconditional_option_test import (
DEFAULT_ROUNDOFF_MULTIPLIER,
delta_hedged_option_returns,
delta_hedged_roundoff_band,
)
StateMode = Literal["raw", "instrument"]
LONG_DIRECTION = 0
SHORT_DIRECTION = 1
DIRECTIONS = ("long_dH", "short_minus_dH")
[docs]
@dataclass(frozen=True, kw_only=True)
class LocalTailProgress:
"""Cumulative one-sided-support counts for a valuation in progress."""
split: str
processed_states: int
total_states: int
states_with_any_long_candidate: int
states_with_any_short_candidate: int
long_only_candidate_states: int
short_only_candidate_states: int
both_direction_candidate_states: int
[docs]
@version("1.2")
@dataclass(frozen=True, kw_only=True)
class LocalTailConfig:
"""Configuration for an exact uniform adaptive-neighbour estimator."""
state_mode: StateMode
neighbors: int = 1_024
instrument_candidate_neighbors: int = 8_192
tail_ranks: tuple[int, ...] = (1, 2, 10, 20, 50)
signal_thresholds: tuple[float, ...] = (2.0, 3.0)
query_batch_size: int = 3_584
instrument_batch_size: int = 20
roundoff_multiplier: float = DEFAULT_ROUNDOFF_MULTIPLIER
state_scale_floor: float = 1.0e-6
def __post_init__(self) -> None:
verify_inp(
self.state_mode in {"raw", "instrument"},
lambda: f"unknown state mode {self.state_mode!r}",
)
for name in (
"neighbors",
"instrument_candidate_neighbors",
"query_batch_size",
"instrument_batch_size",
):
value = getattr(self, name)
verify_inp(
isinstance(value, int) and value > 0,
lambda name=name, value=value: f"'{name}' must be positive, got {value!r}",
)
verify_inp(
self.instrument_candidate_neighbors >= self.neighbors,
lambda: (
"instrument candidate neighbours must be at least the final "
f"neighbour count, got {self.instrument_candidate_neighbors} < {self.neighbors}"
),
)
verify_inp(bool(self.tail_ranks), "at least one local tail rank is required")
verify_inp(
all(isinstance(rank, int) and rank > 0 for rank in self.tail_ranks),
lambda: f"tail ranks must be positive integers, got {self.tail_ranks}",
)
verify_inp(
tuple(sorted(set(self.tail_ranks))) == self.tail_ranks,
lambda: f"tail ranks must be unique and increasing, got {self.tail_ranks}",
)
verify_inp(
bool(self.signal_thresholds)
and all(
math.isfinite(value) and value > 0.0
for value in self.signal_thresholds
)
and tuple(sorted(set(self.signal_thresholds)))
== self.signal_thresholds,
lambda: (
"signal thresholds must be finite, positive, unique, and "
f"increasing, got {self.signal_thresholds}"
),
)
verify_inp(
2 * self.tail_ranks[-1] <= self.neighbors,
lambda: (
f"largest tail rank {self.tail_ranks[-1]} needs at most half of "
f"{self.neighbors} neighbours"
),
)
for name in ("roundoff_multiplier", "state_scale_floor"):
value = getattr(self, name)
verify_inp(
math.isfinite(value) and value > 0.0,
lambda name=name, value=value: f"'{name}' must be finite and positive, got {value}",
)
@property
def local_tail_probabilities(self) -> tuple[float, ...]:
return tuple(rank / self.neighbors for rank in self.tail_ranks)
@property
def zero_event_upper_probability_95(self) -> float:
"""Exact iid binomial upper bound after zero events in ``neighbors`` draws."""
return 1.0 - 0.05 ** (1.0 / self.neighbors)
[docs]
def prepare_delta_hedged_local_data(
universe: Mapping[str, Any],
*,
roundoff_multiplier: float = DEFAULT_ROUNDOFF_MULTIPLIER,
) -> tuple[dict[str, np.ndarray], UniverseGeometry]:
"""Return delta-hedged option returns, roundoff bands, and state fields."""
geometry = validate_universe(universe)
dH = delta_hedged_option_returns(
dC=np.asarray(universe["dC"]),
delta=np.asarray(universe["delta"]),
dS=np.asarray(universe["dS"]),
)
roundoff = delta_hedged_roundoff_band(
C=np.asarray(universe["C"]),
dC=np.asarray(universe["dC"]),
delta=np.asarray(universe["delta"]),
dS=np.asarray(universe["dS"]),
multiplier=roundoff_multiplier,
)
xstate = np.asarray(universe["xstate"], dtype=np.float32)[
:, : geometry.steps
]
log_strikes = np.asarray(universe["log_strikes"], dtype=np.float32).reshape(
geometry.samples, geometry.steps, -1
)
expiries = np.asarray(universe["expiries"], dtype=np.float32).reshape(-1)
expected = (geometry.samples, geometry.steps, geometry.expiries * geometry.strikes)
assert dH.shape == roundoff.shape == log_strikes.shape == expected
assert xstate.shape == (
geometry.samples,
geometry.steps,
geometry.state_variables,
)
if not all(np.isfinite(value).all() for value in (dH, roundoff, xstate, log_strikes)):
raise ValueError("prepared local-tail data contain non-finite values")
if np.any(roundoff <= 0.0):
raise ValueError("local-tail roundoff bands must be positive")
return {
"returns": dH.astype(np.float32, copy=False),
"roundoff": roundoff.astype(np.float32, copy=False),
"xstate": xstate,
"log_strikes": log_strikes,
"expiries": expiries,
}, geometry
[docs]
def reduced_option_state(
data: Mapping[str, np.ndarray],
*,
geometry: UniverseGeometry,
mode: StateMode,
instrument: int = 0,
) -> np.ndarray:
"""Return ``xstate`` or one option's ``(xstate, log-strike, expiry)``."""
verify_inp(mode in {"raw", "instrument"}, lambda: f"unknown state mode {mode!r}")
instruments = geometry.expiries * geometry.strikes
verify_inp(
0 <= instrument < instruments,
lambda: f"instrument {instrument} is outside [0, {instruments})",
)
xstate = np.asarray(data["xstate"], dtype=np.float32)
if mode == "raw":
return xstate
log_strike = np.asarray(data["log_strikes"], dtype=np.float32)[
..., instrument : instrument + 1
]
expiry_index = instrument // geometry.strikes
expiry = np.full(
(*xstate.shape[:-1], 1),
np.asarray(data["expiries"], dtype=np.float32)[expiry_index],
dtype=np.float32,
)
output = np.concatenate((xstate, log_strike, expiry), axis=-1)
assert output.shape[:-1] == xstate.shape[:-1]
assert output.shape[-1] == geometry.state_variables + 2
return output
[docs]
def state_normalization(
training_state: np.ndarray,
*,
scale_floor: float,
path_weights: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Return training-only weighted population location and scale."""
verify_inp(
training_state.ndim == 3 and training_state.shape[-1] > 0,
lambda: f"training state must have shape (S, TAU, F), got {training_state.shape}",
)
verify_inp(
math.isfinite(scale_floor) and scale_floor > 0.0,
lambda: f"scale floor must be finite and positive, got {scale_floor}",
)
flat = training_state.reshape(-1, training_state.shape[-1]).astype(
np.float64, copy=False
)
if path_weights is None:
mean = flat.mean(axis=0, dtype=np.float64)
scale = flat.std(axis=0, dtype=np.float64)
else:
weights = np.asarray(path_weights, dtype=np.float64)
verify_inp(
weights.shape == (training_state.shape[0],),
lambda: (
"path weights must have one value per independent path, got "
f"{weights.shape} for state {training_state.shape}"
),
)
if not np.all(np.isfinite(weights)) or not np.all(weights > 0.0):
raise ValueError("state-normalization path weights must be finite and positive")
observation_weights = np.repeat(weights, training_state.shape[1])
mass = float(observation_weights.sum())
mean = np.sum(observation_weights[:, None] * flat, axis=0) / mass
variance = (
np.sum(observation_weights[:, None] * np.square(flat - mean), axis=0)
/ mass
)
scale = np.sqrt(np.maximum(variance, 0.0))
constant = scale < scale_floor
scale = np.maximum(scale, scale_floor)
return mean.astype(np.float32), scale.astype(np.float32), constant
[docs]
def normalize_state(
state: np.ndarray, *, mean: np.ndarray, scale: np.ndarray
) -> np.ndarray:
"""Normalize a state array without changing its shape or float32 dtype."""
verify_inp(
state.shape[-1] == mean.shape[0] == scale.shape[0],
lambda: f"state/normalization shapes differ: {state.shape}/{mean.shape}/{scale.shape}",
)
if not np.isfinite(state).all() or not np.isfinite(mean).all() or not np.isfinite(scale).all():
raise ValueError("state normalization contains non-finite values")
if np.any(scale <= 0.0):
raise ValueError("state normalization scale must be positive")
return ((state - mean) / scale).astype(np.float32, copy=False)
[docs]
def shared_instrument_metric_diagnostics(
data: Mapping[str, np.ndarray],
*,
training_paths: int,
geometry: UniverseGeometry,
scale_floor: float,
sample_states: int = 4_096,
) -> dict[str, Any]:
"""Test whether every option produces the same standardized strike metric.
When log strikes differ only by instrument-specific constants, their
standardized values are identical. Expiries are constant and therefore
have zero distance. In that case one exact neighbour graph is valid for
every separately estimated option.
"""
log_strikes = np.asarray(data["log_strikes"], dtype=np.float32)[
:training_paths
].reshape(-1, geometry.expiries * geometry.strikes)
mean = log_strikes.mean(axis=0, dtype=np.float64)
scale = log_strikes.std(axis=0, dtype=np.float64)
constant_strikes = scale < scale_floor
safe_scale = np.maximum(scale, scale_floor)
count = min(sample_states, log_strikes.shape[0])
sample_index = np.linspace(0, log_strikes.shape[0] - 1, count, dtype=np.int64)
standardized = (
log_strikes[sample_index].astype(np.float64) - mean
) / safe_scale
deviation = np.max(np.abs(standardized - standardized[:, :1]))
expiries = np.asarray(data["expiries"], dtype=np.float32)
output = {
"sample_states": count,
"max_abs_standardized_log_strike_difference": float(deviation),
"constant_log_strike_instruments": int(constant_strikes.sum()),
"expiry_values": expiries.astype(float).tolist(),
"expiry_is_constant_across_states": True,
"shared_metric": bool(deviation <= 5.0e-5),
}
return output
def _weighted_lower_tail(
values: Tensor,
normalized_weights: Tensor,
*,
probabilities: tuple[float, ...],
) -> tuple[Tensor, Tensor]:
"""Return weighted lower quantiles and CVaRs along the neighbour axis."""
sorted_values, order = torch.sort(values, dim=1)
sorted_weights = torch.gather(normalized_weights, 1, order)
cumulative = torch.cumsum(sorted_weights, dim=1)
cumulative_before = cumulative - sorted_weights
quantiles: list[Tensor] = []
cvars: list[Tensor] = []
for probability in probabilities:
threshold = torch.as_tensor(
probability, dtype=values.dtype, device=values.device
)
index = torch.argmax((cumulative >= threshold).to(torch.int8), dim=1)
quantiles.append(
torch.gather(sorted_values, 1, index[:, None, :]).squeeze(1)
)
included = torch.minimum(
sorted_weights,
torch.relu(threshold - cumulative_before),
)
cvars.append((sorted_values * included).sum(dim=1) / threshold)
return torch.stack(quantiles, dim=1), torch.stack(cvars, dim=1)
[docs]
def local_neighbor_statistics(
neighbor_returns: Tensor,
neighbor_roundoff: Tensor,
*,
tail_ranks: tuple[int, ...],
neighbor_log_weights: Tensor | None = None,
) -> dict[str, Tensor]:
"""Return positive/negative parts and signed lower-tail order statistics.
Inputs have shape ``(queries, neighbours, instruments)``. Direction zero
is the long return and direction one is the negated return.
"""
verify_inp(
neighbor_returns.shape == neighbor_roundoff.shape
and neighbor_returns.ndim == 3,
lambda: (
"neighbour returns/roundoff must share shape (Q, K, J), got "
f"{neighbor_returns.shape}/{neighbor_roundoff.shape}"
),
)
neighbors = neighbor_returns.shape[1]
verify_inp(
bool(tail_ranks)
and tuple(sorted(set(tail_ranks))) == tail_ranks
and 2 * tail_ranks[-1] <= neighbors,
lambda: f"invalid tail ranks {tail_ranks} for {neighbors} neighbours",
)
if not torch.isfinite(neighbor_returns).all().item():
raise ValueError("neighbour returns contain non-finite values")
if not torch.isfinite(neighbor_roundoff).all().item() or not torch.all(
neighbor_roundoff >= 0.0
).item():
raise ValueError("neighbour roundoff is non-finite or negative")
weighted = neighbor_log_weights is not None
if neighbor_log_weights is None:
normalized_weights = torch.full_like(
neighbor_returns,
1.0 / neighbors,
)
else:
verify_inp(
neighbor_log_weights.shape
in {neighbor_returns.shape[:2], neighbor_returns.shape},
lambda: (
"neighbour log weights must have shape (Q,K) or (Q,K,J), got "
f"{neighbor_log_weights.shape} for {neighbor_returns.shape}"
),
)
if not torch.isfinite(neighbor_log_weights).all().item():
raise ValueError("neighbour log weights contain non-finite values")
log_weights = neighbor_log_weights.to(neighbor_returns.dtype)
if log_weights.ndim == 2:
log_weights = log_weights[..., None].expand_as(neighbor_returns)
normalized_weights = torch.softmax(log_weights, dim=1)
positive = torch.relu(neighbor_returns - neighbor_roundoff)
negative = torch.relu(-neighbor_returns - neighbor_roundoff)
positive_event = neighbor_returns > neighbor_roundoff
negative_event = neighbor_returns < -neighbor_roundoff
mean = (normalized_weights * neighbor_returns).sum(dim=1)
second_moment = (
normalized_weights * neighbor_returns.square()
).sum(dim=1)
variance = (second_moment - mean.square()).clamp_min(0.0)
if weighted:
probabilities = tuple(rank / neighbors for rank in tail_ranks)
long_quantile, long_cvar = _weighted_lower_tail(
neighbor_returns,
normalized_weights,
probabilities=probabilities,
)
short_quantile, short_cvar = _weighted_lower_tail(
-neighbor_returns,
normalized_weights,
probabilities=probabilities,
)
else:
maximum_rank = tail_ranks[-1]
lower = torch.topk(
neighbor_returns,
maximum_rank,
dim=1,
largest=False,
sorted=True,
).values
upper = torch.topk(
neighbor_returns,
maximum_rank,
dim=1,
largest=True,
sorted=True,
).values
long_quantile = torch.stack(
[lower[:, rank - 1] for rank in tail_ranks], dim=1
)
short_quantile = torch.stack(
[-upper[:, rank - 1] for rank in tail_ranks], dim=1
)
long_cvar = torch.stack(
[lower[:, :rank].mean(dim=1) for rank in tail_ranks], dim=1
)
short_cvar = torch.stack(
[-upper[:, :rank].mean(dim=1) for rank in tail_ranks], dim=1
)
return {
"mean": mean,
"second_moment": second_moment,
"variance": variance,
"positive_mean": (normalized_weights * positive).sum(dim=1),
"negative_mean": (normalized_weights * negative).sum(dim=1),
"positive_probability": (
normalized_weights * positive_event.to(neighbor_returns.dtype)
).sum(dim=1),
"negative_probability": (
normalized_weights * negative_event.to(neighbor_returns.dtype)
).sum(dim=1),
"positive_count": positive_event.sum(dim=1),
"negative_count": negative_event.sum(dim=1),
"effective_sample_size": normalized_weights.square().sum(dim=1).reciprocal(),
"quantile": torch.stack((long_quantile, short_quantile), dim=1),
"lower_cvar": torch.stack((long_cvar, short_cvar), dim=1),
}
def _tensor_list(value: Tensor) -> list[Any]:
return value.detach().cpu().tolist()
def _append_prefix_sample(
sample: np.ndarray, count: int, values: np.ndarray
) -> int:
"""Append only the prefix which fits in a fixed diagnostic reservoir."""
if count >= sample.size:
return count
take = min(values.size, sample.size - count)
sample[count : count + take] = values[:take]
return count + take
[docs]
@torch.inference_mode()
def valuate_local_tail(
*,
reference_state: Tensor,
reference_returns: Tensor,
reference_roundoff: Tensor,
reference_log_weights: Tensor | None = None,
query_state: Tensor,
query_returns: Tensor,
query_roundoff: Tensor,
query_log_weights: Tensor | None = None,
reference_instrument_feature: Tensor | None = None,
query_instrument_feature: Tensor | None = None,
config: LocalTailConfig,
split: str,
leave_one_out: bool,
leave_one_out_reference_indices: Tensor | None = None,
progress: Callable[[LocalTailProgress], None] | None = None,
progress_interval_seconds: float = 30.0,
) -> dict[str, Any]:
"""Value all query states with an exact adaptive-neighbour box kernel."""
verify_inp(isinstance(split, str) and split != "", "split must be nonempty")
verify_inp(
math.isfinite(progress_interval_seconds) and progress_interval_seconds > 0.0,
lambda: (
"progress_interval_seconds must be finite and positive, got "
f"{progress_interval_seconds}"
),
)
verify_inp(
reference_state.ndim == query_state.ndim == 2,
lambda: f"state tensors must be matrices, got {reference_state.shape}/{query_state.shape}",
)
verify_inp(
reference_returns.shape == reference_roundoff.shape
and query_returns.shape == query_roundoff.shape,
lambda: "return and roundoff shapes must agree",
)
verify_inp(
reference_returns.ndim == query_returns.ndim == 2
and reference_returns.shape[1] == query_returns.shape[1],
lambda: f"return tensors must have matching instrument axes",
)
verify_inp(
reference_state.shape[0] == reference_returns.shape[0]
and query_state.shape[0] == query_returns.shape[0],
lambda: "state and return observation counts must agree",
)
for name, log_weights, observations in (
("reference_log_weights", reference_log_weights, reference_state.shape[0]),
("query_log_weights", query_log_weights, query_state.shape[0]),
):
if log_weights is None:
continue
verify_inp(
log_weights.shape == (observations,)
and log_weights.dtype.is_floating_point,
lambda name=name, log_weights=log_weights, observations=observations: (
f"'{name}' must be floating with shape {(observations,)}, got "
f"{log_weights.shape}/{log_weights.dtype}"
),
)
verify_inp(
config.neighbors < reference_state.shape[0]
if leave_one_out
else config.neighbors <= reference_state.shape[0],
lambda: f"not enough reference observations for {config.neighbors} neighbours",
)
if leave_one_out_reference_indices is not None:
verify_inp(
leave_one_out,
"leave-one-out reference indices require leave_one_out=True",
)
verify_inp(
leave_one_out_reference_indices.shape == (query_state.shape[0],)
and leave_one_out_reference_indices.dtype == torch.int64,
lambda: (
"leave-one-out reference indices must be int64 with shape "
f"{(query_state.shape[0],)}, got "
f"{leave_one_out_reference_indices.shape}/"
f"{leave_one_out_reference_indices.dtype}"
),
)
if (
torch.any(leave_one_out_reference_indices < 0).item()
or torch.any(
leave_one_out_reference_indices >= reference_state.shape[0]
).item()
):
raise ValueError("leave-one-out reference indices are out of range")
elif leave_one_out:
verify_inp(
query_state.shape[0] == reference_state.shape[0],
lambda: (
"implicit leave-one-out requires identical query/reference "
f"counts, got {query_state.shape[0]} and {reference_state.shape[0]}"
),
)
candidate_neighbors = (
config.instrument_candidate_neighbors
if config.state_mode == "instrument"
else config.neighbors
)
verify_inp(
candidate_neighbors < reference_state.shape[0]
if leave_one_out
else candidate_neighbors <= reference_state.shape[0],
lambda: (
f"not enough reference observations for {candidate_neighbors} "
"candidate neighbours"
),
)
if config.state_mode == "instrument":
verify_inp(
reference_instrument_feature is not None
and query_instrument_feature is not None,
"instrument mode requires reference and query instrument features",
)
assert reference_instrument_feature is not None
assert query_instrument_feature is not None
verify_inp(
reference_instrument_feature.shape == reference_returns.shape
and query_instrument_feature.shape == query_returns.shape,
lambda: (
"instrument-feature shapes must match returns, got "
f"{reference_instrument_feature.shape}/{reference_returns.shape} and "
f"{query_instrument_feature.shape}/{query_returns.shape}"
),
)
else:
verify_inp(
reference_instrument_feature is None and query_instrument_feature is None,
"raw mode does not consume instrument features",
)
tensors = (
reference_state,
reference_returns,
reference_roundoff,
query_state,
query_returns,
query_roundoff,
)
if reference_log_weights is not None:
tensors += (reference_log_weights,)
if query_log_weights is not None:
tensors += (query_log_weights,)
if reference_instrument_feature is not None:
tensors += (reference_instrument_feature,)
if query_instrument_feature is not None:
tensors += (query_instrument_feature,)
if leave_one_out_reference_indices is not None:
tensors += (leave_one_out_reference_indices,)
device = reference_state.device
if any(value.device != device for value in tensors):
raise ValueError("all local-tail tensors must be on the same device")
if any(not torch.isfinite(value).all().item() for value in tensors):
raise ValueError("local-tail tensors contain non-finite values")
queries = query_state.shape[0]
instruments = reference_returns.shape[1]
levels = len(config.tail_ranks)
signal_levels = len(config.signal_thresholds)
moment_sum = torch.zeros(4, instruments, dtype=torch.float64, device=device)
query_weight = (
torch.ones(queries, dtype=torch.float64, device=device)
if query_log_weights is None
else torch.exp(
query_log_weights.to(torch.float64)
- query_log_weights.to(torch.float64).max()
)
)
query_weight_sum = query_weight.sum()
query_effective_sample_size = (
query_weight_sum.square() / query_weight.square().sum()
)
support_count = torch.zeros(2, instruments, dtype=torch.int64, device=device)
support_counter = torch.zeros_like(support_count)
support_favorable = torch.zeros_like(support_count)
quantile_count = torch.zeros(2, levels, instruments, dtype=torch.int64, device=device)
quantile_counter = torch.zeros_like(quantile_count)
cvar_count = torch.zeros_like(quantile_count)
cvar_counter = torch.zeros_like(quantile_count)
support_states = torch.zeros(2, dtype=torch.int64, device=device)
exclusive_support_states = torch.zeros(3, dtype=torch.int64, device=device)
support_state_weight = torch.zeros(2, dtype=torch.float64, device=device)
exclusive_support_state_weight = torch.zeros(
3, dtype=torch.float64, device=device
)
quantile_states = torch.zeros(2, levels, dtype=torch.int64, device=device)
cvar_states = torch.zeros_like(quantile_states)
quantile_both = torch.zeros(levels, dtype=torch.int64, device=device)
cvar_both = torch.zeros_like(quantile_both)
signal_count = torch.zeros(
2, signal_levels, instruments, dtype=torch.int64, device=device
)
signal_counter = torch.zeros_like(signal_count)
signal_states = torch.zeros(
2, signal_levels, dtype=torch.int64, device=device
)
signal_both = torch.zeros(signal_levels, dtype=torch.int64, device=device)
signal_max = torch.zeros(instruments, dtype=torch.float64, device=device)
zero_sigma_count = torch.zeros(instruments, dtype=torch.int64, device=device)
support_policy_pnl: list[np.ndarray] = []
support_policy_band: list[np.ndarray] = []
support_policy_instrument: list[np.ndarray] = []
support_policy_direction: list[np.ndarray] = []
support_policy_weight: list[np.ndarray] = []
radius_comparisons = queries * (
instruments if config.state_mode == "instrument" else 1
)
radius_sample = np.empty(min(1_000_000, radius_comparisons), dtype=np.float32)
radius_sample_count = 0
radius_sum = 0.0
radius_minimum = math.inf
radius_maximum = -math.inf
certified_count = torch.zeros(instruments, dtype=torch.int64, device=device)
local_ess_sum = torch.zeros((), dtype=torch.float64, device=device)
local_ess_minimum = math.inf
local_ess_maximum = -math.inf
reference_norm = reference_state.square().sum(dim=1)
started = time.monotonic()
last_progress = -math.inf
for start in range(0, queries, config.query_batch_size):
stop = min(start + config.query_batch_size, queries)
batch_state = query_state[start:stop]
distance = batch_state @ reference_state.T
distance.mul_(-2.0).add_(reference_norm)
distance.add_(batch_state.square().sum(dim=1, keepdim=True))
if leave_one_out:
row = torch.arange(stop - start, device=device)
excluded = (
torch.arange(start, stop, device=device)
if leave_one_out_reference_indices is None
else leave_one_out_reference_indices[start:stop]
)
distance[row, excluded] = torch.inf
nearest_distance, nearest = torch.topk(
distance,
candidate_neighbors,
dim=1,
largest=False,
sorted=True,
)
del distance
nearest_distance.clamp_min_(0.0)
if config.state_mode == "raw":
batch_radius = nearest_distance[:, -1].sqrt()
values = batch_radius.cpu().numpy()
radius_sample_count = _append_prefix_sample(
radius_sample, radius_sample_count, values
)
radius_sum += float(batch_radius.sum(dtype=torch.float64).item())
radius_minimum = min(radius_minimum, float(batch_radius.min().item()))
radius_maximum = max(radius_maximum, float(batch_radius.max().item()))
certified_count += stop - start
batch_query_returns = query_returns[start:stop]
batch_query_roundoff = query_roundoff[start:stop]
batch_query_weight = query_weight[start:stop]
batch_support_states = torch.zeros(
2, stop - start, dtype=torch.bool, device=device
)
batch_quantile_states = torch.zeros(
2, levels, stop - start, dtype=torch.bool, device=device
)
batch_cvar_states = torch.zeros_like(batch_quantile_states)
batch_signal_states = torch.zeros(
2,
signal_levels,
stop - start,
dtype=torch.bool,
device=device,
)
batch_policy_score = torch.full(
(stop - start,), -torch.inf, dtype=query_returns.dtype, device=device
)
batch_policy_pnl = torch.zeros_like(batch_policy_score)
batch_policy_band = torch.zeros_like(batch_policy_score)
batch_policy_instrument = torch.full(
(stop - start,), -1, dtype=torch.int64, device=device
)
batch_policy_direction = torch.full_like(batch_policy_instrument, -1)
for left in range(0, instruments, config.instrument_batch_size):
right = min(left + config.instrument_batch_size, instruments)
if config.state_mode == "instrument":
assert reference_instrument_feature is not None
assert query_instrument_feature is not None
reference_feature = reference_instrument_feature[:, left:right][nearest]
query_feature = query_instrument_feature[
start:stop, left:right
][:, None, :]
augmented_distance = nearest_distance[..., None] + (
reference_feature - query_feature
).square()
selected_distance, selected_local = torch.topk(
augmented_distance,
config.neighbors,
dim=1,
largest=False,
sorted=True,
)
del augmented_distance, reference_feature, query_feature
expanded_nearest = nearest[..., None].expand(
-1, -1, right - left
)
selected = torch.gather(expanded_nearest, 1, selected_local)
del expanded_nearest, selected_local
instrument_index = torch.arange(left, right, device=device).view(
1, 1, -1
)
neighbor_returns = reference_returns[selected, instrument_index]
neighbor_roundoff = reference_roundoff[selected, instrument_index]
neighbor_log_weights = (
None
if reference_log_weights is None
else reference_log_weights[selected]
)
certified = selected_distance[:, -1] < nearest_distance[:, -1, None]
certified_count[left:right] += certified.sum(dim=0)
batch_radius = selected_distance[:, -1].sqrt()
radius_sum += float(batch_radius.sum(dtype=torch.float64).item())
radius_minimum = min(radius_minimum, float(batch_radius.min().item()))
radius_maximum = max(radius_maximum, float(batch_radius.max().item()))
radius_sample_count = _append_prefix_sample(
radius_sample,
radius_sample_count,
batch_radius.flatten().cpu().numpy(),
)
del selected, selected_distance, certified, batch_radius
else:
neighbor_returns = reference_returns[:, left:right][nearest]
neighbor_roundoff = reference_roundoff[:, left:right][nearest]
neighbor_log_weights = (
None
if reference_log_weights is None
else reference_log_weights[nearest]
)
statistics = local_neighbor_statistics(
neighbor_returns,
neighbor_roundoff,
tail_ranks=config.tail_ranks,
neighbor_log_weights=neighbor_log_weights,
)
del neighbor_returns, neighbor_roundoff, neighbor_log_weights
query_value = batch_query_returns[:, left:right]
query_band = batch_query_roundoff[:, left:right]
conditional_mean = statistics["mean"]
conditional_sigma = statistics["variance"].sqrt()
positive_sigma = conditional_sigma > 0.0
signal_max[left:right] = torch.maximum(
signal_max[left:right],
torch.where(
positive_sigma,
conditional_mean.abs() / conditional_sigma,
torch.where(
conditional_mean == 0.0,
torch.zeros_like(conditional_mean),
torch.full_like(conditional_mean, torch.inf),
),
)
.amax(dim=0)
.to(torch.float64),
)
zero_sigma_count[left:right] += (~positive_sigma).sum(dim=0)
local_ess = statistics["effective_sample_size"]
local_ess_sum += local_ess.sum(dtype=torch.float64)
local_ess_minimum = min(
local_ess_minimum, float(local_ess.min().item())
)
local_ess_maximum = max(
local_ess_maximum, float(local_ess.max().item())
)
for moment_index, name in enumerate(
(
"positive_mean",
"negative_mean",
"positive_probability",
"negative_probability",
)
):
moment_sum[moment_index, left:right] += (
batch_query_weight[:, None]
* statistics[name].to(torch.float64)
).sum(dim=0)
support_long = (statistics["negative_count"] == 0) & (
statistics["positive_count"] > 0
)
support_short = (statistics["positive_count"] == 0) & (
statistics["negative_count"] > 0
)
support = torch.stack((support_long, support_short), dim=0)
signed_query = torch.stack((query_value, -query_value), dim=0)
thresholds = torch.as_tensor(
config.signal_thresholds,
dtype=conditional_mean.dtype,
device=device,
).reshape(1, -1, 1)
signal_long = conditional_mean[:, None, :] > (
thresholds * conditional_sigma[:, None, :]
)
signal_short = conditional_mean[:, None, :] < -(
thresholds * conditional_sigma[:, None, :]
)
signal = torch.stack((signal_long, signal_short), dim=0).permute(
0, 2, 1, 3
)
signal_count[:, :, left:right] += signal.sum(dim=2)
signal_counter[:, :, left:right] += (
signal & (signed_query[:, None] < -query_band[None, None])
).sum(dim=2)
batch_signal_states |= signal.any(dim=3)
signal_both += (signal_long & signal_short).sum(dim=(0, 2))
support_count[:, left:right] += support.sum(dim=1)
support_counter[:, left:right] += (
support & (signed_query < -query_band)
).sum(dim=1)
support_favorable[:, left:right] += (
support & (signed_query > query_band)
).sum(dim=1)
batch_support_states |= support.any(dim=2)
predicted_gain = torch.stack(
(statistics["positive_mean"], statistics["negative_mean"]),
dim=0,
)
local_score = torch.where(
support,
predicted_gain,
torch.full_like(predicted_gain, -torch.inf),
).permute(1, 0, 2).reshape(stop - start, -1)
local_best_score, local_flat_index = local_score.max(dim=1)
local_trade = torch.isfinite(local_best_score)
local_direction = torch.div(
local_flat_index, right - left, rounding_mode="floor"
)
local_instrument = local_flat_index % (right - left)
local_pnl = signed_query.permute(1, 0, 2).reshape(
stop - start, -1
).gather(1, local_flat_index[:, None]).squeeze(1)
local_band = query_band[:, None, :].expand(
-1, 2, -1
).reshape(stop - start, -1).gather(
1, local_flat_index[:, None]
).squeeze(1)
replace = local_trade & (local_best_score > batch_policy_score)
batch_policy_score = torch.where(
replace, local_best_score, batch_policy_score
)
batch_policy_pnl = torch.where(replace, local_pnl, batch_policy_pnl)
batch_policy_band = torch.where(replace, local_band, batch_policy_band)
batch_policy_instrument = torch.where(
replace, local_instrument + left, batch_policy_instrument
)
batch_policy_direction = torch.where(
replace, local_direction, batch_policy_direction
)
quantile = statistics["quantile"]
cvar = statistics["lower_cvar"]
tail_candidate = quantile > query_band[:, None, None]
cvar_candidate = cvar > query_band[:, None, None]
tail_candidate = tail_candidate.permute(1, 2, 0, 3)
cvar_candidate = cvar_candidate.permute(1, 2, 0, 3)
signed_counter = (
signed_query[:, None] < -query_band[None, None]
)
quantile_count[:, :, left:right] += tail_candidate.sum(dim=2)
quantile_counter[:, :, left:right] += (
tail_candidate & signed_counter
).sum(dim=2)
cvar_count[:, :, left:right] += cvar_candidate.sum(dim=2)
cvar_counter[:, :, left:right] += (
cvar_candidate & signed_counter
).sum(dim=2)
batch_quantile_states |= tail_candidate.any(dim=3)
batch_cvar_states |= cvar_candidate.any(dim=3)
quantile_both += (
tail_candidate[LONG_DIRECTION] & tail_candidate[SHORT_DIRECTION]
).sum(dim=(1, 2))
cvar_both += (
cvar_candidate[LONG_DIRECTION] & cvar_candidate[SHORT_DIRECTION]
).sum(dim=(1, 2))
support_states += batch_support_states.sum(dim=1)
support_state_weight += (
batch_support_states.to(torch.float64)
* batch_query_weight[None, :]
).sum(dim=1)
long_state = batch_support_states[LONG_DIRECTION]
short_state = batch_support_states[SHORT_DIRECTION]
exclusive_support_states += torch.stack(
(
(long_state & ~short_state).sum(),
(short_state & ~long_state).sum(),
(long_state & short_state).sum(),
)
)
exclusive_support_state_weight += torch.stack(
(
(
(long_state & ~short_state).to(torch.float64)
* batch_query_weight
).sum(),
(
(short_state & ~long_state).to(torch.float64)
* batch_query_weight
).sum(),
(
(long_state & short_state).to(torch.float64)
* batch_query_weight
).sum(),
)
)
quantile_states += batch_quantile_states.sum(dim=2)
cvar_states += batch_cvar_states.sum(dim=2)
signal_states += batch_signal_states.sum(dim=2)
support_policy_pnl.append(batch_policy_pnl.cpu().numpy())
support_policy_band.append(batch_policy_band.cpu().numpy())
support_policy_instrument.append(batch_policy_instrument.cpu().numpy())
support_policy_direction.append(batch_policy_direction.cpu().numpy())
support_policy_weight.append(batch_query_weight.cpu().numpy())
now = time.monotonic()
if progress is not None and (
now - last_progress >= progress_interval_seconds or stop == queries
):
state_counts = support_states.detach().cpu().tolist()
exclusive_counts = exclusive_support_states.detach().cpu().tolist()
progress(
LocalTailProgress(
split=split,
processed_states=stop,
total_states=queries,
states_with_any_long_candidate=state_counts[LONG_DIRECTION],
states_with_any_short_candidate=state_counts[SHORT_DIRECTION],
long_only_candidate_states=exclusive_counts[0],
short_only_candidate_states=exclusive_counts[1],
both_direction_candidate_states=exclusive_counts[2],
)
)
last_progress = now
elapsed = time.monotonic() - started
denominator = float(query_weight_sum.item())
radius_values = radius_sample[:radius_sample_count]
def candidate_block(
count: Tensor,
counter: Tensor,
states: Tensor,
both: Tensor | None = None,
) -> dict[str, Any]:
rate = torch.where(count > 0, counter.to(torch.float64) / count, 0.0)
output: dict[str, Any] = {
"count_by_direction_level_instrument": _tensor_list(count),
"count_by_direction_level": _tensor_list(count.sum(dim=-1)),
"states_with_any_by_direction_level": _tensor_list(states),
"realized_counter_sign_count_by_direction_level": _tensor_list(
counter.sum(dim=-1)
),
"realized_counter_sign_rate_by_direction_level": _tensor_list(
torch.where(
count.sum(dim=-1) > 0,
counter.sum(dim=-1).to(torch.float64) / count.sum(dim=-1),
0.0,
)
),
"realized_counter_sign_rate_by_direction_level_instrument": _tensor_list(
rate
),
}
if both is not None:
output["both_directions_positive_count_by_level"] = _tensor_list(both)
return output
support_rate = torch.where(
support_count > 0,
support_counter.to(torch.float64) / support_count,
0.0,
)
policy_pnl = np.concatenate(support_policy_pnl).astype(np.float64, copy=False)
policy_band = np.concatenate(support_policy_band).astype(np.float64, copy=False)
policy_instrument = np.concatenate(support_policy_instrument)
policy_direction = np.concatenate(support_policy_direction)
policy_weight = np.concatenate(support_policy_weight).astype(
np.float64, copy=False
)
policy_trade = policy_instrument >= 0
policy_loss = policy_trade & (policy_pnl < -policy_band)
policy_gain = policy_trade & (policy_pnl > policy_band)
policy_neutral = policy_trade & ~policy_loss & ~policy_gain
policy_trade_count = int(policy_trade.sum())
policy_quantile_levels = np.asarray(
[0.0, 0.01, 0.05, 0.5, 0.95, 0.99, 1.0], dtype=np.float64
)
policy_selected_count = np.bincount(
policy_instrument[policy_trade], minlength=instruments
)
policy_direction_count = np.bincount(
policy_direction[policy_trade], minlength=2
)
policy_weight_sum = float(policy_weight.sum())
policy_trade_weight = float(policy_weight[policy_trade].sum())
policy_gain_weight = float(policy_weight[policy_gain].sum())
policy_loss_weight = float(policy_weight[policy_loss].sum())
def weighted_policy_quantiles() -> list[float] | None:
if policy_trade_count == 0:
return None
values = policy_pnl[policy_trade]
weights = policy_weight[policy_trade]
order = np.argsort(values)
values = values[order]
weights = weights[order]
cumulative = np.cumsum(weights) / np.sum(weights)
return [
float(values[min(np.searchsorted(cumulative, level), values.size - 1)])
for level in policy_quantile_levels
]
return {
"split": split,
"queries": queries,
"reference_observations": reference_state.shape[0],
"instruments": instruments,
"leave_one_out": leave_one_out,
"neighbors": config.neighbors,
"effective_sample_size": float(
local_ess_sum.item() / (queries * instruments)
),
"local_effective_sample_size": {
"minimum": local_ess_minimum,
"mean": float(local_ess_sum.item() / (queries * instruments)),
"maximum": local_ess_maximum,
},
"query_measure": (
"likelihood-weighted original P"
if query_log_weights is not None
else "uniform empirical"
),
"query_weight_sum": float(query_weight_sum.item()),
"query_effective_sample_size": float(
query_effective_sample_size.item()
),
"tail_ranks": list(config.tail_ranks),
"tail_probabilities": list(config.local_tail_probabilities),
"zero_event_upper_probability_95": (
None
if reference_log_weights is not None
else config.zero_event_upper_probability_95
),
"mean_positive_part_by_instrument": _tensor_list(moment_sum[0] / denominator),
"mean_negative_part_by_instrument": _tensor_list(moment_sum[1] / denominator),
"mean_positive_probability_by_instrument": _tensor_list(moment_sum[2] / denominator),
"mean_negative_probability_by_instrument": _tensor_list(moment_sum[3] / denominator),
"mean_positive_part": float(moment_sum[0].sum().item() / (denominator * instruments)),
"mean_negative_part": float(moment_sum[1].sum().item() / (denominator * instruments)),
"mean_positive_probability": float(moment_sum[2].sum().item() / (denominator * instruments)),
"mean_negative_probability": float(moment_sum[3].sum().item() / (denominator * instruments)),
"zero_adverse": {
"count_by_direction_instrument": _tensor_list(support_count),
"count_by_direction": _tensor_list(support_count.sum(dim=-1)),
"states_with_any_by_direction": _tensor_list(support_states),
"weighted_state_probability_by_direction": _tensor_list(
support_state_weight / query_weight_sum
),
"states_by_direction_exclusivity": {
"long_only": int(exclusive_support_states[0].item()),
"short_only": int(exclusive_support_states[1].item()),
"both_directions": int(exclusive_support_states[2].item()),
"neither_direction": int(
queries - exclusive_support_states.sum().item()
),
},
"weighted_state_probability_by_direction_exclusivity": {
"long_only": float(
exclusive_support_state_weight[0].item() / denominator
),
"short_only": float(
exclusive_support_state_weight[1].item() / denominator
),
"both_directions": float(
exclusive_support_state_weight[2].item() / denominator
),
"neither_direction": float(
1.0
- exclusive_support_state_weight.sum().item() / denominator
),
},
"realized_counter_sign_count_by_direction": _tensor_list(
support_counter.sum(dim=-1)
),
"realized_counter_sign_rate_by_direction": _tensor_list(
torch.where(
support_count.sum(dim=-1) > 0,
support_counter.sum(dim=-1).to(torch.float64)
/ support_count.sum(dim=-1),
0.0,
)
),
"realized_favorable_sign_count_by_direction": _tensor_list(
support_favorable.sum(dim=-1)
),
"realized_favorable_sign_rate_by_direction": _tensor_list(
torch.where(
support_count.sum(dim=-1) > 0,
support_favorable.sum(dim=-1).to(torch.float64)
/ support_count.sum(dim=-1),
0.0,
)
),
"realized_counter_sign_rate_by_direction_instrument": _tensor_list(
support_rate
),
},
"zero_adverse_single_instrument_policy": {
"definition": (
"At each query state, among directions with zero adverse "
"outcomes in the training-neighbour sample, select the one "
"instrument/direction with the largest estimated favorable "
"part and use unit L1 exposure; otherwise do not trade."
),
"query_states": queries,
"trading_states": policy_trade_count,
"trading_state_probability": policy_trade_count / queries,
"weighted_trading_state_probability": (
policy_trade_weight / policy_weight_sum
),
"long_trades": int(policy_direction_count[LONG_DIRECTION]),
"short_trades": int(policy_direction_count[SHORT_DIRECTION]),
"realized_gains_beyond_roundoff": int(policy_gain.sum()),
"realized_losses_beyond_roundoff": int(policy_loss.sum()),
"realized_neutral_within_roundoff": int(policy_neutral.sum()),
"empirical_nonnegative_within_roundoff": bool(not np.any(policy_loss)),
"positive_return_probability_all_queries": (
policy_gain_weight / policy_weight_sum
),
"positive_return_probability_when_trading": (
0.0
if policy_trade_count == 0
else policy_gain_weight / policy_trade_weight
),
"loss_probability_when_trading": (
0.0
if policy_trade_count == 0
else policy_loss_weight / policy_trade_weight
),
"zero_loss_probability_upper_95_when_trading": (
None
if (
policy_trade_count == 0
or np.any(policy_loss)
or query_log_weights is not None
)
else 1.0 - 0.05 ** (1.0 / policy_trade_count)
),
"pnl_mean_all_queries": float(
np.dot(policy_weight, policy_pnl) / policy_weight_sum
),
"pnl_mean_when_trading": (
0.0
if policy_trade_count == 0
else float(
np.dot(
policy_weight[policy_trade], policy_pnl[policy_trade]
)
/ policy_trade_weight
)
),
"pnl_quantile_levels": policy_quantile_levels.tolist(),
"pnl_quantiles_when_trading": (
weighted_policy_quantiles()
),
"selected_count_by_instrument": policy_selected_count.tolist(),
},
"conditional_moment_signal": {
"definition": (
"directional E[dH|z] exceeds g times the population "
"conditional standard deviation estimated from the same "
"uniform k-nearest-neighbour sample"
),
"thresholds": list(config.signal_thresholds),
**candidate_block(
signal_count, signal_counter, signal_states, signal_both
),
"maximum_abs_mean_over_sigma_by_instrument": _tensor_list(
signal_max
),
"maximum_abs_mean_over_sigma": float(signal_max.max().item()),
"zero_sigma_count_by_instrument": _tensor_list(zero_sigma_count),
"zero_sigma_count": int(zero_sigma_count.sum().item()),
},
"quantile_positive": candidate_block(
quantile_count, quantile_counter, quantile_states, quantile_both
),
"lower_cvar_positive": candidate_block(
cvar_count, cvar_counter, cvar_states, cvar_both
),
"neighbor_radius": {
"minimum": radius_minimum,
"mean": radius_sum / float(radius_comparisons),
"quantiles": {
f"{probability:g}": float(np.quantile(radius_values, probability))
for probability in (0.05, 0.5, 0.95)
},
"maximum": radius_maximum,
"quantile_sample_comparisons": radius_sample_count,
"total_comparisons": radius_comparisons,
"metric": "Euclidean distance after training-only componentwise standardization",
},
"neighbor_search": {
"candidate_neighbors": candidate_neighbors,
"final_neighbors": config.neighbors,
"certified_exact_count_by_instrument": _tensor_list(certified_count),
"certified_exact_fraction_by_instrument": _tensor_list(
certified_count.to(torch.float64) / queries
),
"certified_exact_fraction": float(
certified_count.sum().item() / (queries * instruments)
),
"certification": (
"selected augmented kth distance is strictly below the raw "
"candidate-pool boundary"
if config.state_mode == "instrument"
else "exact all-reference search"
),
},
"elapsed_seconds": elapsed,
}
__all__ = [
"DIRECTIONS",
"LONG_DIRECTION",
"LocalTailConfig",
"LocalTailProgress",
"SHORT_DIRECTION",
"local_neighbor_statistics",
"normalize_state",
"prepare_delta_hedged_local_data",
"reduced_option_state",
"shared_instrument_metric_diagnostics",
"state_normalization",
"valuate_local_tail",
]