"""State-conditional direct-arbitrage policies and certification.
The policy sees only the current state ``z``. A softmax chooses a long/short
direction while a separate sigmoid gate chooses gross exposure. Thus long
and short positions are represented separately and
``sum(long + short) <= 1``.
For realized return ``dH`` and effective one-way cost ``c``, net return is
``(long - short) @ dH - (long + short) @ c``.
Population almost-sure nonnegativity cannot be proved from Monte Carlo data.
The certification helpers therefore report every validation loss and use
paths, rather than path-time observations, as independent clusters.
"""
from __future__ import annotations
import math
import os
from collections.abc import Iterator, Mapping
from dataclasses import asdict, dataclass
from pathlib import Path
from statistics import NormalDist
from typing import Any
import numpy as np
import torch
from cdxcore.err import verify_inp
from cdxcore.uniquehash import named_unique_filename48_8
from cdxcore.util import is_filename
from cdxcore.version import version
from filelock import FileLock
from torch import Tensor
from quantdata.torch import BaseEnvironment, BaseModel
from quantdata.torch.ffw import FeedForward, FeedForwardConfig
[docs]
@version("1.5")
@dataclass(frozen=True, kw_only=True)
class DirectArbitrageConfig:
"""Logical policy, economic threshold, and constrained-loss configuration."""
label: str
state_features: int
instruments: int
group: str = ""
user_version: str = "0.1"
epsilon: float = 0.01
delta: float = 0.01
numerical_tolerance: float = 1.0e-4
smooth_temperature: float = 0.0025
tail_fraction: float = 0.01
downside_weight: float = 10.0
tail_weight: float = 100.0
overlap_weight: float = 1.0
mean_reward_weight: float = 0.1
discovery_fraction: float = 0.05
constraint_ramp_fraction: float = 0.45
initial_trade_fraction: float = 0.01
mask_arbitrage_candidates: bool = False
objective: str = "simplex-soft-hit"
cap_weight: float = 1.0
seed: int = 70_771
network_config: FeedForwardConfig = FeedForwardConfig(
num_depth=2,
num_width=100,
activation="LeakyReLU",
use_bias=True,
)
def __post_init__(self) -> None:
verify_inp(
isinstance(self.label, str) and is_filename(self.label),
lambda: f"'label' must be a filename, got {self.label!r}",
)
verify_inp(
isinstance(self.group, str) and not self.group.startswith("/"),
lambda: f"'group' must be relative, got {self.group!r}",
)
verify_inp(
self.state_features > 0,
lambda: f"'state_features' must be positive, got {self.state_features}",
)
verify_inp(
self.instruments > 0,
lambda: f"'instruments' must be positive, got {self.instruments}",
)
for name in (
"epsilon",
"numerical_tolerance",
"smooth_temperature",
"downside_weight",
"tail_weight",
"overlap_weight",
"mean_reward_weight",
"cap_weight",
):
value = getattr(self, name)
verify_inp(
math.isfinite(value) and value > 0.0,
lambda name=name, value=value: (
f"'{name}' must be finite and positive, got {value}"
),
)
for name in (
"delta",
"tail_fraction",
"discovery_fraction",
"constraint_ramp_fraction",
"initial_trade_fraction",
):
value = getattr(self, name)
verify_inp(
math.isfinite(value) and 0.0 < value < 1.0,
lambda name=name, value=value: (
f"'{name}' must be strictly between zero and one, got {value}"
),
)
verify_inp(
self.discovery_fraction + self.constraint_ramp_fraction < 1.0,
"discovery and constraint-ramp fractions must sum to less than one",
)
verify_inp(isinstance(self.seed, int), lambda: f"'seed' is not an int: {self.seed!r}")
verify_inp(
isinstance(self.mask_arbitrage_candidates, bool),
lambda: (
"'mask_arbitrage_candidates' must be bool, got "
f"{self.mask_arbitrage_candidates!r}"
),
)
verify_inp(
self.objective in {"simplex-soft-hit", "radial-l1-v2"},
lambda: f"unknown direct-arbitrage objective {self.objective!r}",
)
@property
def filename(self) -> str:
return named_unique_filename48_8(self.label, self)
[docs]
@version("1.0")
@dataclass(frozen=True, kw_only=True)
class DirectArbitrageTrainConfig:
"""Operational configuration for direct-policy optimization."""
batch_size: int = 1_024
valuation_batch_size: int = 4_096
lr: float = 1.0e-3
final_lr: float = 1.0e-6
weight_decay: float = 1.0e-4
compile: bool = True
value_update: int = 10_000
checkpoint_update: int = 10_000
checkpoint_root: str = "~/.cache/rndysanos/direct-arbitrage"
def __post_init__(self) -> None:
for name in (
"batch_size",
"valuation_batch_size",
"value_update",
"checkpoint_update",
):
value = getattr(self, name)
verify_inp(value > 0, lambda name=name, value=value: f"'{name}' must be positive, got {value}")
for name in ("lr", "final_lr"):
value = getattr(self, name)
verify_inp(
math.isfinite(value) and value > 0.0,
lambda name=name, value=value: f"'{name}' must be finite and positive, got {value}",
)
verify_inp(
math.isfinite(self.weight_decay) and self.weight_decay >= 0.0,
lambda: f"'weight_decay' must be finite and nonnegative, got {self.weight_decay}",
)
[docs]
class LongShortSimplexPolicy(BaseModel):
"""State-only long/short direction and independent scalar exposure gate."""
def __init__(
self,
env: BaseEnvironment,
*,
config: DirectArbitrageConfig,
state_mean: Tensor,
state_scale: Tensor,
) -> None:
super().__init__(env)
expected = (config.state_features,)
verify_inp(
state_mean.shape == state_scale.shape == expected,
lambda: (
f"state normalization must have shape {expected}, found "
f"{state_mean.shape} and {state_scale.shape}"
),
)
if not torch.isfinite(state_mean).all().item():
raise ValueError("state mean contains non-finite values")
if not torch.isfinite(state_scale).all().item() or not torch.all(state_scale > 0.0).item():
raise ValueError("state scale must be finite and positive")
self.config = config
self.network = FeedForward(
env,
num_input=config.state_features,
num_output=2 * config.instruments + 1,
config=config.network_config,
)
self.register_buffer("state_mean", state_mean.detach().clone())
self.register_buffer("state_scale", state_scale.detach().clone())
self._initialize_no_trade()
def _initialize_no_trade(self) -> None:
final = self.network.In[-1] if self.network.In else self.network.I0
assert final.bias is not None
gate_logit = math.log(
self.config.initial_trade_fraction
/ (1.0 - self.config.initial_trade_fraction)
)
with torch.no_grad():
final.weight.zero_()
final.bias.zero_()
final.bias[-1] = gate_logit
[docs]
def forward(
self,
z: Tensor,
excluded_directions: Tensor | None = None,
) -> tuple[Tensor, Tensor, Tensor]:
verify_inp(
z.shape[-1] == self.config.state_features,
lambda: f"state shape {z.shape} has the wrong final dimension",
)
normalized = (z - self.state_mean) / self.state_scale
logits = self.network(normalized)
J = self.config.instruments
direction_logits = logits[..., : 2 * J]
if excluded_directions is not None:
verify_inp(
excluded_directions.shape == direction_logits.shape,
lambda: (
f"excluded-direction shape {excluded_directions.shape} != "
f"{direction_logits.shape}"
),
)
verify_inp(
excluded_directions.dtype == torch.bool,
lambda: (
"excluded directions must have bool dtype, got "
f"{excluded_directions.dtype}"
),
)
direction_logits = direction_logits.masked_fill(
excluded_directions, -torch.inf
)
direction = torch.softmax(direction_logits, dim=-1)
exposure = torch.sigmoid(logits[..., -1])
long = exposure[..., None] * direction[..., :J]
short = exposure[..., None] * direction[..., J:]
no_trade = 1.0 - exposure
self.verify(lambda: torch.isfinite(direction), "policy direction is non-finite")
self.verify(lambda: torch.isfinite(exposure), "policy exposure is non-finite")
return long, short, no_trade
[docs]
class RadialL1Policy(BaseModel):
"""V2.0 policy with one unbounded signed output and radial L1 clipping."""
def __init__(
self,
env: BaseEnvironment,
*,
config: DirectArbitrageConfig,
state_mean: Tensor,
state_scale: Tensor,
) -> None:
super().__init__(env)
expected = (config.state_features,)
verify_inp(
state_mean.shape == state_scale.shape == expected,
lambda: (
f"state normalization must have shape {expected}, found "
f"{state_mean.shape} and {state_scale.shape}"
),
)
if not torch.isfinite(state_mean).all().item():
raise ValueError("state mean contains non-finite values")
if not torch.isfinite(state_scale).all().item() or not torch.all(
state_scale > 0.0
).item():
raise ValueError("state scale must be finite and positive")
self.config = config
self.network = FeedForward(
env,
num_input=config.state_features,
num_output=config.instruments,
config=config.network_config,
)
self.register_buffer("state_mean", state_mean.detach().clone())
self.register_buffer("state_scale", state_scale.detach().clone())
self._initialize_small_trade()
def _initialize_small_trade(self) -> None:
final = self.network.In[-1] if self.network.In else self.network.I0
with torch.no_grad():
final.weight.mul_(self.config.initial_trade_fraction)
if final.bias is not None:
final.bias.zero_()
[docs]
def forward(
self,
z: Tensor,
excluded_directions: Tensor | None = None,
) -> tuple[Tensor, Tensor]:
"""Return the feasible signed action and feasible raw L1 norm."""
verify_inp(
z.shape[-1] == self.config.state_features,
lambda: f"state shape {z.shape} has the wrong final dimension",
)
normalized = (z - self.state_mean) / self.state_scale
raw = self.network(normalized)
if excluded_directions is not None:
expected = raw.shape[:-1] + (2 * self.config.instruments,)
verify_inp(
excluded_directions.shape == expected,
lambda: (
f"excluded-direction shape {excluded_directions.shape} != "
f"{expected}"
),
)
verify_inp(
excluded_directions.dtype == torch.bool,
lambda: (
"excluded directions must have bool dtype, got "
f"{excluded_directions.dtype}"
),
)
J = self.config.instruments
positive = raw.clamp_min(0.0).masked_fill(
excluded_directions[..., :J], 0.0
)
negative = (-raw).clamp_min(0.0).masked_fill(
excluded_directions[..., J:], 0.0
)
raw = positive - negative
raw_l1 = raw.abs().sum(dim=-1)
action = raw / raw_l1.clamp_min(1.0)[..., None]
self.verify(lambda: torch.isfinite(action), "radial policy action is non-finite")
self.verify(lambda: torch.isfinite(raw_l1), "radial policy norm is non-finite")
return action, raw_l1
[docs]
@dataclass
class DirectArbitrageTrainingResult:
"""Trained policy, complete valuations, and restart metadata."""
policy: LongShortSimplexPolicy | RadialL1Policy
training: dict[str, Any]
validation: dict[str, Any] | None
tracker: dict[str, list[Any]]
checkpoint_path: Path
restarted: bool
[docs]
def net_return(
*,
long: Tensor,
short: Tensor,
dH: Tensor,
cost: Tensor,
) -> Tensor:
"""Return net P&L for separate long and short allocations."""
verify_inp(
long.shape == short.shape == dH.shape == cost.shape,
lambda: (
"long, short, return, and cost shapes differ: "
f"{long.shape}, {short.shape}, {dH.shape}, {cost.shape}"
),
)
for name, value in (("long", long), ("short", short), ("dH", dH), ("cost", cost)):
if not torch.isfinite(value).all().item():
raise ValueError(f"'{name}' contains non-finite values")
if not torch.all(long >= 0.0).item() or not torch.all(short >= 0.0).item():
raise ValueError("long and short allocations must be nonnegative")
if not torch.all(cost >= 0.0).item():
raise ValueError("cost must be nonnegative")
return _net_return_unchecked(long=long, short=short, dH=dH, cost=cost)
def _net_return_unchecked(
*, long: Tensor, short: Tensor, dH: Tensor, cost: Tensor
) -> Tensor:
"""Performance path after shapes and signs were established by preparation."""
return ((long - short) * dH - (long + short) * cost).sum(dim=-1)
[docs]
def direct_arbitrage_loss(
*,
long: Tensor,
short: Tensor,
dH: Tensor,
cost: Tensor,
config: DirectArbitrageConfig,
penalty_scale: float | Tensor = 1.0,
) -> tuple[Tensor, dict[str, Tensor]]:
"""Reward threshold hits while heavily penalizing mean and tail downside."""
pnl = _net_return_unchecked(long=long, short=short, dH=dH, cost=cost)
soft_hit = torch.sigmoid(
(pnl - config.epsilon) / config.smooth_temperature
).mean()
mean_reward = pnl.mean() / config.epsilon
downside = (-pnl - config.numerical_tolerance).clamp_min(0.0) / config.epsilon
flat_downside = downside.reshape(-1)
tail_count = max(1, int(math.ceil(flat_downside.numel() * config.tail_fraction)))
tail_downside = torch.topk(flat_downside, k=tail_count, sorted=False).values.mean()
overlap = (long * short).sum(dim=-1).mean()
scale = torch.as_tensor(penalty_scale, dtype=pnl.dtype, device=pnl.device)
loss = (
-soft_hit
- config.mean_reward_weight * mean_reward
+ scale * config.downside_weight * downside.mean()
+ scale * config.tail_weight * tail_downside
+ config.overlap_weight * overlap
)
return loss, {
"pnl": pnl,
"soft_hit_probability": soft_hit,
"mean_reward_scaled": mean_reward,
"downside_mean_scaled": downside.mean(),
"tail_downside_scaled": tail_downside,
"long_short_overlap": overlap,
}
[docs]
def v2_arbitrage_loss(
*,
action: Tensor,
raw_l1: Tensor,
dH: Tensor,
cost: Tensor,
config: DirectArbitrageConfig,
penalty_scale: float | Tensor = 1.0,
) -> tuple[Tensor, dict[str, Tensor]]:
"""V2.0 mean/downside/CVaR loss with radial-output conditioning penalty."""
long = action.clamp_min(0.0)
short = (-action).clamp_min(0.0)
pnl = _net_return_unchecked(long=long, short=short, dH=dH, cost=cost)
downside = (-pnl).clamp_min(0.0)
flat_downside = downside.reshape(-1)
tail_count = max(1, int(math.ceil(flat_downside.numel() * config.tail_fraction)))
tail_downside = torch.topk(
flat_downside, k=tail_count, sorted=False
).values.mean()
cap_excess_sqr = (raw_l1 - 1.0).clamp_min(0.0).square().mean()
scale = torch.as_tensor(penalty_scale, dtype=pnl.dtype, device=pnl.device)
loss = (
-pnl.mean()
+ scale * config.downside_weight * downside.mean()
+ scale * config.tail_weight * tail_downside
+ config.cap_weight * cap_excess_sqr
)
return loss, {
"pnl": pnl,
"pnl_mean": pnl.mean(),
"downside_mean": downside.mean(),
"tail_downside": tail_downside,
"cap_excess_sqr": cap_excess_sqr,
"raw_l1_mean": raw_l1.mean(),
"raw_l1_max": raw_l1.max(),
}
def _policy_allocations(
policy: LongShortSimplexPolicy | RadialL1Policy,
z: Tensor,
excluded_directions: Tensor | None,
) -> tuple[Tensor, Tensor, Tensor, Tensor | None]:
if isinstance(policy, RadialL1Policy):
action, raw_l1 = policy(z, excluded_directions)
long = action.clamp_min(0.0)
short = (-action).clamp_min(0.0)
no_trade = 1.0 - action.abs().sum(dim=-1)
return long, short, no_trade, raw_l1
long, short, no_trade = policy(z, excluded_directions)
return long, short, no_trade, None
def _iter_path_batches(
data: Mapping[str, Tensor], batch_size: int
) -> Iterator[dict[str, Tensor]]:
paths = data["z"].shape[0]
for start in range(0, paths, batch_size):
stop = min(start + batch_size, paths)
yield {name: value[start:stop] for name, value in data.items()}
def _sample_paths(
data: Mapping[str, Tensor],
*,
batch_size: int,
generator: torch.Generator,
) -> dict[str, Tensor]:
paths = data["z"].shape[0]
indices = torch.randint(
paths,
(min(batch_size, paths),),
device=data["z"].device,
generator=generator,
)
return {name: value[indices] for name, value in data.items()}
[docs]
@torch.no_grad()
def state_normalization(
data: Mapping[str, Tensor], *, batch_size: int
) -> tuple[Tensor, Tensor]:
"""Return training-only state mean and population standard deviation."""
features = data["z"].shape[-1]
total = torch.zeros(features, dtype=torch.float64, device=data["z"].device)
total_sqr = torch.zeros_like(total)
observations = 0
for batch in _iter_path_batches(data, batch_size):
z = batch["z"]
total += z.sum(dim=(0, 1), dtype=torch.float64)
total_sqr += z.square().sum(dim=(0, 1), dtype=torch.float64)
observations += z.shape[0] * z.shape[1]
mean = total / float(observations)
variance = (total_sqr / float(observations) - mean.square()).clamp_min(0.0)
scale = variance.sqrt().clamp_min(1.0e-6)
return mean.to(data["z"].dtype), scale.to(data["z"].dtype)
[docs]
@torch.no_grad()
def unconditional_cost(
data: Mapping[str, Tensor],
*,
cost_uncond: float,
batch_size: int,
) -> Tensor:
"""Return the full-training population standard error per instrument."""
verify_inp(
math.isfinite(cost_uncond) and cost_uncond >= 0.0,
lambda: f"'cost_uncond' must be finite and nonnegative, got {cost_uncond}",
)
instruments = data["dH"].shape[-1]
total = torch.zeros(instruments, dtype=torch.float64, device=data["dH"].device)
total_sqr = torch.zeros_like(total)
observations = 0
for batch in _iter_path_batches(data, batch_size):
dH = batch["dH"]
total += dH.sum(dim=(0, 1), dtype=torch.float64)
total_sqr += dH.square().sum(dim=(0, 1), dtype=torch.float64)
observations += dH.shape[0] * dH.shape[1]
mean = total / float(observations)
variance = (total_sqr / float(observations) - mean.square()).clamp_min(0.0)
return (cost_uncond * torch.sqrt(variance / float(observations))).to(
data["dH"].dtype
)
[docs]
@torch.no_grad()
def effective_cost(
data: Mapping[str, Tensor],
*,
bar_c: Tensor,
cost_prop: float,
conditional_variance_model: Any | None,
n_cost: int,
batch_size: int,
) -> Tensor:
"""Materialize ``max(c_cond, bar_c, c_prop)`` without retaining gradients."""
verify_inp(n_cost > 0, lambda: f"'n_cost' must be positive, got {n_cost}")
verify_inp(
math.isfinite(cost_prop) and cost_prop >= 0.0,
lambda: f"'cost_prop' must be finite and nonnegative, got {cost_prop}",
)
H = data["H"]
verify_inp(
bar_c.shape == (H.shape[-1],),
lambda: f"global cost shape {bar_c.shape} != {(H.shape[-1],)}",
)
output = torch.empty_like(H)
start = 0
for batch in _iter_path_batches(data, batch_size):
stop = start + batch["z"].shape[0]
c_prop = cost_prop * batch["H"].abs()
cost = torch.maximum(c_prop, bar_c.reshape(1, 1, -1))
if conditional_variance_model is not None:
_, _, variance = conditional_variance_model.moment_outputs(
batch["z"], batch["dH"]
)
c_cond = torch.sqrt(variance.clamp_min(0.0) / float(n_cost))
cost = torch.maximum(cost, c_cond)
output[start:stop] = cost
start = stop
if not torch.isfinite(output).all().item() or not torch.all(output >= 0.0).item():
raise RuntimeError("effective cost contains invalid values")
return output
def _clustered_lower_bound(path_rates: np.ndarray, confidence: float = 0.95) -> float:
if path_rates.size == 0:
raise ValueError("at least one path is required")
if path_rates.size == 1:
return float(path_rates[0])
z_score = NormalDist().inv_cdf(confidence)
error = path_rates.std(ddof=1) / math.sqrt(path_rates.size)
return float(max(0.0, path_rates.mean() - z_score * error))
def _wilson_upper(successes: int, observations: int, confidence: float = 0.95) -> float:
verify_inp(0 <= successes <= observations, "invalid binomial counts")
verify_inp(observations > 0, "at least one binomial observation is required")
z_score = NormalDist().inv_cdf(confidence)
probability = successes / observations
denominator = 1.0 + z_score**2 / observations
center = probability + z_score**2 / (2.0 * observations)
radius = z_score * math.sqrt(
probability * (1.0 - probability) / observations
+ z_score**2 / (4.0 * observations**2)
)
return float(min(1.0, (center + radius) / denominator))
[docs]
@torch.no_grad()
def valuate_policy(
policy: LongShortSimplexPolicy | RadialL1Policy,
data: Mapping[str, Tensor],
*,
batch_size: int,
split: str,
) -> dict[str, Any]:
"""Value a policy and conservatively certify it using independent paths."""
verify_inp(split in {"training", "validation"}, lambda: f"unknown split {split!r}")
pnl_chunks: list[np.ndarray] = []
gross_chunks: list[np.ndarray] = []
no_trade_chunks: list[np.ndarray] = []
raw_l1_chunks: list[np.ndarray] = []
long_total = torch.zeros(
policy.config.instruments, dtype=torch.float64, device=policy.device
)
short_total = torch.zeros_like(long_total)
overlap_total = torch.zeros_like(long_total)
observations = 0
policy.eval()
for batch in _iter_path_batches(data, batch_size):
excluded = batch.get("excluded_directions")
long, short, no_trade, raw_l1 = _policy_allocations(
policy, batch["z"], excluded
)
pnl = net_return(
long=long,
short=short,
dH=batch["dH"],
cost=batch["cost"],
)
pnl_chunks.append(pnl.cpu().numpy())
gross_chunks.append((long + short).sum(dim=-1).cpu().numpy())
no_trade_chunks.append(no_trade.cpu().numpy())
if raw_l1 is not None:
raw_l1_chunks.append(raw_l1.cpu().numpy())
long_total += long.sum(dim=(0, 1), dtype=torch.float64)
short_total += short.sum(dim=(0, 1), dtype=torch.float64)
overlap_total += (long * short).sum(dim=(0, 1), dtype=torch.float64)
observations += pnl.numel()
pnl = np.concatenate(pnl_chunks, axis=0).astype(np.float64, copy=False)
gross = np.concatenate(gross_chunks, axis=0).astype(np.float64, copy=False)
no_trade = np.concatenate(no_trade_chunks, axis=0).astype(np.float64, copy=False)
raw_l1 = (
None
if not raw_l1_chunks
else np.concatenate(raw_l1_chunks, axis=0).astype(np.float64, copy=False)
)
epsilon_hits = pnl > policy.config.epsilon
negative = pnl < -policy.config.numerical_tolerance
strict_negative = pnl < 0.0
path_hit_rates = epsilon_hits.mean(axis=1)
losing_paths = np.any(negative, axis=1)
hit_probability = float(epsilon_hits.mean())
hit_lower = _clustered_lower_bound(path_hit_rates)
losing_path_count = int(losing_paths.sum())
losing_path_probability_upper = _wilson_upper(losing_path_count, len(losing_paths))
mean_long = (long_total / float(observations)).cpu().numpy()
mean_short = (short_total / float(observations)).cpu().numpy()
mean_abs = mean_long + mean_short
order = np.argsort(-mean_abs)[: min(20, len(mean_abs))]
quantile_levels = np.asarray([0.0, 0.001, 0.01, 0.05, 0.5, 0.95, 0.99, 1.0])
empirical_nonnegative = bool(not np.any(negative))
excluded = data.get("excluded_directions")
return {
"split": split,
"paths": int(pnl.shape[0]),
"steps": int(pnl.shape[1]),
"observations": int(pnl.size),
"epsilon": policy.config.epsilon,
"delta": policy.config.delta,
"numerical_tolerance": policy.config.numerical_tolerance,
"pnl_mean": float(pnl.mean()),
"pnl_std_population": float(pnl.std(ddof=0)),
"pnl_quantile_levels": quantile_levels.tolist(),
"pnl_quantiles": np.quantile(pnl, quantile_levels).tolist(),
"pnl_min": float(pnl.min()),
"pnl_max": float(pnl.max()),
"strict_negative_observations": int(strict_negative.sum()),
"negative_observations_beyond_tolerance": int(negative.sum()),
"empirical_nonnegative_within_tolerance": empirical_nonnegative,
"epsilon_hit_probability": hit_probability,
"epsilon_hit_probability_path_clustered_lower_95": hit_lower,
"losing_paths": losing_path_count,
"losing_path_probability_upper_95": losing_path_probability_upper,
"qualifies_empirically": bool(
empirical_nonnegative and hit_lower > policy.config.delta
),
"gross_exposure_mean": float(gross.mean()),
"gross_exposure_quantiles": np.quantile(gross, quantile_levels).tolist(),
"no_trade_mean": float(no_trade.mean()),
"raw_l1_mean": None if raw_l1 is None else float(raw_l1.mean()),
"raw_l1_max": None if raw_l1 is None else float(raw_l1.max()),
"raw_l1_above_one_probability": (
None if raw_l1 is None else float((raw_l1 > 1.0).mean())
),
"raw_l1_above_two_probability": (
None if raw_l1 is None else float((raw_l1 > 2.0).mean())
),
"boundary_exposure_probability": float((gross > 0.99).mean()),
"excluded_direction_fraction": (
0.0
if excluded is None
else float(excluded.to(torch.float64).mean().item())
),
"mean_long_by_instrument": mean_long.tolist(),
"mean_short_by_instrument": mean_short.tolist(),
"mean_long_short_overlap_by_instrument": (
overlap_total / float(observations)
).cpu().numpy().tolist(),
"top_instruments_by_mean_gross_exposure": [
{
"instrument": int(index),
"mean_long": float(mean_long[index]),
"mean_short": float(mean_short[index]),
"mean_gross": float(mean_abs[index]),
}
for index in order
],
"mc_convention": (
"hit-rate uncertainty clusters all time steps by simulated path; "
"loss certification rejects on any path-time loss beyond tolerance"
),
}
def _checkpoint_path(
config: DirectArbitrageConfig,
train_config: DirectArbitrageTrainConfig,
) -> Path:
root = Path(train_config.checkpoint_root).expanduser()
if config.group:
root = root.joinpath(*Path(config.group).parts)
return root / f"{config.filename}.pt"
def _compact_valuation(value: Mapping[str, Any]) -> dict[str, Any]:
names = (
"pnl_mean",
"pnl_std_population",
"pnl_min",
"pnl_max",
"strict_negative_observations",
"negative_observations_beyond_tolerance",
"epsilon_hit_probability",
"epsilon_hit_probability_path_clustered_lower_95",
"losing_paths",
"losing_path_probability_upper_95",
"qualifies_empirically",
"gross_exposure_mean",
"no_trade_mean",
"raw_l1_mean",
"raw_l1_max",
"raw_l1_above_one_probability",
"raw_l1_above_two_probability",
"boundary_exposure_probability",
"excluded_direction_fraction",
)
return {name: value[name] for name in names}
def _atomic_checkpoint(path: Path, state: Mapping[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
torch.save(dict(state), temporary)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _training_checkpoint_state(
*,
config: DirectArbitrageConfig,
train_config: DirectArbitrageTrainConfig,
policy: LongShortSimplexPolicy | RadialL1Policy,
optimizer: torch.optim.Optimizer,
scheduler: torch.optim.lr_scheduler.LRScheduler,
generator: torch.Generator,
epoch: int,
tracker: Mapping[str, list[Any]],
) -> dict[str, Any]:
state: dict[str, Any] = {
"schema": 1,
"config_version": DirectArbitrageConfig.version.unique_id48,
"policy_version": (
"radial-l1-v2-v1"
if isinstance(policy, RadialL1Policy)
else "long-short-simplex-v3"
),
"user_version": config.user_version,
"config": asdict(config),
"train_config": asdict(train_config),
"epoch": epoch,
"policy_state": policy.state_dict(),
"optimizer_state": optimizer.state_dict(),
"scheduler_state": scheduler.state_dict(),
"generator_state": generator.get_state(),
"torch_rng_state": torch.get_rng_state(),
"tracker": dict(tracker),
}
if policy.device.type == "cuda":
state["cuda_rng_state"] = torch.cuda.get_rng_state(policy.device)
return state
[docs]
def train_direct_policy(
training: Mapping[str, Tensor],
validation: Mapping[str, Tensor] | None,
*,
config: DirectArbitrageConfig,
train_config: DirectArbitrageTrainConfig,
epochs: int,
env: BaseEnvironment,
restart: bool = False,
) -> DirectArbitrageTrainingResult:
"""Train a direct policy and atomically checkpoint resumable optimizer state."""
verify_inp(epochs > 0, lambda: f"'epochs' must be positive, got {epochs}")
required = {"z", "dH", "cost"}
if config.mask_arbitrage_candidates:
required.add("excluded_directions")
verify_inp(
required <= training.keys(),
lambda: f"training data is missing {sorted(required - training.keys())}",
)
if validation is not None:
verify_inp(
required <= validation.keys(),
lambda: f"validation data is missing {sorted(required - validation.keys())}",
)
verify_inp(
training["dH"].shape == training["cost"].shape,
lambda: (
f"training return/cost shapes differ: {training['dH'].shape} and "
f"{training['cost'].shape}"
),
)
for split, data in (("training", training), ("validation", validation)):
if data is None or "excluded_directions" not in data:
continue
excluded = data["excluded_directions"]
expected = data["dH"].shape[:-1] + (2 * config.instruments,)
verify_inp(
excluded.shape == expected and excluded.dtype == torch.bool,
lambda split=split, excluded=excluded, expected=expected: (
f"{split} excluded directions must be bool with shape {expected}, "
f"got {excluded.shape}/{excluded.dtype}"
),
)
if torch.any(excluded.all(dim=-1)).item():
raise ValueError(f"{split} mask excludes every direction at a state")
torch.manual_seed(config.seed)
if env.device.type == "cuda":
torch.cuda.manual_seed(config.seed)
mean, scale = state_normalization(
training, batch_size=train_config.valuation_batch_size
)
policy_class = (
RadialL1Policy
if config.objective == "radial-l1-v2"
else LongShortSimplexPolicy
)
policy = policy_class(
env, config=config, state_mean=mean, state_scale=scale
)
policy_version = (
"radial-l1-v2-v1"
if isinstance(policy, RadialL1Policy)
else "long-short-simplex-v3"
)
optimizer = torch.optim.AdamW(
policy.parameters(),
lr=train_config.lr,
weight_decay=train_config.weight_decay,
)
gamma = (
train_config.final_lr / train_config.lr
) ** (1.0 / float(max(epochs - 1, 1)))
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma)
generator = torch.Generator(device=env.device)
generator.manual_seed(config.seed + 1)
tracker: dict[str, list[Any]] = {
"epochs": [],
"batch_loss": [],
"learning_rate": [],
"gradient_norm": [],
"valuation_epochs": [],
"training_values": [],
"validation_values": [],
}
checkpoint_path = _checkpoint_path(config, train_config)
epoch = 0
restarted = False
lock = FileLock(f"{checkpoint_path}.lock")
with lock:
if checkpoint_path.exists() and not restart:
state = torch.load(
checkpoint_path, map_location=env.device, weights_only=False
)
compatible = (
state.get("schema") == 1
and state.get("config_version")
== DirectArbitrageConfig.version.unique_id48
and state.get("policy_version") == policy_version
and state.get("user_version") == config.user_version
and state.get("config") == asdict(config)
)
if compatible:
policy.load_state_dict(state["policy_state"])
optimizer.load_state_dict(state["optimizer_state"])
scheduler.load_state_dict(state["scheduler_state"])
generator.set_state(state["generator_state"])
torch.set_rng_state(state["torch_rng_state"])
if env.device.type == "cuda" and "cuda_rng_state" in state:
torch.cuda.set_rng_state(state["cuda_rng_state"], env.device)
epoch = int(state["epoch"])
tracker = state["tracker"]
restarted = True
else:
print(
f"Ignoring incompatible direct-arbitrage checkpoint {checkpoint_path}",
flush=True,
)
def loss_step(
batch: Mapping[str, Tensor], penalty_scale: Tensor
) -> Tensor:
excluded = batch.get("excluded_directions")
if isinstance(policy, RadialL1Policy):
action, raw_l1 = policy(batch["z"], excluded)
loss, _ = v2_arbitrage_loss(
action=action,
raw_l1=raw_l1,
dH=batch["dH"],
cost=batch["cost"],
config=config,
penalty_scale=penalty_scale,
)
else:
long, short, _ = policy(batch["z"], excluded)
loss, _ = direct_arbitrage_loss(
long=long,
short=short,
dH=batch["dH"],
cost=batch["cost"],
config=config,
penalty_scale=penalty_scale,
)
return loss
compiled_loss_step = loss_step
if train_config.compile and not policy.debug:
compiled_loss_step = torch.compile(loss_step, dynamic=True)
penalty_scale = torch.ones((), dtype=env.fdtype, device=env.device)
def save() -> None:
_atomic_checkpoint(
checkpoint_path,
_training_checkpoint_state(
config=config,
train_config=train_config,
policy=policy,
optimizer=optimizer,
scheduler=scheduler,
generator=generator,
epoch=epoch,
tracker=tracker,
),
)
for epoch_index in range(epoch, epochs):
policy.train()
optimizer.zero_grad(set_to_none=True)
batch = _sample_paths(
training,
batch_size=train_config.batch_size,
generator=generator,
)
progress = (epoch_index + 1) / float(epochs)
if progress <= config.discovery_fraction:
scale_value = 0.0
elif progress < (
config.discovery_fraction + config.constraint_ramp_fraction
):
scale_value = (
progress - config.discovery_fraction
) / config.constraint_ramp_fraction
else:
scale_value = 1.0
penalty_scale.fill_(scale_value)
loss = compiled_loss_step(batch, penalty_scale)
if not torch.isfinite(loss).item():
raise RuntimeError(
f"non-finite direct-arbitrage loss at epoch {epoch_index + 1}"
)
loss.backward()
gradient_sqr = torch.zeros((), dtype=torch.float64, device=env.device)
nonfinite_gradients = 0
for parameter in policy.parameters():
if parameter.grad is None:
continue
gradient = parameter.grad.detach()
nonfinite_gradients += int(
torch.count_nonzero(~torch.isfinite(gradient)).item()
)
gradient_sqr += torch.nan_to_num(gradient.to(torch.float64)).square().sum()
if nonfinite_gradients:
raise RuntimeError(
f"found {nonfinite_gradients} non-finite gradient elements"
)
optimizer.step()
epoch = epoch_index + 1
if epoch < epochs:
scheduler.step()
tracker["epochs"].append(epoch)
tracker["batch_loss"].append(float(loss.detach().item()))
tracker["learning_rate"].append(float(optimizer.param_groups[0]["lr"]))
tracker["gradient_norm"].append(float(torch.sqrt(gradient_sqr).item()))
should_value = (
epoch == 1
or epoch % train_config.value_update == 0
or epoch == epochs
)
if should_value:
training_value = valuate_policy(
policy,
training,
batch_size=train_config.valuation_batch_size,
split="training",
)
validation_value = (
valuate_policy(
policy,
validation,
batch_size=train_config.valuation_batch_size,
split="validation",
)
if validation is not None
else None
)
tracker["valuation_epochs"].append(epoch)
tracker["training_values"].append(
_compact_valuation(training_value)
)
tracker["validation_values"].append(
None
if validation_value is None
else _compact_valuation(validation_value)
)
print(
f"Epoch {epoch}/{epochs}: loss={loss.item():.7g}, "
f"train hit={training_value['epsilon_hit_probability']:.6g}, "
f"train losses={training_value['negative_observations_beyond_tolerance']}, "
f"validation hit={0.0 if validation_value is None else validation_value['epsilon_hit_probability']:.6g}, "
f"validation losses={0 if validation_value is None else validation_value['negative_observations_beyond_tolerance']}",
flush=True,
)
save()
elif epoch % train_config.checkpoint_update == 0:
save()
training_value = valuate_policy(
policy,
training,
batch_size=train_config.valuation_batch_size,
split="training",
)
validation_value = (
valuate_policy(
policy,
validation,
batch_size=train_config.valuation_batch_size,
split="validation",
)
if validation is not None
else None
)
save()
return DirectArbitrageTrainingResult(
policy=policy,
training=training_value,
validation=validation_value,
tracker=tracker,
checkpoint_path=checkpoint_path,
restarted=restarted,
)
__all__ = [
"DirectArbitrageConfig",
"DirectArbitrageTrainConfig",
"DirectArbitrageTrainingResult",
"LongShortSimplexPolicy",
"RadialL1Policy",
"direct_arbitrage_loss",
"effective_cost",
"net_return",
"state_normalization",
"train_direct_policy",
"unconditional_cost",
"v2_arbitrage_loss",
"valuate_policy",
]