"""Replay a learned arb-tracing portfolio from one exact starting state."""
from __future__ import annotations
import argparse
import gc
import json
import math
import os
from dataclasses import asdict
from pathlib import Path
from typing import Any
import numpy as np
import torch
from cdxcore.err import verify_inp
from cdxcore.verbose import Context
from scipy.spatial import cKDTree
from quantdata.biz_calendar import DAYS_PER_YEAR
from quantdata.torch import BaseEnvironment
from quantdata.torch.ffw import FeedForwardConfig
from .arb_tracing import ArbTracingConfig, OrthogonalArbDirection
from .required_runs import (
RequiredCase,
clopper_pearson_one_sided,
positive_relative_weights,
stratified_path_split,
)
from .run_expiry_arbitrage_experiments import _DEFAULT_CACHE_ROOT, _load_case
from .terminal_universe import expiry_step_offsets
_DEFAULT_TRACE_ROOT = Path(
"quantdata/dysanos/grid/gendysanos/reports/arb-tracing"
)
_DEFAULT_OUTPUT_ROOT = Path(
"quantdata/dysanos/grid/gendysanos/reports/conditional-arb-replay"
)
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
temporary.write_text(
json.dumps(value, indent=2, allow_nan=False), encoding="utf-8"
)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _model_config(value: dict[str, Any]) -> ArbTracingConfig:
fields = dict(value)
fields["network_config"] = FeedForwardConfig(**fields["network_config"])
return ArbTracingConfig(**fields)
@torch.no_grad()
def _frozen_actions(
report: dict[str, Any],
state: np.ndarray,
*,
device: str,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Reconstruct the final sum of frozen stage directions."""
verify_inp(
state.ndim == 3 and state.shape[1] == 10,
lambda: f"candidate states must have shape (S,10,F), got {state.shape}",
)
env = BaseEnvironment(precision=32, device=device, debug=False)
tensor_state = torch.as_tensor(state, dtype=env.fdtype, device=env.device)
total = None
first_mean = None
first_whitener = None
for stage in report["stages"]:
config = _model_config(stage["model_config"])
model = OrthogonalArbDirection(
env,
config=config,
state_mean=torch.zeros(
config.trading_times,
config.state_features,
dtype=env.fdtype,
device=env.device,
),
state_whitener=torch.eye(
config.state_features,
dtype=env.fdtype,
device=env.device,
)[None].repeat(config.trading_times, 1, 1),
)
checkpoint = torch.load(
stage["checkpoint"], map_location=env.device, weights_only=False
)
model.load_state_dict(checkpoint["model"])
model.eval()
direction, _ = model(tensor_state)
scale = torch.as_tensor(
stage["base_update_notional_by_time"],
dtype=env.fdtype,
device=env.device,
)[None, :, None, None]
contribution = scale * direction
total = contribution if total is None else total + contribution
if first_mean is None:
first_mean = model.state_mean.detach().cpu().numpy()
first_whitener = model.state_whitener.detach().cpu().numpy()
del model, checkpoint, direction, contribution
assert total is not None and first_mean is not None and first_whitener is not None
result = total.detach().cpu().numpy().astype(np.float64)
if not np.all(np.isfinite(result)):
raise FloatingPointError("reconstructed frozen action is non-finite")
del tensor_state, total
if torch.device(device).type == "cuda":
torch.cuda.empty_cache()
return result, first_mean, first_whitener
def _portfolio_pnl_band(
action: np.ndarray,
returns: np.ndarray,
costs: np.ndarray,
instrument_band: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
"""Evaluate a fixed portfolio and its propagated float32 band."""
action = np.asarray(action, dtype=np.float64)
returns = np.asarray(returns, dtype=np.float64)
costs = np.asarray(costs, dtype=np.float64)
instrument_band = np.asarray(instrument_band, dtype=np.float64)
verify_inp(
returns.shape == costs.shape == instrument_band.shape
and returns.shape[1:] == action.shape,
lambda: (
"conditional portfolio geometry differs: "
f"{action.shape}/{returns.shape}/{costs.shape}/{instrument_band.shape}"
),
)
absolute = np.abs(action)
pnl = np.sum(action[None] * returns - absolute[None] * costs, axis=(1, 2))
band = np.sum(absolute[None] * instrument_band, axis=(1, 2))
band += 32.0 * np.finfo(np.float32).eps * np.sum(
absolute[None] * (np.abs(returns) + costs), axis=(1, 2)
)
if not np.all(np.isfinite(pnl)) or not np.all(np.isfinite(band)):
raise FloatingPointError("conditional portfolio valuation is non-finite")
return pnl, band
def _weighted_statistics(
pnl: np.ndarray,
band: np.ndarray,
log_weights: np.ndarray,
component: np.ndarray,
) -> dict[str, Any]:
weights = positive_relative_weights(log_weights)
normalized = weights / weights.sum()
loss = pnl < -band
mean = float(np.dot(normalized, pnl))
mean_error = float(
np.sqrt(np.sum(np.square(normalized * (pnl - mean))))
)
probability = float(np.dot(normalized, loss))
probability_error = float(
np.sqrt(np.sum(np.square(normalized * (loss - probability))))
)
direct = component == 0
return {
"paths": int(pnl.size),
"weighted_P_mean": mean,
"weighted_P_mean_standard_error": mean_error,
"weighted_P_loss_probability": probability,
"weighted_P_loss_probability_standard_error": probability_error,
"proposal_loss_count": int(np.count_nonzero(loss)),
"direct_P_paths": int(np.count_nonzero(direct)),
"direct_P_loss_count": int(np.count_nonzero(loss & direct)),
}
def _select_candidate_state(
*,
loaded,
report: dict[str, Any],
query_paths: int,
neighbors: int,
seed: int,
device: str,
) -> dict[str, Any]:
"""Choose the strongest locally screened training state deterministically."""
split = stratified_path_split(
loaded.component,
audit_fraction=0.2,
seed=loaded.state.shape[0] + 97,
)
direct_reference = split.reference[loaded.component[split.reference] == 0]
verify_inp(direct_reference.size > 0, "no direct-P training states are available")
rng = np.random.default_rng(seed)
query = np.sort(
rng.choice(
direct_reference,
size=min(query_paths, direct_reference.size),
replace=False,
)
)
query_action, state_mean, state_whitener = _frozen_actions(
report, loaded.state[query], device=device
)
candidate_times = sorted(
{
int(item["trading_time_index"])
for stage in report["stages"]
for item in stage["selection"]
if item.get("training_near_arbitrage_candidate", False)
}
)
verify_inp(
bool(candidate_times),
"the source report contains no training near-arbitrage trading time",
)
candidates: list[dict[str, Any]] = []
for time_index in candidate_times:
reference_state = np.einsum(
"sf,fg->sg",
loaded.state[split.reference, time_index].astype(np.float64)
- state_mean[time_index],
state_whitener[time_index].astype(np.float64),
)
query_state = np.einsum(
"sf,fg->sg",
loaded.state[query, time_index].astype(np.float64)
- state_mean[time_index],
state_whitener[time_index].astype(np.float64),
)
tree = cKDTree(reference_state)
distance, local = tree.query(
query_state,
k=min(neighbors + 1, split.reference.size),
workers=-1,
)
for query_index, path_index in enumerate(query):
paths = split.reference[np.asarray(local[query_index]).reshape(-1)]
distances = np.asarray(distance[query_index]).reshape(-1)
keep = paths != path_index
paths = paths[keep][:neighbors]
distances = distances[keep][:neighbors]
action = query_action[query_index, time_index]
pnl, band = _portfolio_pnl_band(
action,
loaded.returns[paths, time_index],
loaded.costs[paths, time_index],
loaded.numerical_band[paths, time_index],
)
stats = _weighted_statistics(
pnl,
band,
loaded.log_path_weight[paths],
loaded.component[paths],
)
loss_upper_score = (
stats["weighted_P_loss_probability"]
+ 3.0 * stats["weighted_P_loss_probability_standard_error"]
)
mean_lower = (
stats["weighted_P_mean"]
- 3.0 * stats["weighted_P_mean_standard_error"]
)
candidates.append(
{
"query_path": int(path_index),
"trading_time_index": time_index,
"neighbor_radius": float(distances[-1]),
"action": action,
"action_l1": float(np.abs(action).sum()),
"local_statistics": stats,
"selection_score": (loss_upper_score, -mean_lower),
}
)
selected = min(candidates, key=lambda item: item["selection_score"])
selected["states_screened"] = len(candidates)
selected["candidate_trading_times"] = candidate_times
selected.pop("selection_score")
return selected
def _load_physical_sampler(generator: str, *, seed: int):
"""Load the fitted generator and return its direct-P sampler and expiries."""
if generator == "dysanos":
from .torch_prep import GenDysanosExperiment, _load_generator_components
experiment = GenDysanosExperiment(
num_samples=2,
num_alpha=None,
use_matrix_exp=True,
bandwidth=1.0,
trading_frequency="weekly",
return_horizon="expiry",
num_trading_times=1,
seed=seed,
importance_sampling=None,
)
grid_config, _, _, sampler = _load_generator_components(
experiment, verbose=Context.quiet
)
expiries = np.asarray(grid_config.grid_expiries, dtype=np.float64)
else:
from quantdata.dysanos.implied.decoder import PCATotalVarianceDecoder
from quantdata.dysanos.implied.experiment import ImpliedPCAExperiment
from quantdata.dysanos.implied.market import load_fitted_pca
from quantdata.dysanos.implied.simulator import ImpliedPCASampler
experiment = ImpliedPCAExperiment(
num_samples=2,
num_alpha=None,
use_matrix_exp=True,
bandwidth=1.0,
trading_frequency="weekly",
return_horizon="expiry",
num_trading_times=1,
seed=seed,
importance_sampling=None,
)
_, market, pca, diagnostics = load_fitted_pca(
experiment.fit_config, verbose=Context.quiet
)
PCATotalVarianceDecoder(pca)
sampler = ImpliedPCASampler(
pca=pca,
scores=diagnostics.scores,
time_y=market.time_y,
spot_ref=market.spot_ref,
num_alpha=experiment.num_alpha,
seed=seed,
verbose=Context.quiet,
)
expiries = np.asarray(pca.expiries, dtype=np.float64)
sampler.rng = np.random.default_rng(seed)
return sampler, expiries
def _conditional_delta_pnl(
terminal_spot: np.ndarray,
*,
action: np.ndarray,
spot: float,
option_price: np.ndarray,
delta: np.ndarray,
cash_strike: np.ndarray,
cost_rate: float,
) -> tuple[np.ndarray, np.ndarray]:
"""Value one fixed multi-expiry delta-hedged portfolio."""
terminal_spot = np.asarray(terminal_spot, dtype=np.float64)
option_price = np.asarray(option_price, dtype=np.float64)
delta = np.asarray(delta, dtype=np.float64)
cash_strike = np.asarray(cash_strike, dtype=np.float64)
action = np.asarray(action, dtype=np.float64)
expiries, strikes = option_price.shape
verify_inp(
terminal_spot.ndim == 2
and terminal_spot.shape[1] == expiries
and delta.shape == cash_strike.shape == option_price.shape
and action.shape == (expiries, strikes + 1),
"conditional expiry portfolio has inconsistent geometry",
)
spot_return = terminal_spot - spot
option_return = (
np.maximum(terminal_spot[:, :, None] - cash_strike[None], 0.0)
- option_price[None]
- delta[None] * spot_return[:, :, None]
)
returns = np.concatenate((spot_return[:, :, None], option_return), axis=2)
option_cost = cost_rate * (
np.abs(option_price) + np.abs(delta * spot)
)
costs = np.concatenate(
(
np.full((expiries, 1), cost_rate * abs(spot)),
option_cost,
),
axis=1,
)
spot_scale = np.abs(terminal_spot) + abs(spot)
option_scale = (
np.abs(terminal_spot[:, :, None])
+ np.abs(cash_strike[None])
+ np.abs(option_price[None])
+ np.abs(delta[None])
* (np.abs(terminal_spot[:, :, None]) + abs(spot))
)
scale = np.concatenate((spot_scale[:, :, None], option_scale), axis=2)
instrument_band = 32.0 * np.finfo(np.float32).eps * (
scale + costs[None]
)
return _portfolio_pnl_band(
action, returns, np.broadcast_to(costs, returns.shape), instrument_band
)
def _deterministic_terminal_spot_lower_bound(
*,
action: np.ndarray,
spot: float,
option_price: np.ndarray,
delta: np.ndarray,
cash_strike: np.ndarray,
cost_rate: float,
) -> dict[str, Any]:
"""Minimize the frozen P&L over every positive terminal-spot vector."""
action = np.asarray(action, dtype=np.float64)
option_price = np.asarray(option_price, dtype=np.float64)
delta = np.asarray(delta, dtype=np.float64)
cash_strike = np.asarray(cash_strike, dtype=np.float64)
expiries, strikes = option_price.shape
verify_inp(
action.shape == (expiries, strikes + 1)
and delta.shape == cash_strike.shape == option_price.shape,
"deterministic terminal-spot test has inconsistent geometry",
)
reports: list[dict[str, Any]] = []
total_lower_bound = 0.0
joint_minimizer = np.empty(expiries, dtype=np.float64)
unbounded = False
for expiry in range(expiries):
spot_action = action[expiry, 0]
option_action = action[expiry, 1:]
option_cost = cost_rate * (
np.abs(option_price[expiry])
+ np.abs(delta[expiry] * spot)
)
fixed_cost = (
abs(spot_action) * cost_rate * abs(spot)
+ np.dot(np.abs(option_action), option_cost)
)
candidates = np.unique(
np.concatenate((np.asarray([0.0]), cash_strike[expiry]))
)
option_return = (
np.maximum(
candidates[:, None] - cash_strike[expiry, None], 0.0
)
- option_price[expiry, None]
- delta[expiry, None] * (candidates[:, None] - spot)
)
value = (
spot_action * (candidates - spot)
+ option_return @ option_action
- fixed_cost
)
minimum_index = int(np.argmin(value))
finite_lower_bound = float(value[minimum_index])
tail_slope = float(
spot_action + np.dot(option_action, 1.0 - delta[expiry])
)
tail_tolerance = (
32.0
* np.finfo(np.float64).eps
* max(1.0, float(np.abs(action[expiry]).sum()))
)
tail_unbounded = bool(tail_slope < -tail_tolerance)
lower_bound = -math.inf if tail_unbounded else finite_lower_bound
unbounded = unbounded or tail_unbounded
total_lower_bound = (
-math.inf
if unbounded
else total_lower_bound + finite_lower_bound
)
joint_minimizer[expiry] = candidates[minimum_index]
reports.append(
{
"expiry_index": expiry,
"lower_bound": None if tail_unbounded else finite_lower_bound,
"minimizing_terminal_spot": float(candidates[minimum_index]),
"tail_slope": tail_slope,
"tail_slope_tolerance": tail_tolerance,
"unbounded_below_in_upper_tail": tail_unbounded,
}
)
joint_pnl, joint_band = _conditional_delta_pnl(
joint_minimizer[None],
action=action,
spot=spot,
option_price=option_price,
delta=delta,
cash_strike=cash_strike,
cost_rate=cost_rate,
)
return {
"domain": "all vectors of strictly positive terminal spots; zero is used as the continuous lower-tail limit",
"method": (
"separable piecewise-linear minimization at zero and every cash "
"strike, with a nonnegative upper-tail slope required per expiry"
),
"lower_bound": None if unbounded else total_lower_bound,
"unbounded_below": unbounded,
"finite_joint_test_spot": joint_minimizer.tolist(),
"finite_joint_test_pnl": float(joint_pnl[0]),
"finite_joint_test_numerical_band": float(joint_band[0]),
"nonnegative_for_all_terminal_spots": bool(
not unbounded and total_lower_bound >= -float(joint_band[0])
),
"by_expiry": reports,
}
[docs]
def run(args: argparse.Namespace) -> dict[str, Any]:
case = RequiredCase(
generator=args.generator,
trading_frequency="weekly",
return_horizon="expiry",
hedge="delta",
cost_rate=1.0e-4,
path_count=args.source_paths,
)
trace_path = Path(args.trace_root) / (
f"arb-tracing-powerexp-g1e+02-{case.generator}-{case.path_count}.json"
)
report = json.loads(trace_path.read_text(encoding="utf-8"))
verify_inp(report.get("status") == "complete", "arb-tracing report is incomplete")
loaded = _load_case(
case, cache_root=Path(args.cache_root), need_primitives=True
)
selected = _select_candidate_state(
loaded=loaded,
report=report,
query_paths=args.query_paths,
neighbors=args.neighbors,
seed=args.seed + 17,
device=args.device,
)
path_index = selected["query_path"]
time_index = selected["trading_time_index"]
action = np.asarray(selected.pop("action"), dtype=np.float64)
initial_state = loaded.state[path_index, time_index].astype(np.float64)
spot = float(loaded.spot[path_index, time_index])
option_price = loaded.option_price[path_index, time_index].astype(np.float64)
delta = loaded.delta[path_index, time_index].astype(np.float64)
cash_strike = (
np.exp(loaded.log_strike[path_index, time_index].astype(np.float64))
* spot
)
cached_cost = loaded.costs[path_index, time_index].astype(np.float64)
del loaded, report
gc.collect()
sampler, expiries = _load_physical_sampler(case.generator, seed=args.seed)
offsets, expiry_error = expiry_step_offsets(expiries, dt=1.0 / DAYS_PER_YEAR)
verify_inp(
abs(math.exp(initial_state[0]) - spot)
<= 1.0e-5 * max(1.0, abs(spot)),
lambda: (
"cached spot and selected log-spot differ: "
f"{math.exp(initial_state[0])}/{spot}"
),
)
expected_cost = np.concatenate(
(
np.full((expiries.size, 1), case.cost_rate * abs(spot)),
case.cost_rate
* (np.abs(option_price) + np.abs(delta * spot)),
),
axis=1,
)
verify_inp(
np.allclose(cached_cost, expected_cost, rtol=2.0e-6, atol=1.0e-8),
"conditional replay costs differ from cached delta-hedged costs",
)
deterministic_test = _deterministic_terminal_spot_lower_bound(
action=action,
spot=spot,
option_price=option_price,
delta=delta,
cash_strike=cash_strike,
cost_rate=case.cost_rate,
)
pnl_values: list[np.ndarray] = []
band_values: list[np.ndarray] = []
step_offsets = np.concatenate((np.asarray([0]), offsets))
for start in range(0, args.conditional_paths, args.block_size):
stop = min(start + args.block_size, args.conditional_paths)
state = sampler.sample_conditional_xstates_at_offsets(
initial_state=initial_state,
step_offsets=step_offsets,
num_samples=stop - start,
dt=1.0 / DAYS_PER_YEAR,
)
terminal_spot = np.exp(state[:, 1:, 0])
pnl, band = _conditional_delta_pnl(
terminal_spot,
action=action,
spot=spot,
option_price=option_price,
delta=delta,
cash_strike=cash_strike,
cost_rate=case.cost_rate,
)
pnl_values.append(pnl)
band_values.append(band)
print(
f"[{case.generator}] conditional continuations "
f"{stop:,}/{args.conditional_paths:,}; losses="
f"{sum(np.count_nonzero(x < -y) for x, y in zip(pnl_values, band_values)):,}",
flush=True,
)
pnl = np.concatenate(pnl_values)
band = np.concatenate(band_values)
loss = pnl < -band
gain = pnl > band
loss_count = int(np.count_nonzero(loss))
gain_count = int(np.count_nonzero(gain))
_, loss_upper = clopper_pearson_one_sided(
loss_count, args.conditional_paths, alpha=0.05
)
mean = float(np.mean(pnl))
mean_error = float(np.std(pnl, ddof=0) / math.sqrt(pnl.size))
result = {
"status": "complete",
"experiment": "exact-state conditional independent-continuation replay",
"configuration": {
**asdict(case),
"conditional_paths": args.conditional_paths,
"candidate_query_paths": args.query_paths,
"candidate_neighbors": args.neighbors,
"seed": args.seed,
"block_size": args.block_size,
},
"method": {
"candidate_source": (
"direct-P training states at trading times passing the 100,000-path "
"power-exp training near-arbitrage screen"
),
"candidate_selection": (
"minimum local weighted loss upper score, then maximum local "
"three-standard-error mean lower bound"
),
"portfolio": (
"sum of all frozen arb-tracing stage directions and selected "
"notionals; no parameter is refitted"
),
"continuation_law": (
"100,000 independent exact Gaussian matrix-exponential "
"continuations under P from the identical raw state x_t"
),
"expiry_sampling": (
"the exact Gaussian transition is composed directly between "
"the seven expiry offsets; omitted daily states are integrated out"
),
},
"source_trace": str(trace_path),
"selected_state": {
**selected,
"raw_x": initial_state.tolist(),
"spot": spot,
"action": action.tolist(),
},
"expiry_offsets": offsets.tolist(),
"maximum_expiry_time_error": expiry_error,
"deterministic_terminal_spot_test": deterministic_test,
"conditional_test": {
"paths": args.conditional_paths,
"loss_count_beyond_float32_band": loss_count,
"gain_count_beyond_float32_band": gain_count,
"loss_probability": loss_count / args.conditional_paths,
"loss_probability_one_sided_95_upper": loss_upper,
"mean_pnl": mean,
"mean_pnl_standard_error": mean_error,
"mean_pnl_lower_three_standard_errors": mean - 3.0 * mean_error,
"minimum_pnl_minus_band": float(np.min(pnl - band)),
"pnl_quantiles": {
str(probability): float(np.quantile(pnl, probability))
for probability in (0.0, 0.001, 0.01, 0.05, 0.5, 0.95)
},
"no_observed_loss": loss_count == 0,
"positive_mean_above_three_standard_errors": (
mean - 3.0 * mean_error > 0.0
),
"conditional_near_arbitrage_candidate": (
loss_count == 0 and mean - 3.0 * mean_error > 0.0
),
},
}
output = Path(args.output_root) / (
f"conditional-arb-replay-{case.generator}-{case.path_count}-"
f"n{args.conditional_paths}.json"
)
_atomic_json(output, result)
print(f"Wrote {output}", flush=True)
return result
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--generator", choices=("dysanos", "pca-iv"), required=True)
parser.add_argument("--source-paths", type=int, default=100_000)
parser.add_argument("--conditional-paths", type=int, default=100_000)
parser.add_argument("--query-paths", type=int, default=256)
parser.add_argument("--neighbors", type=int, default=1_024)
parser.add_argument("--block-size", type=int, default=10_000)
parser.add_argument("--seed", type=int, default=2_026_081)
parser.add_argument("--device", default="cuda:2")
parser.add_argument("--cache-root", default=str(_DEFAULT_CACHE_ROOT))
parser.add_argument("--trace-root", default=str(_DEFAULT_TRACE_ROOT))
parser.add_argument("--output-root", default=str(_DEFAULT_OUTPUT_ROOT))
return parser
if __name__ == "__main__":
run(_parser().parse_args())