"""Solve the empirical time-zero minimum-density-cap problem exactly."""
from __future__ import annotations
import argparse
import gc
import json
import os
import time
from dataclasses import asdict
from pathlib import Path
from typing import Any
import numpy as np
from cdxcore.err import verify_inp
from scipy import sparse
from scipy.optimize import linprog
from quantdata.torch import BaseEnvironment
from .required_runs import RequiredCase
from .run_expiry_arbitrage_experiments import _DEFAULT_CACHE_ROOT, _load_case, _to_device
from .run_parallel_direction_comparison import (
_select_times_and_fold,
_to_itm_dollar_portfolio,
)
_DEFAULT_OUTPUT_ROOT = Path(
"quantdata/dysanos/grid/gendysanos/reports/implied-cvar"
)
_DEFAULT_DENSITY_ROOT = Path(
"/media/storage/tmp/quantdata/dysanos/implied-cvar/densities"
)
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
temporary.write_text(
json.dumps(value, indent=2, allow_nan=False), encoding="utf-8"
)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _independent_t0_returns(data) -> np.ndarray:
"""Keep every expiry-specific stock and ITM option return at time zero."""
returns = data.returns[:, 0].double().cpu().numpy()
result = returns.reshape(returns.shape[0], -1)
scale = np.sqrt(np.mean(np.square(result), axis=0))
if not np.all(np.isfinite(result)) or not np.all(scale > 0.0):
raise FloatingPointError("time-zero normalized returns are degenerate")
return result / scale[None]
[docs]
def solve_minimum_density_cap(
returns: np.ndarray,
physical_probability: np.ndarray,
*,
method: str = "highs",
) -> tuple[Any, np.ndarray | None]:
"""Minimize ``max_s q_s/p_s`` subject to all sample moments being zero."""
paths, instruments = returns.shape
probability = np.asarray(physical_probability, dtype=np.float64)
probability /= probability.sum()
objective = np.zeros(paths + 1, dtype=np.float64)
objective[-1] = 1.0
equality = np.empty((instruments + 1, paths + 1), dtype=np.float64)
equality[0, :-1] = 1.0
equality[0, -1] = 0.0
equality[1:, :-1] = returns.T
equality[1:, -1] = 0.0
equality_value = np.zeros(instruments + 1, dtype=np.float64)
equality_value[0] = 1.0
cap_column = sparse.csr_matrix(
(-probability, (np.arange(paths), np.zeros(paths, dtype=np.int64))),
shape=(paths, 1),
)
inequality = sparse.hstack(
(sparse.eye(paths, format="csr"), cap_column), format="csr"
)
started = time.monotonic()
options: dict[str, Any] = {"presolve": True}
if method == "highs-ipm":
options["run_crossover"] = "off"
result = linprog(
objective,
A_ub=inequality,
b_ub=np.zeros(paths, dtype=np.float64),
A_eq=equality,
b_eq=equality_value,
bounds=[(0.0, None)] * paths + [(1.0, None)],
method=method,
options=options,
)
result.elapsed_seconds = time.monotonic() - started
if not result.success:
return result, None
q = np.asarray(result.x[:-1], dtype=np.float64)
density = q / probability
return result, density
[docs]
def solve_strict_sample_arbitrage_margin(returns: np.ndarray) -> Any:
"""Maximize the worst sampled return over signed unit-L1 portfolios."""
paths, instruments = returns.shape
objective = np.zeros(2 * instruments + 1, dtype=np.float64)
objective[-1] = -1.0
path_constraint = np.concatenate(
(-returns, returns, np.ones((paths, 1), dtype=np.float64)), axis=1
)
gross_constraint = np.concatenate(
(np.ones(2 * instruments, dtype=np.float64), np.zeros(1, dtype=np.float64))
)[None]
inequality = np.concatenate((path_constraint, gross_constraint), axis=0)
right_hand_side = np.zeros(paths + 1, dtype=np.float64)
right_hand_side[-1] = 1.0
started = time.monotonic()
result = linprog(
objective,
A_ub=inequality,
b_ub=right_hand_side,
bounds=[(0.0, None)] * (2 * instruments) + [(None, None)],
method="highs",
options={"presolve": True},
)
result.elapsed_seconds = time.monotonic() - started
return result
[docs]
def solve_density_cap_feasibility(
returns: np.ndarray,
physical_probability: np.ndarray,
density_cap: float,
*,
method: str = "highs",
) -> Any:
"""Test exact moment feasibility at one fixed positive density cap."""
paths, instruments = returns.shape
probability = np.asarray(physical_probability, dtype=np.float64)
probability /= probability.sum()
equality = np.concatenate(
(np.ones((1, paths), dtype=np.float64), returns.T), axis=0
)
value = np.zeros(instruments + 1, dtype=np.float64)
value[0] = 1.0
options: dict[str, Any] = {"presolve": True}
if method == "highs-ipm":
# A basic solution is unnecessary for a feasibility certificate. The
# interior-point iterate is audited below, so skip the costly simplex
# crossover on this highly degenerate zero-objective problem.
options["run_crossover"] = "off"
started = time.monotonic()
result = linprog(
np.zeros(paths, dtype=np.float64),
A_eq=equality,
b_eq=value,
bounds=list(zip(np.zeros(paths), density_cap * probability, strict=True)),
method=method,
options=options,
)
result.elapsed_seconds = time.monotonic() - started
return result
[docs]
def run(args: argparse.Namespace) -> dict[str, Any]:
case = RequiredCase(
generator=args.generator,
trading_frequency="weekly",
return_horizon="expiry",
hedge="raw",
cost_rate=0.0,
path_count=100_000,
)
loaded = _load_case(
case,
cache_root=Path(args.cache_root),
need_primitives=True,
)
source_cache = str(loaded.source_cache)
source_cache_uid = loaded.source_cache_uid
env = BaseEnvironment(precision=64, device="cpu", debug=False)
full_data, source_split = _to_device(loaded, case=case, env=env)
del loaded
gc.collect()
data, _ = _select_times_and_fold(
full_data,
deterministic_time=0,
conditional_time=6,
fold=0,
seed=100_097,
)
del full_data
data, normalization = _to_itm_dollar_portfolio(
data,
cost_rate=0.0,
vector_barrier=False,
)
returns = _independent_t0_returns(data)
probability = data.path_weight.double().cpu().numpy()
probability /= probability.sum()
if args.margin_only:
optimized = solve_strict_sample_arbitrage_margin(returns)
report = {
"status": "complete",
"experiment": "strict empirical time-zero arbitrage margin",
"definition": (
"maximize delta subject to R a>=delta on every sampled path "
"and ||a||_1<=1"
),
"configuration": asdict(case),
"source_cache": source_cache,
"source_cache_uid": source_cache_uid,
"source_split_unused_for_fitting": source_split,
"portfolio_normalization": normalization,
"paths": int(returns.shape[0]),
"independent_return_coordinates": int(returns.shape[1]),
"solver": {
"success": bool(optimized.success),
"status": int(optimized.status),
"message": str(optimized.message),
"iterations": int(optimized.nit),
"elapsed_seconds": float(optimized.elapsed_seconds),
"strict_sample_arbitrage_margin": (
float(-optimized.fun) if optimized.success else None
),
},
}
report_path = Path(args.output_root) / (
f"implied-cvar-{args.generator}-weekly-expiry-itm-dollar-"
"t0-strict-margin.json"
)
_atomic_json(report_path, report)
print(json.dumps(report["solver"], indent=2), flush=True)
print(f"Completed strict t0 margin report {report_path}", flush=True)
return report
if args.feasibility_cap is not None:
verify_inp(
args.feasibility_cap > 1.0,
lambda: (
"--feasibility-cap must exceed one so that the reported "
f"lambda is strictly positive; found {args.feasibility_cap}"
),
)
optimized = solve_density_cap_feasibility(
returns,
probability,
args.feasibility_cap,
method=args.solver,
)
density_path: Path | None = None
density_diagnostics: dict[str, float] = {}
if optimized.success:
q = np.asarray(optimized.x, dtype=np.float64)
density = q / probability
moment = q @ returns
normalization_error = abs(float(q.sum()) - 1.0)
maximum_cap_excess = max(
0.0, float(np.max(density - args.feasibility_cap))
)
maximum_negative_mass = max(0.0, float(np.max(-q)))
density_diagnostics = {
"density_minimum": float(density.min()),
"density_maximum": float(density.max()),
"density_weighted_mean": float(np.dot(probability, density)),
"maximum_absolute_scaled_martingale_error": float(
np.max(np.abs(moment))
),
"normalization_error": normalization_error,
"maximum_density_cap_excess": maximum_cap_excess,
"maximum_negative_target_mass": maximum_negative_mass,
"positive_density_path_fraction": float(np.mean(density > 0.0)),
"target_measure_effective_sample_size": float(
np.square(q.sum()) / np.square(q).sum()
),
}
density_path = Path(args.density_root) / (
f"{args.generator}-weekly-expiry-itm-dollar-t0-"
f"cap-{args.feasibility_cap:g}-density.npz"
)
density_path.parent.mkdir(parents=True, exist_ok=True)
np.savez_compressed(
density_path,
density=density,
probability=probability,
)
report = {
"status": "complete",
"experiment": "exact empirical time-zero fixed-cap feasibility",
"configuration": asdict(case),
"source_cache": source_cache,
"source_cache_uid": source_cache_uid,
"portfolio_normalization": normalization,
"paths": int(returns.shape[0]),
"independent_return_coordinates": int(returns.shape[1]),
"density_cap": args.feasibility_cap,
"lambda": args.feasibility_cap - 1.0,
"solver": {
"method": args.solver,
"success": bool(optimized.success),
"status": int(optimized.status),
"message": str(optimized.message),
"iterations": int(optimized.nit),
"crossover_iterations": int(optimized.crossover_nit or 0),
"elapsed_seconds": float(optimized.elapsed_seconds),
**density_diagnostics,
},
"density_path": None if density_path is None else str(density_path),
}
report_path = Path(args.output_root) / (
f"implied-cvar-{args.generator}-weekly-expiry-itm-dollar-t0-"
f"cap-{args.feasibility_cap:g}-feasibility.json"
)
_atomic_json(report_path, report)
print(json.dumps(report["solver"], indent=2), flush=True)
print(f"Completed fixed-cap t0 report {report_path}", flush=True)
return report
optimized, density = solve_minimum_density_cap(returns, probability)
diagnostics: dict[str, Any] = {
"success": bool(optimized.success),
"status": int(optimized.status),
"message": str(optimized.message),
"iterations": int(optimized.nit),
"elapsed_seconds": float(optimized.elapsed_seconds),
}
density_path: Path | None = None
if density is not None:
q = probability * density
moment = q @ returns
cap = float(optimized.fun)
diagnostics.update(
{
"minimum_density_cap": cap,
"minimum_lambda": cap - 1.0,
"density_minimum": float(density.min()),
"density_maximum": float(density.max()),
"density_weighted_mean": float(np.dot(probability, density)),
"maximum_absolute_scaled_martingale_error": float(
np.max(np.abs(moment))
),
"positive_density_path_fraction": float(np.mean(density > 0.0)),
"target_measure_effective_sample_size": float(
np.square(q.sum()) / np.square(q).sum()
),
}
)
density_path = (
Path(args.density_root)
/ f"{args.generator}-weekly-expiry-itm-dollar-t0-density.npz"
)
density_path.parent.mkdir(parents=True, exist_ok=True)
np.savez_compressed(density_path, density=density, probability=probability)
report = {
"status": "complete",
"experiment": "exact empirical time-zero minimum CVaR density cap",
"definition": (
"minimize c subject to q>=0, sum(q)=1, q^T R=0, and "
"q_s<=c p_s; lambda=c-1"
),
"configuration": asdict(case),
"source_cache": source_cache,
"source_cache_uid": source_cache_uid,
"source_split_unused_for_fitting": source_split,
"portfolio_normalization": normalization,
"paths": int(returns.shape[0]),
"independent_return_coordinates": int(returns.shape[1]),
"solver": diagnostics,
"density_path": None if density_path is None else str(density_path),
}
report_path = Path(args.output_root) / (
f"implied-cvar-{args.generator}-weekly-expiry-itm-dollar-t0-exact-lp.json"
)
_atomic_json(report_path, report)
print(json.dumps(report["solver"], indent=2), flush=True)
print(f"Completed exact t0 report {report_path}", flush=True)
return report
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--generator", choices=("pca-iv", "dysanos"), required=True)
parser.add_argument("--cache-root", default=str(_DEFAULT_CACHE_ROOT))
parser.add_argument("--output-root", default=str(_DEFAULT_OUTPUT_ROOT))
parser.add_argument("--density-root", default=str(_DEFAULT_DENSITY_ROOT))
parser.add_argument("--margin-only", action="store_true")
parser.add_argument("--feasibility-cap", type=float)
parser.add_argument(
"--solver", choices=("highs", "highs-ds", "highs-ipm"), default="highs"
)
return parser
if __name__ == "__main__":
run(_parser().parse_args())