"""Test individual options for unconditional one-sided simulated returns.
The trading rule is deliberately independent of ``z``. Raw sign imbalance is
measured on the full universe. Cost-adjusted long and short directions use a
50/50 path split, with every cost calibrated from the training half.
"""
from __future__ import annotations
import argparse
import gc
import json
import os
import time
from pathlib import Path
from typing import Any
import numpy as np
import torch
from quantdata.torch import BaseEnvironment, DeviceLock
from .run_direct_arbitrage import _conditional_cost_model
from .run_experiment import memory_snapshot
from .torch_drift_change import UniverseGeometry, validate_universe
from .torch_prep import RNDysanosExperiment, prep_experiment
from .unconditional_option_test import (
DEFAULT_ROUNDOFF_MULTIPLIER,
FLOAT32_UNIT_ROUNDOFF,
SignAccumulator,
delta_hedged_option_cost_notionals,
delta_hedged_option_returns,
delta_hedged_roundoff_band,
net_return_roundoff_band,
population_standard_error,
rank_imbalance,
rank_no_arbitrage_candidates,
)
_SENSITIVITY_MULTIPLIERS = (0.0, 1.0, 8.0, 32.0, 128.0)
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
try:
temporary.write_text(json.dumps(value, indent=2), encoding="utf-8")
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _load_conditional_cost_model(
*,
report_path: Path,
env: BaseEnvironment,
data_geometry: UniverseGeometry,
) -> tuple[torch.nn.Module, str]:
"""Load a fixed moment model while allowing a new Monte Carlo path count."""
report = json.loads(report_path.read_text(encoding="utf-8"))
checkpoint_geometry = UniverseGeometry(
samples=int(report["paths"]),
steps=data_geometry.steps,
expiries=data_geometry.expiries,
strikes=data_geometry.strikes,
state_variables=data_geometry.state_variables,
)
return _conditional_cost_model(
report_path=report_path,
env=env,
geometry=checkpoint_geometry,
)
def _path_slices(start: int, stop: int, batch_size: int):
for left in range(start, stop, batch_size):
yield slice(left, min(left + batch_size, stop))
def _option_batch(
universe: dict[str, np.ndarray], paths: slice
) -> tuple[np.ndarray, np.ndarray]:
dH = delta_hedged_option_returns(
dC=universe["dC"][paths],
delta=universe["delta"][paths],
dS=universe["dS"][paths],
)
H = delta_hedged_option_cost_notionals(
C=universe["C"][paths],
delta=universe["delta"][paths],
spot=universe["spot"][paths],
)
return dH, H
def _raw_signs(
universe: dict[str, np.ndarray],
*,
geometry: UniverseGeometry,
batch_size: int,
) -> dict[str, Any]:
accumulators = {
multiplier: SignAccumulator(geometry.expiries * geometry.strikes)
for multiplier in _SENSITIVITY_MULTIPLIERS
}
positive_by_step = np.zeros(geometry.steps, dtype=np.int64)
negative_by_step = np.zeros(geometry.steps, dtype=np.int64)
neutral_by_step = np.zeros(geometry.steps, dtype=np.int64)
for paths in _path_slices(0, geometry.samples, batch_size):
dH = delta_hedged_option_returns(
dC=universe["dC"][paths],
delta=universe["delta"][paths],
dS=universe["dS"][paths],
)
for multiplier, accumulator in accumulators.items():
tolerance = delta_hedged_roundoff_band(
C=universe["C"][paths],
dC=universe["dC"][paths],
delta=universe["delta"][paths],
dS=universe["dS"][paths],
multiplier=multiplier,
)
accumulator.update(dH, tolerance)
if multiplier == DEFAULT_ROUNDOFF_MULTIPLIER:
positive_by_step += np.sum(dH > tolerance, axis=(0, 2))
negative_by_step += np.sum(dH < -tolerance, axis=(0, 2))
neutral_by_step += np.sum(
np.abs(dH) <= tolerance, axis=(0, 2)
)
observations_by_step = geometry.samples * geometry.expiries * geometry.strikes
return {
"definition": "imbalance = abs(count(dH < -e) - count(dH > e))",
"recommended_multiplier": DEFAULT_ROUNDOFF_MULTIPLIER,
"recommended_top": rank_imbalance(
accumulators[DEFAULT_ROUNDOFF_MULTIPLIER],
strikes=geometry.strikes,
count=geometry.expiries * geometry.strikes,
),
"by_transition": [
{
"transition": step,
"positive": int(positive_by_step[step]),
"negative": int(negative_by_step[step]),
"neutral": int(neutral_by_step[step]),
"positive_fraction": (
float(positive_by_step[step]) / observations_by_step
),
"negative_fraction": (
float(negative_by_step[step]) / observations_by_step
),
}
for step in range(geometry.steps)
],
"sensitivity": {
f"{multiplier:g}": rank_imbalance(
accumulator,
strikes=geometry.strikes,
count=20,
)
for multiplier, accumulator in accumulators.items()
},
}
def _unconditional_standard_error(
universe: dict[str, np.ndarray],
*,
stop: int,
geometry: UniverseGeometry,
batch_size: int,
) -> np.ndarray:
J = geometry.expiries * geometry.strikes
total = np.zeros(J, dtype=np.float64)
total_sqr = np.zeros(J, dtype=np.float64)
observations = 0
for paths in _path_slices(0, stop, batch_size):
dH, _ = _option_batch(universe, paths)
dH64 = dH.astype(np.float64, copy=False)
total += dH64.sum(axis=(0, 1), dtype=np.float64)
total_sqr += np.square(dH64).sum(axis=(0, 1), dtype=np.float64)
observations += dH.shape[0] * dH.shape[1]
expected = stop * geometry.steps
if observations != expected:
raise RuntimeError(f"cost observations {observations} != {expected}")
return population_standard_error(total, total_sqr, observations)
def _conditional_variance(
model: torch.nn.Module,
universe: dict[str, np.ndarray],
paths: slice,
*,
geometry: UniverseGeometry,
env: BaseEnvironment,
option_dH: np.ndarray,
) -> np.ndarray:
S = option_dH.shape[0]
TAU = geometry.steps
xstate = torch.as_tensor(
universe["xstate"][paths, :TAU], dtype=env.fdtype, device=env.device
)
log_strikes = torch.as_tensor(
universe["log_strikes"][paths].reshape(S, TAU, -1),
dtype=env.fdtype,
device=env.device,
)
expiries = torch.as_tensor(
universe["expiries"].reshape(1, 1, geometry.expiries),
dtype=env.fdtype,
device=env.device,
).expand(S, TAU, geometry.expiries)
z = torch.cat((xstate, log_strikes, expiries), dim=-1)
spot_dH = torch.as_tensor(
universe["dS"][paths, :, None], dtype=env.fdtype, device=env.device
)
options = torch.as_tensor(option_dH, dtype=env.fdtype, device=env.device)
dH = torch.cat((spot_dH, options), dim=-1)
with torch.inference_mode():
_, _, variance = model.moment_outputs(z, dH)
output = variance[..., 1:].cpu().numpy()
if not np.isfinite(output).all() or np.any(output < 0.0):
raise RuntimeError("conditional variance is non-finite or negative")
return output
def _direction_report(
training_long: SignAccumulator,
training_short: SignAccumulator,
validation_long: SignAccumulator,
validation_short: SignAccumulator,
*,
strikes: int,
) -> dict[str, Any]:
count = 2 * training_long.instruments
training = rank_no_arbitrage_candidates(
training_long, training_short, strikes=strikes, count=count
)
validation = rank_no_arbitrage_candidates(
validation_long, validation_short, strikes=strikes, count=count
)
validation_by_key = {
(item["option"], item["direction"]): item for item in validation
}
paired = [
{
"training": item,
"validation": validation_by_key[(item["option"], item["direction"])],
}
for item in training[:20]
]
return {
"selection_rule": (
"training fewest negative observations, then fewest losing paths, "
"then largest path-clustered 95% lower mean"
),
"training_selected_with_validation": paired,
"training_empirical_no_loss_count": sum(
item["negative"] == 0 and item["mean_lower_95"] > 0.0
for item in training
),
"validation_empirical_no_loss_count": sum(
item["negative"] == 0 and item["mean_lower_95"] > 0.0
for item in validation
),
"training_selected_validated_count": sum(
pair["training"]["negative"] == 0
and pair["training"]["mean_lower_95"] > 0.0
and pair["validation"]["negative"] == 0
and pair["validation"]["mean_lower_95"] > 0.0
for pair in paired
),
}
def _costed_signs(
universe: dict[str, np.ndarray],
*,
geometry: UniverseGeometry,
batch_size: int,
conditional_model: torch.nn.Module,
env: BaseEnvironment,
cost_prop: float,
) -> dict[str, Any]:
split = geometry.samples // 2
J = geometry.expiries * geometry.strikes
n_cost = split * geometry.steps
bar_c = _unconditional_standard_error(
universe,
stop=split,
geometry=geometry,
batch_size=batch_size,
)
accumulators = {
mode: {
name: SignAccumulator(J)
for name in ("training_long", "training_short", "validation_long", "validation_short")
}
for mode in ("unconditional", "conditional")
}
binding_cond = np.zeros(J, dtype=np.int64)
binding_prop = np.zeros(J, dtype=np.int64)
binding_uncond = np.zeros(J, dtype=np.int64)
binding_prop_uncond = np.zeros(J, dtype=np.int64)
conditional_observations = 0
for split_name, start, stop in (
("training", 0, split),
("validation", split, geometry.samples),
):
for paths in _path_slices(start, stop, batch_size):
dH, H = _option_batch(universe, paths)
proportional = cost_prop * np.abs(H)
unconditional = np.maximum(
proportional, bar_c.reshape(1, 1, -1)
)
variance = _conditional_variance(
conditional_model,
universe,
paths,
geometry=geometry,
env=env,
option_dH=dH,
)
conditional_component = np.sqrt(variance / float(n_cost))
conditional = np.maximum(proportional, conditional_component)
binding_uncond += (
bar_c.reshape(1, 1, -1) >= proportional
).sum(axis=(0, 1), dtype=np.int64)
binding_prop_uncond += (
proportional >= bar_c.reshape(1, 1, -1)
).sum(axis=(0, 1), dtype=np.int64)
binding_cond += (conditional_component >= proportional).sum(
axis=(0, 1), dtype=np.int64
)
binding_prop += (proportional >= conditional_component).sum(
axis=(0, 1), dtype=np.int64
)
conditional_observations += dH.shape[0] * dH.shape[1]
for mode, cost in (
("unconditional", unconditional),
("conditional", conditional),
):
tolerance = net_return_roundoff_band(dH, cost)
accumulators[mode][f"{split_name}_long"].update(dH - cost, tolerance)
accumulators[mode][f"{split_name}_short"].update(-dH - cost, tolerance)
report: dict[str, Any] = {
"n_cost": n_cost,
"training_paths": split,
"validation_paths": geometry.samples - split,
"cost_prop": cost_prop,
"unconditional_bar_c": bar_c.tolist(),
"conditional_binding_fraction": (
binding_cond / float(conditional_observations)
).tolist(),
"proportional_binding_fraction_in_conditional_mode": (
binding_prop / float(conditional_observations)
).tolist(),
"unconditional_binding_fraction": (
binding_uncond / float(conditional_observations)
).tolist(),
"proportional_binding_fraction_in_unconditional_mode": (
binding_prop_uncond / float(conditional_observations)
).tolist(),
}
for mode, values in accumulators.items():
report[mode] = _direction_report(
values["training_long"],
values["training_short"],
values["validation_long"],
values["validation_short"],
strikes=geometry.strikes,
)
return report
[docs]
def run(args: argparse.Namespace) -> dict[str, Any]:
if args.paths <= 0 or args.paths % 2:
raise ValueError("--paths must be a positive even number")
experiment = RNDysanosExperiment(
num_samples=args.paths,
pcnt_validation=0.5,
cnt_universe=1,
universe_seed_offset=args.universe_seed_offset,
)
started = time.monotonic()
universe = prep_experiment(experiment, plot_sampler=False)[0]
geometry = validate_universe(universe)
if geometry.samples != args.paths:
raise RuntimeError(f"prepared {geometry.samples} paths, expected {args.paths}")
prepared_seconds = time.monotonic() - started
raw_started = time.monotonic()
raw = _raw_signs(universe, geometry=geometry, batch_size=args.batch_size)
raw_seconds = time.monotonic() - raw_started
raw_only = getattr(args, "raw_only", False)
if raw_only:
checkpoint = None
costed = None
cost_seconds = 0.0
memory_before = None
memory_after = None
device = None
else:
with DeviceLock() as device_lock:
env = BaseEnvironment(device=device_lock.device, debug=False)
conditional_model, checkpoint = _load_conditional_cost_model(
report_path=Path(args.conditional_report),
env=env,
data_geometry=geometry,
)
memory_before = memory_snapshot(env.device)
cost_started = time.monotonic()
costed = _costed_signs(
universe,
geometry=geometry,
batch_size=args.batch_size,
conditional_model=conditional_model,
env=env,
cost_prop=args.cost_prop,
)
if env.device.type == "cuda":
torch.cuda.synchronize(env.device)
cost_seconds = time.monotonic() - cost_started
memory_after = memory_snapshot(env.device)
device = str(env.device)
del conditional_model
del universe
gc.collect()
report = {
"method": "state-independent single-option long/short sign test",
"return_semantics": "delta-hedged, non-cash-normalized option dH",
"paths": args.paths,
"observations": args.paths * geometry.steps,
"universe_seed_offset": args.universe_seed_offset,
"geometry": {
"samples": geometry.samples,
"steps": geometry.steps,
"expiries": geometry.expiries,
"strikes": geometry.strikes,
"state_variables": geometry.state_variables,
},
"float_dtype": "float32",
"unit_roundoff": FLOAT32_UNIT_ROUNDOFF,
"roundoff_multiplier": DEFAULT_ROUNDOFF_MULTIPLIER,
"roundoff_definition": (
"raw: 32*u*max(1, abs(C_tp1)+abs(C_t)+abs(delta*dS)); "
"net: 32*u*max(1, abs(dH)+cost)"
),
"conditional_checkpoint": checkpoint,
"device": device,
"preparation_seconds": prepared_seconds,
"raw_seconds": raw_seconds,
"cost_seconds": cost_seconds,
"memory_before": memory_before,
"memory_after": memory_after,
"raw": raw,
"costed": costed,
"limitations": (
"Raw-only pre-network sign gate; no conditional-moment estimator "
"was loaded."
if raw_only
else (
"The frozen conditional-moment estimator was trained previously "
"on an independent universe and is reused at both sample sizes "
"to isolate the effect of the new Monte Carlo universe and "
"n_cost scaling."
)
),
}
_atomic_json(Path(args.report), report)
return report
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--paths", type=int, required=True)
parser.add_argument("--universe-seed-offset", type=int, required=True)
parser.add_argument("--batch-size", type=int, default=4_096)
parser.add_argument("--cost-prop", type=float, default=0.0001)
parser.add_argument(
"--conditional-report",
default=(
"quantdata/dysanos/grid/rndysanos/reports/"
"full-200000-e1000000-up-to10-per-cap-conditional-same.json"
),
)
parser.add_argument("--report", required=True)
parser.add_argument(
"--raw-only",
action="store_true",
help="stop after the pre-network raw sign-imbalance gate",
)
return parser
[docs]
def main() -> None:
report = run(_parser().parse_args())
summary = {
"report": report["paths"],
"worst_raw": report["raw"]["recommended_top"][:5],
"by_transition": report["raw"]["by_transition"],
}
if report["costed"] is not None:
summary.update(
unconditional=report["costed"]["unconditional"],
conditional=report["costed"]["conditional"],
)
print(json.dumps(summary, indent=2), flush=True)
if __name__ == "__main__":
main()