Source code for quantdata.dysanos.grid.gendysanos.run_required_experiments

"""Execute the staged experiments required by :mod:`GENDYSANOS.md`."""

from __future__ import annotations

import argparse
import gc
import hashlib
import json
import math
import os
import resource
import subprocess
import sys
import time
from dataclasses import asdict
from pathlib import Path
from typing import Any

import numpy as np
import torch
from cdxcore.verbose import Context
from scipy.stats import norm

from quantdata.dysanos.grid.torch_simulator import GaussianSpotImportanceConfig

from .case_store import DerivedCaseStore
from .conditional import run_time_separated_local_tail
from .matched_cache import (
    MatchedUniverseBlockCache,
    migrate_legacy_option_settlement_cache,
)
from .required_runs import (
    REPORTED_COST_RATES,
    REPORTED_PATH_COUNTS,
    OptionCoverageAccumulator,
    RequiredCase,
    TimeSeparatedUnconditionalAccumulator,
    audit_constant_basket,
    certify_unconditional_directions,
    convergence_report,
    derive_case_arrays,
    float64_replay_report,
    functional_gate_report,
    required_cases,
    solve_constant_basket,
    stratified_path_split,
)
from .static_spread_strategy import StateDependentStaticSpreadAccumulator

_BASE_SCHEMA_VERSION = "1.3"
_REPORT_SCHEMA_VERSION = "2.8"
_DEFAULT_CACHE_ROOT = "/media/storage/tmp/quantdata/dysanos/required-runs"
_DEFAULT_REPORT_ROOT = "quantdata/dysanos/grid/gendysanos/reports/required"


def _required_importance_config() -> GaussianSpotImportanceConfig:
    """Return the common proposal spanning all ten action cohorts."""

    return GaussianSpotImportanceConfig(
        spot_std_multiplier=8.0,
        base_probability=0.1,
        target_action_count=10,
    )


def _json_safe(value: Any) -> Any:
    if isinstance(value, np.ndarray):
        return [_json_safe(item) for item in value.tolist()]
    if isinstance(value, np.generic):
        return _json_safe(value.item())
    if isinstance(value, float) and not math.isfinite(value):
        return None
    if isinstance(value, dict):
        return {str(name): _json_safe(item) for name, item in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_safe(item) for item in value]
    return value


def _atomic_json(path: Path, value: dict[str, Any]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    try:
        temporary.write_text(
            json.dumps(_json_safe(value), indent=2, allow_nan=False),
            encoding="utf-8",
        )
        os.replace(temporary, path)
    finally:
        temporary.unlink(missing_ok=True)


def _memory_snapshot() -> dict[str, Any]:
    maximum_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1_024
    gpu_allocated = 0
    gpu_reserved = 0
    if torch.cuda.is_available():
        gpu_allocated = int(torch.cuda.max_memory_allocated())
        gpu_reserved = int(torch.cuda.max_memory_reserved())
    return {
        "maximum_resident_bytes": int(maximum_rss),
        "maximum_cuda_allocated_bytes": gpu_allocated,
        "maximum_cuda_reserved_bytes": gpu_reserved,
    }


def _base_configuration(
    *, generator: str, frequency: str, paths: int, seed: int, block_size: int
) -> dict[str, Any]:
    return {
        "schema_version": _BASE_SCHEMA_VERSION,
        "generator": generator,
        "trading_frequency": frequency,
        "num_samples": paths,
        "num_trading_times": 10,
        "daily_intermediate_path": True,
        "return_horizons_derived": ["expiry", "incremental"],
        "hedges_derived": ["delta", "raw"],
        "cost_rates_derived": list(REPORTED_COST_RATES),
        "surface_first": True,
        "surface_state_dimension": 20,
        "num_alpha": None,
        "training_grid_strikes": 100,
        "traded_strikes_per_expiry": 20,
        "use_matrix_exp": True,
        "bandwidth": 1.0,
        "seed": seed,
        "block_size": block_size,
        "importance_sampling": asdict(_required_importance_config()),
    }


def _base_uid(configuration: dict[str, Any]) -> str:
    encoded = json.dumps(configuration, sort_keys=True).encode("utf-8")
    digest = hashlib.sha256(encoded).hexdigest()[:16]
    return (
        f"{configuration['generator']}-{configuration['trading_frequency']}-"
        f"{configuration['num_samples']}-{digest}"
    )


def _generate_base_cache(
    *,
    configuration: dict[str, Any],
    cache_root: Path,
    verbose: Context,
) -> MatchedUniverseBlockCache:
    """Generate one path sample shared by all twelve derived cases."""

    generator = configuration["generator"]
    frequency = configuration["trading_frequency"]
    paths = configuration["num_samples"]
    cache = MatchedUniverseBlockCache(
        cache_root / "universes",
        cache_uid=_base_uid(configuration),
        configuration=configuration,
        num_samples=paths,
    )
    if cache.is_complete:
        return cache
    for source_schema_version in ("1.2", "1.0"):
        legacy_configuration = dict(configuration)
        legacy_configuration["schema_version"] = source_schema_version
        legacy_uid = _base_uid(legacy_configuration)
        legacy_manifest = cache_root / "universes" / legacy_uid / "manifest.json"
        if legacy_manifest.exists():
            verbose.write(
                lambda: (
                    f"Migrating verified matched cache {legacy_uid} to "
                    f"{cache.cache_uid}"
                )
            )

            def migration_progress(block: int, total: int) -> None:
                if block == 1 or block == total or block % 10 == 0:
                    verbose.write(
                        lambda block=block, total=total: (
                            f"Migrated matched cache blocks {block:,}/{total:,}"
                        )
                    )

            return migrate_legacy_option_settlement_cache(
                root=cache_root / "universes",
                source_uid=legacy_uid,
                destination_uid=cache.cache_uid,
                destination_configuration=configuration,
                num_samples=paths,
                source_schema_version=source_schema_version,
                progress=migration_progress,
            )
    cache.begin()
    importance = _required_importance_config()
    if generator == "dysanos":
        from .torch_prep import (
            DysanosUniverseStreamDiagnostics,
            GenDysanosExperiment,
            finalize_dysanos_stream_diagnostics,
            iter_dysanos_matched_universe_blocks,
        )

        experiment = GenDysanosExperiment(
            num_samples=paths,
            trading_frequency=frequency,
            return_horizon="expiry",
            num_trading_times=10,
            num_alpha=None,
            use_matrix_exp=True,
            bandwidth=1.0,
            seed=configuration["seed"],
            block_size=configuration["block_size"],
            importance_sampling=importance,
        )
        diagnostics = DysanosUniverseStreamDiagnostics()
        for block in iter_dysanos_matched_universe_blocks(
            experiment, diagnostics=diagnostics, verbose=verbose
        ):
            cache.write(block)
        path_diagnostics = finalize_dysanos_stream_diagnostics(diagnostics)
        generated_diagnostics: dict[str, Any] = {
            "decoder": {
                "name": "ML-SANOS",
                "static_arbitrage_repair": False,
                "cross_section_static_arbitrage_free_by_construction": True,
                "incremental_call_roundoff_clip_count": (
                    diagnostics.incremental_call_roundoff_clip_count
                ),
                "minimum_unclipped_incremental_call": (
                    diagnostics.minimum_unclipped_incremental_call
                ),
                "roundoff_stabilization": (
                    "alive incremental normalized calls in [-32 eps32,0) are "
                    "set to the exact nonnegative boundary; larger negatives fail"
                ),
            },
            "sampling": asdict(path_diagnostics),
        }
    elif generator == "pca-iv":
        from quantdata.dysanos.implied.decoder import PCATotalVarianceDecoder
        from quantdata.dysanos.implied.experiment import ImpliedPCAExperiment
        from quantdata.dysanos.implied.market import load_fitted_pca
        from quantdata.dysanos.implied.simulator import ImpliedPCASampler
        from quantdata.dysanos.implied.static_arbitrage import (
            StaticArbitrageAccumulator,
        )
        from quantdata.dysanos.implied.universe import (
            PCAUniverseStreamDiagnostics,
            default_static_strike_mesh,
            finalize_stream_diagnostics,
            iter_pca_matched_universe_blocks,
        )

        experiment = ImpliedPCAExperiment(
            num_samples=paths,
            trading_frequency=frequency,
            return_horizon="expiry",
            num_trading_times=10,
            num_alpha=None,
            use_matrix_exp=True,
            bandwidth=1.0,
            seed=configuration["seed"],
            block_size=configuration["block_size"],
            importance_sampling=importance,
        )
        _, market, pca, pca_diagnostics = load_fitted_pca(
            experiment.fit_config, verbose=verbose
        )
        decoder = PCATotalVarianceDecoder(pca)
        sampler = ImpliedPCASampler(
            pca=pca,
            scores=pca_diagnostics.scores,
            time_y=market.time_y,
            spot_ref=market.spot_ref,
            num_alpha=experiment.num_alpha,
            seed=experiment.seed,
            verbose=verbose,
        )
        static_strikes = default_static_strike_mesh(
            experiment.static_mesh_points
        )
        static = StaticArbitrageAccumulator(
            strikes=static_strikes,
            expiries=pca.expiries,
            roundoff_multipliers=experiment.roundoff_multipliers,
            seed=experiment.seed + 911,
        )
        diagnostics = PCAUniverseStreamDiagnostics()
        for block in iter_pca_matched_universe_blocks(
            experiment,
            decoder=decoder,
            sampler=sampler,
            diagnostics=diagnostics,
            static_accumulator=static,
            static_strikes=static_strikes,
            verbose=verbose,
        ):
            cache.write(block)
        path_diagnostics = finalize_stream_diagnostics(diagnostics)
        generated_diagnostics = {
            "decoder": {
                "name": "PCA-IV",
                "static_arbitrage_repair": False,
                "sample_rejection": False,
            },
            "sampling": asdict(path_diagnostics),
            "static_arbitrage": static.report(),
        }
    else:
        raise ValueError(f"unknown generator {generator!r}")
    cache.complete(diagnostics=_json_safe(generated_diagnostics))
    return cache


def _importance_gate(
    sampling: dict[str, Any],
    *,
    paths: int,
    unconditional: dict[str, Any],
) -> dict[str, Any]:
    importance = sampling.get("importance_sampling")
    if importance is None:
        return {"passed": False, "reason": "importance sampling is absent"}
    path = importance["path_weights"]
    mean = float(path["mean_density"])
    ess = float(path["independent_path_effective_sample_size"])
    second = mean * mean * paths / ess
    standard_error = math.sqrt(max(second - mean * mean, 0.0) / paths)
    controls = importance["log_spot_control_at_target_steps"]
    targets = max(1, len(importance["target_steps"]))
    quantile = float(norm.ppf(1.0 - 0.05 / (2.0 * targets)))
    mean_tolerance = quantile * math.sqrt(1.0 / paths + 1.0 / ess)
    variance_tolerance = quantile * math.sqrt(
        2.0 / max(paths - 1, 1) + 2.0 / max(ess - 1.0, 1.0)
    )
    return_items = [
        item
        for time_report in unconditional["times"]
        for direction in ("long", "short")
        for item in time_report["directions"][direction]
    ]
    return_quantile = float(
        norm.ppf(1.0 - 0.05 / (2.0 * max(1, len(return_items))))
    )
    return_failures = 0
    maximum_return_z = 0.0
    for item in return_items:
        error = math.hypot(
            float(item["weighted_P_path_standard_error"]),
            float(item["direct_P_path_standard_error"]),
        )
        difference = abs(
            float(item["weighted_P_mean"]) - float(item["direct_P_mean"])
        )
        z_score = difference / error if error > 0.0 else (math.inf if difference else 0.0)
        maximum_return_z = max(maximum_return_z, z_score)
        return_failures += int(difference > return_quantile * error)
    checks = {
        "density_normalization": abs(mean - 1.0) <= 3.0 * standard_error,
        "target_log_spot_mean_recovery": float(
            controls["maximum_mean_difference_in_direct_P_std"]
        )
        <= mean_tolerance,
        "target_log_spot_variance_recovery": float(
            controls["maximum_relative_variance_difference"]
        )
        <= variance_tolerance,
        "positive_effective_sample_size": 1.0 < ess <= paths,
        "direct_P_and_weighted_return_mean_recovery": return_failures == 0,
    }
    return {
        "passed": bool(all(checks.values())),
        "checks": checks,
        "mean_density": mean,
        "mean_density_standard_error": standard_error,
        "effective_sample_size": ess,
        "simultaneous_mean_tolerance_in_std": mean_tolerance,
        "simultaneous_relative_variance_tolerance": variance_tolerance,
        "return_mean_simultaneous_normal_quantile": return_quantile,
        "maximum_return_mean_difference_z": maximum_return_z,
        "return_mean_recovery_failures": return_failures,
    }


def _coverage_gate(
    report: dict[str, Any],
    *,
    support: dict[str, Any],
    target: int,
    return_horizon: str,
    hedge: str,
) -> dict[str, Any]:
    minimum = math.inf
    for time_report in report["times"]:
        for direction in ("long", "short"):
            for item in time_report["directions"][direction]:
                if item["kind"] != "option":
                    continue
                minimum = min(
                    minimum,
                    item["proposal_loss_count"],
                    item["proposal_gain_count"],
                )
    minimum = int(minimum) if math.isfinite(minimum) else 0
    required_support = [
        f"{hedge}_positive",
        f"{hedge}_negative",
    ]
    if return_horizon == "expiry":
        required_support.extend(("crossing", "noncrossing"))
    applicable_counts = {
        name: int(support["minimum_counts"][name]) for name in required_support
    }
    return {
        "passed": minimum >= target
        and all(value >= target for value in applicable_counts.values()),
        "target_count_per_cost_adjusted_sign_side": target,
        "minimum_cost_adjusted_sign_count": minimum,
        "applicable_support_counts": applicable_counts,
        "strike_crossing_required": return_horizon == "expiry",
        "minimum_strike_and_precost_sign_counts": support["minimum_counts"],
    }


def _conditional_gate(report: dict[str, Any]) -> dict[str, Any]:
    radii: list[float] = []
    effective_sizes: list[float] = []
    for item in report["times"]:
        for split in ("reference", "audit"):
            radii.append(float(item[split]["neighbor_radius"]["mean"]))
            effective_sizes.append(
                float(item[split]["local_effective_sample_size"]["mean"])
            )
    finite = np.all(np.isfinite(radii)) and np.all(np.isfinite(effective_sizes))
    return {
        "passed": bool(
            finite
            and np.all(np.asarray(radii) > 0.0)
            and np.all(np.asarray(effective_sizes) > 1.0)
        ),
        "mean_radii_by_time_and_split": radii,
        "mean_effective_sizes_by_time_and_split": effective_sizes,
    }


def _dynamic_arbitrage_score(
    *,
    gate: dict[str, Any],
    candidate_certification: dict[str, Any],
    constant_baskets: list[dict[str, Any]],
    static_spread_strategy: dict[str, Any],
) -> dict[str, Any]:
    """Summarize independently audited candidates without hiding uncertainty."""

    conditional = candidate_certification.get("conditional_regions", {})
    local_baskets = candidate_certification.get(
        "local_conditional_baskets", {}
    )
    surviving_by_class = {
        "unconditional_unit_direction": len(
            candidate_certification.get("surviving", [])
        ),
        "unconditional_constant_basket": sum(
            item.get("certified") is True for item in constant_baskets
        ),
        "conditional_unit_region": len(conditional.get("surviving", [])),
        "conditional_local_basket": len(local_baskets.get("surviving", [])),
        "state_dependent_static_spread": int(
            static_spread_strategy.get("has_pathwise_dynamic_arbitrage") is True
        ),
    }
    unresolved_by_class = {
        "conditional_unit_region": len(conditional.get("unresolved", [])),
        "conditional_local_basket": len(local_baskets.get("unresolved", [])),
    }
    surviving = sum(surviving_by_class.values())
    unresolved = sum(unresolved_by_class.values())
    diagnostics_complete = (
        gate.get("passed") is True
        and conditional.get("status") == "complete"
        and local_baskets.get("status") == "complete"
    )
    if surviving:
        score: bool | None = True
        assessment = "dynamic arbitrage detected within a tested strategy class"
    elif diagnostics_complete and not unresolved:
        score = False
        assessment = (
            "no per-trading-time conditional arbitrage detected within the "
            "tested strategy classes and finite-sample resolution"
        )
    else:
        score = None
        assessment = "unresolved"
    return {
        "has_dynamic_arbitrage": score,
        "assessment": assessment,
        "strategy_classes": tuple(surviving_by_class),
        "surviving_candidates_by_class": surviving_by_class,
        "unresolved_candidates_by_class": unresolved_by_class,
        "functional_gate_passed": gate.get("passed") is True,
        "all_required_candidate_audits_resolved": diagnostics_complete
        and not unresolved,
        "qualification": (
            "the score concerns the tested strategy classes only and is not a "
            "proof over all predictable self-financing strategies"
        ),
    }


def _timing_replay(block, *, frequency: str) -> dict[str, Any]:
    """Replay matched timestamp and exact-expiry identities for one block."""

    expiry = block.expiry
    incremental = block.incremental
    stride = 5 if frequency == "weekly" else 1
    expiries = np.asarray(expiry["expiries"], dtype=np.float64).reshape(-1)
    offsets = np.rint(expiries * 252.0).astype(np.int64)
    expected_expired = offsets <= stride
    checks = {
        "ten_expiry_cohorts": expiry["C"].shape[1] == 10,
        "nine_incremental_intervals": incremental["C"].shape[1] == 9,
        "shared_trading_states": np.array_equal(
            incremental["xstate"], expiry["xstate"][:, :9]
        ),
        "shared_entry_prices": np.array_equal(
            incremental["C"], expiry["C"][:, :9]
        ),
        "shared_entry_deltas": np.array_equal(
            incremental["delta"], expiry["delta"][:, :9]
        ),
        "incremental_spot_endpoint": np.allclose(
            incremental["dS"],
            expiry["spot"][:, 1:] - expiry["spot"][:, :9],
            rtol=0.0,
            atol=32.0
            * np.finfo(np.float32).eps
            * np.maximum(np.abs(expiry["spot"][:, 1:]), 1.0e-30),
        ),
        "incremental_option_hedge_endpoint": np.allclose(
            incremental["option_dS"],
            np.where(
                expected_expired[None, None, :],
                expiry["expiry_dS"][:, :9],
                incremental["dS"][..., None],
            ),
            rtol=0.0,
            atol=32.0
            * np.finfo(np.float32).eps
            * np.maximum(np.abs(expiry["spot"][:, 1:, None]), 1.0e-30),
        ),
        "incremental_option_settlement_endpoint": np.allclose(
            incremental["option_settlement_dS"],
            np.where(
                expected_expired[None, None, :],
                expiry["expiry_dS"][:, :9],
                incremental["dS"][..., None],
            ),
            rtol=0.0,
            atol=32.0
            * np.finfo(np.float32).eps
            * np.maximum(np.abs(expiry["spot"][:, 1:, None]), 1.0e-30),
        ),
        "inside_interval_expiry_marker": np.array_equal(
            np.asarray(incremental["option_expired"]).reshape(-1),
            expected_expired,
        ),
    }
    return {"passed": bool(all(checks.values())), "checks": checks, "stride": stride}


def _run_case(
    *,
    case: RequiredCase,
    cache: MatchedUniverseBlockCache,
    cache_root: Path,
    report_root: Path,
    device: torch.device,
    run_conditional: bool,
    run_baskets: bool,
) -> dict[str, Any]:
    label = (
        f"{case.generator}-{case.trading_frequency}-{case.path_count}-"
        f"{case.return_horizon}-{case.hedge}-{case.cost_rate:.0e}"
    )
    report_path = report_root / f"{label}.json"
    report: dict[str, Any] = {
        "status": "running",
        "schema_version": _REPORT_SCHEMA_VERSION,
        "configuration": asdict(case),
        "base_cache": str(cache.directory),
        "base_cache_uid": cache.cache_uid,
        "return_definition": case.return_horizon,
        "hedge_definition": (
            "raw option return"
            if case.hedge == "raw"
            else (
                "option return minus entry delta times spot return to the "
                "option close: exact expiry inside an interval, otherwise "
                "the next trading time"
            )
        ),
        "cost_definition": (
            "spot q*abs(S); raw option q*abs(C); delta-hedged option "
            "q*(abs(C)+abs(delta*S)); no option/spot netting"
        ),
        "numerical_tolerance": (
            "eta=32*finfo(float32).eps*max(unreduced arithmetic scale+cost, "
            "finfo(float32).tiny); no cash floor"
        ),
    }
    _atomic_json(report_path, report)
    accumulator = TimeSeparatedUnconditionalAccumulator()
    support_accumulator = OptionCoverageAccumulator()
    static_spread_accumulator = (
        StateDependentStaticSpreadAccumulator()
        if case.return_horizon == "expiry" and case.hedge == "raw"
        else None
    )
    replay_passed = True
    replay_has_expiry_payoff = False
    replay_max_return = replay_max_cost = replay_max_expiry = 0.0
    components = np.empty(case.path_count, dtype=np.int16)
    timing_checks: list[dict[str, Any]] = []
    for block in cache.iter_blocks():
        timing_checks.append(
            _timing_replay(block, frequency=case.trading_frequency)
        )
        universe = getattr(block, case.return_horizon)
        derived = derive_case_arrays(
            universe, hedge=case.hedge, cost_rate=case.cost_rate
        )
        if static_spread_accumulator is not None:
            static_spread_accumulator.update(universe, derived)
        log_weight = np.asarray(
            universe.get("log_path_weight", np.zeros(block.stop - block.start))
        )
        component = np.asarray(
            universe.get(
                "importance_component", np.zeros(block.stop - block.start)
            )
        )
        accumulator.update(
            derived,
            log_path_weights=log_weight,
            importance_components=component,
        )
        support_accumulator.update(universe, log_path_weights=log_weight)
        components[block.start : block.stop] = component
        replay = float64_replay_report(universe, derived)
        replay_passed &= replay["passed"]
        replay_max_return = max(
            replay_max_return, replay["maximum_absolute_return_identity_error"]
        )
        replay_max_cost = max(
            replay_max_cost, replay["maximum_absolute_cost_identity_error"]
        )
        replay_expiry = replay["maximum_absolute_expiry_payoff_identity_error"]
        if replay_expiry is not None:
            replay_has_expiry_payoff = True
            replay_max_expiry = max(replay_max_expiry, replay_expiry)
    unconditional = accumulator.report()
    option_coverage = support_accumulator.report()
    split = stratified_path_split(components, seed=case.path_count + case.gpu)
    split_report = {
        "reference_paths": int(split.reference.size),
        "audit_paths": int(split.audit.size),
        "disjoint": np.intersect1d(split.reference, split.audit).size == 0,
        "mixture_stratified": True,
        "component_counts": split.component_counts,
    }
    float64_replay = {
        "passed": replay_passed,
        "maximum_absolute_return_identity_error": replay_max_return,
        "maximum_absolute_cost_identity_error": replay_max_cost,
        "maximum_absolute_expiry_payoff_identity_error": (
            replay_max_expiry if replay_has_expiry_payoff else None
        ),
    }
    timing_replay = {
        "passed": bool(all(item["passed"] for item in timing_checks)),
        "blocks_checked": len(timing_checks),
        "failed_blocks": [
            index for index, item in enumerate(timing_checks) if not item["passed"]
        ],
        "checks": timing_checks[0]["checks"] if timing_checks else {},
    }
    report.update(
        unconditional=unconditional,
        option_coverage=option_coverage,
        split=split_report,
        float64_replay=float64_replay,
        timing_replay=timing_replay,
    )
    static_spread_strategy = (
        static_spread_accumulator.report()
        if static_spread_accumulator is not None
        else {
            "applicability": "raw option returns held to expiry",
            "status": "not applicable to this return/hedge case",
            "has_pathwise_dynamic_arbitrage": False,
        }
    )
    report["state_dependent_static_spread_strategy"] = static_spread_strategy
    base_diagnostics = cache.manifest()["diagnostics"]
    sampling = base_diagnostics["sampling"]
    coverage_check = _coverage_gate(
        unconditional,
        support=option_coverage,
        target=1_000,
        return_horizon=case.return_horizon,
        hedge=case.hedge,
    )
    importance_check = _importance_gate(
        sampling,
        paths=case.path_count,
        unconditional=unconditional,
    )
    prerequisite_checks_passed = (
        coverage_check["passed"]
        and importance_check["passed"]
        and float64_replay["passed"]
    )

    conditional: dict[str, Any] = {"status": "not-run"}
    baskets: list[dict[str, Any]] = []
    stored_case = None
    stored_log_weight = None
    stored_components = None
    store = None
    if run_conditional or run_baskets:
        store = DerivedCaseStore(
            cache_root / "case-work",
            case=case,
            source_cache_uid=cache.cache_uid,
        )
        if not store.is_complete:
            store.build(cache.iter_blocks())
        state, stored_case, log_weight, stored_components = store.arrays()
        stored_log_weight = log_weight
        if run_conditional:
            neighbors = 4_096 if case.path_count >= 1_000_000 else 1_024
            conditional = run_time_separated_local_tail(
                state=state,
                case=stored_case,
                log_path_weights=log_weight,
                importance_components=stored_components,
                split=split,
                neighbors=neighbors,
                query_paths_per_split=min(1_024, split.audit.size),
                instrument_block_size=20,
                seed=case.path_count + 101,
                device=device,
                certification_prerequisites_passed=(
                    prerequisite_checks_passed
                ),
                basket_solver_workers=min(12, max(1, (os.cpu_count() or 1) // 2)),
            )
        if run_baskets:
            for time_index in range(stored_case.returns.shape[1]):
                baskets.append(
                    solve_constant_basket(
                        stored_case,
                        time_index=time_index,
                        reference_paths=split.reference,
                        log_path_weights=log_weight,
                    )
                )
    report["conditional"] = conditional
    report["trained_strategy_class"] = {
        "individual": "unit long and short directions",
        "constant_basket": "state-independent L1-bounded linear program",
        "conditional_unit": "single-instrument k-nearest-neighbour state policy",
        "conditional_basket": (
            "independent all-instrument L1-bounded linear program at every "
            "queried state and trading time"
        ),
    }
    if (
        baskets
        and stored_case is not None
        and stored_log_weight is not None
        and stored_components is not None
    ):
        baskets = [
            audit_constant_basket(
                solution,
                stored_case,
                split=split,
                log_path_weights=stored_log_weight,
                importance_components=stored_components,
                prerequisite_checks_passed=prerequisite_checks_passed,
            )
            for solution in baskets
        ]
    report["constant_baskets"] = baskets
    if (
        stored_case is not None
        and stored_log_weight is not None
        and stored_components is not None
    ):
        candidate_certification = certify_unconditional_directions(
            stored_case,
            split=split,
            log_path_weights=stored_log_weight,
            importance_components=stored_components,
            seed=case.path_count + 313,
            prerequisite_checks_passed=prerequisite_checks_passed,
        )
        conditional_surviving: list[dict[str, Any]] = []
        conditional_rejected: list[dict[str, Any]] = []
        conditional_unresolved: list[dict[str, Any]] = []
        basket_surviving: list[dict[str, Any]] = []
        basket_rejected: list[dict[str, Any]] = []
        basket_unresolved: list[dict[str, Any]] = []
        for time_report in conditional.get("times", []):
            audit = time_report["frozen_candidate_region_audit"]
            conditional_surviving.extend(audit["surviving"])
            conditional_rejected.extend(audit["rejected"])
            conditional_unresolved.extend(audit["unresolved"])
            local_basket = time_report["local_conditional_baskets"]
            basket_surviving.extend(local_basket["surviving"])
            basket_rejected.extend(local_basket["rejected"])
            basket_unresolved.extend(local_basket["unresolved"])
        candidate_certification["conditional_regions"] = {
            "status": (
                "complete" if conditional.get("status") == "complete" else "not-run"
            ),
            "surviving": conditional_surviving,
            "rejected": conditional_rejected,
            "unresolved": conditional_unresolved,
        }
        candidate_certification["local_conditional_baskets"] = {
            "status": (
                "complete" if conditional.get("status") == "complete" else "not-run"
            ),
            "surviving": basket_surviving,
            "rejected": basket_rejected,
            "unresolved": basket_unresolved,
        }
    else:
        candidate_certification = {
            "status": "not-run",
            "surviving": [],
            "rejected": [],
        }
    report["candidate_certification"] = candidate_certification
    memory = _memory_snapshot()
    gate = functional_gate_report(
        {
            "finite_arrays": True,
            "timing_identities": timing_replay,
            "float64_accounting_replay": float64_replay,
            "decoder_identity": {
                "passed": base_diagnostics["decoder"]["static_arbitrage_repair"]
                is False,
                **base_diagnostics["decoder"],
            },
            "strike_and_sign_boundary_coverage": coverage_check,
            "importance_normalization_and_moment_recovery": importance_check,
            "stratified_disjoint_split": {
                "passed": split_report["disjoint"]
                and split.reference.size == 4 * case.path_count // 5,
                **split_report,
            },
            "conditional_radius_and_ess": (
                _conditional_gate(conditional)
                if conditional.get("status") == "complete"
                else {"passed": False, "reason": "conditional diagnostics not run"}
            ),
            "extremes_replayed": float64_replay,
            "memory_within_limits": {
                "passed": memory["maximum_resident_bytes"] < 900 * 2**30
                and memory["maximum_cuda_reserved_bytes"] < 92 * 2**30,
                **memory,
            },
        }
    )
    if case.path_count == 1_000:
        gate["reported_stage_applicability"] = (
            "mechanical smoke only; coverage thresholds are intentionally not "
            "relaxed and this gate does not authorize a reported stage"
        )
    dynamic_arbitrage = _dynamic_arbitrage_score(
        gate=gate,
        candidate_certification=candidate_certification,
        constant_baskets=baskets,
        static_spread_strategy=static_spread_strategy,
    )
    if store is not None:
        del state, log_weight, stored_case, stored_log_weight, stored_components
        gc.collect()
        store.cleanup()
        report["temporary_case_store"] = {
            "path": str(store.directory),
            "removed_after_diagnostics": True,
        }
    report.update(
        status="complete",
        functional_gate=gate,
        dynamic_arbitrage=dynamic_arbitrage,
        peak_memory=memory,
        sampling_diagnostics=sampling,
        generator_diagnostics=base_diagnostics,
    )
    _atomic_json(report_path, report)
    return report


[docs] def run_worker( *, generator: str, frequency: str, paths: int, horizons: tuple[str, ...], hedges: tuple[str, ...], costs: tuple[float, ...], cache_root: Path, report_root: Path, seed: int, block_size: int, run_conditional: bool, run_baskets: bool, device: torch.device, ) -> list[dict[str, Any]]: """Run one generator-frequency worker on its assigned visible GPU.""" configuration = _base_configuration( generator=generator, frequency=frequency, paths=paths, seed=seed, block_size=block_size, ) cache = _generate_base_cache( configuration=configuration, cache_root=cache_root, verbose=Context.all, ) reports = [] for horizon in horizons: for hedge in hedges: for cost in costs: case = RequiredCase( generator=generator, trading_frequency=frequency, return_horizon=horizon, hedge=hedge, cost_rate=cost, path_count=paths, ) completed = _load_completed_case_report( report_root=report_root, case=case, base_cache_uid=cache.cache_uid, require_conditional=run_conditional, require_baskets=run_baskets, ) if completed is not None: reports.append(completed) continue reports.append( _run_case( case=case, cache=cache, cache_root=cache_root, report_root=report_root, device=device, run_conditional=run_conditional, run_baskets=run_baskets, ) ) return reports
def _case_report_path(report_root: Path, case: RequiredCase) -> Path: return report_root / ( f"{case.generator}-{case.trading_frequency}-{case.path_count}-" f"{case.return_horizon}-{case.hedge}-{case.cost_rate:.0e}.json" ) def _load_completed_case_report( *, report_root: Path, case: RequiredCase, base_cache_uid: str, require_conditional: bool = False, require_baskets: bool = False, ) -> dict[str, Any] | None: """Return a configuration-matched completed report for restart recovery.""" path = _case_report_path(report_root, case) if not path.exists(): return None try: report = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None conditional = report.get("conditional", {}) certification = report.get("candidate_certification", {}) conditional_times = conditional.get("times", []) has_current_conditional = ( conditional.get("status") == "complete" and certification.get("conditional_regions", {}).get("status") == "complete" and certification.get("local_conditional_baskets", {}).get("status") == "complete" and len(conditional_times) == case.return_times and all( "searched_query_states" in item.get("local_conditional_baskets", {}) for item in conditional_times ) ) has_current_baskets = len(report.get("constant_baskets", [])) == ( case.return_times ) if ( report.get("status") != "complete" or report.get("schema_version") != _REPORT_SCHEMA_VERSION or report.get("configuration") != asdict(case) or report.get("base_cache_uid") != base_cache_uid or "dynamic_arbitrage" not in report or (require_conditional and not has_current_conditional) or (require_baskets and not has_current_baskets) or ( case.cost_rate == 0.0 and ( not report.get("constant_baskets") or not all( item.get("zero_cost_signed_action_formulation") == "free-signed-action-v2" for item in report.get("constant_baskets", []) ) ) ) or ( case.path_count in REPORTED_PATH_COUNTS and not report.get("functional_gate", {}).get("passed", False) ) ): return None return report def _stage_passed( *, report_root: Path, paths: int, horizon: str, hedge: str, cost: float, ) -> bool: for generator, frequency in ( ("dysanos", "daily"), ("dysanos", "weekly"), ("pca-iv", "daily"), ("pca-iv", "weekly"), ): case = RequiredCase( generator=generator, trading_frequency=frequency, return_horizon=horizon, hedge=hedge, cost_rate=cost, path_count=paths, ) path = _case_report_path(report_root, case) if not path.exists(): return False report = json.loads(path.read_text(encoding="utf-8")) if report.get("status") != "complete" or not report.get( "functional_gate", {} ).get("passed", False): return False return True
[docs] def run_four_gpu_stage( *, paths: int, horizon: str, hedge: str, costs: tuple[float, ...], cache_root: Path, report_root: Path, seed: int, block_size: int, run_conditional: bool, run_baskets: bool, ) -> None: """Run the four prescribed generator-frequency workers concurrently.""" processes: list[subprocess.Popen] = [] for gpu, (generator, frequency) in enumerate( ( ("dysanos", "daily"), ("dysanos", "weekly"), ("pca-iv", "daily"), ("pca-iv", "weekly"), ) ): command = [ sys.executable, "-m", "quantdata.dysanos.grid.gendysanos.run_required_experiments", "--worker", "--generator", generator, "--frequency", frequency, "--paths", str(paths), "--horizons", horizon, "--hedges", hedge, "--costs", *(str(value) for value in costs), "--cache-root", str(cache_root), "--report-root", str(report_root), "--seed", str(seed), "--block-size", str(block_size), "--device", "cuda", ] if run_conditional: command.append("--conditional") if run_baskets: command.append("--baskets") environment = os.environ.copy() environment["CUDA_VISIBLE_DEVICES"] = str(gpu) processes.append(subprocess.Popen(command, env=environment)) failures = [process.wait() for process in processes] if any(code != 0 for code in failures): raise RuntimeError(f"four-GPU stage failed with exit codes {failures}")
[docs] def run_required_sequence(args: argparse.Namespace) -> None: """Run smoke, 100,000, and gated one-million stages in required order.""" cache_root = Path(args.cache_root) report_root = Path(args.report_root) for horizon in ("expiry", "incremental"): for hedge in ("delta", "raw"): run_four_gpu_stage( paths=1_000, horizon=horizon, hedge=hedge, costs=(1.0e-4,), cache_root=cache_root, report_root=report_root, seed=args.seed, block_size=min(args.block_size, 1_000), run_conditional=True, run_baskets=True, ) run_four_gpu_stage( paths=100_000, horizon=horizon, hedge=hedge, costs=(1.0e-4,), cache_root=cache_root, report_root=report_root, seed=args.seed, block_size=args.block_size, run_conditional=True, run_baskets=True, ) if not _stage_passed( report_root=report_root, paths=100_000, horizon=horizon, hedge=hedge, cost=1.0e-4, ): raise RuntimeError( "functional gate failed; the one-million stage was not launched" ) run_four_gpu_stage( paths=1_000_000, horizon=horizon, hedge=hedge, costs=(1.0e-4,), cache_root=cache_root, report_root=report_root, seed=args.seed, block_size=args.block_size, run_conditional=True, run_baskets=True, ) for paths in (100_000, 1_000_000): for horizon in ("expiry", "incremental"): for hedge in ("delta", "raw"): run_four_gpu_stage( paths=paths, horizon=horizon, hedge=hedge, costs=(0.0, 1.0e-3), cache_root=cache_root, report_root=report_root, seed=args.seed, block_size=args.block_size, run_conditional=True, run_baskets=True, ) write_aggregate_report(report_root)
[docs] def write_aggregate_report(report_root: Path) -> dict[str, Any]: """Write a complete matrix report and 100k-to-one-million comparisons.""" reports: dict[int, list[dict[str, Any]]] = {} missing: dict[int, list[dict[str, Any]]] = {} for paths in (100_000, 1_000_000): reports[paths] = [] missing[paths] = [] for case in required_cases(paths): path = _case_report_path(report_root, case) if path.exists(): reports[paths].append(json.loads(path.read_text(encoding="utf-8"))) else: missing[paths].append(asdict(case)) convergence: list[dict[str, Any]] = [] large_by_key = { _comparison_key(item["configuration"]): item for item in reports[1_000_000] } for small in reports[100_000]: key = _comparison_key(small["configuration"]) large = large_by_key.get(key) if large is None: continue small_metrics = _convergence_metrics(small) large_metrics = _convergence_metrics(large) if small_metrics is None or large_metrics is None: convergence.append( {"case": key, "status": "unresolved-incomplete-diagnostics"} ) continue convergence.append( { "case": key, "status": "complete", "assessment": convergence_report( small_metrics, large_metrics ), "candidate_rate_100000": small_metrics["candidate_rate"], "candidate_rate_1000000": large_metrics["candidate_rate"], } ) summaries = { str(paths): [ _aggregate_case_summary(item, report_root=report_root) for item in reports[paths] ] for paths in (100_000, 1_000_000) } score_counts = { str(paths): { "has_dynamic_arbitrage": sum( item["dynamic_arbitrage"]["has_dynamic_arbitrage"] is True for item in reports[paths] ), "no_dynamic_arbitrage_detected": sum( item["dynamic_arbitrage"]["has_dynamic_arbitrage"] is False for item in reports[paths] ), "unresolved": sum( item["dynamic_arbitrage"]["has_dynamic_arbitrage"] is None for item in reports[paths] ), } for paths in (100_000, 1_000_000) } aggregate = { "status": ( "complete" if not missing[100_000] and not missing[1_000_000] else "incomplete" ), "expected_cases_per_path_count": 48, "completed_cases": {str(key): len(value) for key, value in reports.items()}, "missing_cases": missing, "qualification": ( "finite Monte Carlo can report no per-trading-time conditional " "arbitrage detected within the tested strategy class and resolution; " "it cannot prove mathematical absence of arbitrary dynamic arbitrage" ), "score_counts": score_counts, "cases": summaries, "convergence": convergence, } _atomic_json(report_root / "aggregate.json", aggregate) return aggregate
def _aggregate_case_summary( report: dict[str, Any], *, report_root: Path ) -> dict[str, Any]: """Return compact comparison fields while retaining the detailed report.""" configuration = report["configuration"] case = RequiredCase(**configuration) static = ( report.get("generator_diagnostics", {}) .get("static_arbitrage", {}) .get("primary", {}) .get("aggregate") ) importance = report.get("functional_gate", {}).get("checks", {}).get( "importance_normalization_and_moment_recovery" ) return { "status": report.get("status"), "configuration": configuration, "report_file": str(_case_report_path(report_root, case)), "base_cache_uid": report.get("base_cache_uid"), "functional_gate_passed": report.get("functional_gate", {}).get("passed"), "dynamic_arbitrage": report.get("dynamic_arbitrage"), "importance": importance, "static_arbitrage_primary": static, "peak_memory": report.get("peak_memory"), } def _comparison_key(configuration: dict[str, Any]) -> str: """Return the path-count-independent identity of a reported case.""" return "|".join( str(configuration[name]) for name in ( "generator", "trading_frequency", "return_horizon", "hedge", "cost_rate", ) ) def _convergence_metrics(report: dict[str, Any]) -> dict[str, float] | None: """Extract rate-based metrics used by the required convergence check.""" conditional = report.get("conditional", {}) if conditional.get("status") != "complete": return None sampling = report["sampling_diagnostics"]["importance_sampling"] path = sampling["path_weights"] paths = float(report["configuration"]["path_count"]) weight_mean = float(path["mean_density"]) effective_size = float(path["independent_path_effective_sample_size"]) second = weight_mean * weight_mean * paths / effective_size weight_error = math.sqrt( max(second - weight_mean * weight_mean, 0.0) / paths ) mean_errors: list[float] = [] for time_report in report["unconditional"]["times"]: for direction in ("long", "short"): mean_errors.extend( float(item["weighted_P_path_standard_error"]) for item in time_report["directions"][direction] ) radii: list[float] = [] local_sizes: list[float] = [] queries = 0 screened = 0 for time_report in conditional["times"]: audit = time_report["audit"] radii.append(float(audit["neighbor_radius"]["mean"])) local_sizes.append(float(audit["local_effective_sample_size"]["mean"])) queries += int(audit["queries"]) screened += int( time_report["frozen_candidate_region_audit"][ "reference_screened_candidates" ] ) return { "paths": paths, "weight_mean": weight_mean, "weight_mean_standard_error": weight_error, "effective_sample_size": effective_size, "mean_standard_error": float(np.median(mean_errors)), "conditional_radius": float(np.mean(radii)), "conditional_effective_sample_size": float(np.mean(local_sizes)), "candidate_rate": screened / max(queries, 1), } def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--worker", action="store_true") parser.add_argument("--generator", choices=("dysanos", "pca-iv")) parser.add_argument("--frequency", choices=("daily", "weekly")) parser.add_argument("--paths", type=int, default=1_000) parser.add_argument( "--horizons", nargs="+", choices=("expiry", "incremental"), default=("expiry",), ) parser.add_argument( "--hedges", nargs="+", choices=("delta", "raw"), default=("delta",) ) parser.add_argument( "--costs", nargs="+", type=float, default=(1.0e-4,) ) parser.add_argument("--conditional", action="store_true") parser.add_argument("--baskets", action="store_true") parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") parser.add_argument("--seed", type=int, default=700_000) parser.add_argument("--block-size", type=int, default=10_000) parser.add_argument("--cache-root", default=_DEFAULT_CACHE_ROOT) parser.add_argument("--report-root", default=_DEFAULT_REPORT_ROOT) parser.add_argument("--aggregate-only", action="store_true") return parser
[docs] def main() -> None: args = _parser().parse_args() if args.aggregate_only: write_aggregate_report(Path(args.report_root)) return if args.worker: if args.generator is None or args.frequency is None: raise ValueError("worker mode requires --generator and --frequency") invalid_costs = set(args.costs) - set(REPORTED_COST_RATES) if invalid_costs: raise ValueError(f"unsupported cost rates {sorted(invalid_costs)}") device = torch.device( "cuda:0" if args.device == "cuda" and torch.cuda.is_available() else "cpu" ) run_worker( generator=args.generator, frequency=args.frequency, paths=args.paths, horizons=tuple(args.horizons), hedges=tuple(args.hedges), costs=tuple(args.costs), cache_root=Path(args.cache_root), report_root=Path(args.report_root), seed=args.seed, block_size=args.block_size, run_conditional=args.conditional, run_baskets=args.baskets, device=device, ) return run_required_sequence(args)
if __name__ == "__main__": started = time.monotonic() main() print(f"required-run command completed in {time.monotonic() - started:.1f}s")