Source code for quantdata.dysanos.grid.gendysanos.conditional

"""Time-resolved conditional support diagnostics for generated markets."""

from __future__ import annotations

import math
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict
from typing import Any

import numpy as np
import torch
from cdxcore.err import verify_inp
from scipy.spatial import cKDTree
from scipy.stats import norm

from quantdata.dysanos.grid.rndysanos.local_tail import (
    LocalTailConfig,
    normalize_state,
    state_normalization,
)
from quantdata.dysanos.implied.conditional_diagnostics import (
    _cost_adjusted_local_directions,
)

from .required_runs import (
    DerivedCaseArrays,
    PathSplit,
    clopper_pearson_one_sided,
    positive_relative_weights,
    propagated_basket_band,
    solve_constant_basket,
)


def _sample_paths(
    candidates: np.ndarray,
    *,
    maximum: int,
    seed: int,
) -> np.ndarray:
    """Select deterministic query paths without changing their split."""

    candidates = np.asarray(candidates, dtype=np.int64)
    verify_inp(candidates.ndim == 1, "query candidates must be one-dimensional")
    if candidates.size <= maximum:
        return np.sort(candidates)
    rng = np.random.default_rng(seed)
    return np.sort(rng.choice(candidates, size=maximum, replace=False))


def _audit_frozen_balls(
    *,
    candidates: list[dict[str, Any]],
    centers: np.ndarray,
    audit_state: np.ndarray,
    audit_returns: np.ndarray,
    audit_costs: np.ndarray,
    audit_bands: np.ndarray,
    audit_log_weights: np.ndarray,
    audit_components: np.ndarray,
    time_index: int,
    family_size: int,
    prerequisite_checks_passed: bool,
    minimum_direct_paths: int = 20,
    minimum_effective_size: float = 20.0,
) -> dict[str, Any]:
    """Evaluate frozen reference balls using actual audit outcomes."""

    adjusted_alpha = 0.05 / family_size
    z = float(norm.ppf(1.0 - adjusted_alpha))
    audit_log_weights = np.asarray(audit_log_weights, dtype=np.float64)
    components = np.asarray(audit_components)
    surviving: list[dict[str, Any]] = []
    rejected: list[dict[str, Any]] = []
    unresolved: list[dict[str, Any]] = []
    for candidate in candidates:
        center = centers[int(candidate["query_index"])]
        radius = float(candidate["neighbor_radius"])
        squared_distance = np.sum(
            np.square(audit_state.astype(np.float64, copy=False) - center),
            axis=1,
        )
        region = squared_distance <= radius * radius
        instrument = int(candidate["instrument"])
        sign = 1.0 if candidate["direction"] == "long" else -1.0
        value = (
            sign * audit_returns[:, instrument] - audit_costs[:, instrument]
        ).astype(np.float64, copy=False)
        band = audit_bands[:, instrument].astype(np.float64, copy=False)
        regional_paths = int(np.count_nonzero(region))
        region_weight = (
            positive_relative_weights(audit_log_weights[region])
            if regional_paths
            else np.empty(0, dtype=np.float64)
        )
        direct_region = region & (components == 0)
        direct_paths = int(np.count_nonzero(direct_region))
        mass = float(region_weight.sum())
        square_mass = float(np.square(region_weight).sum())
        effective_size = mass * mass / square_mass if square_mass > 0.0 else 0.0
        base = {
            **candidate,
            "trading_time_index": time_index,
            "state_region": {
                "kind": "closed normalized Euclidean ball",
                "center": center.astype(float).tolist(),
                "radius": radius,
            },
            "audit_paths_in_region": regional_paths,
            "direct_P_audit_paths_in_region": direct_paths,
            "regional_effective_sample_size": effective_size,
        }
        if (
            direct_paths < minimum_direct_paths
            or effective_size < minimum_effective_size
        ):
            unresolved.append(
                {
                    **base,
                    "status": "unresolved",
                    "reason": "insufficient direct-P regional paths or weighted ESS",
                    "certified": False,
                }
            )
            continue
        loss = value < -band
        gain = value > band
        direct_value = value[direct_region]
        direct_loss = int(np.count_nonzero(loss[direct_region]))
        direct_gain = int(np.count_nonzero(gain[direct_region]))
        gain_lower, _ = clopper_pearson_one_sided(
            direct_gain, direct_paths, alpha=adjusted_alpha
        )
        _, loss_upper = clopper_pearson_one_sided(
            direct_loss, direct_paths, alpha=adjusted_alpha
        )
        direct_mean = float(np.mean(direct_value))
        direct_error = float(
            np.std(direct_value, ddof=0) / math.sqrt(direct_paths)
        )
        direct_mean_lower = direct_mean - z * direct_error
        region_value = value[region]
        region_gain = gain[region].astype(np.float64)
        weighted_mean = float(np.dot(region_weight, region_value) / mass)
        weighted_gain = float(np.dot(region_weight, region_gain) / mass)
        weighted_mean_error = float(
            np.sqrt(
                np.dot(
                    np.square(region_weight),
                    np.square(region_value - weighted_mean),
                )
            )
            / mass
        )
        weighted_gain_error = float(
            np.sqrt(
                np.dot(
                    np.square(region_weight),
                    np.square(region_gain - weighted_gain),
                )
            )
            / mass
        )
        weighted_mean_lower = weighted_mean - z * weighted_mean_error
        weighted_gain_lower = weighted_gain - z * weighted_gain_error
        conditions = {
            "no_audit_loss_beyond_band": int(np.count_nonzero(loss[region])) == 0,
            "adjusted_direct_P_gain_lower_positive": gain_lower > 0.0,
            "adjusted_direct_P_mean_lower_positive": direct_mean_lower > 0.0,
            "adjusted_weighted_P_gain_lower_positive": weighted_gain_lower > 0.0,
            "adjusted_weighted_P_mean_lower_positive": weighted_mean_lower > 0.0,
            "importance_coverage_and_replay_pass": prerequisite_checks_passed,
        }
        item = {
            **base,
            "status": "evaluated",
            "audit_loss_count": int(np.count_nonzero(loss[region])),
            "audit_gain_count": int(np.count_nonzero(gain[region])),
            "direct_P_loss_count": direct_loss,
            "direct_P_gain_count": direct_gain,
            "adjusted_direct_P_loss_probability_upper": loss_upper,
            "adjusted_direct_P_gain_probability_lower": gain_lower,
            "direct_P_mean": direct_mean,
            "adjusted_direct_P_mean_lower": direct_mean_lower,
            "weighted_P_gain_probability": weighted_gain,
            "adjusted_weighted_P_gain_probability_lower": weighted_gain_lower,
            "weighted_P_mean": weighted_mean,
            "adjusted_weighted_P_mean_lower": weighted_mean_lower,
            "conditions": conditions,
            "certified": bool(all(conditions.values())),
        }
        (surviving if item["certified"] else rejected).append(item)
    return {
        "reference_screened_candidates": len(candidates),
        "multiplicity": {
            "method": "Bonferroni familywise correction",
            "family_size": family_size,
            "adjusted_alpha": adjusted_alpha,
        },
        "minimum_direct_P_paths": minimum_direct_paths,
        "minimum_regional_effective_sample_size": minimum_effective_size,
        "surviving": surviving,
        "rejected": rejected,
        "unresolved": unresolved,
    }


def _fit_and_audit_local_baskets(
    *,
    centers: np.ndarray,
    reference_neighbor_indices: np.ndarray,
    neighbor_radius: np.ndarray,
    audit_state: np.ndarray,
    case: DerivedCaseArrays,
    split: PathSplit,
    log_path_weights: np.ndarray,
    importance_components: np.ndarray,
    time_index: int,
    neighbors: int,
    family_size: int,
    prerequisite_checks_passed: bool,
    progress_interval_seconds: float,
    solver_workers: int = 1,
) -> dict[str, Any]:
    """Fit an all-instrument L1 basket at every query and audit its state ball."""

    centers = np.asarray(centers, dtype=np.float64)
    reference_neighbor_indices = np.asarray(
        reference_neighbor_indices, dtype=np.int64
    )
    neighbor_radius = np.asarray(neighbor_radius, dtype=np.float64)
    queries = centers.shape[0]
    verify_inp(
        centers.ndim == 2
        and reference_neighbor_indices.ndim == 2
        and reference_neighbor_indices.shape[0] == queries
        and neighbor_radius.shape == (queries,),
        "local basket query geometry is inconsistent",
    )
    verify_inp(solver_workers >= 1, "basket solver workers must be positive")
    if queries == 0:
        return {
            "searched_query_states": 0,
            "reference_candidates": 0,
            "zero_reference_solutions": 0,
            "surviving": [],
            "rejected": [],
            "unresolved": [],
        }
    adjusted_alpha = 0.05 / family_size
    z = float(norm.ppf(1.0 - adjusted_alpha))
    audit_log_weights = np.asarray(log_path_weights, dtype=np.float64)[split.audit]
    audit_components = np.asarray(importance_components)[split.audit]
    audit_tree = cKDTree(np.asarray(audit_state, dtype=np.float64))
    audit_region_rows = audit_tree.query_ball_point(
        centers,
        neighbor_radius,
        return_sorted=True,
        workers=-1,
    )
    surviving: list[dict[str, Any]] = []
    rejected: list[dict[str, Any]] = []
    unresolved: list[dict[str, Any]] = []
    zero_reference_solutions = 0
    reference_candidates = 0
    started = time.monotonic()
    last_progress = -math.inf

    def report_progress(query_index: int) -> None:
        nonlocal last_progress
        now = time.monotonic()
        if (
            now - last_progress >= progress_interval_seconds
            or query_index + 1 == queries
        ):
            print(
                f"All-instrument local baskets t={time_index}: "
                f"{query_index + 1:,}/{queries:,}; "
                f"reference candidates={reference_candidates:,}, "
                f"surviving={len(surviving):,}",
                flush=True,
            )
            last_progress = now

    def solve_query(query_index: int) -> dict[str, Any]:
        local_rows = reference_neighbor_indices[query_index]
        return solve_constant_basket(
            case,
            time_index=time_index,
            reference_paths=split.reference[local_rows],
            log_path_weights=log_path_weights,
        )

    if solver_workers == 1:
        solutions = [solve_query(query_index) for query_index in range(queries)]
    else:
        print(
            f"All-instrument local baskets t={time_index}: solving "
            f"{queries:,} independent LPs on {solver_workers} CPU workers",
            flush=True,
        )
        with ThreadPoolExecutor(max_workers=solver_workers) as executor:
            solutions = list(executor.map(solve_query, range(queries)))

    for query_index, solution in enumerate(solutions):
        center = centers[query_index]
        action = np.asarray(solution["action"], dtype=np.float64)
        base = {
            "query_index": query_index,
            "trading_time_index": time_index,
            "state_region": {
                "kind": "closed normalized Euclidean ball",
                "center": center.astype(float).tolist(),
                "radius": float(neighbor_radius[query_index]),
            },
            "strategy_class": "local all-instrument state-independent L1 basket",
            "reference_solution": solution,
        }
        if np.abs(action).sum() <= 1.0e-14 or solution["gain_count"] == 0:
            zero_reference_solutions += 1
            report_progress(query_index)
            continue
        reference_candidates += 1
        region_rows = np.asarray(audit_region_rows[query_index], dtype=np.int64)
        regional_paths = int(region_rows.size)
        direct = audit_components[region_rows] == 0
        direct_paths = int(np.count_nonzero(direct))
        region_weights = (
            positive_relative_weights(audit_log_weights[region_rows])
            if regional_paths
            else np.empty(0, dtype=np.float64)
        )
        mass = float(region_weights.sum())
        square_mass = float(np.square(region_weights).sum())
        effective_size = mass * mass / square_mass if square_mass > 0.0 else 0.0
        if direct_paths < 20 or effective_size < 20.0:
            unresolved.append(
                {
                    **base,
                    "status": "unresolved",
                    "audit_paths_in_region": regional_paths,
                    "direct_P_audit_paths_in_region": direct_paths,
                    "regional_effective_sample_size": effective_size,
                    "certified": False,
                }
            )
            report_progress(query_index)
            continue
        regional_case_rows = split.audit[region_rows]
        returns = case.returns[regional_case_rows, time_index].astype(
            np.float64, copy=False
        )
        costs = case.costs[regional_case_rows, time_index].astype(
            np.float64, copy=False
        )
        bands = case.numerical_band[regional_case_rows, time_index].astype(
            np.float64, copy=False
        )
        pnl = returns @ action - costs @ np.abs(action)
        band = propagated_basket_band(action, returns, costs, bands)
        loss = pnl < -band
        gain = pnl > band
        direct_loss = int(np.count_nonzero(loss[direct]))
        direct_gain = int(np.count_nonzero(gain[direct]))
        gain_lower, _ = clopper_pearson_one_sided(
            direct_gain, direct_paths, alpha=adjusted_alpha
        )
        _, loss_upper = clopper_pearson_one_sided(
            direct_loss, direct_paths, alpha=adjusted_alpha
        )
        direct_mean = float(np.mean(pnl[direct]))
        direct_mean_lower = direct_mean - z * float(
            np.std(pnl[direct], ddof=0) / math.sqrt(direct_paths)
        )
        weighted_mean = float(np.dot(region_weights, pnl) / mass)
        weighted_gain = float(
            np.dot(region_weights, gain.astype(np.float64)) / mass
        )
        weighted_error = float(
            np.sqrt(
                np.dot(
                    np.square(region_weights),
                    np.square(pnl - weighted_mean),
                )
            )
            / mass
        )
        weighted_gain_error = float(
            np.sqrt(
                np.dot(
                    np.square(region_weights),
                    np.square(gain.astype(np.float64) - weighted_gain),
                )
            )
            / mass
        )
        weighted_mean_lower = weighted_mean - z * weighted_error
        weighted_gain_lower = weighted_gain - z * weighted_gain_error
        conditions = {
            "no_audit_loss_beyond_band": int(np.count_nonzero(loss)) == 0,
            "adjusted_direct_P_gain_lower_positive": gain_lower > 0.0,
            "adjusted_direct_P_mean_lower_positive": direct_mean_lower > 0.0,
            "adjusted_weighted_P_gain_lower_positive": weighted_gain_lower > 0.0,
            "adjusted_weighted_P_mean_lower_positive": weighted_mean_lower > 0.0,
            "importance_coverage_and_replay_pass": prerequisite_checks_passed,
        }
        item = {
            **base,
            "status": "evaluated",
            "audit_paths_in_region": regional_paths,
            "direct_P_audit_paths_in_region": direct_paths,
            "regional_effective_sample_size": effective_size,
            "audit_loss_count": int(np.count_nonzero(loss)),
            "audit_neutral_count": int(np.count_nonzero(~loss & ~gain)),
            "audit_gain_count": int(np.count_nonzero(gain)),
            "minimum_audit_economic_pnl": float(np.min(pnl)),
            "maximum_audit_economic_pnl": float(np.max(pnl)),
            "minimum_audit_pnl_plus_numerical_band": float(np.min(pnl + band)),
            "direct_P_loss_count": direct_loss,
            "direct_P_gain_count": direct_gain,
            "adjusted_direct_P_loss_probability_upper": loss_upper,
            "adjusted_direct_P_gain_probability_lower": gain_lower,
            "direct_P_mean": direct_mean,
            "adjusted_direct_P_mean_lower": direct_mean_lower,
            "weighted_P_gain_probability": weighted_gain,
            "adjusted_weighted_P_gain_probability_lower": weighted_gain_lower,
            "weighted_P_mean": weighted_mean,
            "adjusted_weighted_P_mean_lower": weighted_mean_lower,
            "conditions": conditions,
            "certified": bool(all(conditions.values())),
        }
        (surviving if item["certified"] else rejected).append(item)
        report_progress(query_index)
    return {
        "definition": (
            "maximize the local physical mean over every L1-bounded basket of "
            "all spot and option instruments, subject to nonnegative economic "
            "P&L on every reference neighbor"
        ),
        "searched_query_states": queries,
        "searched_all_query_states_independent_of_individual_screens": True,
        "instruments": int(case.returns.shape[-1]),
        "neighbors": int(neighbors),
        "reference_candidates": reference_candidates,
        "zero_reference_solutions": zero_reference_solutions,
        "multiplicity": {
            "method": "Bonferroni familywise correction",
            "family_size": family_size,
            "adjusted_alpha": adjusted_alpha,
        },
        "surviving": surviving,
        "rejected": rejected,
        "unresolved": unresolved,
        "elapsed_seconds": time.monotonic() - started,
    }


[docs] def run_time_separated_local_tail( *, state: np.ndarray, case: DerivedCaseArrays, log_path_weights: np.ndarray, importance_components: np.ndarray, split: PathSplit, neighbors: int, query_paths_per_split: int, instrument_block_size: int, seed: int, device: torch.device, tail_ranks: tuple[int, ...] = (1, 2, 10, 20, 50), signal_thresholds: tuple[float, ...] = (2.0, 3.0), progress_interval_seconds: float = 30.0, certification_prerequisites_passed: bool = False, basket_solver_workers: int = 1, ) -> dict[str, Any]: """Fit and value a separate conditional law at every trading time. Only reference paths enter normalization and nearest-neighbour fitting. Queries are drawn from the direct-P component. A reference query excludes its complete path; because time is never pooled, this is exactly one row. """ state = np.asarray(state, dtype=np.float32) log_path_weights = np.asarray(log_path_weights, dtype=np.float64) importance_components = np.asarray(importance_components) S, times, features = state.shape verify_inp( case.returns.shape[:2] == (S, times), "conditional state and return geometry differ", ) verify_inp( log_path_weights.shape == importance_components.shape == (S,), "conditional likelihood arrays have invalid shape", ) verify_inp( np.intersect1d(split.reference, split.audit).size == 0 and split.reference.size + split.audit.size == S, "conditional reference/audit split is not disjoint and exhaustive", ) verify_inp( basket_solver_workers >= 1, "basket solver workers must be positive", ) reference_components = importance_components[split.reference] audit_components = importance_components[split.audit] reference_direct_rows = np.flatnonzero(reference_components == 0) audit_direct_rows = np.flatnonzero(audit_components == 0) verify_inp( reference_direct_rows.size > 0 and audit_direct_rows.size > 0, "conditional valuation requires direct-P queries in both splits", ) training_rows = _sample_paths( reference_direct_rows, maximum=query_paths_per_split, seed=seed + 11, ) audit_rows = _sample_paths( audit_direct_rows, maximum=query_paths_per_split, seed=seed + 29, ) effective_neighbors = min(neighbors, split.reference.size - 1) verify_inp( effective_neighbors >= 2, "conditional valuation requires at least three reference paths", ) effective_tail_ranks = tuple( rank for rank in tail_ranks if 2 * rank <= effective_neighbors ) if not effective_tail_ranks: effective_tail_ranks = (1,) config = LocalTailConfig( state_mode="raw", neighbors=effective_neighbors, instrument_candidate_neighbors=effective_neighbors, tail_ranks=effective_tail_ranks, signal_thresholds=signal_thresholds, query_batch_size=min(query_paths_per_split, 3_584), instrument_batch_size=instrument_block_size, state_scale_floor=1.0e-6, ) def tensor(value: np.ndarray) -> torch.Tensor: return torch.as_tensor( np.array(value, dtype=np.float32, order="C", copy=True), dtype=torch.float32, device=device, ) reports: list[dict[str, Any]] = [] for time_index in range(times): reference_state_raw = state[split.reference, time_index] reference_weights = positive_relative_weights( log_path_weights[split.reference] ) mean, scale, constant = state_normalization( reference_state_raw[:, None, :], scale_floor=config.state_scale_floor, path_weights=reference_weights, ) reference_state = normalize_state( reference_state_raw[:, None, :], mean=mean, scale=scale )[:, 0] audit_state = normalize_state( state[split.audit, time_index][:, None, :], mean=mean, scale=scale, )[:, 0] reference_return = case.returns[split.reference, time_index] audit_return = case.returns[split.audit, time_index] reference_cost = case.costs[split.reference, time_index] audit_cost = case.costs[split.audit, time_index] reference_band = case.numerical_band[split.reference, time_index] audit_band = case.numerical_band[split.audit, time_index] reference_log_weight = log_path_weights[split.reference] training = _cost_adjusted_local_directions( reference_state=tensor(reference_state), reference_returns=tensor(reference_return), reference_cost=tensor(reference_cost), reference_roundoff=tensor(reference_band), reference_log_weights=tensor(reference_log_weight), query_state=tensor(reference_state[training_rows]), query_returns=tensor(reference_return[training_rows]), query_cost=tensor(reference_cost[training_rows]), query_roundoff=tensor(reference_band[training_rows]), config=config, split=f"reference-t{time_index}", leave_one_out_indices=torch.as_tensor( training_rows, dtype=torch.int64, device=device ), progress_interval_seconds=progress_interval_seconds, return_neighbor_indices=True, ) reference_neighbor_indices = np.asarray( training.pop("_neighbor_indices"), dtype=np.int64 ) reference_neighbor_radius = np.asarray( training.pop("_neighbor_radius"), dtype=np.float64 ) audit = _cost_adjusted_local_directions( reference_state=tensor(reference_state), reference_returns=tensor(reference_return), reference_cost=tensor(reference_cost), reference_roundoff=tensor(reference_band), reference_log_weights=tensor(reference_log_weight), query_state=tensor(audit_state[audit_rows]), query_returns=tensor(audit_return[audit_rows]), query_cost=tensor(audit_cost[audit_rows]), query_roundoff=tensor(audit_band[audit_rows]), config=config, split=f"audit-t{time_index}", leave_one_out_indices=None, progress_interval_seconds=progress_interval_seconds, ) candidate_audit = _audit_frozen_balls( candidates=training["screened_candidates"], centers=reference_state[training_rows], audit_state=audit_state, audit_returns=audit_return, audit_costs=audit_cost, audit_bands=audit_band, audit_log_weights=log_path_weights[split.audit], audit_components=importance_components[split.audit], time_index=time_index, family_size=max(1, 48 * times * query_paths_per_split), prerequisite_checks_passed=certification_prerequisites_passed, ) local_baskets = _fit_and_audit_local_baskets( centers=reference_state[training_rows], reference_neighbor_indices=reference_neighbor_indices, neighbor_radius=reference_neighbor_radius, audit_state=audit_state, case=case, split=split, log_path_weights=log_path_weights, importance_components=importance_components, time_index=time_index, neighbors=config.neighbors, family_size=max(1, 48 * times * query_paths_per_split), prerequisite_checks_passed=certification_prerequisites_passed, progress_interval_seconds=progress_interval_seconds, solver_workers=basket_solver_workers, ) reports.append( { "trading_time_index": time_index, "state_normalization": { "reference_only_mean": mean.tolist(), "reference_only_population_scale": scale.tolist(), "constant_feature_indices": np.flatnonzero(constant).tolist(), }, "reference": training, "audit": audit, "frozen_candidate_region_audit": candidate_audit, "local_conditional_baskets": local_baskets, } ) return { "status": "complete", "device": str(device), "configuration": asdict(config), "state_definition": "raw z_t=(log-spot, surface state)", "time_pooling": False, "split": { "reference_paths": int(split.reference.size), "audit_paths": int(split.audit.size), "mixture_stratified": True, "reference_queries": int(training_rows.size), "audit_queries": int(audit_rows.size), "queries_from_direct_P_component": True, "reference_queries_leave_one_complete_path_out": True, "component_counts": split.component_counts, }, "times": reports, }
__all__ = ["run_time_separated_local_tail"]