Source code for quantdata.dysanos.grid.rndysanos.quadratic_action

"""Regularized conditional-action objectives for drift diagnostics.

This module deliberately does not construct a risk-neutral density.  For

``Y = a(z)' dH - kappa * ||a(z)||**2``

is evaluated through the optimized-certainty-equivalent objective

``sup_{a,y} E[u(Y+y)-y]``.

the first-order condition is

``E[u'(Y) * (dH - 2*kappa*a(z)) | z] = 0``.

Here the OCE first-order condition in ``y`` normalizes marginal utility to one.
The action condition consequently leaves conditional drift
``2*kappa*a(z)`` under that marginal-utility density rather than zero.  The
quadratic term makes the optimization coercive and is useful for learning
conditional drift without relying on an output cap.
"""

from __future__ import annotations

import math
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from typing import Any

import numpy as np
import torch
from cdxcore.err import verify_inp
from cdxcore.version import version
from torch import Tensor

from quantdata.torch import BaseEnvironment, BaseModel
from quantdata.torch.ffw import FeedForward, FeedForwardConfig
from quantdata.torch.utility import torch_utility

from .torch_drift_change import ScaledTanhNetwork


[docs] @version("1.0") def downside_oce_utility( x: Tensor, y: Tensor, *, scale: float, power: int, ) -> tuple[Tensor, Tensor]: r"""Return ``-scale*(-(x+y))_+**power-y`` and its derivative in ``x``.""" verify_inp( math.isfinite(scale) and scale > 1.0, lambda: f"'scale' must be finite and greater than one, got {scale}", ) verify_inp(power >= 1, lambda: f"'power' must be positive, got {power}") xbar = x + y downside = torch.clamp(-xbar, min=0.0) utility = -scale * downside.pow(power) - y if power == 1: # Select the right derivative zero at the CVaR cutoff. marginal = torch.where( xbar < 0.0, torch.full_like(xbar, scale), torch.zeros_like(xbar), ) else: marginal = scale * float(power) * downside.pow(power - 1) return utility, marginal
[docs] @version("1.3", dependencies=[downside_oce_utility, torch_utility]) @dataclass(frozen=True, kw_only=True) class QuadraticActionConfig: """Logical configuration of the regularized action experiment.""" utility: str = "entropy" lmbda: float = 1.0 downside_power: int = 2 quadratic_coefficient: float = 0.5 bounded_action: bool = False initial_action_scale: float = 1.0 network_config: FeedForwardConfig = FeedForwardConfig( num_depth=2, num_width=100, activation="LeakyReLU", use_bias=True, ) def __post_init__(self) -> None: verify_inp( self.utility in {"entropy", "powexp", "downside"}, lambda: ( "'utility' must be 'entropy', 'powexp', or 'downside', got " f"{self.utility!r}" ), ) verify_inp( isinstance(self.downside_power, int) and self.downside_power >= 1, lambda: ( "'downside_power' must be a positive integer, got " f"{self.downside_power!r}" ), ) verify_inp( math.isfinite(self.lmbda) and self.lmbda > 0.0, lambda: f"'lmbda' must be finite and positive, got {self.lmbda}", ) verify_inp( math.isfinite(self.quadratic_coefficient) and self.quadratic_coefficient > 0.0, lambda: ( "'quadratic_coefficient' must be finite and positive, got " f"{self.quadratic_coefficient}" ), ) verify_inp( math.isfinite(self.initial_action_scale) and self.initial_action_scale > 0.0, lambda: ( "'initial_action_scale' must be finite and positive, got " f"{self.initial_action_scale}" ), )
[docs] @version("1.1") class DirectActionNetwork(BaseModel): """Unbounded feed-forward action with a neutral zero initialization.""" def __init__( self, env: BaseEnvironment, *, num_input: int, num_output: int, config: FeedForwardConfig, ) -> None: super().__init__(env) self.network = FeedForward( env, num_input=num_input, num_output=num_output, config=config, ) self.zero_output_layer()
[docs] @torch.no_grad() def zero_output_layer(self) -> None: """Set the last affine map to zero without changing hidden features.""" output_layer = ( self.network.In[-1] if len(self.network.In) > 0 else self.network.I0 ) output_layer.weight.zero_() if output_layer.bias is not None: output_layer.bias.zero_()
[docs] def forward(self, z: Tensor) -> Tensor: action = self.network(z) self.verify(lambda: torch.isfinite(action), "direct action is non-finite") return action
[docs] @version( "1.3", dependencies=[ DirectActionNetwork, QuadraticActionConfig, ScaledTanhNetwork, ], ) class QuadraticActionModel(BaseModel): """Conditional portfolio with a quadratic position charge inside utility.""" def __init__( self, env: BaseEnvironment, *, num_input: int, num_instruments: int, config: QuadraticActionConfig, ) -> None: super().__init__(env) verify_inp(num_input > 0, "'num_input' must be positive") verify_inp(num_instruments > 0, "'num_instruments' must be positive") self.config = config if config.bounded_action: self.action = ScaledTanhNetwork( env, num_input=num_input, num_output=num_instruments, config=config.network_config, initial_scale=config.initial_action_scale, scale_per_output=True, ) self.action.zero_output_layer() else: self.action = DirectActionNetwork( env, num_input=num_input, num_output=num_instruments, config=config.network_config, ) self.cash_offset = torch.nn.Parameter( torch.zeros((), dtype=env.fdtype, device=env.device) ) if config.utility == "downside": self.utility = None self.utility_derivative = None self.utility_label = ( f"downside-oce-p{config.downside_power}@{config.lmbda:.2g}" ) else: self.utility, self.utility_derivative, self.utility_label = torch_utility( config.utility, config.lmbda, )
[docs] def outputs(self, data: Mapping[str, Tensor]) -> dict[str, Tensor]: z, dH = data["z"], data["dH"] verify_inp( z.shape[:-1] == dH.shape[:-1], lambda: f"state/return leading shapes differ: {z.shape} and {dH.shape}", ) action = self.action(z) verify_inp( action.shape == dH.shape, lambda: f"action shape {action.shape} != return shape {dH.shape}", ) penalty = self.config.quadratic_coefficient * action.square().sum(dim=-1) gross_pnl = (action * dH).sum(dim=-1) pnl = gross_pnl - penalty if self.config.utility == "downside": utility, marginal = downside_oce_utility( pnl, self.cash_offset, scale=1.0 + self.config.lmbda, power=self.config.downside_power, ) else: assert self.utility is not None assert self.utility_derivative is not None utility = self.utility(pnl, self.cash_offset) marginal = self.utility_derivative(pnl, self.cash_offset) residual = marginal[..., None] * ( dH - 2.0 * self.config.quadratic_coefficient * action ) self.verify(lambda: torch.isfinite(pnl), "quadratic-action P&L is non-finite") self.verify( lambda: torch.isfinite(utility), "quadratic-action utility is non-finite", ) self.verify( lambda: torch.isfinite(marginal) & (marginal >= 0.0), "quadratic-action marginal utility is invalid", ) return { "action": action, "gross_pnl": gross_pnl, "penalty": penalty, "pnl": pnl, "utility": utility, "marginal_utility": marginal, "first_order_residual": residual, "cash_offset": self.cash_offset, }
[docs] def forward(self, data: Mapping[str, Tensor]) -> Tensor: output = self.outputs(data) weight = data.get("sampling_weight") if weight is None: weight = torch.ones( output["pnl"].shape[0], dtype=output["pnl"].dtype, device=output["pnl"].device, ) verify_inp( weight.shape == (output["pnl"].shape[0],), lambda: f"sampling-weight shape {weight.shape} is invalid", ) loss = -(weight[:, None] * output["utility"]).mean() self.verify(lambda: torch.isfinite(loss), "quadratic-action loss is non-finite") return loss
[docs] def iter_path_batches( data: Mapping[str, Tensor], batch_size: int | None ) -> Iterator[dict[str, Tensor]]: """Yield complete-path batches without changing any within-path reduction.""" paths = data["z"].shape[0] size = paths if batch_size is None else batch_size verify_inp(size > 0, lambda: f"'batch_size' must be positive, got {size}") for start in range(0, paths, size): stop = min(start + size, paths) yield {name: value[start:stop] for name, value in data.items()}
[docs] def objective_gradient( model: QuadraticActionModel, data: Mapping[str, Tensor], *, batch_size: int | None, ) -> dict[str, float | int]: """Differentiate the exact full-split empirical objective in path batches.""" parameters = [parameter for parameter in model.parameters() if parameter.requires_grad] if not parameters: raise ValueError("the model has no trainable parameters") for parameter in parameters: parameter.grad = None paths = data["z"].shape[0] total_loss = 0.0 for batch in iter_path_batches(data, batch_size): fraction = batch["z"].shape[0] / float(paths) loss = model(batch) * fraction loss_value = float(loss.detach().cpu().item()) if not math.isfinite(loss_value): raise FloatingPointError("quadratic-action objective became non-finite") loss.backward() total_loss += loss_value squared_norm = torch.zeros((), dtype=torch.float64, device=parameters[0].device) nonfinite = 0 for parameter in parameters: if parameter.grad is None: continue gradient = parameter.grad.detach() nonfinite += int(torch.count_nonzero(~torch.isfinite(gradient)).item()) squared_norm += torch.nan_to_num(gradient.to(torch.float64)).square().sum() return { "loss": total_loss, "gradient_norm": float(torch.sqrt(squared_norm).cpu().item()), "nonfinite_gradient_count": nonfinite, }
[docs] @torch.no_grad() def valuate_quadratic_action( model: QuadraticActionModel, data: Mapping[str, Tensor], *, batch_size: int | None, ) -> dict[str, Any]: """Return saturation, first-order, and induced-density diagnostics.""" S, TAU, TNX = data["dH"].shape device = data["dH"].device zero_ti = lambda: torch.zeros((TAU, TNX), dtype=torch.float64, device=device) sum_weight = torch.zeros(TAU, dtype=torch.float64, device=device) sum_marginal = torch.zeros(TAU, dtype=torch.float64, device=device) sum_density_dH = zero_ti() sum_density_action = zero_ti() sum_residual, sum_residual_sqr = zero_ti(), zero_ti() sum_action, sum_action_sqr, max_action = zero_ti(), zero_ti(), zero_ti() saturation = zero_ti() total_utility = torch.zeros((), dtype=torch.float64, device=device) total_pnl = torch.zeros_like(total_utility) total_penalty = torch.zeros_like(total_utility) bounded = isinstance(model.action, ScaledTanhNetwork) for batch in iter_path_batches(data, batch_size): output = model.outputs(batch) weight = batch.get("sampling_weight") if weight is None: weight = torch.ones( batch["z"].shape[0], dtype=batch["z"].dtype, device=device, ) w = weight[:, None] wi = weight[:, None, None] marginal_weight = w * output["marginal_utility"] sum_weight += w.sum(dim=0, dtype=torch.float64).expand(TAU) sum_marginal += marginal_weight.sum(dim=0, dtype=torch.float64) sum_density_dH += ( marginal_weight[..., None] * batch["dH"] ).sum(dim=0, dtype=torch.float64) sum_density_action += ( marginal_weight[..., None] * output["action"] ).sum(dim=0, dtype=torch.float64) weighted_residual = wi * output["first_order_residual"] sum_residual += weighted_residual.sum(dim=0, dtype=torch.float64) # The Monte Carlo estimator is formed under the proposal measure. # Its sampling variance is therefore the variance of ``weight*residual``. sum_residual_sqr += weighted_residual.square().sum( dim=0, dtype=torch.float64 ) sum_action += (wi * output["action"]).sum(dim=0, dtype=torch.float64) sum_action_sqr += ( wi * output["action"].square() ).sum(dim=0, dtype=torch.float64) max_action = torch.maximum( max_action, output["action"].abs().amax(dim=0).to(torch.float64), ) if bounded: cap = model.action.scale.detach()[None, :] saturation += ( wi * (output["action"].abs() >= 0.99 * cap) ).sum(dim=0, dtype=torch.float64) total_utility += (w * output["utility"]).sum(dtype=torch.float64) total_pnl += (w * output["pnl"]).sum(dtype=torch.float64) total_penalty += (w * output["penalty"]).sum(dtype=torch.float64) denominator = sum_weight[:, None].clamp_min(torch.finfo(torch.float64).tiny) marginal_denominator = sum_marginal[:, None].clamp_min( torch.finfo(torch.float64).tiny ) residual_mean = sum_residual / denominator residual_variance = ( sum_residual_sqr / denominator - residual_mean.square() ).clamp_min(0.0) residual_error = torch.sqrt(residual_variance / float(S)) normalized_residual = torch.where( residual_error > 0.0, residual_mean / residual_error, torch.zeros_like(residual_mean), ) action_mean = sum_action / denominator action_std = ( sum_action_sqr / denominator - action_mean.square() ).clamp_min(0.0).sqrt() density_mean_dH = sum_density_dH / marginal_denominator density_mean_action = sum_density_action / marginal_denominator density_identity_residual = density_mean_dH - ( 2.0 * model.config.quadratic_coefficient * density_mean_action ) observations = sum_weight.sum().clamp_min(torch.finfo(torch.float64).tiny) def cpu(value: Tensor) -> np.ndarray: return value.detach().cpu().numpy() result: dict[str, Any] = { "utility": float((total_utility / observations).cpu().item()), "pnl_mean": float((total_pnl / observations).cpu().item()), "quadratic_penalty_mean": float( (total_penalty / observations).cpu().item() ), "action_mean": cpu(action_mean), "action_std": cpu(action_std), "action_max_abs": cpu(max_action), "first_order_residual_mean": cpu(residual_mean), "first_order_residual_error": cpu(residual_error), "first_order_residual_z": cpu(normalized_residual), "first_order_residual_max_abs_z": float( normalized_residual.abs().amax().cpu().item() ), "marginal_density_mean_dH": cpu(density_mean_dH), "marginal_density_mean_action": cpu(density_mean_action), "marginal_density_identity_residual": cpu(density_identity_residual), "marginal_density_identity_max_abs": float( density_identity_residual.abs().amax().cpu().item() ), "marginal_density_martingale_max_abs": float( density_mean_dH.abs().amax().cpu().item() ), "cash_offset": float(model.cash_offset.detach().cpu().item()), } if bounded: result["action_cap"] = cpu(model.action.scale) result["action_cap_saturation_rate"] = cpu(saturation / denominator) result["action_cap_saturation_rate_max"] = float( (saturation / denominator).amax().cpu().item() ) result["cash_offset_first_order_residual"] = float( (sum_marginal.sum() / sum_weight.sum() - 1.0).cpu().item() ) for name, value in result.items(): if isinstance(value, float) and not math.isfinite(value): raise FloatingPointError(f"valuation output {name!r} is non-finite") if isinstance(value, np.ndarray) and not np.isfinite(value).all(): raise FloatingPointError(f"valuation output {name!r} is non-finite") return result
[docs] def conditional_gaussian_entropy_action( mean: Tensor, covariance: Tensor, *, lmbda: float, quadratic_coefficient: float, ) -> Tensor: r"""Return ``(2*kappa*I + lambda*Sigma)^{-1} mean``.""" verify_inp(mean.ndim >= 1, lambda: f"mean shape {mean.shape} is invalid") verify_inp( covariance.shape == mean.shape + (mean.shape[-1],), lambda: ( f"covariance shape {covariance.shape} is incompatible with " f"mean shape {mean.shape}" ), ) verify_inp( math.isfinite(lmbda) and lmbda > 0.0, lambda: f"'lmbda' must be finite and positive, got {lmbda}", ) verify_inp( math.isfinite(quadratic_coefficient) and quadratic_coefficient > 0.0, lambda: ( "'quadratic_coefficient' must be finite and positive, got " f"{quadratic_coefficient}" ), ) identity = torch.eye( mean.shape[-1], dtype=mean.dtype, device=mean.device ) system = ( 2.0 * quadratic_coefficient * identity + lmbda * covariance ) action = torch.linalg.solve(system, mean.unsqueeze(-1)).squeeze(-1) if not torch.isfinite(action).all().item(): raise FloatingPointError("conditional Gaussian action is non-finite") return action
[docs] def parameter_norm(parameters: Sequence[Tensor]) -> float: """Return the finite Euclidean norm of a parameter sequence.""" total = torch.zeros((), dtype=torch.float64) for parameter in parameters: total += parameter.detach().cpu().to(torch.float64).square().sum() result = float(torch.sqrt(total).item()) if not math.isfinite(result): raise FloatingPointError("parameter norm is non-finite") return result
__all__ = [ "DirectActionNetwork", "QuadraticActionConfig", "QuadraticActionModel", "conditional_gaussian_entropy_action", "downside_oce_utility", "iter_path_batches", "objective_gradient", "parameter_norm", "valuate_quadratic_action", ]