"""
SANOS (Jax)
=============
Fitting the DLVs of SANOS.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
import warnings as warnings
import numpy as np
import cvxpy as cp
from cvxpy.error import SolverError
from dataclasses import dataclass, field
from cdxcore.err import verify, verify_inp
from cdxcore.verbose import Context
from cdxcore.version import version
from cdxcore.bs import bs
from cdxcore.pretty import PrettyObject
import scipy
from sympy import true
from quantdata.biz_calendar import DAYS_PER_YEAR, ONE_MINUTE
from quantdata.utils.backbone import Backbone, BackboneNumpy
import math
from quantdata.sanos.sanos_tools import SanosPricer, SanosLPSolver, warn_and_fix_density, make_variances
from quantdata.sanos.sanos import SanosFitter
from quantdata.sanos.tridiag import solve_sanos_tridiagonal
from quantdata.jax.bbjax import BackboneJax
_bbjax =BackboneJax()
import jax
import jax.numpy as jnp
[docs]
@version("0.1", dependencies=[SanosFitter])
def fit_dlvsqs( self : SanosFitter,
expiries : np.ndarray,
strikes : list[np.ndarray],
mids : list[np.ndarray],
xstrikes : list[np.ndarray],*,
weights : list[np.ndarray]|None = None,
vols : np.ndarray|float|None = None, # forward volatilities per expiry, used for the strike expansion step. If a float is provided, it is used for all expiries. If None, it is estimated from the data as the median of the implied vols.
init_dlvs : list[np.ndarray]|None = None, # initial DLVs for the optimization. If None, they are initialized to 0.2.
message_context: str|callable[[], str] | None = None,
debug_do_not_optimize : bool = False, # if True, the function will return a pricer with the initial DLVs without optimizing them. This is useful for debugging and testing.
debug_print_info : bool = False,
verbose : Context = Context.all ) -> SanosPricer:
"""
Fits an implied surface for the given prices and strikes by directly optimizing for the DLVs.
"""
def message(with_cols : bool = True) -> str:
if message_context is None:
return ""
if callable(message_context):
msg = message_context()
else:
msg = str(message_context)
if msg == "":
return ""
msg = msg[0].upper() + msg[1:]
if not with_cols:
return msg
return msg + ": "
txt_smth = "linear" if self.smoothness is None else f"{self.smoothness:.2f}-smooth"
T = len(expiries)
verify_inp( len(strikes) == T, lambda : f"{message()}Expected number of strike arrays {T} to match number of expiries {len(expiries)}" )
verify_inp( len(mids) == T, lambda : f"{message()}Expected number of mids arrays {T} to match number of expiries {len(expiries)}" )
verify_inp( weights is None or len(weights) == T, lambda : f"{message()}Expected number of weights arrays {T} to match number of expiries {len(expiries)}" )
verify_inp( len(xstrikes) == T, lambda : f"{message()}Expected number of xstrike arrays {T} to match number of expiries {len(expiries)}" )
verify_inp( init_dlvs is None or len(init_dlvs) == T, lambda : f"{message()}Expected number of init_dlvs arrays {T} to match number of expiries {len(expiries)}" )
verify_inp( expiries[0] > 0. and (len(expiries)==1 or np.min(np.diff(expiries)) > 0.), lambda : f"{message()}Expiries must be positive and strictly increasing, but found {expiries}" )
assert not debug_do_not_optimize or not init_dlvs is None, f"{message()}If 'debug_do_not_optimize' is True, 'init_dlvs' must be provided to initialize the DLVs, but found init_dlvs={init_dlvs} and debug_do_not_optimize={debug_do_not_optimize}"
variances = make_variances(expiries, strikes, mids, vols=vols, message_context=message_context)
del vols
tme = verbose.timer()
jdtype = jnp.float64
jdevice = jax.devices("cpu")[0]
max_dlvsq = self.max_dlv**2 if not self.max_dlv is None else None
min_dlvsq = self.min_dlv**2
class JaxObjective:
def __init__( jself ):
jself.iw_minus = []
jself.iw_plus = []
jself.L = []
jself.CX = []
jself.mids = []
jself.weights = None
jself.xdensity0 = None
# x
jself.num_dlvs = None
jself.dlv_slices = []
def verify( jself, condition: bool|Backbone.Array, message : str|callable, exception : Exception = RuntimeError ):
def callback( condition ):
if jnp.all( condition ):
return
raise exception( message() if callable(message) else message )
jax.debug.callback( callback, condition )
def __call__( jself, rawdlvs : jnp.ndarray, training : bool = True ) -> jnp.ndarray | tuple[jnp.ndarray, list[jnp.ndarray], list[jnp.ndarray]] :
q_tm1 = jself.xdensity0
error = 0.
wsum = 0.
assert rawdlvs.shape == (jself.num_dlvs,), f"Expected rawdlvs to have shape {(jself.num_dlvs,)}, but found {rawdlvs.shape}"
rawdlvs = [ rawdlvs[slice_] for slice_ in jself.dlv_slices ]
xdensities = [] if not training else None
calls = [] if not training else None
for t, expiry in enumerate(expiries):
dlvinp_t = rawdlvs[t]
iw_minus_t = jself.iw_minus[t]
iw_plus_t = jself.iw_plus[t]
L_t = jself.L[t]
CX_t = jself.CX[t]
mids_t = jself.mids[t]
weights_t = jself.weights[t] if not jself.weights is None else 1.
# move density forward
if t>0:
q_tm1 = L_t @ q_tm1
else:
q_tm1 = jself.xdensity0
jself.verify( jnp.min( q_tm1 ) >= -1E-8, lambda : f"{message()}Negative density values found at expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d, before solving DLV equation: min density value is {np.min(q_tm1)}. This may indicate an issue with the strike expansion step. Consider providing a better 'vols' input to improve the strike expansion, or check the input data for inconsistencies {jnp.min( q_tm1 )}." )
# solve DLV equation
dlvsqs_t = ( jax.nn.sigmoid(dlvinp_t) * (max_dlvsq - min_dlvsq) + min_dlvsq ) if not max_dlvsq is None else\
( jnp.exp(dlvinp_t) + min_dlvsq )
w_minus_t = dlvsqs_t * iw_minus_t
w_plus_t = dlvsqs_t * iw_plus_t
q_t = solve_sanos_tridiagonal( w_minus=w_minus_t, w_plus=w_plus_t, rhs=q_tm1, backbone=_bbjax )
assert q_t.shape == xstrikes[t].shape, f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: shape error in DLV equation solution, expected {xstrikes[t].shape} but found {q_t.shape}"
# compute market call prices
C_t = CX_t @ q_t
assert C_t.shape == strikes[t].shape, f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: shape of model call prices C_t must match shape of strikes_t, but found {C_t.shape} and {strikes[t].shape}"
# add errors
error_t = weights_t * ( C_t - mids_t )
error_t = error_t**2 #jnp.minimum( error_t, 1. ) + 0.5 * jnp.maximum( error_t-1., 0 )**2
error += jnp.sum( error_t )
wsum += jnp.sum( weights_t )
if not training:
xdensities.append( q_t )
calls.append( C_t )
q_tm1 = q_t
error = error / wsum
if training:
return error
return error, xdensities, calls
jself = JaxObjective()
# initial density
# ----------------
jself.xdensity0 = jax.device_put( np.diff( np.diff( np.maximum( 0., 1. - xstrikes[0] ), prepend=1. ) / np.diff(xstrikes[0], prepend=0.), append=0. ), device=jdevice )
jself.num_dlvs = 0
init_dlvsraw = None if init_dlvs is None else []
for t, expiry in enumerate(expiries):
def local_message():
msg = message(False)
ext = f"expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d"
if msg == "":
msg = ext[0].upper() + ext[1:]
else:
msg += " " + ext
return msg
# fit
def np_call( spots, strikes ):
if self.smoothness is None or self.smoothness == 0. or expiry < ONE_MINUTE:
return np.maximum( 0., spots - strikes )
expiry_ = np.clip( expiry, expiries[0], expiries[-1] )
vol = math.sqrt( self.smoothness * np.interp( expiry_, expiries, variances ) / expiry_ )
assert np.isfinite(vol), f"Internal error: {message()}infinite vol"
if vol == 0.:
return np.maximum( 0., spots - strikes )
return bs.price( k=strikes/spots, vol=vol, sqrtT=math.sqrt(expiry), eps=self._eps_price )*spots
strikes_t = strikes[t]
mids_t = mids[t]
xstrikes_t = xstrikes[t]
weights_t = weights[t] if not weights is None else None
verify_inp( len(strikes_t) > 1 and strikes_t[0] > 0. and jnp.min( jnp.diff(strikes_t) ) > 0., f"{local_message()}: 'strikes' must be positive and increasing.")
verify_inp( len(xstrikes_t ) > 1 and xstrikes_t[0] > 0. and jnp.min( jnp.diff(xstrikes_t) ) > 0., f"{local_message()}: 'xstrikes' must be positive and increasing.")
verify_inp( strikes_t.shape == mids_t.shape, f"{local_message()}: 'strikes' and 'mids' must have the same shape, found {strikes_t.shape} and {mids_t.shape}")
if not weights_t is None: verify_inp( strikes_t.shape == weights_t.shape, f"{local_message()}: 'strikes' and 'weights' must have the same shape, found {strikes_t.shape} and {weights_t.shape}")
if t>0:
assert strikes_t[0] <= strikes[t-1][0], f"{local_message()}: 'strikes' must be increasing across expiries, but found {strikes[t-1][0]:.4g} and {strikes_t[0]:.4g} for current and previous expiry."
assert strikes_t[-1] >= strikes[t-1][-1], f"{local_message()}: 'strikes' must be increasing across expiries, but found {strikes[t-1][-1]:.4g} and {strikes_t[-1]:.4g} for current and previous expiry."
Nj = len(xstrikes_t)-2
jself.dlv_slices.append( slice(jself.num_dlvs, jself.num_dlvs+Nj) )
jself.num_dlvs += Nj
if not init_dlvs is None:
init_dlvsqs_t = init_dlvs[t]**2
assert len(init_dlvsqs_t) == Nj, f"{message()} init_dlvs for expiry #{t} must have length {Nj}, but found {len(init_dlvsqs_t)}"
if not max_dlvsq is None:
ddlvsq = max_dlvsq - min_dlvsq
init_dlvsqs_t = np.clip( init_dlvsqs_t -min_dlvsq, min=1E-8, max=ddlvsq-1E-8 ) / ddlvsq
assert np.min( init_dlvsqs_t) > 0. and np.max( init_dlvsqs_t ) < 1. , f"{message()} init_dlvs for expiry #{t} must be between 0 and 1 found min {np.min(init_dlvsqs_t)} and max {np.max(init_dlvsqs_t)}"
init_dlvsqs_t = np.log( init_dlvsqs_t / ( 1. - init_dlvsqs_t ) )
else:
init_dlvsqs_t = np.clip( init_dlvsqs_t, min=min_dlvsq+1E-8 ) - min_dlvsq
init_dlvsqs_t = np.log( init_dlvsqs_t )
init_dlvsraw.append( init_dlvsqs_t )
del Nj
# compute transition 1
# --------------------
dt = expiry - ( expiries[t-1] if t>0 else 0. )
box = xstrikes_t[1:-1]**2 * dt / ( xstrikes_t[2:] - xstrikes_t[:-2])
w_plus = box / ( xstrikes_t[2:] - xstrikes_t[1:-1] )
w_minus = box / ( xstrikes_t[1:-1] - xstrikes_t[:-2] )
assert np.min( w_plus ) >= 0., f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: computed 'w_plus' must be non-negative, but found min {np.min(w_plus)}"
assert np.min( w_minus ) >= 0., f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: computed 'w_minus' must be non-negative, but found min {np.min(w_minus)}"
jself.iw_minus.append( jax.device_put( w_minus, device=jdevice ) )
jself.iw_plus.append( jax.device_put( w_plus, device=jdevice ) )
# interpolation matrx L
# https://arxiv.org/pdf/2601.11209 (22)
# we compute this in higher precision
if t>0:
xstrikes_tm1 = xstrikes[t-1]
L = np.maximum( 0., xstrikes_tm1[None,:] - xstrikes_t[:,None] )
L = np.diff( np.diff( L, axis=0, prepend=1. ) / np.diff( xstrikes_t, prepend=0. )[:,None], axis=0, append=0. )
assert L.shape == (len(xstrikes_t), len(xstrikes_tm1)), f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: expected shape {(len(xstrikes_t), len(xstrikes_tm1))} but found {L.shape} for interpolation matrix L"
jself.L.append( jax.device_put( L, device=jdevice ) )
else:
jself.L.append( None )
# objective
# ---------
jself.CX.append( jax.device_put( np_call( spots=xstrikes_t[None,:], strikes=strikes_t[:,None] ), device=jdevice ) )
jself.weights.append( jax.device_put( weights_t, device=jdevice ) ) if not weights_t is None else None
jself.mids.append( jax.device_put( mids_t, device=jdevice ) )
# DLVs
# ----
dlvsraw = jnp.full((jself.num_dlvs,), 0., dtype=jdtype, device=jdevice) if init_dlvs is None else\
jax.device_put( np.concatenate( init_dlvsraw ), device=jdevice )
_ = jself(dlvsraw)
# compile
# -------
with verbose.write_t(lambda : f"{message()}compiling optimizer for {jself.num_dlvs} {txt_smth} DLVs on {jdevice}... ", end='') as tme:
jself.value_grad = jax.jit( jax.value_and_grad( lambda x : jself.__call__(x, training=True), has_aux=False ) ).lower(dlvsraw).compile() if not debug_do_not_optimize else None
jself.value_and_data = jax.jit( lambda x : jself.__call__(x, training=False) ).lower(dlvsraw).compile()
verbose.write(f"done; this took {tme}.", head=False)
# compile
# -------
if debug_do_not_optimize:
result_raw = dlvsraw
else:
with verbose.write_t(lambda : f"{message()}optimizing {jself.num_dlvs} {txt_smth} DLVs on {jdevice}... ", end='') as tme:
res = scipy.optimize.minimize(
jself.value_grad,
dlvsraw,
jac=True,
method="L-BFGS-B",
options={
"maxiter": 2000,
"maxfun": 10000,
"maxls": 100,
"maxcor": 20,
# Make stopping less dominated by tiny relative f improvement.
"ftol": 1e-14,
# This is the projected-gradient tolerance.
"gtol": 1e-8,
},
)
if not res.success :
raise RuntimeError( f"{message()}Optimizer failed with message: {res.message}" )
verbose.write(f"done; optimizer returned '{res.message}'. This took {tme}.", head=False)
if debug_print_info:
print("success:", res.success)
print("message:", res.message)
print("nit:", res.nit)
print("nfev:", res.nfev)
print("fun:", res.fun)
print("|grad|_inf:", np.max(np.abs(res.jac)))
print("|grad|_2:", np.linalg.norm(res.jac))
result_raw = res.x
# evaluate
# --------
if debug_print_info:
rawdlvsqs = ( jax.nn.sigmoid(result_raw) * (max_dlvsq - min_dlvsq) + min_dlvsq ) if not max_dlvsq is None else\
( jnp.exp(result_raw) + min_dlvsq )
rawdlvs = [ np.sqrt(rawdlvsqs[slice_]) for slice_ in jself.dlv_slices ]
for t, expiry in enumerate(expiries):
print(f"Expiry #{t} {expiry*DAYS_PER_YEAR:.1f}d: raw DLVs: {rawdlvs[t]}")
err, xdensities, calls = jself.value_and_data( dlvsraw )
err = jax.device_get( err )
xdensities = [ np.clip( jax.device_get( xd ), min=0. ) for xd in xdensities ]
calls = [ np.clip( jax.device_get( c ), min=0. ) for c in calls ]
if debug_print_info:
print(f"Final root mean squared error: {err:.4g} in price space")
return SanosPricer( smoothness=self.smoothness,
expiries=expiries,
variances=variances,
xstrikes=xstrikes,
xdensities=xdensities,
min_dlv=self.min_dlv,
max_dlv=self.max_dlv,)