"""
Options Filter
==============
Provides daily option data subject to some filter and optional normalization.
"""
from __future__ import annotations
from quantdata.widb.day import enrich_day, SCHEMA_ENRICHED_DAY, RawDayFilter, ExpiryFilter, SubDir, _processing
from quantdata.widb.arbfree import MkArbFree, mk_arb_free_day, SCHEMA_ARB_FREE_DAY_EXPIRY, SCHEMA_ARB_FREE_DAY_SURFACE
from cdxcore.version import version
from cdxcore.verbose import Context
from cdxcore.subdir import CacheMode
from cdxcore.util import fmt_date, fmt_list, fmt_big_byte_number, fmt_dict
from cdxcore.uniquehash import named_unique_filename48_8
from quantdata.utils.pd import create_empty_df, verify_schema
from cdxcore.err import verify_inp, verify
import datetime as dt
import numpy as np
import pandas as pd
import math as math
from dataclasses import dataclass, field
import warnings as warnings
# ======================================================================
# Sample days for a given ticker
# ======================================================================
[docs]
@version("0.62")
@dataclass(frozen=True)
class OptionsFilter:
"""
Create a time series with a maximum number of valid options per time step.
"""
normalize : bool = True # Normalize output data:
# * Drop ``spot``
# * Add ``spot_ret``: the return of the spot price compared to the previous day, i.e. ``spot / spot_prev - 1``.
# * Divide the following fields by spot: ``spot_prev``, ``call_ask``, ``call_bid``, ``fwd``, ``strike``, ``mkt_call_bid``, ``mkt_put_bid``, ``mkt_call_ask``, ``mkt_put_ask``.
max_num_options: int = 1000 #: Maximum number of options. Options are sorted by NMS5, expiry and call/put, and only the top max_num_options are kept. This is applied after all other filters.#
max_business_days_to_expiry: int = 252*3 #: Maximum business days to expiry.
max_nms: float|tuple[float,float] = (-5.,2.) #: Maximum |NMS5| or a tuple with NMS5_lower<0<NMS5_upper.
nms_use_atm : bool = True #: Whether to use NMS5 with the ATM volatility instead of the actual NMS5.
min_vegaSqrtVar: float = 0.001 #: Minimum vega * sqrt(tte), i.e. the derivative in sqrtVar: BSsq(K,sqrtVar) = BSvol(K,sqrtVar/sqrt(t),t). This is a measure of how much the option price moves with volatility, normalized by time to expiry.
min_log_volume: float = 1.0 #: Minimum log volume.
min_log_open_interest: float = 1.0 #: Minimum log open interest.
remove_am: bool = False #: Whether to remove options that settled at the AM price.
min_opt_per_expiry: int = 10 #: Minimum number of options per expiry. This is applied *before* the max_num_options filter, and can lead to fewer than max_num_options being returned if there are many expiries with few options.
def __post_init__(self):
verify_inp(self.max_num_options >= 10, lambda : f"'max_num_options' must be at least 10, found {self.max_num_options}")
verify_inp(self.max_business_days_to_expiry > 0, lambda : f"'max_business_days_to_expiry' must be positive, found {self.max_business_days_to_expiry}")
if isinstance(self.max_nms, tuple):
verify_inp(len(self.max_nms) == 2, lambda : f"'max_nms' tuple must have length 2, found {len(self.max_nms)}")
verify_inp(self.max_nms[0] < 0 < self.max_nms[1], lambda : f"'max_nms' tuple must have negative first element and positive second element, found {self.max_nms}")
else:
verify_inp(self.max_nms > 0, lambda : f"'max_nms' must be positive, found {self.max_nms}")
verify_inp(self.min_vegaSqrtVar >= 0, lambda : f"'min_vegaSqrtVar' must be non-negative, found {self.min_vegaSqrtVar}")
verify_inp(self.min_log_volume >= 0, lambda : f"'min_log_volume' must be non-negative, found {self.min_log_volume}")
verify_inp(self.min_log_open_interest >= 0, lambda : f"'min_log_open_interest' must be non-negative, found {self.min_log_open_interest}")
verify_inp(self.min_opt_per_expiry > 0, lambda : f"'min_opt_per_expiry' must be positive, found {self.min_opt_per_expiry}")
def __str__(self):#by convention use a some what readable string with comma separated key=value pairs
return f"nopt={self.max_num_options},bdte={self.max_business_days_to_expiry},nms{'at' if self.nms_use_atm else ''}={self.max_nms},lvolu={self.min_log_volume},loi={self.min_log_open_interest},rmam={self.remove_am},vegaSV={self.min_vegaSqrtVar},mxpp={self.min_opt_per_expiry}"
def _filter(self, ticker : str, d : dt.date, df : pd.DataFrame) -> pd.DataFrame:
"""
Filter data frame
"""
assert set(df.columns) >= set(SCHEMA_ENRICHED_DAY), f"Internal error {ticker} {d}: DataFrame has missing columns. Missing: {fmt_list(set(SCHEMA_ENRICHED_DAY) - set(df.columns))}. Please contact the authors with an example."
assert not df.isna().any().any(), f"Internal error {ticker} {d}: filter resulted in NaN values. Please contact the authors with an example."
n_init = len(df)
assert not df.empty, "Internal error - this should have been filtered earlier"
df = df[df['bdte'] <= self.max_business_days_to_expiry]
if self.normalize:
df = normalize_day(ticker, d, df)
min_nms = -self.max_nms if isinstance(self.max_nms, (int, float)) else self.max_nms[0]
max_nms = self.max_nms if isinstance(self.max_nms, (int, float)) else self.max_nms[1]
if self.nms_use_atm:
df = df[(df['nms_atm'] >= min_nms) & (df['nms_atm'] <= max_nms)]
else:
df = df[(df['nms'] >= min_nms) & (df['nms'] <= max_nms)]
df = df[df['log_volume'] >= self.min_log_volume] if self.min_log_volume > 0. else df
df = df[df['log_open_interest'] >= self.min_log_open_interest] if self.min_log_open_interest > 0. else df
if self.remove_am:
df = df[~df['is_am']]
# filter expiries
if not df.empty:
expiry_counts = df['expiry_id'].value_counts()
valid_expiries = expiry_counts[expiry_counts >= self.min_opt_per_expiry].index
df = df[df['expiry_id'].isin(valid_expiries)]
if df.empty:
expiry_counts = expiry_counts.to_dict()
expiry_counts = { d: expiry_counts[d] for d in sorted(expiry_counts) }
warnings.warn(f"{ticker} {d}: after filtering no expiry had the minimum of {self.min_opt_per_expiry} options per expiry left. This means this sample is not eligible. Here are the options per filtered expiry: {fmt_dict(expiry_counts)}.")
return df
# remove excess options while keeping min_opt_per_expiry per expiry, and the closes to ATM
if not df.empty and len(df) > self.max_num_options:
nms = df['nms'].to_numpy()
abs_nms = np.abs(nms)
# Reuse per-expiry positional indices instead of repeatedly filtering the full frame.
reserved_positions = []
remaining_positions = []
for expiry_id, expiry_positions in df.groupby('expiry_id', sort=False).indices.items():
expiry_positions = np.asarray(expiry_positions, dtype=np.intp)
assert len(expiry_positions) >= self.min_opt_per_expiry, f"Internal error: expiry_id {expiry_id} has fewer options than 'min_opt_per_expiry' after filtering. This should not happen, as we already filtered out expiries with too few options. Please contact the authors with an example."
if len(expiry_positions) == self.min_opt_per_expiry:
reserved_positions.append(expiry_positions)
continue
expiry_nms = nms[expiry_positions]
selected_mask = np.zeros((len(expiry_positions),), dtype=bool)
selected_positions = []
# Add smallest positive NMS.
pos_local = np.flatnonzero(expiry_nms > 0.)
if len(pos_local) > 0:
best_pos_local = pos_local[np.argmin(expiry_nms[pos_local])]
selected_mask[best_pos_local] = True
selected_positions.append(expiry_positions[best_pos_local:best_pos_local+1])
# Add highest negative NMS.
neg_local = np.flatnonzero(expiry_nms < 0.)
if len(neg_local) > 0:
best_neg_local = neg_local[np.argmax(expiry_nms[neg_local])]
if not selected_mask[best_neg_local]:
selected_mask[best_neg_local] = True
selected_positions.append(expiry_positions[best_neg_local:best_neg_local+1])
# Add (min_opt_per_expiry - 2) remaining options with smallest abs(NMS).
remaining_count = self.min_opt_per_expiry - np.sum(selected_mask)
if remaining_count > 0:
remaining_local = np.flatnonzero(~selected_mask)
if len(remaining_local) > 0:
remaining_order = remaining_local[np.argsort(abs_nms[expiry_positions[remaining_local]], kind='stable')]
extra_local = remaining_order[:remaining_count]
selected_mask[extra_local] = True
selected_positions.append(expiry_positions[extra_local])
reserved_positions.append(np.concatenate(selected_positions) if selected_positions else np.empty((0,), dtype=np.intp))
expiry_remaining = expiry_positions[~selected_mask]
if len(expiry_remaining) > 0:
remaining_positions.append(expiry_remaining)
reserved_positions = np.concatenate(reserved_positions) if reserved_positions else np.empty((0,), dtype=np.intp)
remaining_positions = np.concatenate(remaining_positions) if remaining_positions else np.empty((0,), dtype=np.intp)
# Sort remaining by abs_nms and limit.
remaining_slots = max(0, self.max_num_options - len(reserved_positions))
if remaining_slots > 0 and len(remaining_positions) > 0:
remaining_positions = remaining_positions[np.argsort(abs_nms[remaining_positions], kind='stable')[:remaining_slots]]
else:
remaining_positions = np.empty((0,), dtype=np.intp)
# Combine reserved and remaining.
df = df.iloc[np.concatenate([reserved_positions, remaining_positions])]
assert len(df) <= self.max_num_options, f"Internal error {ticker} {d}: after filtering by max_num_options, the number of options exceeds max_num_options. Please contact the authors with an example."
if df.empty:
raise RuntimeError(f"Internal error {ticker} {d}: filter resulted in an empty data set. Please contact the authors with an example.")
assert not df.isna().any().any(), f"Internal error {ticker} {d}: filter resulted in NaN values. Please contact the authors with an example."
return df
[docs]
@version("0.3", dependencies=[enrich_day] )
def normalize_day(ticker : str, d : dt.date, df : pd.DataFrame) -> pd.DataFrame:
"""
Create a normalized version of a day.
* Drop ``spot``
* Add ``spot_ret``: the return of the spot price compared to the previous day, i.e. ``spot / spot_prev - 1``.
* Divide the following fields by spot: ``spot_prev``, ``call_ask``, ``call_bid``, ``fwd``, ``strike``, ``mkt_call_bid``, ``mkt_put_bid``, ``mkt_call_ask``, ``mkt_put_ask``.
"""
df = df.copy()
spot = df['spot'].to_numpy()
spot_prev = df['spot_prev'].to_numpy()
if np.min(spot) <= 0.:
raise ValueError(f"{ticker} {d}: Expected positive spot, got {np.min(spot):.4g} for ticker {ticker}.")
if np.min(spot_prev) <= 0.:
raise ValueError(f"{ticker} {d}: Expected positive spot_prev, got {np.min(spot_prev):.4g} for ticker {ticker}.")
df.drop("spot", axis=1, inplace=True)
df['spot_ret'] = spot / spot_prev - 1.
df['spot_ref'] = spot
df['spot_prev'] /= spot
df['call_ask'] /= spot
df['call_bid'] /= spot
df['fwd'] /= spot
df['strike'] /= spot
df['mkt_call_bid'] /= spot
df['mkt_put_bid'] /= spot
df['mkt_call_ask'] /= spot
df['mkt_put_ask'] /= spot
return df
[docs]
def SCHEMA_OPTION_DAY( mk_arb_free: MkArbFree|None, normalized : bool ) -> list[str]:
"""
Returns the schema for an option day series object depending on the :class:`MkArbFree` configuration and whether it is normalized or not.
"""
if mk_arb_free is None:
source_schema = SCHEMA_ENRICHED_DAY
elif mk_arb_free.no_arb_mode == "expiry":
source_schema = SCHEMA_ARB_FREE_DAY_EXPIRY
elif mk_arb_free.no_arb_mode == "surface":
source_schema = SCHEMA_ARB_FREE_DAY_SURFACE
else:
raise ValueError(f"Unknown no_arb_mode: {mk_arb_free.no_arb_mode}")
if normalized:
source_schema = dict(source_schema)
del source_schema['spot']
source_schema['spot_ref'] = np.float32
source_schema['spot_ret'] = np.float32
return source_schema
[docs]
@version( "0.2", dependencies=[enrich_day, mk_arb_free_day, OptionsFilter, normalize_day] )
def filter_options_day(
ticker: str,
d: dt.date,
source_data : pd.DataFrame | None = None,
*,
raw_day_filter: RawDayFilter = RawDayFilter(),
expiry_filter: ExpiryFilter = ExpiryFilter(),
mk_arb_free: MkArbFree = MkArbFree(),
options_filter : OptionsFilter = OptionsFilter(),
# operational
raise_if_bad_day: bool = True,
# pass on caching parameters
override_cache_mode: CacheMode | None = None,
cache_generate_only : bool = False,
# verbose
verbose: Context = Context.all,
) -> pd.DataFrame | tuple[pd.DataFrame, dict]:
"""
Filter options according to ``options_filter``.
This includes filtering by moneyness, time to expiry, volume, open interest, and other criteria defined in :class:`OptionsFilter`.
This function itself is not cached, but it since it is basically a thin layer on top of :func:`mk_arb_free_day` and the filtering logic is fast,
caching is implicit.
"""
verify_inp(isinstance(options_filter, OptionsFilter), lambda : f"'options_filter' must be an instance of OptionsFilter, found {type(options_filter)}")
verify_inp(mk_arb_free is None or isinstance(mk_arb_free, MkArbFree), lambda : f"'mk_arb_free' must be None or an instance of MkArbFree, found {type(mk_arb_free)}")
verify_inp(isinstance(raw_day_filter, RawDayFilter), lambda : f"'raw_day_filter' must be an instance of RawDayFilter, found {type(raw_day_filter)}")
verify_inp(isinstance(expiry_filter, ExpiryFilter), lambda : f"'expiry_filter' must be an instance of ExpiryFilter, found {type(expiry_filter)}")
schema = SCHEMA_OPTION_DAY(mk_arb_free, options_filter.normalize)
if not source_data is None:
verify_inp( isinstance(source_data, pd.DataFrame), f"{ticker} {d}: expected 'source_data' to be a pandas DataFrame or None, got {type(source_data)}.")
data = source_data
else:
data = mk_arb_free_day(
ticker=ticker, d=d,
raw_day_filter=raw_day_filter,
expiry_filter=expiry_filter,
mk_arb_free=mk_arb_free,
raise_if_bad_day=raise_if_bad_day,
verbose=verbose,
# pass on caching parameters
override_cache_mode=override_cache_mode,
cache_generate_only=cache_generate_only )
if data is None:
assert raise_if_bad_day, "Internal error: empty data should only be returned when 'raise_if_bad_day' is False. Please check the enriched day data for details."
return None
if data.empty:
return create_empty_df(schema)
data = options_filter._filter(ticker, d, data)
verify_schema(data, schema, lambda : f"{ticker} {d}")
return data
@_processing(fmt=SubDir.PICKLE).cache(
"0.1",
dependencies=[filter_options_day],
uid=lambda ticker, d, raw_day_filter, expiry_filter, mk_arb_free, options_filter :\
f"options/{named_unique_filename48_8(f'{str(expiry_filter)},{str(raw_day_filter)},{str(mk_arb_free)},{str(options_filter)}', expiry_filter, raw_day_filter, mk_arb_free, options_filter)}"+\
f"/{ticker}/{d}"
)
def _filter_options_day_num_options( *,
ticker: str,
d: dt.date,
raw_day_filter: RawDayFilter = RawDayFilter(),
expiry_filter: ExpiryFilter = ExpiryFilter(),
mk_arb_free: MkArbFree = MkArbFree(),
options_filter : OptionsFilter = OptionsFilter(),
# operational
raise_if_bad_day: bool = True,
# pass on caching parameters
override_cache_mode: CacheMode | None = None,
cache_generate_only : bool = False,
# verbose
verbose: Context|None = Context.quiet
) -> int:
"""
Dummy wrapper around :func:`filter_options_day` to cache the number of options after filtering.
"""
r = filter_options_day(
ticker=ticker, d=d,
raw_day_filter=raw_day_filter, expiry_filter=expiry_filter, mk_arb_free=mk_arb_free, options_filter=options_filter,
raise_if_bad_day=raise_if_bad_day,
override_cache_mode=override_cache_mode, cache_generate_only=cache_generate_only,
verbose=verbose if not verbose is None else Context.quiet
)
return len(r) if not r is None else None