"""
Implied Day
^^^^^^^^^^^
This file generates sample data sets of selected options quote time series for a given day, indexed by expiry.
"""
from datetime import date
from cdxcore.jcpool import JCPool
from cdxcore.pretty import PrettyObject
from cdxcore.subdir import CacheMode, SubDir
from cdxcore.util import fmt_list
from cdxcore.uniquehash import named_unique_filename48_8
from cdxcore.bs import bs
import polars as pl
from quantdata.biz_calendar import NMS5, DateRange, prev_business_day, session_open_close, DAYS_PER_YEAR, count_business_days_between, to_date
from quantdata.massive.client import MassiveClient
from quantdata.pool import PoolConfig
from quantdata.massive.options.contracts import ContractsFilter, filter_contracts, REJECTED
from quantdata.massive.processing import align_trades_to_quotes, align_quotes_to_trades, filter_valid_quotes, filter_valid_trades, TradeQuoteAlignmentConfig
from quantdata.utils.pl import verify_schema, create_empty_df
from cdxcore.version import version
from cdxcore.verbose import Context
from dataclasses import asdict, dataclass, field
from cdxcore.uniquehash import unique_hash8, named_unique_filename48_8
import numpy as np
import math
import warnings
from quantdata.ncde.implied import _locations_ncde_implied
from quantdata.widb.bad import BAD
_cache = _locations_ncde_implied.processing(fmt=SubDir.POLARS_PARQUET)
OPT_TICK_SIZE = 0.01 # not quite true - this is the tick size for small priced options
[docs]
@version("0.21", dependencies=[ContractsFilter])
@dataclass(frozen=True)
class OptionQuotesDayConfig:
"""
Configuration for managing option quotes day data.
"""
max_num_options_per_expiry : int|None = 1000
contract_filter : ContractsFilter = ContractsFilter()
alignment_config : TradeQuoteAlignmentConfig = TradeQuoteAlignmentConfig()
overnight_fraction : float = 0.2
dte_exclude_half_days : bool = False
min_vol = 0.01
max_vol = 5.
def __str__(self):
return f"mxn={self.max_num_options_per_expiry}{',exchl' if self.dte_exclude_half_days else ''},{self.contract_filter},{self.alignment_config}"
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",dependencies=[align_quotes_to_trades],
uid=lambda ticker, d, alignment_config: f"trades/{named_unique_filename48_8(str(alignment_config),alignment_config)}/{ticker}/{d}/{ticker}"
)
def _ticker_trade_series( ticker : str, d : date, alignment_config : TradeQuoteAlignmentConfig = TradeQuoteAlignmentConfig(), verbose : Context|None = Context.quiet) -> pl.DataFrame:
""" Cached ticker series """
verbose = verbose if verbose is not None else Context.quiet
with verbose.write_t(lambda : f"{ticker} {d}: Fetching time series for {ticker} {d}... ", end='') as tme:
df = align_quotes_to_trades( ticker, d, config = alignment_config, within_session=True, drop_unmatched=False )
verbose.write(lambda : f"done; found {len(df)} quotes; this took {tme}.", head=False)
return df.sort("ts_ns")
OPT_QUOTE_BY_EXPIRY_SCHEMA = {
# option quotes
# option quotes may not have trades
"t_d": pl.Float64, # time since open in days
"mid_price": pl.Float64, # mid price (divded by prev_spot_close)
"half_spread": pl.Float64, # half spread (divided by prev_spot_close)
"opt_tick_size": pl.Float64, # option tick size (divided by prev_spot_close)
"size": pl.Float64, # total size
"bid_size_pcnt": pl.Float64, # bid size as percentage of total size
"mid_ret": pl.Float64, # pcnt return in mid
"dt_d": pl.Float64, # time diff in days since last quote
"dte": pl.Float64, # dte in days computed for each option quote, adjusted for time of day: ref_dte - (t_ms / (6.5 * 3600 * 1000)) * (1 - overnight_fraction)
"nms5": pl.Float64, # normalized strike
"intrinsic": pl.Float64, # intrinsic value (divided by prev_spot_close)
"is_intrinsic": bool, # whether the option is near the money, defined as having intrinsic value at least 10% of the mid price
"bid_vol": pl.Float64, # bid vol
"ask_vol": pl.Float64, # ask vol
"mid_vol": pl.Float64, # mid vol
"bid_delta": pl.Float64, # bid delta
"ask_delta": pl.Float64, # ask delta
"mid_delta": pl.Float64, # mid delta
"bid_gamma": pl.Float64, # bid gamma
"ask_gamma": pl.Float64, # ask gamma
"mid_gamma": pl.Float64, # mid gamma
"bid_vega": pl.Float64, # bid vega
"ask_vega": pl.Float64, # ask vega
"mid_vega": pl.Float64, # mid vega
"bid_theta": pl.Float64, # bid theta
"ask_theta": pl.Float64, # ask theta
"mid_theta": pl.Float64, # mid theta
# option trades
"trade_t_d": pl.Float64, # time diff in days since last trade
"trade_price": pl.Float64, # trade price (divided by prev_spot_close)
"trade_size": pl.Float64, # trade size
"trade_direction": pl.Float64, # trade direction: (trade_price - mid_price) / half_spread
"trade_ret": pl.Float64, # trade return: (trade_price - prev_trade_price) / prev_trade_price
"trade_dt_d": pl.Float64, # time diff in days since last trade
"trade_dte": pl.Float64, # dte in days computed for each option trade, adjusted for time of day: ref_dte - (trade_t_ms / (6.5 * 3600 * 1000)) * (1 - overnight_fraction)
"trade_vol": pl.Float64, # trade price (divided by prev_spot_close)
"trade_delta": pl.Float64, # trade delta
"trade_gamma": pl.Float64, # trade gamma
"trade_vega": pl.Float64, # trade vega
"trade_theta": pl.Float64, # trade theta
# spot quotes
# all spot quotes have trades
"spot_quote_t_d": pl.Float64, # time diff in days since last spot quote
"spot_mid_price": pl.Float64, # spot mid price (divided by prev_spot_close)
"spot_half_spread": pl.Float64, # spot half spread (divided by prev_spot_close)
"spot_size": pl.Float64, # spot total size
"spot_bid_size_pcnt": pl.Float64, # spot bid size as percentage of
"spot_mid_ret": pl.Float64, # spot mid price return: (spot_mid_price - prev_spot_mid_price) / prev_spot_mid_price
"spot_quote_dt_d": pl.Float64, # time diff in days since last spot quote
# spot trades
"spot_trade_t_d": pl.Float64, # time diff in days since last spot trade
"spot_trade_price": pl.Float64, # spot trade price (divided by prev_spot_close)
"spot_trade_size": pl.Float64, # spot trade size
"spot_trade_direction": pl.Float64, # spot trade direction: (spot_trade_price - spot_mid_price) / spot_half_spread
"spot_trade_ret": pl.Float64, # spot trade return: (spot_trade_price - prev_spot_trade_price) / prev_spot_trade_price
"spot_trade_dt_d": pl.Float64, # time diff in days since last spot trade
# ref data
"ticker": str,
"opt_ticker": str,
"paired_opt_ticker": str, # call for put and put for call ...
"date": pl.Date,
"exp_date": pl.Date,
"is_call": bool,
"is_american": bool,
"strike": pl.Float64, # divided by prev_spot_close
"prev_spot_close": pl.Float64,
}
""" Schema returned by _opt_expiry_quote_time_series """
TRADE_COLUMNS = {'trade_t_d', 'trade_price', 'trade_size', 'trade_direction', 'trade_ret', 'trade_dt_d', 'trade_dte'}
""" Columns which are null if there was no trade """
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.4",
dependencies=[align_trades_to_quotes,_ticker_trade_series, OptionQuotesDayConfig],
uid=lambda ticker, opt_ticker, d, overnight_fraction, dte_exclude_half_days, alignment_config:\
"opt_quotes/"+\
f"{named_unique_filename48_8(f'onf={overnight_fraction}{',xclhlf' if dte_exclude_half_days else ''},{alignment_config}',dte_exclude_half_days,overnight_fraction,alignment_config)}/"+\
f"{ticker}/{d}/{named_unique_filename48_8(opt_ticker)}"
)
def _opt_quote_time_series( ticker : str, opt_ticker : str, d : date,
overnight_fraction : float = 0.2,
dte_exclude_half_days : bool = True,
alignment_config : TradeQuoteAlignmentConfig = TradeQuoteAlignmentConfig(), *, verbose : Context|None = Context.quiet) -> pl.DataFrame:
"""
Option quote time series aligned with spot trades.
"""
verbose = verbose if verbose is not None else Context.quiet
spot_trades = _ticker_trade_series( ticker, d, alignment_config=alignment_config, verbose=verbose )
if len(spot_trades) == 0:
warnings.warn(f"{ticker} {d} option {opt_ticker}: No valid spot trades found for {ticker} on {d} -- skipping this option.")
return create_empty_df( OPT_QUOTE_BY_EXPIRY_SCHEMA )
assert spot_trades["price"].is_not_null().all(), f"{ticker} {d} option {opt_ticker}: Found null values 'price' after joining quotes with spot trades...?"
"""
spot_trades schema:
SCHEMA_ALIGN_QUOTES_TO_TRADES = {
"ts_ns": np.dtype("datetime64[ns]"),
"price": np.float64,
"size": np.float64,
"trade_code": np.uint32,
"quote_ts_ns": np.dtype("datetime64[ns]"),
"bid_price": np.float64,
"bid_size": np.float64,
"ask_price": np.float64,
"ask_size": np.float64,
}
"""
with verbose.write_t(lambda : f"{ticker} {d}: Fetching time series for option {opt_ticker} {d}... ", end='') as tme:
opt_quotes = align_trades_to_quotes( opt_ticker, d, config=alignment_config, within_session=True, drop_unmatched=False ).sort("ts_ns")
if len(opt_quotes) == 0:
client = MassiveClient()
num_quotes = len( client.fetch_quotes(opt_ticker, d) )
num_trades = len( client.fetch_trades(opt_ticker, d) )
num_valid_quotes = len( filter_valid_quotes( opt_ticker, d))
num_valid_trades = len( filter_valid_trades( opt_ticker, d))
warnings.warn(f"{ticker} {d} option {opt_ticker}: No valid quotes were aligned with valid trades. Number of quotes: {num_quotes}, valid quotes: {num_valid_quotes}, number of trades: {num_trades}, valid trades: {num_valid_trades} -- skipping this option.")
return create_empty_df( OPT_QUOTE_BY_EXPIRY_SCHEMA )
min_spot_ts = spot_trades["ts_ns"].min()
opt_quotes = opt_quotes.filter(pl.col("ts_ns") >= min_spot_ts)
if len(opt_quotes) <= 1:
warnings.warn(f"{ticker} {d} option {opt_ticker}: {'no quotes' if len(opt_quotes) == 0 else 'just one quote'} past the first spot trade (first spot trade ts: {min_spot_ts}) -- skipping this option.")
return create_empty_df( OPT_QUOTE_BY_EXPIRY_SCHEMA )
verbose.write(lambda : f"done; found {len(opt_quotes)} quotes; this took {tme}.", head=False)
"""
opt_quotes schema:
SCHEMA_ALIGN_TRADES_TO_QUOTES = {
"ts_ns": np.dtype("datetime64[ns]"),
"bid_price": np.float64,
"bid_size": np.float64,
"ask_price": np.float64,
"ask_size": np.float64,
"trade_ts_ns": np.dtype("datetime64[ns]"),
"price": np.float64,
"size": np.float64,
"trade_code": np.uint32,
}
"""
spot_cols = set(spot_trades.columns)
spot_trades = spot_trades.rename({k: "spot_" + k for k in spot_cols})
assert len(spot_trades) > 0 and spot_trades["spot_price"].is_not_null().all(), f"{ticker} {d} option {opt_ticker}: Found null values in 'spot_price'...?"
assert not 'spot_price' in opt_quotes.columns, f"{ticker} {d} option {opt_ticker}: 'spot_price' should not be in opt_quotes columns...?"
result = opt_quotes.join_asof(
spot_trades,
left_on="ts_ns",
right_on="spot_ts_ns",
strategy="backward" )
result = result.with_columns( *[ pl.col("spot_" + c).forward_fill().backward_fill() for c in spot_cols ] )
result = result.filter(pl.col("bid_price").is_not_null())
assert len(result) > 1, f"{ticker} {d} option {opt_ticker}: Nor just one aligned quotes found for option {opt_ticker} on {d} after joining {len(opt_quotes)} quotes with {len(spot_trades)} spot trades...?"
assert result["spot_price"].is_not_null().all(), f"{ticker} {d} option {opt_ticker}: Found null values in 'spot_price' after joining quotes with spot trades...?"
# convert
# -------
client = MassiveClient()
ref_data = client.option_contract_overview( opt_ticker, d )
prev_close = client.consistent_prev_close( ticker, d, exclude_half_days=False )
strike = float( ref_data["strike_price"] )
expiry = to_date( ref_data["expiration_date"] )
is_american = ref_data["exercise_style"].lower().startswith("a")
is_call = ref_data["contract_type"].lower().startswith("c")
open, close,\
is_half_day = session_open_close(d)
diff_oc_us = (close - open).total_seconds() * 1_000_000
dte = float( count_business_days_between( d, expiry, exclude_half_days=dte_exclude_half_days ) )
result = ( result.
rename(dict(price="trade_price", size="trade_size", spot_price="spot_trade_price", spot_size="spot_trade_size", spot_ts_ns="spot_trade_ts_ns")).
with_columns(
( pl.col("ask_price") / prev_close ).alias("ask_price"),
( pl.col("bid_price") / prev_close ).alias("bid_price"),
( pl.col("trade_price") / prev_close ).alias("trade_price"),
( pl.col("spot_ask_price") / prev_close ).alias("spot_ask_price"),
( pl.col("spot_bid_price") / prev_close ).alias("spot_bid_price"),
( pl.col("spot_trade_price") / prev_close ).alias("spot_trade_price"),
).
with_columns(
( pl.col("ts_ns").sub(open).dt.total_microseconds() / diff_oc_us ).alias("t_d"),
( pl.col("trade_ts_ns").sub(open).dt.total_microseconds() / diff_oc_us ).alias("trade_t_d"),
( pl.col("spot_trade_ts_ns").sub(open).dt.total_microseconds() / diff_oc_us ).alias("spot_trade_t_d"),
( pl.col("spot_quote_ts_ns").sub(open).dt.total_microseconds() / diff_oc_us ).alias("spot_quote_t_d"),
( (pl.col("ask_price") + pl.col("bid_price")) / 2.0 ).alias("mid_price"),
( (pl.col("ask_price") - pl.col("bid_price")) / 2.0 ).alias("half_spread"),
( (pl.col("spot_ask_price") + pl.col("spot_bid_price")) / 2.0 ).alias("spot_mid_price"),
( (pl.col("spot_ask_price") - pl.col("spot_bid_price")) / 2.0 ).alias("spot_half_spread"),
( pl.col("bid_size") + pl.col("ask_size") ).alias("size"),
( pl.col("bid_size") / (pl.col("bid_size") + pl.col("ask_size")).clip(lower_bound=1) ).alias("bid_size_pcnt"),
( pl.col("spot_bid_size") + pl.col("spot_ask_size") ).alias("spot_size"),
( pl.col("spot_bid_size") / (pl.col("spot_bid_size") + pl.col("spot_ask_size")).clip(lower_bound=1) ).alias("spot_bid_size_pcnt"),
).
with_columns(
( (pl.col("trade_price") - pl.col("mid_price")) / pl.col("half_spread") ).alias("trade_direction"),
( (pl.col("spot_trade_price") - pl.col("spot_mid_price")) / pl.col("spot_half_spread") ).alias("spot_trade_direction"),
).
with_columns(
pl.col("mid_price").pct_change().alias("mid_ret"),
pl.col("trade_price").pct_change().alias("trade_ret"),
pl.col("spot_trade_price").pct_change().alias("spot_trade_ret"),
pl.col("spot_mid_price").pct_change().alias("spot_mid_ret"),
( dte + ( 1. - pl.col("t_d") ) * (1. - overnight_fraction)).alias("dte"),
pl.col("t_d").diff().alias("dt_d"),
pl.col("trade_t_d").diff().alias("trade_dt_d"),
( dte + ( 1. - pl.col("trade_t_d") ) * (1. - overnight_fraction)).alias("trade_dte"),
pl.col("spot_trade_t_d").diff().alias("spot_trade_dt_d"),
pl.col("spot_quote_t_d").diff().alias("spot_quote_dt_d"),
).
with_columns(
( ( strike/prev_close / pl.col("spot_mid_price") ).log() * (( 1.+NMS5.EPS )/( pl.col("dte")/DAYS_PER_YEAR + NMS5.EPS )).sqrt() / NMS5.SCALE ).alias("nms5"),
pl.lit(expiry).alias("exp_date"),
pl.lit(d).alias("date"),
pl.lit(strike/prev_close).alias("strike"),
pl.lit(ticker).alias("ticker"),
pl.lit(opt_ticker).alias("opt_ticker"),
pl.lit(is_american).alias("is_american"),
pl.lit(is_call).alias("is_call"),
pl.lit(prev_close).alias("prev_spot_close"),
pl.lit(OPT_TICK_SIZE/prev_close).alias("opt_tick_size"),
).
drop("bid_price", "ask_price", "spot_ask_price", "spot_bid_price",
"ts_ns", "trade_ts_ns", "spot_trade_ts_ns", "spot_quote_ts_ns",
"bid_size", "ask_size", "spot_bid_size", "spot_ask_size",
"trade_code", "spot_trade_code")
)
df_free = result.filter( pl.col("mid_ret").is_finite() )
assert len(df_free) == len(result)-1, f"{ticker} {d} option {opt_ticker}: Expected only one non-finite mid_ret, found {len(result)-len(df_free)} non-finite mid_ret values...?"
result = df_free
del df_free
opt_mid = result["mid_price"].to_numpy()
opt_half_spread = result["half_spread"].to_numpy()
opt_tick_size = result["opt_tick_size"].to_numpy()
tte_y = result["dte"].to_numpy() / DAYS_PER_YEAR
spot = result["spot_mid_price"].to_numpy()
strike = result["strike"].to_numpy()
is_call = result["is_call"].to_numpy()
assert len(spot) > 0, f"{ticker} {d} option {opt_ticker}: Found no valid spot prices after conversion...?"
assert np.all( np.isfinite(opt_mid) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in mid_price after conversion...?"
assert np.all( np.isfinite(opt_half_spread) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in half_spread after conversion...?"
assert np.all( np.isfinite(tte_y) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in tte_y after conversion...?"
assert np.all( np.isfinite(spot) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in spot after conversion...?"
assert np.all( np.isfinite(strike) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in strike after conversion...?"
assert np.all( opt_half_spread >= opt_tick_size/2.-1E-8 ), f"{ticker} {d} option {opt_ticker}: half spread should not be below half tick size, but found half_spread {np.min(np.abs(opt_half_spread-opt_tick_size/2.)):.4g}"
opt_ask = opt_mid + opt_half_spread
opt_bid = opt_mid - opt_half_spread
assert np.all( opt_bid >= opt_tick_size/2.-1E-8 ), f"{ticker} {d} option {opt_ticker}: half spread should not be below half tick size, but found half_spread {np.min(opt_bid-opt_tick_size/2.):.4g}"
intrinsic = np.where( is_call, np.maximum(0., spot - strike), np.maximum(0., strike - spot) )
is_intrinsic = intrinsic + opt_tick_size*1.1 >= opt_bid
result = result.with_columns(
pl.Series("intrinsic", intrinsic),
pl.Series("is_intrinsic", is_intrinsic) )
sqrtT = np.sqrt(tte_y)
what = dict(bid=opt_bid, ask=opt_ask, mid=opt_mid)
rel_k = strike / spot
for w, price in what.items():
vols = bs.implied( k=rel_k, prices=price, is_call=is_call, sqrtT=sqrtT, on_exceed_bounds="warn", vol_min=OptionQuotesDayConfig.min_vol, vol_max=OptionQuotesDayConfig.max_vol )
r = bs(k=rel_k, vol=vols, sqrtT=sqrtT, is_call=is_call, what=bs.VEGA|bs.DELTA|bs.GAMMA|bs.THETA)
r.vega *= spot
r.theta *= spot
r.gamma /= spot
r.vol = vols
result = result.with_columns( [ pl.Series(f"{w}_{k}", v) for k, v in r.items() ] )
del vols
# Assert all trade_columns fields are non-null at the same time
trade_not_null = result['trade_t_d'].is_not_null() & result['trade_price'].is_not_null() & result['trade_size'].is_not_null() & result['trade_direction'].is_not_null() & result['trade_dte'].is_not_null()
trade_all_null = result['trade_t_d'].is_null() & result['trade_price'].is_null() & result['trade_size'].is_null() & result['trade_direction'].is_null() & result['trade_dte'].is_null()
assert (trade_not_null | trade_all_null).all(), f"{ticker} {d} option {opt_ticker}: trade_columns fields must all be null or all non-null together"
trade_ret_not_null = result['trade_ret'].is_not_null() & result['trade_dt_d'].is_not_null()
trade_ret_all_null = result['trade_ret'].is_null() & result['trade_dt_d'].is_null()
assert (trade_ret_not_null | trade_ret_all_null).all(), f"{ticker} {d} option {opt_ticker}: trade_ret fields must all be null or all non-null together"
trade_not_null = trade_not_null.to_numpy()
trade_vol_full = np.full_like( trade_not_null, dtype=np.float64, fill_value=np.nan )
trade_delta = np.full_like( trade_not_null, dtype=np.float64, fill_value=np.nan )
trade_gamma = np.full_like( trade_not_null, dtype=np.float64, fill_value=np.nan )
trade_vega = np.full_like( trade_not_null, dtype=np.float64, fill_value=np.nan )
trade_theta = np.full_like( trade_not_null, dtype=np.float64, fill_value=np.nan )
if np.any(trade_not_null):
trade_prices = result['trade_price'].to_numpy()[trade_not_null]
trade_spot = result['spot_trade_price'].to_numpy()[trade_not_null]
trade_rel_k = strike[trade_not_null] / trade_spot
assert np.all( np.isfinite(trade_prices) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in trade_prices...?"
trade_vols = bs.implied( k=trade_rel_k, prices=trade_prices, is_call=is_call[trade_not_null], sqrtT=sqrtT[trade_not_null], on_exceed_bounds="warn", vol_min=OptionQuotesDayConfig.min_vol, vol_max=OptionQuotesDayConfig.max_vol )
assert np.all( np.isfinite(trade_vols) ), f"{ticker} {d} option {opt_ticker}: Found non-finite values in trade_vols after conversion...?"
r = bs(k=trade_rel_k, vol=trade_vols, sqrtT=sqrtT[trade_not_null], is_call=is_call[trade_not_null], what=bs.VEGA|bs.DELTA|bs.GAMMA|bs.THETA)
r.gamma /= trade_spot
r.vega *= trade_spot
r.theta *= trade_spot
trade_vol_full[trade_not_null] = trade_vols
trade_delta[trade_not_null] = r.delta
trade_gamma[trade_not_null] = r.gamma
trade_vega[trade_not_null] = r.vega
trade_theta[trade_not_null] = r.theta
result = result.with_columns( [
pl.Series("trade_vol", trade_vol_full),
pl.Series("trade_delta", trade_delta),
pl.Series("trade_gamma", trade_gamma),
pl.Series("trade_vega", trade_vega),
pl.Series("trade_theta", trade_theta) ])
for c in result.columns:
if not c in TRADE_COLUMNS:
assert result[c].is_not_null().all(), f"{ticker} {d} option {opt_ticker}: Found null values in {c} after conversion...?"
my_schema = dict(OPT_QUOTE_BY_EXPIRY_SCHEMA)
my_schema.pop("paired_opt_ticker")
verify_schema( result, my_schema, header=lambda : f"{ticker} {d} expiry {expiry} {opt_ticker}: Schema validation failed for enriched option quote time series" )
return result
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.17",
dependencies=[_opt_quote_time_series, filter_contracts],
uid=lambda ticker, d, expiry, opt_ticker_data, option_quotes_day_config: \
f"opt_expiry_quotes/{named_unique_filename48_8(str(option_quotes_day_config),option_quotes_day_config)}/{ticker}/{d}/{expiry}/tickers {unique_hash8(set(opt_ticker_data['opt_ticker'].to_list()))}"
)
def _opt_expiry_quote_time_series( ticker : str,
d : date, expiry : date,
opt_ticker_data : pl.DataFrame,
option_quotes_day_config : OptionQuotesDayConfig, *,
exclude_half_days_from_prev_business_day : bool = True,
pool_config : PoolConfig = PoolConfig.off(),
verbose : Context|None = Context.quiet) -> pl.DataFrame:
"""
Option quote time series aligned with spot trades, for all options of a given expiry.
"""
verbose = verbose if verbose is not None else Context.quiet
opt_tickers = set(opt_ticker_data['opt_ticker'].to_list())
open, close,\
is_half_day = session_open_close(d)
pool = pool_config.pool()
with verbose.write_t(lambda : f"{ticker} {d} expiry {expiry}: Computing enriched time series for {len(opt_tickers)} options:", end='') as tme:
is_single = pool.is_no_pool or pool.is_threading
cnt = 0
tme = verbose.timer()
results = []
excluded = []
for opt_ticker, _ in pool.parallel( { opt_ticker: pool.delayed( _opt_quote_time_series )( ticker,
opt_ticker,
d,
dte_exclude_half_days=option_quotes_day_config.dte_exclude_half_days,
overnight_fraction=option_quotes_day_config.overnight_fraction,
alignment_config=option_quotes_day_config.alignment_config,
verbose=verbose(1) if is_single else None, cache_generate_only=True ) for opt_ticker in opt_tickers } ):
cnt += 1
r = _opt_quote_time_series( ticker, opt_ticker, d,
dte_exclude_half_days=option_quotes_day_config.dte_exclude_half_days,
overnight_fraction=option_quotes_day_config.overnight_fraction,
alignment_config=option_quotes_day_config.alignment_config,
verbose=Context.quiet, override_cache_mode="cacheonly" )
if len(r) == 0:
verbose.write(lambda : f"\r{ticker} {d} expiry {expiry}: Computing enriched time series for {len(opt_tickers)} options: processed {opt_ticker} {cnt}/{len(opt_tickers)}: this option had no valid quotes -- {math.ceil(float(cnt)/float(len(opt_tickers))*100)}% done; so far this took {tme}...", end='')
excluded.append(opt_ticker)
else:
if cnt==1 or tme.interval_test(1.):
verbose.write(lambda : f"\r{ticker} {d} expiry {expiry}: Computing enriched time series for {len(opt_tickers)} options: processed {opt_ticker} {cnt}/{len(opt_tickers)} ... {math.ceil(float(cnt)/float(len(opt_tickers))*100)}% done; so far this took {tme}...", end='')
assert r['opt_ticker'].n_unique() == 1 and r['opt_ticker'].unique()[0] == opt_ticker, f"{ticker} {d} expiry {expiry} {opt_ticker}: Expected all rows to have the same opt_ticker {opt_ticker}, but found multiple opt_tickers in the result...?"
paired_opt_ticker = opt_ticker_data.filter( pl.col("opt_ticker") == opt_ticker )["paired_opt_ticker"].unique()
assert len(paired_opt_ticker) == 1, f"{ticker} {d} expiry {expiry} {opt_ticker}: Expected exactly one paired_opt_ticker, but found {len(paired_opt_ticker)}: {fmt_list(paired_opt_ticker.to_list())}"
r = r.with_columns( pl.lit(paired_opt_ticker[0]).alias("paired_opt_ticker") )
verify_schema( r, OPT_QUOTE_BY_EXPIRY_SCHEMA, header=lambda : f"{ticker} {d} expiry {expiry} {opt_ticker}: Schema validation failed for enriched option quote time series" )
results.append(r)
del r
results = pl.concat(results).sort("t_d", "opt_ticker")
verbose.write(lambda : f"\r{ticker} {d}: Computed enriched time series at expiry {expiry} for {len(opt_tickers)-len(excluded)} out of {len(opt_tickers)} options; this took {tme}.")
return results
[docs]
@version("0.11", dependencies=[_opt_quote_time_series, filter_contracts])
def generate_day( ticker : str, d : date,
option_quotes_day_config : OptionQuotesDayConfig, *,
pair_options : bool = False,
pool_config : PoolConfig = PoolConfig.off(),
cache_generate_only : bool = False,
override_cache_mode : CacheMode|str|None = None,
verbose : Context = Context.quiet
):
"""
Generate polars data frame per expiry for all eligible options.
"""
verbose.write(lambda : f"{ticker} {d}: Starting data generation with config {option_quotes_day_config}:")
# contract selection
# ------------------
df = filter_contracts( ticker, d=d, exclude_half_days=True, drop_no_trades_today=True,
contracts_filter=option_quotes_day_config.contract_filter, pool_config=pool_config, verbose=verbose )
# add "paired_ticker"
df = (
df.with_columns(
pl.when(pl.col("contract_type") == "call")
.then(pl.lit("put"))
.when(pl.col("contract_type") == "put")
.then(pl.lit("call"))
.otherwise(pl.lit(None))
.alias("_paired_contract_type")
)
.join(
df.select(
"exp_date",
"strike_price",
pl.col("contract_type").alias("_paired_contract_type"),
pl.col("ticker").alias("paired_ticker"),
)
.unique(keep="any"),
on=["exp_date", "strike_price", "_paired_contract_type"],
how="left",
)
.sort("exp_date", "strike_price", "contract_type")
.drop("_paired_contract_type")
)
unpaired_df = df.filter(pl.col("paired_ticker").is_null())
if len(unpaired_df) > 0:
warnings.warn( f"{ticker} {d}: Removing {len(unpaired_df)} used options without matching call/put pair: {fmt_list(unpaired_df['ticker'].to_list())}. (Assuming these are not regular options - please amend code if assumption is not correct)" )
df = df.filter(pl.col("paired_ticker").is_not_null())
# defensive test to check we didn't double-match.
pair_validation = (
df.group_by("ticker")
.agg(
pl.col("paired_ticker").n_unique().alias("num_paired_tickers"),
pl.col("paired_ticker").unique().sort().alias("paired_tickers"),
)
.filter(pl.col("num_paired_tickers") != 1)
)
assert len(pair_validation) == 0, (
f"{ticker} {d}: Expected exactly one paired_ticker for each ticker, but found invalid mappings for "
f"{fmt_list(pair_validation['ticker'].to_list())}: {pair_validation.to_dicts()}"
)
del pair_validation
if pair_options:
used = df.filter(pl.col("reject_code") == REJECTED.USED)
selected_tickers = (
pl.concat(
[
used.select(pl.col("ticker").alias("ticker")),
used.select(pl.col("paired_ticker").alias("ticker")),
],
how="vertical",
)
.unique(subset=["ticker"], keep="first")
)
df = (
df.join(selected_tickers, on="ticker", how="semi")
.sort("exp_date", "strike_price", "contract_type")
)
del used, unpaired_df, selected_tickers
else:
df = df.filter(pl.col("reject_code") == REJECTED.USED)
if len(df) == 0:
warnings.warn(f"{ticker} {d}: No eligible options remained after contract selection.")
return {}
# reference data
# --------------
expiries = df["exp_date"].unique().sort().to_list()
prev_spot = float( df["spot_ref_prev_close"].unique()[0] )
ref_data_cols = [
pl.col("ticker").alias("opt_ticker"),
pl.col("paired_ticker").alias("paired_opt_ticker"),
pl.col("exp_date"),
pl.col("nms5").alias("ref_nms5"),
pl.col("exercise_style")
.str.to_lowercase()
.str.starts_with("a")
.alias("is_american"),
pl.col("contract_type")
.str.to_lowercase()
.str.starts_with("c")
.alias("is_call"),
( pl.col("strike_price")/prev_spot ).alias("strike"),
pl.col("days_to_exp").alias("ref_dte"),
pl.lit(prev_spot).alias("prev_spot"),
]
ref_data = ( df.with_columns(
pl.format("{} {} DTE {} {} {} @ {} ({}%)",
pl.col("ticker"),
pl.col("exp_date"),
pl.col("days_to_exp"),
pl.col("exercise_style"),
pl.col("contract_type"),
pl.col("strike_price").round(2),
(100.0 * pl.col("strike_price")/prev_spot).round(2)).alias("opt_str") ).
select( *ref_data_cols ).with_columns( pl.lit(ticker).alias("ticker") )
)
ref_data = { expiry: ref_data.filter(pl.col("exp_date") == expiry).drop("exp_date") for expiry in expiries }
if option_quotes_day_config.max_num_options_per_expiry is not None:
verbose.write(lambda : f"{ticker} {d}: Limiting to at most {option_quotes_day_config.max_num_options_per_expiry} options per expiry, selected by smallest absolute nms5:")
for expiry, ref_df in ref_data.items():
n_cnt = len(ref_df)
ref_df = (
ref_df
.with_columns(abs_nms5=pl.col("ref_nms5").abs())
.sort("abs_nms5")
.head(option_quotes_day_config.max_num_options_per_expiry)
.drop("abs_nms5")
)
ref_data[expiry] = ref_df
min_nms5 = ref_df["ref_nms5"].min()
max_nms5 = ref_df["ref_nms5"].max()
verbose.report(1,lambda : f"{ticker} {d} expiry {expiry}: Selected {len(ref_df)}/{n_cnt} options, sorted by smallest absolute nms5. Resulting options range from nms5 {min_nms5:.3f} to {max_nms5:.3f}.")
# time series
# -----------
with verbose.write_t(lambda : f"{ticker} {d}: Starting time series generation for expiries {fmt_list(expiries)} with config {option_quotes_day_config} and pool config {pool_config}:") as tme:
results = {}
for expiry in expiries:
opt_ticker_data = ref_data[expiry]
r = _opt_expiry_quote_time_series( ticker, d, expiry, opt_ticker_data, option_quotes_day_config, pool_config=pool_config,
verbose=verbose, cache_generate_only=cache_generate_only,
override_cache_mode=override_cache_mode )
results[expiry] = r
verbose.write(lambda : f"{ticker} {d}: Completed {'paired option ' if pair_options else ''}time series generation for expiries {fmt_list(expiries)}; this took {tme}.")
return results
# ================================================================
# Command-line interface
# ================================================================
import click
from quantdata.widb import _cli_defaults
import datetime as dt
from cdxcore.util import verify_inp, fmt_date
@click.command()
@click.argument("ticker", required=True, type=str)
@click.option("--period",nargs=2, type=click.DateTime(formats=["%Y-%m-%d"]), default=_cli_defaults.period, metavar="START END", help="Period YYYY-MM-DD YYYY-MM-DD")
@click.option('--workers', default=_cli_defaults.workers, help='Number of workers', type=int)
@click.option('--threading', is_flag=True, help="Use only threading, no multiprocessing")
@click.option('--refresh', is_flag=True, help="Refresh data (this will only refresh the top level files)")
def main(ticker : str,
period : list[dt.datetime],
workers : int,
threading : bool,
refresh : bool):
verify_inp( period is not None and len(period) == 2, "A start and end date must be specified")
start, end = period
date_range = DateRange(start=to_date(start), end=to_date(end), exclude_half_days=True )
dates = sorted( set(date_range.get_business_days())-set(BAD.DAYS_ALL) )
pool_config = PoolConfig(num_workers=workers, threading=threading)
for d in dates:
_ = generate_day(ticker, d, OptionQuotesDayConfig(), pool_config=pool_config, verbose=Context.all, cache_generate_only=True )
if __name__ == "__main__":
main()