Source code for quantdata.massive.processing

"""
quantdata.massive.processing
^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Some simple tools to make life easier when working with Massive data.
These are not meant to be comprehensive utilities, but rather simple examples of how to filter and align trades and quotes.

Function in this file:

* :func:`filter_valid_trades` removes trades with invalid timestamps, prices, sizes, or trade codes.
* :func:`filter_valid_quotes` removes quotes with invalid timestamps, prices, sizes, or crossed/locked markets.
* :func:`align_quotes_to_trades` matches each trade to a nearby valid quote based on price and timestamp criteria.
* :func:`align_trades_to_quotes` matches each quote to a nearby valid trade based on price and timestamp criteria.
"""

from quantdata import version
import quantdata
from quantdata.biz_calendar import session_open_close
from quantdata.massive.client import MassiveClient
from quantdata.massive.trade_conds import NORMAL_TRADE_EXCLUDE_FLAGS
import numpy as np
import polars as pl
from datetime import date
from cdxcore.err import verify_inp, verify
from dataclasses import dataclass, field

[docs] @version("0.1", dependencies=[quantdata.biz_calendar.session_open_close, quantdata.massive.client.MassiveClient.fetch_trades]) def filter_valid_trades( ticker: str, d: date, within_session: bool = True, ) -> pl.DataFrame: """ Returns trades for a given underlying and date within session hours (optionally). Removes trades with NORMAL_TRADE_EXCLUDE_FLAGS and price or size zero or negative. For trades at the sime sip time stamp, computes aggregate trade size. Returns a polars data frame with :attr:`MassiveClient.SCHEMA_TRADES` schema with the following columns: ts_ns Datetime[ns,tz] price Float64 size Float64 trade_code UInt32 (semantic mask) sequence_number UInt64. Sequence number of the trade for a given timestamp, useful for ordering trades with the same timestamp. """ client = MassiveClient() trades = client.fetch_trades(ticker, d) assert set(trades.columns) >= {"ts_ns", "price", "size", "trade_code", "sequence_number"}, ( "trades must have ts_ns, price, size, trade_code, sequence_number", trades.columns, ) if not within_session: trades = trades.filter( pl.col("ts_ns").is_not_null() & pl.col("price").is_not_null() & pl.col("size").is_not_null() & pl.col("price").is_finite() & pl.col("size").is_finite() & ((pl.col("trade_code") & int(NORMAL_TRADE_EXCLUDE_FLAGS)) == 0) & (pl.col("price") > 0) & (pl.col("size") > 0) ) else: open_dt, close_dt, _ = session_open_close(d) trades = trades.filter( (pl.col("ts_ns") >= pl.lit(open_dt)) & (pl.col("ts_ns") <= pl.lit(close_dt)) & pl.col("price").is_not_null() & pl.col("size").is_not_null() & pl.col("price").is_finite() & pl.col("size").is_finite() & ((pl.col("trade_code") & int(NORMAL_TRADE_EXCLUDE_FLAGS)) == 0) & (pl.col("price") > 0) & (pl.col("size") > 0) ) # Aggregate trades at same ts_ns: sum sizes and compute weighted average price trades = trades.group_by("ts_ns", maintain_order=True).agg( price=pl.when(pl.col("size").sum() > 0) .then((pl.col("price") * pl.col("size")).sum() / pl.col("size").sum()) .otherwise(pl.col("price").first()), size=pl.col("size").sum(), trade_code=pl.col("trade_code").last(), sequence_number=pl.col("sequence_number").last(), ).sort("ts_ns") return trades
[docs] @version("0.1", dependencies=[quantdata.biz_calendar.session_open_close, quantdata.massive.client.MassiveClient.fetch_quotes]) def filter_valid_quotes( ticker: str, d: date, within_session: bool = True, ) -> pl.DataFrame: """ Returns quotes for a given underlying and date within session hours (optionally). Removes quotes with invalid timestamps, prices, sizes, or crossed/locked markets. Returns a polars data frame with :attr:`MassiveClient.SCHEMA_QUOTES` schema with the following columns: ts_ns Datetime[ns,tz] bid_price Float64 bid_size Float64 ask_price Float64 ask_size Float64 """ client = MassiveClient() quotes = client.fetch_quotes(ticker, d) assert set(quotes.columns) >= {"ts_ns", "bid_price", "bid_size", "ask_price", "ask_size"}, ( "quotes must have ts_ns, bid_price, bid_size, ask_price, ask_size", quotes.columns, ) valid_quotes = ( pl.col("ts_ns").is_not_null() & pl.col("bid_price").is_not_null() & pl.col("ask_price").is_not_null() & pl.col("bid_price").is_finite() & pl.col("ask_price").is_finite() & (pl.col("bid_price") > 0.0) & (pl.col("ask_price") > 0.0) & (pl.col("ask_price") > pl.col("bid_price")) & pl.col("bid_size").is_not_null() & pl.col("ask_size").is_not_null() & pl.col("bid_size").is_finite() & pl.col("ask_size").is_finite() & (pl.col("bid_size") >= 0.0) & (pl.col("ask_size") >= 0.0) & ((pl.col("bid_size") + pl.col("ask_size")) > 0.0) ) if within_session: open_dt, close_dt, _ = session_open_close(d) valid_quotes = ( (pl.col("ts_ns") >= pl.lit(open_dt)) & (pl.col("ts_ns") <= pl.lit(close_dt)) & valid_quotes ) # Aggregate quotes at same ts_ns: keep only the last one by sequence_number quotes = ( quotes.filter(valid_quotes) .group_by("ts_ns", maintain_order=True) .last() ) return quotes
[docs] @version("0.1") @dataclass(frozen=True) class TradeQuoteAlignmentConfig: """ Configuration for matching trades to nearby valid quotes. Times are in milliseconds. """ lookback_ms: int = 250 # how many milliseconds to look back lookfwd_ms: int = 250 # how many milliseconds to look forward def __post_init__(self): verify_inp(self.lookback_ms >= 0, f"lookback_ms must be non-negative, found {self.lookback_ms}") verify_inp(self.lookfwd_ms >= 0, f"lookfwd_ms must be non-negative, found {self.lookfwd_ms}") def __str__(self): return f"bck={self.lookback_ms},fwd={self.lookfwd_ms})"
def _ts_ns(df: pl.DataFrame) -> np.ndarray: return df.select(pl.col("ts_ns").cast(pl.Int64)).to_series().to_numpy() def _find_quote( bid_price: np.ndarray, ask_price: np.ndarray, price: float, left: int, right: int, prefer_last: bool, ) -> int: if left == right: return -1 valid = np.flatnonzero((bid_price[left:right] <= price) & (price <= ask_price[left:right])) if len(valid) == 0: return -1 return left + (valid[-1] if prefer_last else valid[0]) def _find_trade( trade_price: np.ndarray, bid: float, ask: float, left: int, right: int, prefer_last: bool, ) -> int: if left == right: return -1 valid = np.flatnonzero((bid <= trade_price[left:right]) & (trade_price[left:right] <= ask)) if len(valid) == 0: return -1 return left + (valid[-1] if prefer_last else valid[0]) SCHEMA_ALIGN_QUOTES_TO_TRADES = { "ts_ns": np.dtype("datetime64[ns]"), "price": np.float64, "size": np.float64, "trade_code": np.uint32, "quote_ts_ns": np.dtype("datetime64[ns]"), "bid_price": np.float64, "bid_size": np.float64, "ask_price": np.float64, "ask_size": np.float64, }
[docs] @version("0.1", dependencies=[filter_valid_quotes, filter_valid_trades, TradeQuoteAlignmentConfig]) def align_quotes_to_trades( ticker: str, d: date, drop_unmatched: bool, config: TradeQuoteAlignmentConfig = TradeQuoteAlignmentConfig(), *, within_session: bool = True, ) -> pl.DataFrame: """ Fetch valid trades and quotes for ``ticker`` on ``d`` and align them on a common timeline. For each trade, this first finds the latest quote between ``config.lookback_ms`` and the trade timestamp where ``bid_price <= price <= ask_price``. If no such quote exists, it finds the first matching quote up to ``config.lookfwd_ms`` after the trade. The output keeps all trade columns and adds the matched quote columns. If ``drop_unmatched`` is False, unmatched trades have null quote fields. Parameters ---------- ticker : str The underlying ticker to fetch trades and quotes for. d : :class:`datetime.date` The date to fetch trades and quotes for. drop_unmatched : bool If True, only keep trades with a matched quote. If False, keep all trades and have null quote fields for unmatched trades. config : :class:`TradeQuoteAlignmentConfig`, default ``TradeQuoteAlignmentConfig()`` Configuration for matching trades to nearby valid quotes. within_session : bool, default True Whether to only consider trades and quotes within session hours when aligning. """ trades = filter_valid_trades(ticker, d, within_session=within_session).sort("ts_ns").drop("sequence_number") quotes = filter_valid_quotes(ticker, d, within_session=within_session).sort("ts_ns").drop("sequence_number") trade_ts = _ts_ns(trades) quote_ts = _ts_ns(quotes) trade_price = trades["price"].to_numpy() if "price" in trades.columns else np.array([], dtype=float) bid_price = quotes["bid_price"].to_numpy() if "bid_price" in quotes.columns else np.array([], dtype=float) ask_price = quotes["ask_price"].to_numpy() if "ask_price" in quotes.columns else np.array([], dtype=float) matched_rows = np.full(len(trades), -1, dtype=np.int64) lookback_ns = int(config.lookback_ms) * 1_000_000 lookfwd_ns = int(config.lookfwd_ms) * 1_000_000 for i, (ts, price) in enumerate(zip(trade_ts, trade_price)): quote_row = _find_quote( bid_price=bid_price, ask_price=ask_price, price=price, left=np.searchsorted(quote_ts, ts - lookback_ns, side="left"), right=np.searchsorted(quote_ts, ts, side="right"), prefer_last=True, ) if quote_row < 0: quote_row = _find_quote( bid_price=bid_price, ask_price=ask_price, price=price, left=np.searchsorted(quote_ts, ts, side="left"), right=np.searchsorted(quote_ts, ts + lookfwd_ns, side="right"), prefer_last=False, ) if quote_row < 0: continue matched_rows[i] = quote_row out = ( trades.with_row_index("_trade_row") .with_columns( pl.Series("_quote_row", matched_rows), ) .join( quotes.with_row_index("_quote_row") .with_columns(pl.col("_quote_row").cast(pl.Int64)) .rename({"ts_ns": "quote_ts_ns"}), on="_quote_row", how="left", ) .sort("_trade_row") .drop("_trade_row", "_quote_row") ) assert out["ts_ns"].is_not_null().all(), "Found null values in 'ts_ns' after filtering unmatched quotes." assert out["price"].is_not_null().all(), "Found null values in 'price' after filtering unmatched quotes." assert out["size"].is_not_null().all(), "Found null values in 'size' after filtering unmatched quotes." assert out["trade_code"].is_not_null().all(), "Found null values in 'trade_code' after filtering unmatched quotes." if drop_unmatched: out = out.filter(pl.col("quote_ts_ns").is_not_null()) assert out["quote_ts_ns"].is_not_null().all(), "Found null values in 'ts_ns' after filtering unmatched quotes." assert out["bid_price"].is_not_null().all(), "Found null values in 'bid_price' after filtering unmatched quotes." assert out["ask_price"].is_not_null().all(), "Found null values in 'ask_price' after filtering unmatched quotes." assert out["bid_size"].is_not_null().all(), "Found null values in 'bid_size' after filtering unmatched quotes." assert out["ask_size"].is_not_null().all(), "Found null values in 'ask_size' after filtering unmatched quotes." verify(set(out.columns) == set(SCHEMA_ALIGN_QUOTES_TO_TRADES), f"Output is missing expected columns: {sorted(set(SCHEMA_ALIGN_QUOTES_TO_TRADES) - set(out.columns))} and has superfluous columns: {sorted(set(out.columns) - set(SCHEMA_ALIGN_QUOTES_TO_TRADES))}") return out
SCHEMA_ALIGN_TRADES_TO_QUOTES = { "ts_ns": np.dtype("datetime64[ns]"), "bid_price": np.float64, "bid_size": np.float64, "ask_price": np.float64, "ask_size": np.float64, "trade_ts_ns": np.dtype("datetime64[ns]"), "price": np.float64, "size": np.float64, "trade_code": np.uint32, }
[docs] @version("0.1", dependencies=[filter_valid_quotes, filter_valid_trades]) def align_trades_to_quotes( ticker: str, d: date, drop_unmatched: bool, config: TradeQuoteAlignmentConfig = TradeQuoteAlignmentConfig(), *, within_session: bool = True ) -> pl.DataFrame: """ Fetch valid trades and quotes for ``ticker`` on ``d`` and align trades onto quotes. For each quote, this first finds the latest trade between ``config.lookback_ms`` and the quote timestamp where ``bid_price <= price <= ask_price``. If no such trade exists, it finds the first matching trade up to ``config.lookfwd_ms`` after the quote. The output keeps all quote columns and adds matched trade columns. If ``drop_unmatched`` is False, unmatched quotes have null trade fields. Parameters ---------- ticker : str The underlying ticker to fetch trades and quotes for. d : :class:`datetime.date` The date to fetch trades and quotes for. drop_unmatched : bool If True, only keep quotes with a matched trade. If False, keep all quotes and have null trade fields for unmatched quotes. config : :class:`TradeQuoteAlignmentConfig`, default ``TradeQuoteAlignmentConfig()`` Configuration for matching trades to nearby valid quotes. within_session : bool, default True Whether to only consider trades and quotes within session hours when aligning. """ trades = filter_valid_trades(ticker, d, within_session=within_session).sort("ts_ns").drop("sequence_number") quotes = filter_valid_quotes(ticker, d, within_session=within_session).sort("ts_ns").drop("sequence_number") trade_ts = _ts_ns(trades) quote_ts = _ts_ns(quotes) trade_price = trades["price"].to_numpy() if "price" in trades.columns else np.array([], dtype=float) bid_price = quotes["bid_price"].to_numpy() if "bid_price" in quotes.columns else np.array([], dtype=float) ask_price = quotes["ask_price"].to_numpy() if "ask_price" in quotes.columns else np.array([], dtype=float) matched_rows = np.full(len(quotes), -1, dtype=np.int64) lookback_ns = int(config.lookback_ms) * 1_000_000 lookfwd_ns = int(config.lookfwd_ms) * 1_000_000 for i, (ts, bid, ask) in enumerate(zip(quote_ts, bid_price, ask_price)): trade_row = _find_trade( trade_price=trade_price, bid=bid, ask=ask, left=np.searchsorted(trade_ts, ts - lookback_ns, side="left"), right=np.searchsorted(trade_ts, ts, side="right"), prefer_last=True, ) if trade_row < 0: trade_row = _find_trade( trade_price=trade_price, bid=bid, ask=ask, left=np.searchsorted(trade_ts, ts, side="left"), right=np.searchsorted(trade_ts, ts + lookfwd_ns, side="right"), prefer_last=False, ) if trade_row < 0: continue matched_rows[i] = trade_row out = ( quotes.with_row_index("_quote_row") .with_columns( pl.Series("_trade_row", matched_rows), ) .join( trades.with_row_index("_trade_row") .with_columns(pl.col("_trade_row").cast(pl.Int64)) .rename({"ts_ns": "trade_ts_ns"}), on="_trade_row", how="left", ) .sort("_quote_row") .drop("_quote_row", "_trade_row") ) if drop_unmatched: out = out.filter(pl.col("trade_ts_ns").is_not_null()) verify(set(out.columns) == set(SCHEMA_ALIGN_TRADES_TO_QUOTES), f"Output is missing expected columns: {sorted(set(SCHEMA_ALIGN_TRADES_TO_QUOTES) - set(out.columns))} and has superfluous columns: {sorted(set(out.columns) - set(SCHEMA_ALIGN_TRADES_TO_QUOTES))}") return out