Source code for quantdata.massive.tickers.data

"""
quantdata.massive.tickers.data
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Build underlying tick-level data from Massive: unadjusted trades, weighted mid-price
tape, and 1-second trade bars.
"""

from __future__ import annotations

import math
from datetime import date, datetime, timedelta

import polars as pl
from cdxcore.err import verify
from cdxcore.util import fmt_date
from cdxcore.version import version

from quantdata.biz_calendar import TZ_NY, full_second_grid_ns, session_open_close
from quantdata.massive import MassiveClient
from quantdata.tickers import SubDir, _locations_tickers

_data_cache = _locations_tickers


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


@version("0.1")
def _second_bin_ix_expr(dt_ns_expr: pl.Expr, open_dt: datetime) -> pl.Expr:
    base = pl.lit(open_dt).cast(pl.Datetime("ns", TZ_NY))
    return ((dt_ns_expr - base).dt.total_seconds()).cast(pl.Int32).alias("second_bin_ix")


@version("0.1")
def _bin_type_series(n: int) -> pl.Series:
    # bin_type: 1 = opening bin, 0 = middle bin, 2 = closing bin
    if n <= 0:
        return pl.Series("bin_type", [], dtype=pl.Int8)
    if n == 1:
        return pl.Series("bin_type", [1], dtype=pl.Int8)
    return pl.Series("bin_type", [1] + [0] * (n - 2) + [2], dtype=pl.Int8)


@version("0.1")
def _map_close_inclusive_to_last_bin(ts_col: str, close_dt: datetime) -> pl.Expr:
    """
    Map any timestamp exactly at ``close_dt`` to the last bin key ``close_dt - 1s``,
    implementing the semantics:

    * all bins except last: ``[t, t+1s)``
    * last bin: ``[close_dt-1s, close_dt]``
    """
    last_bin_dt = close_dt - timedelta(seconds=1)
    return (
        pl.when(pl.col(ts_col) == pl.lit(close_dt))
        .then(pl.lit(last_bin_dt))
        .otherwise(pl.col(ts_col))
    )


# ---------------------------------------------------------------------------
# Unadjusted trades
# ---------------------------------------------------------------------------


[docs] @_data_cache(fmt=SubDir.POLARS_PARQUET).cache( "0.1", uid=lambda underlying, d: f"unadjusted_trades/{underlying}/{fmt_date(d)}", exclude_args=["client"], dependencies=[MassiveClient.fetch_trades, session_open_close], ) def build_unadjusted_trades( underlying: str, d: date, *, client: MassiveClient | None = None, ) -> pl.DataFrame: """ Unadjusted trades for ``underlying`` on ``d``, filtered to RTH session ``[open, close]``. Returns ------- pl.DataFrame Columns: ts_ns (Datetime[ns, NY]), price (Float32), size (Float32), trade_code (UInt32) """ client = MassiveClient() if client is None else client open_dt, close_dt, _sec, _hd = session_open_close(d) trades = client.fetch_trades(underlying, d) assert set(trades.columns) >= {"ts_ns", "price", "size"}, trades.columns verify(not trades.is_empty(), f"No trades for {underlying} on {d}") trades = trades.filter( (pl.col("ts_ns") >= pl.lit(open_dt)) & (pl.col("ts_ns") <= pl.lit(close_dt)) ) return trades
# --------------------------------------------------------------------------- # Underlying spot (weighted mid-price) # ---------------------------------------------------------------------------
[docs] @_data_cache(fmt=SubDir.POLARS_PARQUET).cache( "0.1", uid=lambda underlying, d: f"spot_wmid/{underlying}/{fmt_date(d)}", exclude_args=["trades", "client"], dependencies=[ MassiveClient.fetch_quotes, MassiveClient.fetch_day_summary, build_unadjusted_trades, ], ) def build_unadjusted_spot_wmid( underlying: str, d: date, *, trades: pl.DataFrame | None = None, client: MassiveClient | None = None, ) -> pl.DataFrame: """ Weighted mid-price tape for ``underlying`` on ``d``, suitable for asof-joins. The tape is initialised at session open and close with the unadjusted open/close from ``/v1/open-close`` so a spot price is always available. Quote wmid and trade prices are interleaved; trades win on identical timestamps. Returns ------- pl.DataFrame Columns: u_ts_ns (Datetime[ns, NY]), spot (Float32), is_trade (Boolean) """ client = MassiveClient() if client is None else client open_dt, close_dt, _sec, _hd = session_open_close(d) oc = client.fetch_day_summary(underlying, d, adjusted=False) open_px = float(oc["open"]) close_px = float(oc["close"]) assert math.isfinite(open_px) and open_px > 0.0, f"Bad open price {underlying} {d}: {open_px}" assert math.isfinite(close_px) and close_px > 0.0, ( f"Bad close price {underlying} {d}: {close_px}" ) schema = { "u_ts_ns": pl.Datetime("ns", TZ_NY), "spot": pl.Float32, "is_trade": pl.Boolean, } init = pl.DataFrame( {"u_ts_ns": [open_dt, close_dt], "spot": [open_px, close_px], "is_trade": [False, False]}, schema=schema, ) q = client.fetch_quotes(underlying, d) if not q.is_empty(): q = ( q.filter((pl.col("ts_ns") >= pl.lit(open_dt)) & (pl.col("ts_ns") <= pl.lit(close_dt))) .with_columns((pl.col("bid_size") + pl.col("ask_size")).alias("_sz_sum")) .with_columns( pl.when(pl.col("_sz_sum") > 0) .then( ( pl.col("bid_price") * pl.col("ask_size") + pl.col("ask_price") * pl.col("bid_size") ) / pl.col("_sz_sum") ) .otherwise(None) .cast(pl.Float32) .alias("spot") ) .select(pl.col("ts_ns").alias("u_ts_ns"), "spot") .drop_nulls() .with_columns(pl.lit(False).cast(pl.Boolean).alias("is_trade")) .cast(schema) .sort("u_ts_ns") ) else: q = pl.DataFrame(schema=schema) trades = build_unadjusted_trades(underlying, d, client=client) if trades is None else trades t = ( trades.filter((pl.col("ts_ns") >= pl.lit(open_dt)) & (pl.col("ts_ns") <= pl.lit(close_dt))) .select(pl.col("ts_ns").alias("u_ts_ns"), pl.col("price").cast(pl.Float32).alias("spot")) .with_columns(pl.lit(True).cast(pl.Boolean).alias("is_trade")) .cast(schema) .sort("u_ts_ns") ) out = ( pl.concat([init, q, t], how="vertical") .sort(["u_ts_ns", "is_trade"], descending=[False, True]) .unique(subset=["u_ts_ns"], keep="first") .sort("u_ts_ns") ) verify(out.height >= 1, lambda: f"Spot wmid tape empty for {underlying} on {d}") return out
# --------------------------------------------------------------------------- # 1-second trade bars # ---------------------------------------------------------------------------
[docs] @_data_cache(fmt=SubDir.POLARS_PARQUET).cache( "0.1", uid=lambda underlying, d: f"u1s_bars/{underlying}/{fmt_date(d)}", exclude_args=["trades", "client"], dependencies=[ MassiveClient.fetch_trades, session_open_close, full_second_grid_ns, build_unadjusted_trades, ], ) def build_1s_trade_bars( underlying: str, d: date, *, trades: pl.DataFrame | None = None, client: MassiveClient | None = None, ) -> pl.DataFrame: """ Per-second OHLCV trade bars for ``underlying`` on ``d``. Bin semantics: * all bins except last: ``[t, t+1s)`` * last bin: ``[close_dt-1s, close_dt]`` (inclusive close) Returns ------- pl.DataFrame Columns: dt_ns, second_bin_ix, bin_type, open, close, high, low, last_trade, num_trades, vwap, log_cash_volume, log_volume, date, underlying, is_half_day """ client = MassiveClient() if client is None else client open_dt, close_dt, _sec, is_half_day = session_open_close(d) grid = full_second_grid_ns(open_dt, close_dt) verify( grid.to_list()[-1] == close_dt - timedelta(seconds=1), lambda: f"Grid does not end at close_dt-1s: {grid.to_list()[-1]} vs {close_dt}", ) u = pl.DataFrame({"dt_ns": grid}).with_columns( _second_bin_ix_expr(pl.col("dt_ns"), open_dt), _bin_type_series(len(grid)), ) trades = build_unadjusted_trades(underlying, d, client=client) if trades is None else trades trades = ( trades.with_columns( pl.col("ts_ns").dt.truncate("1s").alias("sec_ns_raw"), (pl.col("price") * pl.col("size")).alias("cash"), ) .with_columns(_map_close_inclusive_to_last_bin("sec_ns_raw", close_dt).alias("sec_ns")) .drop("sec_ns_raw") .sort("ts_ns") .with_columns( pl.col("ts_ns").cast(pl.Datetime("ns", TZ_NY)), pl.col("price").cast(pl.Float32), pl.col("size").cast(pl.Float32), pl.col("cash").cast(pl.Float32), pl.col("sec_ns").cast(pl.Datetime("ns", TZ_NY)), ) ) agg = ( trades.group_by("sec_ns") .agg( pl.col("price").first().alias("open"), pl.col("price").last().alias("close"), pl.col("price").max().alias("high"), pl.col("price").min().alias("low"), pl.col("size").sum().fill_null(0.0).alias("volume"), pl.col("cash").sum().fill_null(0.0).alias("cash_volume"), pl.len().cast(pl.Int64).alias("num_trades"), ) .sort("sec_ns") .with_columns( pl.when(pl.col("volume") > 0) .then(pl.col("cash_volume") / pl.col("volume")) .otherwise(None) .alias("vwap"), (pl.col("volume") + 1.0).log10().alias("log_volume"), (pl.col("cash_volume") + 1.0).log10().alias("log_cash_volume"), ) .drop(["volume", "cash_volume"]) ) u = ( u.join(agg.rename({"sec_ns": "dt_ns"}), on="dt_ns", how="left") .with_columns( pl.col("close").forward_fill().cast(pl.Float32).alias("last_trade"), pl.col("open").cast(pl.Float32), pl.col("close").cast(pl.Float32), pl.col("high").cast(pl.Float32), pl.col("low").cast(pl.Float32), pl.col("vwap").cast(pl.Float32), pl.col("num_trades").fill_null(0).cast(pl.Int64), pl.col("log_cash_volume").fill_null(0).cast(pl.Float32), pl.col("log_volume").fill_null(0).cast(pl.Float32), ) .with_columns( pl.lit(underlying).alias("underlying"), pl.lit(d).alias("date").cast(pl.Date), pl.lit(is_half_day).alias("is_half_day").cast(pl.Boolean), ) ) return u