"""
Fetch minute bars from Massive (Polygon) API for a given ticker and date.
Key behaviors:
- Uses adjusted minute aggregates (1/minute) from Polygon/Massive.
- Computes overnight returns based on two-day data.
- Converts timestamps to America/New_York and normalizes datetime to microseconds.
- Filters strictly to regular-session minutes:
* Full day: 09:30 <= t < 16:00 => minute_idx in [0, 389] (390 minutes)
* Half day: 09:30 <= t < 13:00 => minute_idx in [0, 209] (210 minutes)
- Includes sanity checks (warnings) for missing minutes, duplicates, or unexpected counts.
"""
from __future__ import annotations
from datetime import date, time
from typing import Optional
import polars as pl
from cdxcore.err import verify_inp
from cdxcore.subdir import version
from cdxcore.util import fmt_date
from quantdata import Context, env
from quantdata.biz_calendar import get_half_days, prev_business_day
from quantdata.massive import MassiveClient
_a1m_cache = env.locations.raw("volume_curves")("a1m")
# Trading hours constants (ET)
FULL_DAY_OPEN = time(9, 30)
FULL_DAY_CLOSE = time(16, 0)
HALF_DAY_CLOSE = time(13, 0)
# Expected regular-session minute counts
FULL_DAY_MINUTES = 390
HALF_DAY_MINUTES = 210
def _empty_minute_df() -> pl.DataFrame:
"""Return empty DF with stable schema (Datetime[ms, America/New_York])."""
return pl.DataFrame(
{
"datetime": pl.Series(
[], dtype=pl.Datetime(time_unit="ms", time_zone="America/New_York")
),
"date": pl.Series([], dtype=pl.Date),
"minute_idx": pl.Series([], dtype=pl.Int32),
"open": pl.Series([], dtype=pl.Float64),
"close": pl.Series([], dtype=pl.Float64),
"n": pl.Series([], dtype=pl.Int64),
"volume": pl.Series([], dtype=pl.Float64),
"vwap": pl.Series([], dtype=pl.Float64),
}
)
def _warn(verbose: Context, msg: str) -> None:
try:
verbose.write(f"WARNING: {msg}", head=False)
except Exception:
print(f"WARNING: {msg}")
def _sanity_check_minutes(
df: pl.DataFrame,
ticker: str,
dt: date,
is_half_day: bool,
verbose: Context,
) -> None:
"""
Relaxed sanity checks:
- warn on duplicated minute_idx (structural bug / bad API behavior)
- warn if datetime decreases after sorting (structural bug)
- DO NOT warn on missing minutes or count != expected (illiquid tickers)
"""
if df.height == 0:
return
dup_count = int(df.select(pl.col("minute_idx")).is_duplicated().sum())
if dup_count > 0:
_warn(verbose, f"{ticker} {dt:%Y-%m-%d}: {dup_count} duplicated minute_idx values.")
not_mono = df.select((pl.col("datetime") < pl.col("datetime").shift(1)).any()).item()
if not_mono:
_warn(
verbose,
f"{ticker} {dt:%Y-%m-%d}: datetime decreased after minute_idx sort (unexpected).",
)
[docs]
@_a1m_cache.cache(
"0.1",
uid=lambda ticker, dt: f"{ticker}/{fmt_date(dt)}",
dependencies=[MassiveClient],
exclude_args=["client", "verbose"],
)
def fetch_minute_bars(
ticker: str,
dt: date,
is_half_day: bool | None = None,
*,
prev_dt: date | None = None,
prev_is_half_day: bool | None = None,
client: Optional[MassiveClient] = None,
verbose: Context = Context.quiet,
) -> pl.DataFrame:
"""
Fetch adjusted minute bars for a given ticker and date from Massive (Polygon) API.
Note this function returns the adjusted bars; for this reason it also returns
the last minute bar of the previous day (for computing adjusted overnight returns).
Returns a Polars DataFrame with columns:
- datetime: pl.Datetime[ms, America/New_York]
- minute_idx: pl.Int16 (dt minutes: 0..389/209; prev close row: -1)
- open: pl.Float32
- close: pl.Float32
- n: pl.Int32
- volume: pl.Int64
- vwap: pl.Float32
"""
client = MassiveClient() if client is None else client
prev_dt = prev_dt if prev_dt is not None else prev_business_day(dt)
verify_inp(
prev_dt < dt, lambda: f"'prev_dt' {fmt_date(prev_dt)} must be before 'dt' {fmt_date(dt)}"
)
if is_half_day is None or prev_is_half_day is None:
half_days = set(get_half_days(prev_dt, dt))
is_half_day = (dt in half_days) if is_half_day is None else is_half_day
prev_is_half_day = (prev_dt in half_days) if prev_is_half_day is None else prev_is_half_day
url = f"{MassiveClient.BASE_URL}/v2/aggs/ticker/{ticker}/range/1/minute/{prev_dt:%Y-%m-%d}/{dt:%Y-%m-%d}"
with verbose.write_t(
lambda: f"Fetching minute bars for {ticker} on {fmt_date(dt)}... ", end=""
) as tme:
rows = client.paginate(url, params={"adjusted": "true", "limit": 50000, "sort": "asc"})
if not rows:
verbose.write("no data found.", head=False)
return _empty_minute_df()
df = pl.DataFrame(rows, infer_schema_length=None)
if "vw" not in df.columns:
df = df.with_columns(pl.lit(None).alias("vw"))
df = df.with_columns(
[
pl.col("t").cast(pl.Int64, strict=False).alias("ts_ms"),
pl.col("o").cast(pl.Float64, strict=False).alias("open"),
pl.col("c").cast(pl.Float64, strict=False).alias("close"),
pl.col("n").cast(pl.Int64, strict=False).alias("n"),
pl.col("v").cast(pl.Float64, strict=False).alias("volume"),
pl.col("vw").cast(pl.Float64, strict=False).alias("vwap"),
]
)
df = df.with_columns(
pl.from_epoch("ts_ms", time_unit="ms")
.dt.replace_time_zone("UTC")
.dt.convert_time_zone("America/New_York")
.alias("datetime")
).with_columns(pl.col("datetime").dt.date().alias("date"))
dt_open = pl.datetime(
dt.year, dt.month, dt.day, 9, 30, 0, time_unit="ms", time_zone="America/New_York"
)
prev_open = pl.datetime(
prev_dt.year,
prev_dt.month,
prev_dt.day,
9,
30,
0,
time_unit="ms",
time_zone="America/New_York",
)
dt_minutes = HALF_DAY_MINUTES if is_half_day else FULL_DAY_MINUTES
prev_minutes = HALF_DAY_MINUTES if prev_is_half_day else FULL_DAY_MINUTES
df = df.with_columns(
pl.when(pl.col("date") == pl.lit(dt))
.then((pl.col("datetime") - dt_open).dt.total_minutes().floor())
.when(pl.col("date") == pl.lit(prev_dt))
.then((pl.col("datetime") - prev_open).dt.total_minutes().floor())
.otherwise(None)
.cast(pl.Int32)
.alias("minute_idx")
)
df_in_session = df.filter(
((pl.col("date") == pl.lit(dt)) & (pl.col("minute_idx").is_between(0, dt_minutes - 1)))
| (
(pl.col("date") == pl.lit(prev_dt))
& (pl.col("minute_idx").is_between(0, prev_minutes - 1))
)
)
prev_last = (
df_in_session.filter(pl.col("date") == pl.lit(prev_dt)).sort("minute_idx").tail(1)
)
df_dt = (
df_in_session.filter(pl.col("date") == pl.lit(dt))
.sort("datetime")
.unique(subset=["minute_idx"], keep="last")
.sort("minute_idx")
)
df_dt = df_dt.with_columns(
[
pl.col("minute_idx").cast(pl.Int16),
pl.col("open").cast(pl.Float32),
pl.col("close").cast(pl.Float32),
pl.col("vwap").cast(pl.Float32),
pl.col("n").cast(pl.Int32),
pl.col("volume").cast(pl.Int64),
]
)
df_dt = df_dt.select(["datetime", "minute_idx", "open", "close", "n", "volume", "vwap"])
if prev_last.height == 1:
prev_row = (
prev_last.select(["datetime", "open", "close", "n", "volume", "vwap"])
.with_columns(
[
pl.lit(-1).cast(pl.Int16).alias("minute_idx"),
pl.col("open").cast(pl.Float32),
pl.col("close").cast(pl.Float32),
pl.col("vwap").cast(pl.Float32),
pl.col("n").cast(pl.Int32),
pl.col("volume").cast(pl.Int64),
]
)
.select(["datetime", "minute_idx", "open", "close", "n", "volume", "vwap"])
)
df_out = pl.concat([prev_row, df_dt], how="vertical")
else:
df_out = df_dt
_sanity_check_minutes(
df_out.filter(pl.col("minute_idx") >= 0),
ticker=ticker,
dt=dt,
is_half_day=is_half_day,
verbose=verbose,
)
verbose.write(f"found {df_out.height} rows. This took {tme}.", head=False)
return df_out