"""
quantdata.massive.client
^^^^^^^^^^^^^^^^^^^^^^^^
A simple client wrapper for the `www.massive.com <https://www.massive.com>`_ API.
Access
======
You will require a `massive.com token <https://www.massive.com/docs/rest/authentication>`_ to access the data.
Set the token in the ``MASSIVE_TOKEN`` environment variable, e.g. by adding the following line to your ``~/.bashrc`` or ``~/.zshrc`` file:
.. code-block:: bash
export MASSIVE_TOKEN="your_token"
Once installed, use is straightforward, e.g. to fetch trades and quotes for Apple 150 call expiring on 2023-06-16::
from quantdata.massive import MassiveClient
client = MassiveClient()
trades = client.fetch_trades("AAPL230616C00150000", date(2023,6,16))
quotes = client.fetch_quotes("AAPL230616C00150000", date(2023,6,16))
...
"""
from __future__ import annotations
import os as os
import time as pytime
import warnings as warnings
from datetime import date
import polars as pl
import requests as requests
from cdxcore.err import verify
from cdxcore.subdir import SubDir, version
from cdxcore.util import fmt_date, verify_inp
from cdxcore.uniquehash import named_unique_filename48_8
from massive import RESTClient
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from quantdata import env, Locations
from quantdata.biz_calendar import TZ_NY, add_business_days
from quantdata.massive.trade_conds import pack_conditions
_massive_locations = env.locations(
"massive",
fmt_raw=SubDir.PICKLE,
fmt_processing=SubDir.PANDAS_PARQUET,
fmt_training=SubDir.PYTREE_HDF5,
)
_cache = _massive_locations.raw
# --------------------------------------------------------------------
# Low-level HTTP helpers
# --------------------------------------------------------------------
[docs]
@version("0.1")
class MassiveClient(object):
"""
Wrapper around massive.com connectivity.
Standard usage::
from quantdata.massive import MassiveClient
client = MassiveClient()
trades = client.fetch_trades("AAPL230616C00150000", date(2023,6,16))
quotes = client.fetch_quotes("AAPL230616C00150000", date(2023,6,16))
**Temporary Caching**
This client supports in-session caching. Unlike the overall caching framework for :mod:`quantdata`, this caching
scheme is temporary and is not guaranteed to persist between machine reboots (e.g. in Linux the root temporary directory is ``/tmp/massive``).
This can be useful for speeding up repeated requests during an active session
but should not be relied upon for long-term storage of data.
To enable temporary caching in a, say, ``"work"`` context::
handle = MassiveClient("work")
"""
locations = _massive_locations
""" :class:`quantdata.Locations` for massive data. """
_MASSIVE_TOKEN = None
_HTTP_SESSION = None
BASE_URL = "https://api.massive.com" #: Base URL for requests.
START_DATE = date(
2022, 3, 7
)
""" Start date for quote and trade requests for options. `See massive.com <https://massive.com/docs/rest/options/trades-quotes/quotes>`_ """
TIME_SPANS = {
"second",
"minute",
"hour",
"day",
"week",
"month",
"quarter",
"year",
}
""" Standard `massive time spans <https://massive.com/docs/rest/stocks/aggregates/custom-bars>`_ """
_PREV_CLOSE_CACHE = {}
"""
Static members
"""
[docs]
@staticmethod
def get_token():
"""Reads massive token from the ``MASSIVE_TOKEN`` environment variable"""
if MassiveClient._MASSIVE_TOKEN is None:
verify( not env.test_mode, "Attempted to connect to massive.com in test mode. Set QUANTDATA_TEST_MODE=0 to enable connections." )
MassiveClient._MASSIVE_TOKEN = os.environ.get("MASSIVE_TOKEN", None)
verify(
not MassiveClient._MASSIVE_TOKEN is None,
"Must specify environment variable 'MASSIVE_TOKEN' to access `massive.com <https://www.massive.com>`_ data",
exception=RuntimeError,
)
return MassiveClient._MASSIVE_TOKEN
[docs]
@staticmethod
def get_rest_client():
"""
Returns a :class:`massive.RESTClient` instance with the current token, defined by the ``MASSIVE_TOKEN`` environment variable.
"""
return RESTClient(MassiveClient.get_token())
[docs]
@staticmethod
def get_session():
"""
Return an HTTP session per process.
"""
if MassiveClient._HTTP_SESSION is None:
MassiveClient._HTTP_SESSION = requests.Session()
MassiveClient._HTTP_SESSION.mount(
"https://",
HTTPAdapter(
pool_connections=50,
pool_maxsize=50,
max_retries=Retry(
total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]
),
),
)
MassiveClient._HTTP_SESSION.mount(
"http://",
HTTPAdapter(
pool_connections=50,
pool_maxsize=50,
max_retries=Retry(
total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]
),
),
)
return MassiveClient._HTTP_SESSION
"""
Object
"""
def __init__(self, temp_caching_topic: str | None = None, fmt: SubDir.Format = SubDir.PICKLE):
"""Create client with temporary caching topic"""
self._session = None
self._temp_dir = (
MassiveClient.locations.temp("massive_api")(temp_caching_topic, fmt=fmt)
if not temp_caching_topic is None
else None
)
self.get = (
self._temp_dir.cache(label="massive_get", exclude_args=["self"])(self._get)
if not self._temp_dir is None
else self._get
)
@version("0.1")
def _get(self, url: str, params: dict | None = None) -> dict:
"""`
GET wrapper that injects apiKey and raises for HTTP errors.
Do not call this function, call ``self.get`` which is optionally
cached in a temporary directory for speed
"""
MASSIVE_token = MassiveClient.get_token()
if params is None:
params = {"apiKey": MASSIVE_token}
else:
params = {**params, "apiKey": MASSIVE_token}
cnt = 1
while True:
e_ = None
try:
r = self.session.get(url, params=params, timeout=60 * cnt)
r.raise_for_status()
return r.json()
except requests.exceptions.RetryError as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.ReadTimeout as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.ConnectTimeout as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.JSONDecodeError as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.ChunkedEncodingError as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.ConnectionError as e:
if cnt > 5:
raise e
e_ = e
except requests.exceptions.HTTPError as e:
status = e.response.status_code if e.response is not None else None
# transient server-side error → retry
if status is None or not (500 <= status < 600):
raise e
e_ = e
warnings.warn(f"**** Massive error: {e_} ...sleeping for {cnt} seconds ***")
pytime.sleep(cnt)
cnt += 1
[docs]
@version("0.1", dependencies=[_get])
def __call__(self, url: str, params: dict | None = None) -> dict:
"""Short-cut for ``self.get``"""
return self.get(url=url, params=params)
[docs]
@version("0.1", dependencies=[_get])
def paginate(self, initial_url: str, params: dict | None = None) -> list[dict]:
"""
Handle `v3-style pagination <https://massive.com/blog/api-pagination-patterns>`_ (quotes, trades, contracts), c.f.
"""
results: list[dict] = []
data = self.get(initial_url, params or {})
results.extend(data.get("results", []) or [])
next_url = data.get("next_url")
while next_url:
if next_url and next_url.startswith("/"):
next_url = self.BASE_URL + next_url
data = self.get(next_url, None)
results.extend(data.get("results", []) or [])
next_url = data.get("next_url")
pytime.sleep(0.02) # be gentle
return results
@property
def session(self):
"""Return the HTTP session for this MassiveClient. Note that a session is shared within a process"""
if self._session is None:
self._session = MassiveClient.get_session()
return self._session
[docs]
def clear_temp(self):
"""Clear the temporary directory for this MassiveClient."""
if self._temp_dir is not None:
self._temp_dir.delete_all_content(delete_self=True)
# =============================================================================
# Polygon fetch: v3 trades / quotes (SIP). Keep ts_ns in ns.
# =============================================================================
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1",
label=lambda ticker, d: f"has_any_quotes_or_trades/{ticker}/{fmt_date(d)}",
exclude_args=["self"],
)
def has_any_quotes_or_trades(self, ticker: str, d: date) -> tuple[bool, bool]:
"""Whether ``ticker`` has any quotes or trades on date ``d``. Returns tuple (has_quotes, has_trades)."""
url = f"{MassiveClient.BASE_URL}/v3/quotes/{ticker}"
response = self.get(
url,
params={
"timestamp": d.strftime("%Y-%m-%d"), # YYYY-MM-DD
"limit": 1,
"order": "asc",
"sort": "timestamp",
},
)
has_quotes = len(response["results"]) > 0
url = f"{MassiveClient.BASE_URL}/v3/trades/{ticker}"
response = self.get(
url,
params={
"timestamp": d.strftime("%Y-%m-%d"), # YYYY-MM-DD
"limit": 1,
"order": "asc",
"sort": "timestamp",
},
)
has_trades = len(response["results"]) > 0
return has_quotes, has_trades
[docs]
def has_any_quotes(self, ticker: str, d: date) -> bool:
"""Whether ``ticker`` has any quotes on date ``d``."""
return self.has_any_quotes_or_trades(ticker, d)[0]
[docs]
def has_any_trades(self, ticker: str, d: date) -> bool:
"""Whether ``ticker`` has any trades on date ``d``."""
return self.has_any_quotes_or_trades(ticker, d)[1]
SCHEMA_TRADES = {
"ts_ns": pl.Datetime("ns", TZ_NY),
"price": pl.Float64,
"size": pl.Float64,
"trade_code": pl.UInt32,
"sequence_number": pl.UInt64,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`fetch_trades`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.11", label=lambda ticker, d: f"trades/{ticker}/{fmt_date(d)}", exclude_args=["self"]
)
def fetch_trades(self, ticker: str, d: date) -> pl.DataFrame:
"""
Gets trades from /v3/trades/{ticker} sorted by sip_timestamp asc.
Used both for options and underlyings.
The returned :class:`polars.DataFrame` has the following columns:
* ts_ns: Datetime[ns, tz]: time zone information with nanonsecond resolutuin.
* price: Float64
* size: Float64
* trade_code: UInt32. Compressed & Packed trade conditions, see :func:`quantdata.massive.trade_conds.pack_conditions`.
It is recommended to remove trades with uncommon conditions, see :attr:`quantdata.massive.trade_conds.NORMAL_TRADE_EXCLUDE_FLAGS`.
* sequence_number: UInt64. Sequence number of the trade for a given timestamp, useful for ordering trades with the same timestamp.
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL230616C00150000" for Apple 150 call expiring on 2023-06-16.
d : date
Date for which to fetch trades.
Returns
-------
Result : :class:`polars.DataFrame`
Returns a data frame with schema :attr:`MassiveClient.SCHEMA_TRADES`.
The dataframe can have length zero (but it will have the correct data schema).
* ts_ns: Datetime[ns, tz]
* price: Float64
* size: Float64
* trade_code: UInt32. Compressed & Packed trade conditions, see :func:`quantdata.massive.trade_conds.pack_conditions`.
* sequence_number: UInt64. Sequence number of the trade for a given timestamp. Trades will be sorted by (ts_ns, sequence_number) asc.
"""
url = f"{self.BASE_URL}/v3/trades/{ticker}"
params = {
"timestamp": d.strftime("%Y-%m-%d"),
"order": "asc",
"limit": 50000,
"sort": "sip_timestamp",
}
rows = self.paginate(url, params)
rows = [
dict(
price=float(r["price"]),
size=float(r["size"]),
sip_timestamp=r["sip_timestamp"],
trade_code=pack_conditions(r.get("conditions", [])),
sequence_number=int(r["sequence_number"]),
)
for r in rows
]
if not rows:
return pl.DataFrame(schema=MassiveClient.SCHEMA_TRADES)
df = pl.DataFrame(rows, schema={
'price': pl.Float64,
'size': pl.Float64,
'sip_timestamp': pl.Int64,
'trade_code': pl.UInt32,
'sequence_number': pl.UInt64,
}, infer_schema_length=None)
out = df.select(
pl.from_epoch(pl.col("sip_timestamp").cast(pl.Int64), time_unit="ns")
.dt.convert_time_zone(TZ_NY.key)
.alias("ts_ns"),
pl.col("price").cast(pl.Float64),
pl.col("size").cast(pl.Float64),
pl.col("trade_code").cast(pl.UInt32),
pl.col("sequence_number").cast(pl.UInt64)
).sort("ts_ns", "sequence_number")
assert set(out.columns) == set(MassiveClient.SCHEMA_TRADES), (
f"Internal error: expected columns {set(MassiveClient.SCHEMA_TRADES)} but got {set(out.columns)}. Sanity check the request and contact the authors if you need help."
)
return out
SCHEMA_QUOTES = {
"ts_ns": pl.Datetime("ns", TZ_NY),
"bid_price": pl.Float64,
"bid_size": pl.Float64,
"ask_price": pl.Float64,
"ask_size": pl.Float64,
"sequence_number": pl.UInt64,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`fetch_quotes`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.11", label=lambda ticker, d: f"quotes/{ticker}/{fmt_date(d)}", exclude_args=["self"]
)
def fetch_quotes(self, ticker: str, d: date) -> pl.DataFrame:
"""
Fetches quotes from "/v3/quotes/{ticker}" sorted by sip_timestamp (and sequence_number) asc.
Used both for options and underlyings.
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL230616C00150000" for Apple 150 call expiring on 2023-06-16.
d : date
Date for which to fetch quotes.
Returns
-------
Result : :class:`polars.DataFrame`
Returns a data frame with schema :attr:`MassiveClient.SCHEMA_QUOTES`.
The dataframe can have length zero (but it will have the correct data schema).
* ts_ns: Datetime[ns, tz]
* bid_price: Float64
* bid_size: Float64
* ask_price: Float64
* ask_size: Float64
* sequence_number: UInt64. Sequence number of the quote for a given timestamp. Quotes will be sorted by (ts_ns, sequence_number) asc.
"""
url = f"{self.BASE_URL}/v3/quotes/{ticker}"
params = {
"timestamp": d.strftime("%Y-%m-%d"),
"order": "asc",
"limit": 50000,
"sort": "sip_timestamp",
}
rows = self.paginate(url, params)
if not rows:
return pl.DataFrame(schema=MassiveClient.SCHEMA_QUOTES)
df = pl.DataFrame(rows, schema={
# https://massive.com/docs/rest/options/trades-quotes/quotes#
"bid_price": pl.Float64,
"bid_size": pl.Float64,
"ask_price": pl.Float64,
"ask_size": pl.Float64,
"sip_timestamp": pl.Int64,
"sequence_number": pl.UInt64
}, infer_schema_length=None)
out = df.select(
pl.from_epoch(pl.col("sip_timestamp").cast(pl.Int64), time_unit="ns")
.dt.convert_time_zone(TZ_NY.key)
.alias("ts_ns"),
pl.col("bid_price").cast(pl.Float64),
pl.col("bid_size").cast(pl.Float64),
pl.col("ask_price").cast(pl.Float64),
pl.col("ask_size").cast(pl.Float64),#
pl.col("sequence_number").cast(pl.UInt64)
).sort("ts_ns", "sequence_number")
assert set(out.columns) == set(MassiveClient.SCHEMA_QUOTES), (
f"Internal error: expected columns {set(MassiveClient.SCHEMA_QUOTES)} but got {set(out.columns)}. Sanity check the request and contact the authors if you need help."
)
return out
SCHEMA_BARS = {
"ts_ms": pl.Datetime("ms", TZ_NY),
"close": pl.Float64,
"open": pl.Float64,
"high": pl.Float64,
"low": pl.Float64,
"volume": pl.Float64,
"vwap": pl.Float64,
"num_trades": pl.Int64,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`fetch_bars`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",
uid=lambda ticker, start, end, adjusted, timespan: (
f"bars/{timespan}/{'adj' if adjusted else 'raw'}/{ticker}/{fmt_date(start)}_{fmt_date(end)}_{timespan}"
),
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def fetch_bars(
self, ticker: str, start: date, end: date, adjusted: bool, timespan: str
) -> pl.DataFrame:
"""
Download daily bars for a ticker accross a number of dates; convert timestamp to datetime in NY/ns.
See `custom bars documentation<https://massive.com/docs/rest/stocks/aggregates/custom-bars>`_
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL" for Apple.
start : :class:`datetime.date`
Start date for bars.
end : :class:`datetime.date`
End date for bars.
adjusted : bool
Whether to fetch adjusted bars.
timespan : str
Timespan for the bars, e.g. "day", "week", "month".
Must be one of 'second', 'minute', 'hour', 'day', 'week', 'month', 'quarter', 'year'.
Returns
-------
Result : :class:`polars.DataFrame`
Data frame wiwh schema :attr:`MassiveClient.SCHEMA_BARS`. The dataframe can have length zero (but it will have the correct data schema).
- ts_ms: Datetime[ms, tz]: time zone information with millisecond precision
- close: Float64
- open: Float64
- high: Float64
- low: Float64
- volume: Float64
- vwap: Float64
- num_trades: Int64
"""
if timespan not in self.TIME_SPANS:
raise ValueError(
f"Cannot retrieve bars for {ticker} between {start} and {end}: invalid timespan '{timespan}'. Must be one of {sorted(self.TIME_SPANS)}"
)
if start > end:
raise ValueError(
f"Cannot retrieve bars for {ticker} between {start} and {end}: start date must be before end date."
)
url = f"{self.BASE_URL}/v2/aggs/ticker/{ticker}/range/1/{timespan}/{start:%Y-%m-%d}/{end:%Y-%m-%d}"
rows = self.paginate(
url,
params={"adjusted": "false" if not adjusted else "true", "limit": 50000, "sort": "asc"},
)
if not rows:
return pl.DataFrame(schema=MassiveClient.SCHEMA_BARS)
df = pl.DataFrame(rows, infer_schema_length=None, schema={
# https://massive.com/docs/rest/stocks/aggregates/custom-bars
'c': pl.Float64,
'h': pl.Float64,
'l': pl.Float64,
'n': pl.Int64,
'o': pl.Float64,
't': pl.Int64,
'v': pl.Float64,
'vw': pl.Float64,
})
out = df.select(
pl.from_epoch(pl.col("t").cast(pl.Int64), time_unit="ms")
.dt.replace_time_zone("UTC")
.dt.convert_time_zone(TZ_NY.key)
.alias("ts_ms"),
pl.col("c").cast(pl.Float64).alias("close"),
pl.col("o").cast(pl.Float64).alias("open"),
pl.col("h").cast(pl.Float64).alias("high"),
pl.col("l").cast(pl.Float64).alias("low"),
pl.col("v").cast(pl.Float64).alias("volume"),
pl.col("vw").cast(pl.Float64).alias("vwap"),
pl.col("n").cast(pl.Int64).alias("num_trades"),
).sort("ts_ms")
assert set(out.columns) == set(MassiveClient.SCHEMA_BARS), (
f"Internal error: expected columns {set(MassiveClient.SCHEMA_BARS)} but got {set(out.columns)}. Sanity check the request and contact the authors if you need help."
)
return out
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1",
label=lambda ticker, d, adjusted, can_be_none: (
f"open-close/{ticker}/{fmt_date(d)} {'a1' if adjusted else 'a0'} {'c0' if can_be_none else 'c1'}"
),
exclude_args=["self"],
)
def fetch_day_summary(
self, ticker: str, d: date, *, adjusted: bool, can_be_none: bool = False
) -> dict | None:
"""
Fetch open/close/volume/high/low for ``ticker`` on date ``d``.
Returns dict with keys: open, close, low, high, volume, adjusted and optionally afterHours and preMarket if available.
See `daily ticker summary documentation <https://massive.com/docs/rest/options/aggregates/daily-ticker-summary>`_.
If no trade was recorded raise an exception except of ``can_be_none`` is True,
in which case return ``None``.
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL230616C00150000" for Apple 150 call expiring on 2023-06-16.
d : date
Date for which to fetch open/close/volume.
adjusted : bool
Whether to fetch adjusted or unadjusted open/close/volume.
can_be_none : bool, default False
If True, return None if no trade was recorded for the day. If False, raise an exception if no trade was recorded for the day.
Returns
-------
Result : dict
Dict with keys:
* ``open [float]``
* ``close [float]``
* ``low [float]``
* ``high [float]``
* ``volume [float]``
* ``adjusted [bool]``
* ``afterHours [float|None]`` (optional)
* ``preMarket [float|None]`` (optional)
"""
url = f"{self.BASE_URL}/v1/open-close/{ticker}/{d:%Y-%m-%d}"
try:
data = self.get(url, {"adjusted": "false" if not adjusted else "true"})
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404 and can_be_none:
return None
raise requests.exceptions.HTTPError(
f"No open/close/volume for {ticker} on {d} url {url}: {e.response.text}",
response=e.response,
request=e.request,
) from e
# Polygon open-close has "open","close","volume" (and more)
assert not "next_url" in data, (
f"Internal error: pagination not expected for open-close endpoint, url '{url}' response '{data}'. Sanity check request & contact the authors."
)
open = data.get("open")
close = data.get("close")
volume = data.get("volume")
high = data.get("high")
low = data.get("low")
has_data = (
open is not None
and close is not None
and volume is not None
and high is not None
and low is not None
)
if not has_data:
if can_be_none:
return None
raise RuntimeError(
f"Cannot get {'adjusted' if adjusted else 'unadjusted'} open/close/volume/high/low for {ticker} on {d}. Massive URL used: '{url}'."
)
def get_optional(item, type=float):
x = data.get(item, None)
return x if x is None else type(x)
return {
"open": float(open),
"close": float(close),
"volume": float(volume),
"high": float(high),
"low": float(low),
"afterHours": get_optional("afterHours"),
"preMarket": get_optional("preMarket"),
"adjusted": adjusted,
}
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1",
label=lambda underlying, d, max_dte: (
f"all_contracts/max_dte={max_dte if not max_dte is None else 'all'}/{underlying}/{fmt_date(d)}"
),
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def list_all_contracts_expiring_after_day(
self, underlying: str, d: date, max_dte: int = None
) -> list[dict]:
"""
Download ALL option contracts with expiration_date at or past ``d``, with an optional upper bound.
Parameters
----------
underlying : str
Underlying ticker symbol, e.g. "AAPL" for Apple.
d : date
Date for which to fetch contracts. Only contracts with expiration_date >= d are returned.
max_dte : int | None, default ``None``
If not None, only return contracts with ``expiration_date <= d + max_dte`` in business days.
This is computed using :func:`quantdata.biz_calendar.add_business_days` with the NYSE calendar *including half days*.
Returns:
-------
Result : dict
See `v3/reference/options/contracts documentation <https://massive.com/docs/rest/options/contracts>`_.
"""
url = f"{self.BASE_URL}/v3/reference/options/contracts"
params = {
"underlying_ticker": underlying,
"as_of": d.strftime("%Y-%m-%d"),
"expiration_date.gte": d.strftime("%Y-%m-%d"), # at or after today
"limit": 1000,
"order": "asc",
"sort": "expiration_date",
}
if not max_dte is None:
max_expiry = add_business_days(d, max_dte, exclude_half_days=False)
verify_inp(
max_expiry >= d,
f"'max_dte' {max_dte} leads to max_expiry {max_expiry} before date {d}",
)
params["expiration_date.lte"] = max_expiry.strftime("%Y-%m-%d")
return self.paginate(url, params)
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1",
uid=lambda opt_ticker, as_of: f"opt_ref_date/{named_unique_filename48_8(str(opt_ticker) if as_of is None else f'{opt_ticker} {as_of}')}",
)
def option_contract_overview( self, opt_ticker: str, as_of : date | None = None ) -> dict:
"""
Download option `contract overview data <https://massive.com/docs/rest/options/contracts/contract-overview>`_
for a given options ticker.
Parameters
----------
opt_ticker : str
Option ticker symbol, e.g. "AAPL230616C00150000" for Apple 150 call expiring on 2023-06-16.
as_of : :class:`datetime.date` | None, default None
If not None, fetch the contract overview as of this date. Otherwise, fetch the latest.
Returns
-------
Result : dict
See `v3/reference/options/contract overview documentation <https://massive.com/docs/rest/options/contracts/contract-overview>`_.
"""
url = f"{self.BASE_URL}/v3/reference/options/contracts/{opt_ticker}{'' if as_of is None else f'?as_of={as_of.isoformat()}'}"
r = self.get(url)
return r['results']
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1",
uid=lambda underlying, d, expiry: f"expiry_contracts/{underlying}/{fmt_date(d)}/{expiry}"
)
def list_all_contracts_for_expiry(
self, underlying: str, d: date, expiry: date
) -> list[dict]:
"""
Download ALL option contracts with expiration_date at or past ``d``, with an optional upper bound.
Parameters
----------
underlying : str
Underlying ticker symbol, e.g. "AAPL" for Apple.
d : date
Date for which to fetch contracts. Only contracts with expiration_date >= d are returned.
expiry : date
Expiration date for which to fetch contracts. Only contracts with expiration_date equal to expiry are returned.
Returns:
-------
Result : dict
See `v3/reference/options/contracts documentation <https://massive.com/docs/rest/options/contracts>`_.
"""
url = f"{self.BASE_URL}/v3/reference/options/contracts"
params = {
"underlying_ticker": underlying,
"as_of": d.strftime("%Y-%m-%d"),
"expiration_date.gte": expiry.strftime("%Y-%m-%d"),
"expiration_date.lte": expiry.strftime("%Y-%m-%d"),
"limit": 1000,
"order": "asc",
"sort": "expiration_date",
}
return self.paginate(url, params)
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",
uid=lambda ticker, start, end: f"adjfactors/{ticker}/{fmt_date(start)}_{fmt_date(end)}",
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def adjustment_factors(self, ticker: str, start: date, end: date) -> pl.DataFrame:
"""
Downloads daily adjusted and unadjusted close, and computes the adjustment factor as ``adj_close / unadj_close``.
See `custom bars documentation <https://massive.com/docs/rest/stocks/aggregates/custom-bars>`_
Returns a Polars frame with:
* date
* close_unadj
* close_adj
* factor_close = close_adj / close_unadj
Notes:
* Uses NY timezone to derive the trading date from 't'.
* Uses only 'close' as requested.
* If you later want volume/cash-vol invariance checks, also download 'v' and compute factors similarly.
"""
verify_inp(
start <= end,
lambda: (
f"Cannot find adjustment factors for {ticker}: dates must be in order; found start {fmt_date(start)} vs end {fmt_date(end)}"
),
)
unadjusted = self.fetch_bars(ticker, start, end, adjusted=False, timespan="day")
adjusted = self.fetch_bars(ticker, start, end, adjusted=True, timespan="day")
if len(unadjusted) == 0 or len(adjusted) == 0:
raise RuntimeError(
f"Cannot find adjustment factors for {ticker} between {fmt_date(start)} and {fmt_date(end)}: no bars found for {'adjusted' if len(adjusted) == 0 else 'unadjusted'} data. Please check that the ticker is correct and that there were trades for this ticker between {fmt_date(start)} and {fmt_date(end)}. If you need help, please contact the authors."
)
unadjusted = (
unadjusted.with_columns(pl.col("ts_ms").dt.date().alias("date"))
.rename({"close": "unadj_close"})
.select(["date", "unadj_close"])
)
adjusted = (
adjusted.with_columns(pl.col("ts_ms").dt.date().alias("date"))
.rename({"close": "adj_close"})
.select(["date", "adj_close"])
)
# Join + factor
out = (
unadjusted.join(adjusted, on="date", how="outer")
.sort("date")
.with_columns(
(pl.col("adj_close") / pl.col("unadj_close")).cast(pl.Float32).alias("adjfactor")
)
)
err = out.filter(pl.col("adjfactor").is_infinite() | pl.col("adjfactor").is_nan())
if err.height > 0:
print(err.head()) # print stats to stdout
raise RuntimeError(
f"Unexpected data error: found infinite/NaN adjustment factor for ticker {ticker} between {fmt_date(start)} and {fmt_date(end)}. This likely indicates a data error in the underlying bars, e.g. a missing unadjusted close price. Please check the underlying bars and contact the authors if you need help."
)
out = out.select(["date", "adjfactor"])
return out
[docs]
@_cache(fmt=SubDir.PICKLE).cache(
"0.1", uid=lambda ticker: f"reference/{ticker}", exclude_args=["self"]
) # 'self' is excluded fromm the arguments as it does not affect the output.
def ticker_reference_data(self, ticker: str) -> dict:
"""
Return reference data for a given ticker.
See `ticker overview documentation <https://massive.com/docs/rest/stocks/tickers/ticker-overview>`_ for details.
This function converts boolean, integer, floats and dates.
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL230616C00150000" for Apple 150 call expiring on 2023-06-16.
Returns
-------
Result : dict
Dict with reference data for the ticker. See `ticker overview documentation <https://massive.com/docs/rest/stocks/tickers/ticker-overview>`_ for details.
The following conversions will be performed:
* active [bool|None]
* list_date [date|None]
* delisted_utc [date|None]
* market_cap [float|None]
* round_lot [int|None]
* total_employees [int|None]
* weighted_shares_outstanding [int|None]
* share_class_shares_outstanding [int|None]
Raises
------
Ticker not found: :class:`FileNotFoundError`
If the request for reference data from massive did not yield any results.
"""
url = f"{self.BASE_URL}/v3/reference/tickers/{ticker}"
response = self.get(url)
results = response.get("results", {})
if len(results) == 0:
raise FileNotFoundError(
f"No reference data found for ticker {ticker} on massive.com. URL {url}."
)
bool_fields = ["active"]
date_fields = ["list_date", "delisted_utc"]
float_fields = ["market_cap"]
int_fields = [
"round_lot",
"total_employees",
"weighted_shares_outstanding",
"share_class_shares_outstanding",
"sic_code",
]
for f in bool_fields:
if f in results:
results[f] = bool(results[f])
for f in date_fields:
if f in results and results[f] is not None:
results[f] = date.fromisoformat(results[f])
for f in float_fields:
if f in results and results[f] is not None:
results[f] = float(results[f])
for f in int_fields:
if f in results and results[f] is not None:
results[f] = int(results[f])
return results
# ----------------------------
# corporate events etc
# ----------------------------
def _process_massive(
self,
what,
rows: list[dict],
schema: dict[str, pl.DataType],
massive_schema : dict[str, pl.DataType], # massive data schema; not necessarily the same as output schema
sort_by: list[str] | str,
drop_ticker: bool,
) -> pl.DataFrame:
"""
Helper function to process corporate events data (dividends, splits) and convert to Polars DataFrame with correct schema.
"""
if not rows:
return pl.DataFrame(schema=schema)
df = pl.DataFrame(rows, massive_schema, infer_schema_length=None)
if drop_ticker:
df = df.drop("ticker")
for k, t in schema.items():
if k not in df.columns:
df = df.with_columns(pl.lit(None).cast(t).alias(k))
elif t == pl.Date:
df = df.with_columns(pl.col(k).str.strptime(pl.Date, "%Y-%m-%d").alias(k))
else:
df = df.with_columns(pl.col(k).cast(t).alias(k))
df = df.sort(sort_by)
assert set(df.columns) == set(schema), (
f"Internal error {what()}: expected columns {set(schema)} but got {set(df.columns)}. Sanity check the request and contact the authors if you need help."
)
return df
SCHEMA_DIVIDENDS = {
"cash_amount": pl.Float64,
"currency": pl.String,
"declaration_date": pl.Date,
"distribution_type": pl.String,
"ex_dividend_date": pl.Date,
"frequency": pl.Int16,
"historical_adjustment_factor": pl.Float64,
"id": pl.String,
"pay_date": pl.Date,
"record_date": pl.Date,
"split_adjusted_cash_amount": pl.Float64,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`get_dividends`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",
uid=lambda ticker, start, end: f"dividends/{ticker}/{fmt_date(start)}_{fmt_date(end)}",
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def get_dividends(self, ticker: str, start: date, end: date) -> pl.DataFrame:
"""
Retrieve dividends which go ex div between two dates from `/stocks/v1/dividends < https://massive.com/docs/rest/stocks/corporate-actions/dividends>`_.
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL" for Apple.
start : :class:`datetime.date`
Start date for dividends.
end : :class:`datetime.date`
End date for dividends.
Returns
-------
:class:`polars.DataFrame`
DataFrame with dividend information using schema :class:`MassiveClient.SCHEMA_DIVIDENDS`.
The dataframe can have length zero (but it will have the correct data schema).
"""
verify_inp(
start <= end,
lambda: (
f"Cannot find adjustment factors for {ticker}: dates must be in order; found start {fmt_date(start)} vs end {fmt_date(end)}"
),
)
url = f"{self.BASE_URL}/stocks/v1/dividends"
params = {
"ticker": ticker,
"ex_dividend_date.gte": start.strftime("%Y-%m-%d"),
"ex_dividend_date.lte": end.strftime("%Y-%m-%d"),
"limit": 1000,
"order": "asc",
"sort": "ex_dividend_date",
}
rows = self.paginate(url, params)
return self._process_massive(
what=lambda: f"dividends for {ticker} between {fmt_date(start)} and {fmt_date(end)}",
rows=rows,
schema=MassiveClient.SCHEMA_DIVIDENDS,
massive_schema={
# https://massive.com/docs/rest/stocks/corporate-actions/dividends
'cash_amount': pl.Float64,
'currency': pl.String,
'declaration_date': pl.String,
'distribution_type': pl.Int64,
'ex_dividend_date': pl.String,
'frequency': pl.Int16,
'historical_adjustment_factor': pl.Float64,
'id': pl.String,
'pay_date': pl.String,
'record_date': pl.String,
'split_adjusted_cash_amount': pl.Float64,
'ticker': pl.String,
},
sort_by="ex_dividend_date",
drop_ticker=True,
)
SCHEMA_SPLITS = {
"adjustment_type": pl.String,
"execution_date": pl.Date,
"historical_adjustment_factor": pl.Float64,
"id": pl.String,
"split_from": pl.Int32,
"split_to": pl.Int32,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`get_splits`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",
uid=lambda ticker, start, end: f"splits/{ticker}/{fmt_date(start)}_{fmt_date(end)}",
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def get_splits(self, ticker: str, start: date, end: date) -> pl.DataFrame:
"""Retrieve stock splits between two dates from `/stocks/v1/splits <https://massive.com/docs/rest/stocks/corporate-actions/splits>`_."""
url = f"{self.BASE_URL}/stocks/v1/splits"
params = {
"ticker": ticker,
"execution_date.gte": start.strftime("%Y-%m-%d"),
"execution_date.lte": end.strftime("%Y-%m-%d"),
"limit": 1000,
"order": "asc",
"sort": "execution_date",
}
rows = self.paginate(url, params)
return self._process_massive(
what=lambda: f"splits for {ticker} between {fmt_date(start)} and {fmt_date(end)}",
rows=rows,
schema=MassiveClient.SCHEMA_SPLITS,
massive_schema={
# https://massive.com/docs/rest/stocks/corporate-actions/splits
'adjustment_type': pl.String,
'execution_date': pl.Date,
'historical_adjustment_factor': pl.Float64,
'id': pl.String,
'split_from': pl.Int32,
'split_to': pl.Int32,
'ticker': pl.String,
},
sort_by="execution_date",
drop_ticker=True,
)
SCHEMA_HAS_TICKER_CHANGES = {
"date": pl.Date,
"has_ticker_change": pl.Boolean,
}
""" Schema of the :class:`polars.DataFrame` returned by :meth:`has_ticker_changes`. """
[docs]
@_cache(fmt=SubDir.POLARS_PARQUET).cache(
"0.1",
uid=lambda ticker, start, end: f"ticker_changes/{ticker}/{fmt_date(start)}_{fmt_date(end)}",
exclude_args=["self"],
) # 'self' is excluded fromm the arguments as it does not affect the output.
def has_ticker_changes(self, ticker: str, start: date, end: date) -> pl.DataFrame:
"""
Find dates at which an underlying had ticker changes.
If the endpoint is not available return an empty frame.
At the moment ticker changes are `the only events massive.com supports <https://massive.com/docs/rest/stocks/corporate-actions/ticker-events>`_.
"""
client = self.get_rest_client()
events = client.get_ticker_events(ticker)
events = events.events
if len(events) == 0:
return pl.DataFrame(schema=MassiveClient.SCHEMA_HAS_TICKER_CHANGES)
df = pl.DataFrame(events, schema={
#https://massive.com/docs/rest/stocks/corporate-actions/ticker-events
"date": pl.String,
"event_type": pl.String,
"ticker_change": pl.Object
})
event_types = set(df.select(pl.col("type").unique()).get_column("type").to_list())
assert event_types == {"ticker_change"}, (
f"Internal error retrieving ticker changes for {ticker} between {fmt_date(start)} and {fmt_date(end)}: expected event type 'ticker_change' but got {sorted(event_types)}. Please update code. Ref: https://massive.com/docs/rest/stocks/corporate-actions/ticker-events"
)
df = df.select(
pl.col("date").str.slice(0, 10).str.strptime(pl.Date, "%Y-%m-%d").alias("date")
)
# filter by requested window
df = df.filter((pl.col("date") >= start) & (pl.col("date") <= end))
if df.is_empty():
return pl.DataFrame(schema=MassiveClient.SCHEMA_HAS_TICKER_CHANGES)
df = (
df.unique(subset=["date"])
.sort("date")
.with_columns(pl.lit(True).alias("has_ticker_change"))
)
df = df.sort("date")
assert df.height > 0
assert set(df.columns) == set(MassiveClient.SCHEMA_HAS_TICKER_CHANGES), (
f"Internal error retrieving ticker changes for {ticker} between {fmt_date(start)} and {fmt_date(end)}: expected columns {set(MassiveClient.SCHEMA_HAS_TICKER_CHANGES)} but got {set(df.columns)}. Please update code. Ref: https://massive.com/docs/rest/stocks/corporate-actions/ticker-events"
)
return df
[docs]
@version("0.1")
def consistent_prev_close( self, ticker : str, d : date, *, prev_d : date|None = None, exclude_half_days: bool = True ) -> float:
r"""
Computes the previous day close consistent with today's unadjusted prices.
It computes
$$
S_{\mathrm{t-1}} := S^{\mathrm{adjusted}}_{\mathrm{t-1}} \cdot \frac{S^{\mathrm{unadjusted}}_{\mathrm{t}}}{S^{\mathrm{adjusted}}_{\mathrm{t}}}
$$
Parameters
----------
ticker : str
Ticker symbol, e.g. "AAPL".
d : :class:`datetime.date`
Date for which to compute the previous close.
prev_d : :class:`datetime.date` | None, default None
If not None, use this date as the previous day instead of computing it as the previous business day.
exclude_half_days : bool, default ``True``
Whether to exclude half days when computing the previous business day.
This is used if ``prev_d`` is ``None``.
Returns
-------
Close : ``float``
The previous day's close, consistent with today's unadjusted prices.
"""
prev_d = prev_d if not prev_d is None else add_business_days( d, -1, exclude_half_days=exclude_half_days ) # previous business day (NB: if d is a Monday then this is the previous Friday, so we correctly skip the weekend)
r = self._PREV_CLOSE_CACHE.get((ticker, d, prev_d), None)
if r is not None:
return r
close_adj_prev = self.fetch_day_summary( ticker, prev_d, adjusted=True, can_be_none=False )["close"]
close_unadj = self.fetch_day_summary( ticker, d, adjusted=False, can_be_none=False )["close"]
close_adj = self.fetch_day_summary(ticker, d, adjusted=True, can_be_none=False)[ "close" ]
prev_adj_close = close_adj_prev / close_adj * close_unadj
self._PREV_CLOSE_CACHE[(ticker, d, prev_d)] = prev_adj_close
return prev_adj_close