Source code for quantdata.dysanos.grid.rndysanos.torch_prep

"""
Whole training setup
"""

import gc
from dataclasses import dataclass
from datetime import date

import numpy as np
from cdxcore.verbose import Context
from prettytable import PrettyTable


[docs] @dataclass(frozen=True) class RNDysanosExperiment: grid_strikes : int = 100 #: The number of strikes per expiry we are matching the market with num_nms : int = 20 #: The nunber of strikes per expiry we are simulating num_alpha : int|None = 5 use_matrix_exp : bool = False bandwidth : float = 0.2 num_steps : int = 10 num_samples : int = 100_000 pcnt_validation : float = 0.1 clip_grad_norm : float|None = None cnt_universe : int = 1 universe_seed_offset : int = 0 def __str__(self): return f"{self.num_samples},steps={self.num_steps},nms={self.num_nms},gs={self.grid_strikes},uni={self.cnt_universe},seed_offset={self.universe_seed_offset}.alpha={self.num_alpha},matrix_exp={self.use_matrix_exp},bandwidth={self.bandwidth},pcnt_validation={self.pcnt_validation},clip_grad_norm={self.clip_grad_norm}"
# ``create_universe`` below already caches each explicit RNG stream. Do not # cache this aggregate function: a ten-stage experiment would duplicate every # source universe in a second multi-universe HDF5 file, and parallel regimes # would race to write that tens-of-gigabytes aggregate.
[docs] def prep_experiment( experiment : RNDysanosExperiment, *, plot_sampler : bool = True ): gc.collect() verbose = Context.all with verbose.write_t("Importing torch...", end='') as tme: import torch del torch verbose.write(f"done; this took {tme}.", head=False) # load market data # ---------------- from quantdata.dysanos.grid.torch_market import ( DateRange, GridConfig, GridMarketDefinition, PoolConfig, ) gc.collect() verbose.write("Starting to load market data...") grid_config = GridConfig(grid_num_strikes=experiment.grid_strikes) grid_market_def = GridMarketDefinition( ticker="SPX", date_range=DateRange(start=date(2020, 1, 1), end=date(2025, 8, 30)), grid_config=grid_config ) market = grid_market_def.load( pool_config = PoolConfig(num_workers=4, threading=False ), override_cache_mode="cacheonly", verbose=verbose(1) ) verbose.write(f"Loaded market data: found {len(market['dates'])} dates.") # load trained model # ------------------ from quantdata.dysanos.grid.torch_gfx import VizConfig from quantdata.dysanos.grid.torch_model import ( BaseEnvironment, FeedForward, SanosDecoderConfig, SanosModelConfig, ) from quantdata.dysanos.grid.torch_train import TrainConfig, train from quantdata.pool import PoolConfig mdl_config = SanosModelConfig( label="ac_state20", user_version="0.2", group="sm25_100", encoder_config = FeedForward.Config( activation = "SELU", num_depth = 2, num_width = 100, use_bias = True, ), decoder_config = SanosDecoderConfig( smoothness = 0.25, num_state = 20, num_nms = experiment.num_nms, nn_config = FeedForward.Config( activation = "SELU", num_depth = 2, num_width = 100, use_bias = True, ), ), ) model = None torch_data = None env = BaseEnvironment(device="cpu", debug=False) viz_config = VizConfig( num_viz_dates=4 ) trn_config = TrainConfig( num_epochs=100_000, lr=1E-4, final_lr=1E-6, weight_decay=1e-4, batch_size=400, compile=True, epoch_interval=10000, clip_grad_norm=experiment.clip_grad_norm ) with verbose.write_t("Loading trained model...", end='') as tme: model_output,\ model,\ torch_data = train( market, mdl_config=mdl_config, trn_config=trn_config, viz_config=viz_config, just_load="plot", env=env ) torch_data = mdl_config.torch_data( env = env, market = market, verbose = Context.quiet ) if torch_data is None else torch_data del model_output, mdl_config, viz_config, env gc.collect() verbose.write("loaded; this took {tme}.", head=False) # AR(1) PCA # --------=- from quantdata.dysanos.grid.torch_simulator import ARSampler, ARSamplerConfig from quantdata.dysanos.grid.torch_simulator import plot_sampler as plot_ar_sampler verbose.write("Computing AR1(1) PCA") ar_config = ARSamplerConfig( num_alpha=experiment.num_alpha, use_matrix_exp=experiment.use_matrix_exp, ) sampler = ARSampler( market, model, torch_data, config=ar_config, verbose=verbose(1) ) gc.collect() verbose.write("AR(1) PCA computed:") sampler.report_system() if plot_sampler: plot_ar_sampler(sampler) # simulate universe # ----------------- from quantdata.dysanos.grid.torch_universe import create_universe universes = [] for i in range(experiment.cnt_universe): verbose.write(f"Simulating universe {i+1}/{experiment.cnt_universe}") # A cache hit in create_universe does not advance sampler.rng. Assign # an explicit per-universe stream so every cache key and path sample is # distinct even when all preceding universes came from cache. sampler.rng = np.random.default_rng( seed=sampler.config.seed + experiment.universe_seed_offset + i ) S = experiment.num_samples TAU = experiment.num_steps-1 T,_ = grid_config.TN N = experiment.num_nms XS = sampler.num_xstate universe = create_universe( model=model, sampler=sampler, market=market, reset_rng=False, num_steps=experiment.num_steps, num_samples=experiment.num_samples, block_size=100000, bandwidth=experiment.bandwidth, verbose=verbose(1)) print(f"Found universe['C_tp1'].shape={universe['C_tp1'].shape}", flush=True) # reduce universe = dict( dC = universe["C_tp1"]-universe["C_t"], dS = universe["spots"][:,1:]-universe["spots"][:,:-1], C = universe["C_t"], spot = universe["spots"][:,:-1], delta = universe["delta_t"], expiries = universe["expiries"][None,None,:].astype(np.float32), log_strikes = np.log( universe["strikes_t"] ), xstate = universe['xstate'].astype(np.float32), ) assert universe["dC"].shape == (S, TAU, T, N), f"universe['dC'].shape={universe['dC'].shape} != {(S, TAU, T, N)}" assert universe["C"].shape == (S, TAU, T, N), f"universe['C'].shape={universe['C'].shape} != {(S, TAU, T, N)}" assert universe["delta"].shape == (S, TAU, T, N), f"universe['delta'].shape={universe['delta'].shape} != {(S, TAU, T, N)}" assert universe["dS"].shape == (S, TAU,), f"universe['dS'].shape={universe['dS'].shape} != {(S, TAU, )}" assert universe["spot"].shape == (S, TAU,), f"universe['spot'].shape={universe['spot'].shape} != {(S, TAU, )}" assert universe["expiries"].shape == (1, 1, T), f"universe['expiries'].shape={universe['expiries'].shape} != {(1, 1, T)}" assert universe["log_strikes"].shape == (S, TAU, T, N), f"universe['log_strikes'].shape={universe['log_strikes'].shape} != {(S, TAU, T, N)}" xstate = universe['xstate'][:,:-1,...] assert xstate.shape == (S, TAU, XS), f"xstate.shape={xstate.shape} != {(S, TAU, XS)}" fields = sorted(universe) tbl = PrettyTable(field_names=["field", "shape", "nbytes", "dtype"]) for k in fields: v = universe[k] tbl.add_row([k, v.shape, v.nbytes, v.dtype]) verbose.write(f"Simulated universe {i+1}/{experiment.cnt_universe}") verbose.report(1,tbl) universes.append(universe) gc.collect() return universes