Source code for quantdata.torch.ffw

"""
Fast Forward Network
"""

from __future__ import annotations
from quantdata.torch import BaseModel, BaseEnvironment, torch, tnn
from collections.abc import Callable
from dataclasses import dataclass
from cdxcore.err import verify_inp
import math as math
import numpy as nnp

[docs] @dataclass(frozen=True) class FeedForwardConfig: """ Configuration for FeedForward. This is a separate class so it can be imported without importing torch. The default is a one-layer linear network with a width of 16. """ activation : str = 'Softplus' # Activation module; see https://docs.pytorch.org/docs/stable/nn.html num_depth : int = 0 # Depth; if zero then the network becomes a single linear layer. num_width : int = 16 # Inner width; use a negative number as multiplier of ``num_output`` (output dimension). If ``num_depth`` is zero, then this is ignored. use_bias : bool = True # Whether to use bias in the inner layers. If ``num_depth`` is zero, then this is ignored. def __str__(self) -> str: return f"FF(d={self.num_depth},w={self.num_width},a={self.activation},b={self.use_bias})" def __post_init__(self): verify_inp( not isinstance(self.activation,str) or hasattr(torch.nn, self.activation), f"Unknown torchh activation function 'torch.nn.{self.activation}' see https://docs.pytorch.org/docs/stable/nn.html") verify_inp( self.num_depth >= 0, "'num_depth' must be non-negative" ) verify_inp( self.num_width != 0, "'num_width' cannot be zero")
[docs] def get_activation(self) -> Callable: """ Retrieves the requested activation function """ if not isinstance(self.activation, str): verify_inp( callable(self.activation), "'activation' must be a string" ) return self.activation f = getattr(torch.nn, self.activation, None) verify_inp( not f is None, f"Cannot find activation function 'torch.nn.{self.activation}'" ) return f()
[docs] @staticmethod def linear_config( activation : str = 'Identity' ) -> FeedForwardConfig: """ Returns a config for a simple one-layer linear network with the specified activation. """ return FeedForwardConfig( activation=activation, num_depth=0, num_width=16, use_bias=True )
[docs] class FeedForward(BaseModel): """ Dense network applies to list of inputs (instead of a stack). Essentially this function does:: y: input as list, implicitly y=stack(y,axis=-1) x = L0( y ) for d in num_depth: x = activation( x ) x = Ld( x ) if use_final_activation: x = activation( x ) x = mask & x where ``Li`` is a set of linear layers applied to each element in the list. (c)copyright 2019, Hans Buehler Parameters ---------- env : :class:`quantdata.torch.BaseEnvironment` Environment of the model which includes the random generator, default floating point dtype, and whether to run invasive debugging. num_input : int Number of inputs, which will be expected as list of the form ``[ (A,...,B) ]``. Numerically equivalent to a stack of dimension ``(A,....B,num_input)``. num_output : int | tuple[int] Last dimension of output ie output is ``(A,...,B,num_output)``. Set to zero to return ``(A,...,B)``. Accepts a tuple for multi-dimensional output, see :class:`torch.nn.Linear`. config : :class:`quantdata.torch.ffw.FeedForwardConfig`, default is a one-layer linear network. User config of the network. use_final_bias : bool, default ``True`` whether the final linear layer adds a bias. use_final_activation : bool, default ``False`` Whether to apply activation after the final linear layer. """ Config = FeedForwardConfig def __init__(self, env : BaseEnvironment, num_input : int, # number of inputs [ (A,...,B) ] ~ (A,....B,num_input) num_output : int|tuple[int], # last dimension of output ie output is (A,...,B,num_output). Set to zero to return (A,...,B). Can handle tuple for multi-dimensional output. config : FeedForwardConfig = FeedForwardConfig(), *, # config, see above use_final_bias : bool = True, # whether the final linear layer adds a bias use_final_activation : bool = False # whether to apply activation after the final linear layer ): super().__init__( env=env ) verify_inp( num_input > 0, "num_input must be positive" ) num_output = num_output if isinstance(num_output, int) else tuple(num_output) if isinstance(num_output,int): verify_inp( num_output >= 0, "num_output must be non-negative" ) else: verify_inp( all( n > 0 for n in num_output ), lambda : f"'num_output' must be a non-negative integer or a tuple with all dimensions positive. Found {num_output}" ) self.num_output = num_output self.num_input = num_input self.activation = config.get_activation() self.use_final_activation = use_final_activation output_dim = num_output if isinstance(num_output, int) else math.prod(num_output) if config.num_depth > 0: num_width = config.num_width if config.num_width > 0 else ( abs(config.num_width) * max(1, output_dim) ) else: # for a zero depth network, width is just the output dimension # note that num_output==0 means that the output dimension is 1 # but it will be reduced to zero at the end of the forward pass num_width = output_dim if output_dim > 0 else 1 use_bias = config.use_bias if config.num_depth > 0 else use_final_bias self.I0 = tnn.Linear( in_features=num_input, out_features=num_width, bias=use_bias, device=env.device, dtype=env.fdtype) ##OLD self.I0 = tnn.ModuleList( [ tnn.Linear( in_features=1, out_features=num_width, bias=use_bias, device=env.device, dtype=env.fdtype) for i in range(num_input) ] ) if config.num_depth > 0: In = [ tnn.Linear( in_features=num_width, out_features=num_width, bias=use_bias, device=env.device, dtype=env.fdtype ) for d in range(config.num_depth-1) ] In.append( tnn.Linear( in_features=num_width, out_features=output_dim if output_dim > 0 else 1, bias=use_final_bias, device=env.device, dtype=env.fdtype ) ) self.In = tnn.ModuleList( In ) #del In else: self.In = []
[docs] def forward( self, input : list[torch.Tensor]|torch.Tensor, mask : torch.Tensor|None = None ) -> torch.Tensor: """ Compute masked fast forward network. Parameters ---------- input : list[torch.Tensor] | torch.Tensor Either a list with ``num_input`` elements ``(A,...,B)``, or a ``torch.Tensor`` of dimension ``(A,...,B,num_input)``. mask : torch.Tensor | None: A mask of shape ``(A,...,B)`` to apply to the input, or ``None``. If provided, the mask is applied to each input element as ``input = torch.where(mask[...,None], input, 0)``. This can be used to zero out certain inputs without changing the shape of the output. """ def lin_has_finite_weights( lin : tnn.Linear ) -> bool: if lin.bias is None: return torch.isfinite(lin.weight).all() return torch.isfinite(lin.weight).all() & torch.isfinite(lin.bias).all() if isinstance(input, torch.Tensor ): assert input.shape[-1] == self.num_input, f"Input shape mismatch: expected last dimension to be {self.num_input}, but found {input.shape}" x = torch.where( mask[...,None], input, 0.) if not mask is None else input else: assert not isinstance(input, nnp.ndarray), f"Input of type {type(input)} is not supported" assert len(input) == self.num_input, f"Input length mismatch: expected {self.num_input} elements in the input list, but found {len(input)}" shape = torch.broadcast_shapes( *[x.shape for x in input] ) x = torch.stack([ torch.where(mask, x, 0.).expand(shape) for x in input], dim=-1) if not mask is None else\ torch.stack([ x.expand(shape) for x in input], dim=-1) self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: not finite input 'x' after masking and stacking" ) x = self.I0(x) """ OLD input = [ input[...,i] for i in range(input.shape[-1]) ] if isinstance(input, torch.Tensor ) else input verify_inp( len(input) == self.num_input, lambda : f"Input length error: expected {self.num_input}, got {len(input)}" ) self.verify( lambda : torch.any(mask), lambda : f"{FeedForward}: Mask is all False, no valid input to process" ) if not mask is None else None x = None for iinp, (inp, I0) in enumerate(zip(input, self.I0)): inp = inp if mask is None else torch.where( mask, inp, 0. ) self.verify( lambda : torch.isfinite(inp), lambda : f"{FeedForward}: Input #{iinp} is not finite" ) self.verify( lambda : lin_has_finite_weights(I0), lambda : f"{FeedForward}: Input #{iinp} weights or bias are not finite" ) inp = I0( inp[...,None] ) self.verify( lambda :torch.isfinite(inp).all(), lambda : f"{FeedForward}: Iteration #{iinp} activation 'inp' is not finite" ) x = inp if x is None else (x+inp) #del inp #del input self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Composed input 'x' is not finite" ) """ for iI, I in enumerate(self.In): x = self.activation(x) self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Iteration #{iI} activation 'x' is not finite" ) self.verify( lambda : lin_has_finite_weights(I), lambda : f"{FeedForward}: Input #{iI} weights or bias are not finite" ) x = I(x) self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Iteration #{iI} 'x' is not finite" ) if self.use_final_activation: x = self.activation(x) self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Final activation 'x' is not finite" ) if self.num_output == 0: assert x.shape[-1] == 1, ("Internal error: output shape mismatch:", x.shape) x = x[...,0] x = torch.where(mask, x, 0.) if not mask is None else x self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Output 'x' is not finite (num_output=0)" ) elif isinstance(self.num_output, int): assert x.shape[-1] == self.num_output, ("Internal error: output shape mismatch:", x.shape, "vs", self.num_output) x = torch.where(mask[...,None], x, 0.) if not mask is None else x self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Output 'x' is not finite (num_output={self.num_output})" ) else: x = x.reshape( x.shape[:-1] + self.num_output ) if not mask is None: m = mask.reshape( mask.shape + (1,)*(len(x.shape)-len(mask.shape)) ) x = torch.where(m, x, 0.) if not mask is None else x self.verify( lambda : torch.isfinite(x), lambda : f"{FeedForward}: Output 'x' is not finite (num_output={self.num_output})" ) return x