modelzoo.transformers.pytorch.gptj.gptj_model.GPTJModel

class modelzoo.transformers.pytorch.gptj.gptj_model.GPTJModel

Bases: torch.nn.Module

Methods

forward
get_input_embeddings
get_output_embeddings
reset_parameters
tie_weights

__call__(*args: Any, **kwargs: Any) → Any

Call self as a function.

__init__(hidden_size=768, vocab_size=50257, max_position_embeddings=1024, embd_pdrop=0.1, share_embedding_weights=True, position_embedding_type='rotary', rotary_dim=None, rope_theta=10000, num_relative_attention_buckets=32, num_hidden_layers=12, filter_size=3072, dropout_rate=0.1, nonlinearity='gelu', norm_type='layernorm', layer_norm_epsilon=1e-05, use_ffn_bias=True, use_untied_layer_norm=False, num_heads=12, attention_module='aiayn_attention', extra_attention_params={}, attention_type='scaled_dot_product', attention_dropout_rate=0.1, attention_softmax_fp32=True, use_projection_bias_in_attention=False, use_ffn_bias_in_attention=False, initializer_range=0.02, use_bias_in_output=False, norm_first=True, embedding_initializer=None, attention_initializer=None, output_layer_initializer=None, alibi_trainable_slopes=False, pos_scaling_factor=1.0)

static __new__(cls, *args: Any, **kwargs: Any) → Any

previous

modelzoo.transformers.pytorch.gptj.gptj_model

next

modelzoo.transformers.pytorch.gptj.model