attn_name: torch_attn
ffn_type: swiglu
hidden_dim: 64
max_seq_len: 128
n_heads: 2
n_layers: 2
norm_eps: 1.0e-05
norm_type: default_layer_norm
positional_embedding_type: rotary
post_embed_norm: false
qk_norm: false
resume_from_checkpoint: null
resume_weights_only: false
type: transformer
vocab_size: 50432
weight_tying: false