config.json

{
  "architectures": [
    "GPTForCausalLM"
  ],
  "model_type": "gpt2",
  "vocab_size": 65536,
  "n_positions": 2048,
  "n_ctx": 2048,
  "n_embd": 1280,
  "n_head": 10,
  "n_layer": 20,
  "n_kv_head": 10,
  "rotary": true,
  "rotary_base": 10000,
  "tie_word_embeddings": false
}