# Copyright 2023–2025 Google LLC

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# https://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# model config for qwen3-14b-base

base_emb_dim: 5120

base_num_query_heads: 40

base_num_kv_heads: 8

base_mlp_dim: 17408

base_num_decoder_layers: 40

head_dim: 128

mlp_activations: ["silu", "linear"] # "hidden_act": "silu" implies SwiGLU

vocab_size: 151936

decoder_block: "qwen3"

normalization_layer_epsilon: 1.0e-6

rope_max_timescale: 1000000

use_qk_norm: True

logits_via_embedding: False # different from 0.6 and 4B variants, "tie_word_embeddings": false

normalize_embedding_logits: False

tokenizer_type: "huggingface"