# Copyright 2023–2026 Google LLC

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# https://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# model config for gemma4-26b MoE

base_num_decoder_layers: 30

base_emb_dim: 2816

base_num_query_heads: 16

base_num_kv_heads: 8

base_mlp_dim: 2112

base_moe_mlp_dim: 704

mlp_activations: ["gelu","linear"]

head_dim: 256

global_head_dim: 512

global_num_kv_heads: 2

vocab_size: 262144

decoder_block: "gemma4"

normalization_layer_epsilon: 1e-6

logits_via_embedding: True

sliding_window_size: 1024

use_post_attn_norm: true

use_post_ffw_norm: true

share_kv_projections: true

local_rope_max_timescale: 10000

global_rope_max_timescale: 1000000

rope_max_timescale: 1000000

global_rope_proportion: 0.25

local_rope_proportion: 1.0

v_norm_with_scale: false

final_logits_soft_cap: 30.0

# MoE configuration

num_experts: 128

num_experts_per_tok: 8

shared_experts: 1

norm_topk_prob: true

load_balance_loss_weight: 0.001

# Multimodal flags (need to set use_multimodal=true)

rope_theta_for_vit: 100

image_size_for_vit: [672, 960]

num_channels_for_vit: 3

patch_size_for_vit: 16

conv_stride_for_vit: 16

hidden_size_for_vit: 1152

intermediate_size_for_vit: 4304

num_hidden_layers_for_vit: 27

num_attention_heads_for_vit: 16

image_placeholder: "<|image|>"

vision_output_length: 280

num_position_embeddings_for_vit: 10240