project: "mix_chord"

name: "test_mix_chord_toolace"

checkpoint_root_dir: ${oc.env:TRINITY_CHECKPOINT_ROOT_DIR,./checkpoints}

algorithm:

algorithm_type: mix_chord

repeat_times: 8

optimizer:

lr: 1e-6

kl_loss_fn_args:

kl_coef: 0.0

entropy_loss_fn: mix

sample_strategy_args:

expert_data_ratio: 0.11 # 96/864

policy_loss_fn_args: # feel free to change, we encourage you to try out different hyperparameters

mu_warmup_steps: 0 # 0 for chord-mu and chord-phi

mu_decay_steps: 0 # 20 for chord-mu and 0 for chord-phi

mu_peak: 0.1 # 0.6 for chord-mu and 0.1 for chord-phi

mu_valley: 0.1 # 0.0 for chord-mu and 0.1 for chord-phi

enable_phi_function: true # false for chord-mu and true for chord-phi

clip_range: 0.2

sft_loss_agg_mode: "token-mean"

use_dynamic_bsz: true

ppo_mini_batch_size: 864 # 864 = 96 * 8 + 96

ppo_micro_batch_size_per_gpu: 4

ngpus_trainer: 4

train_batch_size_expert: 96 # used to balance the expert data batch weight

train_batch_size_usual: 768 # used to balance the rl data batch weight, 96 * 8

model:

model_path: ${oc.env:TRINITY_MODEL_PATH,LLM-Research/Llama-3.2-3B-Instruct} # fill in your model path here

max_response_tokens: 24576

max_model_len: 28672

cluster:

node_num: 1

gpu_per_node: 8

buffer:

total_steps: 100

batch_size: 96

train_batch_size: 864

explorer_input:

taskset:

name: toolace_rl_split

storage_type: file

path: ${oc.env:TRINITY_TASKSET_PATH,datajuicer/Trinity-ToolAce-RL-split}

rollout_args:

temperature: 1.0

logprobs: 0

eval_tasksets: [] # you can add your own eval tasksets here

default_workflow_type: 'toolcall_workflow'

trainer_input:

experience_buffer:

name: math_buffer

storage_type: queue

path: 'sqlite:///test_mix_chord.db'

auxiliary_buffers:

sft_dataset:

total_epochs: 32

name: SFT_data

storage_type: file

schema_type: sft

path: ${oc.env:TRINITY_SFT_DATASET_PATH,datajuicer/Trinity-ToolAce-SFT-split}

split: 'train'

format:

prompt_type: messages

messages_key: 'messages'

explorer:

eval_interval: 10

runner_per_model: 8

rollout_model:

engine_num: 4

tensor_parallel_size: 1

enable_prefix_caching: false

enforce_eager: true

dtype: bfloat16

seed: 42

synchronizer:

sync_method: 'nccl'

sync_interval: 1

sync_timeout: 1200

trainer:

save_interval: 50

grad_clip: 1.0

use_dynamic_bsz: true

max_token_len_per_gpu: 25600

ulysses_sequence_parallel_size: 2

monitor:

monitor_type: wandb