***************************************************** llama-cli log (working, no problem) ***************************************************** !echo "Hello!\nJust wanted to say hello.\n/exit\n" | ./llama.cpp/build/bin/llama-cli -m "Nemotron-Nano-3-30B-A3B-Q4_K_M.gguf" ggml_cuda_init: found 1 CUDA devices: Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0, VMM: yes Loading model... ▄▄ ▄▄ ██ ██ ██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ ██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ ██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ ██ ██ ▀▀ ▀▀ build : b7640-e443fbcfa model : Nemotron-Nano-3-30B-A3B-Q4_K_M.gguf modalities : text available commands: /exit or Ctrl+C stop or exit /regen regenerate the last response /clear clear the chat history /read add a text file > Hello! The user says "Hello!". We should respond politely. No special instructions. Just greet back. Hello! How can I help you today? [ Prompt: 106.7 t/s | Generation: 219.4 t/s ] > Just wanted to say hello. The user says "Just wanted to say hello." We can respond politely, maybe ask how they're doing. No constraints. Just respond friendly. Hello! Nice to meet you. How’s your day going? [ Prompt: 24.0 t/s | Generation: 268.8 t/s ] > /exit Exiting... llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | llama_memory_breakdown_print: | - CUDA0 (RTX 5090) | 32100 = 1050 + (29932 = 23140 + 4531 + 2260) + 1118 | llama_memory_breakdown_print: | - Host | 1730 = 231 + 0 + 1499 | ***************************************************** from llama_cpp import Llama llm = Llama( #https://huggingface.co/ggml-org/Nemotron-Nano-3-30B-A3B-GGUF model_path="Nemotron-Nano-3-30B-A3B-Q4_K_M.gguf", n_gpu_layers=100, n_ctx=48*1024, verbose=True ) ***************************************************** ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no ggml_cuda_init: found 1 CUDA devices: Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0, VMM: yes llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 5090) (0000:08:00.0) - 31072 MiB free llama_model_loader: loaded meta data with 48 key-value pairs and 401 tensors from ../../DATA/OPT.Models/GGUF/Nemotron-Nano-3-30B-A3B-Q4_K_M.gguf (version GGUF V3 (latest)) llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. llama_model_loader: - kv 0: general.architecture str = nemotron_h_moe llama_model_loader: - kv 1: general.type str = model llama_model_loader: - kv 2: general.sampling.top_p f32 = 1.000000 llama_model_loader: - kv 3: general.sampling.temp f32 = 1.000000 llama_model_loader: - kv 4: general.name str = Nano v3 Llm_Vv1.0.1 Ga llama_model_loader: - kv 5: general.size_label str = 128x2.4B llama_model_loader: - kv 6: nemotron_h_moe.block_count u32 = 52 llama_model_loader: - kv 7: nemotron_h_moe.context_length u32 = 1048576 llama_model_loader: - kv 8: nemotron_h_moe.embedding_length u32 = 2688 llama_model_loader: - kv 9: nemotron_h_moe.feed_forward_length arr[i32,52] = [0, 1856, 0, 1856, 0, 0, 1856, 0, 185... llama_model_loader: - kv 10: nemotron_h_moe.attention.head_count u32 = 32 llama_model_loader: - kv 11: nemotron_h_moe.attention.head_count_kv arr[i32,52] = [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, ... llama_model_loader: - kv 12: nemotron_h_moe.rope.freq_base f32 = 10000.000000 llama_model_loader: - kv 13: nemotron_h_moe.attention.layer_norm_rms_epsilon f32 = 0.000010 llama_model_loader: - kv 14: nemotron_h_moe.attention.layer_norm_epsilon f32 = 0.000010 llama_model_loader: - kv 15: nemotron_h_moe.expert_used_count u32 = 6 llama_model_loader: - kv 16: nemotron_h_moe.expert_group_count u32 = 1 llama_model_loader: - kv 17: nemotron_h_moe.expert_group_used_count u32 = 1 llama_model_loader: - kv 18: nemotron_h_moe.vocab_size u32 = 131072 llama_model_loader: - kv 19: nemotron_h_moe.rope.dimension_count u32 = 84 llama_model_loader: - kv 20: nemotron_h_moe.ssm.conv_kernel u32 = 4 llama_model_loader: - kv 21: nemotron_h_moe.ssm.state_size u32 = 128 llama_model_loader: - kv 22: nemotron_h_moe.ssm.group_count u32 = 8 llama_model_loader: - kv 23: nemotron_h_moe.ssm.inner_size u32 = 4096 llama_model_loader: - kv 24: nemotron_h_moe.ssm.time_step_rank u32 = 64 llama_model_loader: - kv 25: nemotron_h_moe.rope.scaling.finetuned bool = false llama_model_loader: - kv 26: nemotron_h_moe.attention.key_length u32 = 128 llama_model_loader: - kv 27: nemotron_h_moe.attention.value_length u32 = 128 llama_model_loader: - kv 28: nemotron_h_moe.expert_feed_forward_length u32 = 1856 llama_model_loader: - kv 29: nemotron_h_moe.expert_shared_feed_forward_length u32 = 3712 llama_model_loader: - kv 30: nemotron_h_moe.expert_count u32 = 128 llama_model_loader: - kv 31: nemotron_h_moe.expert_shared_count u32 = 1 llama_model_loader: - kv 32: nemotron_h_moe.expert_weights_norm bool = true llama_model_loader: - kv 33: nemotron_h_moe.expert_weights_scale f32 = 2.500000 llama_model_loader: - kv 34: tokenizer.ggml.model str = gpt2 llama_model_loader: - kv 35: tokenizer.ggml.pre str = pixtral llama_model_loader: - kv 36: tokenizer.ggml.tokens arr[str,131072] = ["", "~~", "~~", "[INST]", "[... llama_model_loader: - kv 37: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ... Exception ignored on calling ctypes callback function: Traceback (most recent call last): File "/usr/local/lib/python3.11/dist-packages/llama_cpp/_logger.py", line 39, in llama_log_callback print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) ^^^^^^^^^^^^^^^^^^^^ UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc4 in position 128: invalid continuation byte llama_model_loader: - kv 39: tokenizer.ggml.bos_token_id u32 = 1 llama_model_loader: - kv 40: tokenizer.ggml.eos_token_id u32 = 11 llama_model_loader: - kv 41: tokenizer.ggml.unknown_token_id u32 = 0 llama_model_loader: - kv 42: tokenizer.ggml.padding_token_id u32 = 0 llama_model_loader: - kv 43: tokenizer.ggml.add_bos_token bool = false llama_model_loader: - kv 44: tokenizer.ggml.add_eos_token bool = false llama_model_loader: - kv 45: tokenizer.chat_template str = {% macro render_extra_keys(json_dict,... llama_model_loader: - kv 46: general.quantization_version u32 = 2 llama_model_loader: - kv 47: general.file_type u32 = 15 llama_model_loader: - type f32: 237 tensors llama_model_loader: - type q5_0: 110 tensors llama_model_loader: - type q8_0: 25 tensors llama_model_loader: - type q4_K: 29 tensors print_info: file format = GGUF V3 (latest) print_info: file type = Q4_K - Medium print_info: file size = 22.82 GiB (6.21 BPW) init_tokenizer: initializing tokenizer for type 2 load: control token: 216 '' is not marked as EOG load: control token: 408 '' is not marked as EOG load: control token: 845 '' is not marked as EOG load: control token: 555 '' is not marked as EOG load: control token: 257 '' is not marked as EOG load: control token: 474 '' is not marked as EOG load: control token: 241 '' is not marked as EOG load: control token: 646 '' is not marked as EOG load: control token: 601 '' is not marked as EOG load: control token: 905 '' is not marked as EOG load: control token: 168 '' is not marked as EOG load: control token: 37 '' is not marked as EOG load: control token: 125 '' is not marked as EOG load: control token: 95 '' is not marked as EOG load: control token: 304 '' is not marked as EOG load: control token: 706 '' is not marked as EOG load: control token: 423 '' is not marked as EOG load: control token: 6 '[/AVAILABLE_TOOLS]' is not marked as EOG load: control token: 592 '' is not marked as EOG load: control token: 409 '' is not marked as EOG load: control token: 231 '' is not marked as EOG load: control token: 607 '' is not marked as EOG load: control token: 7 '[TOOL_RESULTS]' is not marked as EOG load: control token: 138 '' is not marked as EOG load: control token: 455 '' is not marked as EOG load: control token: 801 '' is not marked as EOG load: control token: 288 '' is not marked as EOG load: control token: 619 '' is not marked as EOG load: control token: 133 '' is not marked as EOG load: control token: 827 '' is not marked as EOG load: control token: 771 '' is not marked as EOG load: control token: 723 '' is not marked as EOG load: control token: 494 '' is not marked as EOG load: control token: 444 '' is not marked as EOG load: control token: 422 '' is not marked as EOG load: control token: 441 '' is not marked as EOG load: control token: 372 '' is not marked as EOG load: control token: 874 '' is not marked as EOG load: control token: 137 '' is not marked as EOG load: control token: 250 '' is not marked as EOG load: control token: 840 '' is not marked as EOG load: control token: 846 '' is not marked as EOG load: control token: 465 '' is not marked as EOG load: control token: 425 '' is not marked as EOG load: control token: 984 '' is not marked as EOG load: control token: 328 '' is not marked as EOG load: control token: 186 '' is not marked as EOG load: control token: 973 '' is not marked as EOG load: control token: 394 '' is not marked as EOG load: control token: 522 '' is not marked as EOG load: control token: 385 '' is not marked as EOG load: control token: 496 '' is not marked as EOG load: control token: 498 '' is not marked as EOG load: control token: 489 '' is not marked as EOG load: control token: 767 '' is not marked as EOG load: control token: 916 '' is not marked as EOG load: control token: 818 '' is not marked as EOG load: control token: 894 '' is not marked as EOG load: control token: 668 '' is not marked as EOG load: control token: 566 '' is not marked as EOG load: control token: 898 '' is not marked as EOG load: control token: 490 '' is not marked as EOG load: control token: 977 '' is not marked as EOG load: control token: 900 '' is not marked as EOG load: control token: 541 '' is not marked as EOG load: control token: 21 '' is not marked as EOG load: control token: 787 '' is not marked as EOG load: control token: 722 '' is not marked as EOG load: control token: 803 '' is not marked as EOG load: control token: 210 '' is not marked as EOG load: control token: 205 '' is not marked as EOG load: control token: 485 '' is not marked as EOG load: control token: 40 '' is not marked as EOG load: control token: 57 '' is not marked as EOG load: control token: 115 '' is not marked as EOG load: control token: 798 '' is not marked as EOG load: control token: 700 '' is not marked as EOG load: control token: 961 '' is not marked as EOG load: control token: 101 '' is not marked as EOG load: control token: 279 '' is not marked as EOG load: control token: 139 '' is not marked as EOG load: control token: 903 '' is not marked as EOG load: control token: 625 '' is not marked as EOG load: control token: 147 '' is not marked as EOG load: control token: 837 '' is not marked as EOG load: control token: 438 '' is not marked as EOG load: control token: 500 '' is not marked as EOG load: control token: 371 '' is not marked as EOG load: control token: 31 '' is not marked as EOG load: control token: 27 '' is not marked as EOG load: control token: 948 '' is not marked as EOG load: control token: 72 '' is not marked as EOG load: control token: 53 '' is not marked as EOG load: control token: 70 '' is not marked as EOG load: control token: 737 '' is not marked as EOG load: control token: 864 '' is not marked as EOG load: control token: 623 '' is not marked as EOG load: control token: 421 '' is not marked as EOG load: control token: 431 '' is not marked as EOG load: control token: 589 '' is not marked as EOG load: control token: 926 '' is not marked as EOG load: control token: 743 '' is not marked as EOG load: control token: 484 '' is not marked as EOG load: control token: 721 '' is not marked as EOG load: control token: 940 '' is not marked as EOG load: control token: 971 '' is not marked as EOG load: control token: 136 '' is not marked as EOG load: control token: 445 '' is not marked as EOG load: control token: 142 '' is not marked as EOG load: control token: 934 '' is not marked as EOG load: control token: 316 '' is not marked as EOG load: control token: 690 '' is not marked as EOG load: control token: 887 '' is not marked as EOG load: control token: 531 '' is not marked as EOG load: control token: 482 '' is not marked as EOG load: control token: 560 '' is not marked as EOG load: control token: 333 '' is not marked as EOG load: control token: 574 '' is not marked as EOG load: control token: 734 '' is not marked as EOG load: control token: 491 '' is not marked as EOG load: control token: 199 '' is not marked as EOG load: control token: 449 '' is not marked as EOG load: control token: 704 '' is not marked as EOG load: control token: 618 '' is not marked as EOG load: control token: 516 '' is not marked as EOG load: control token: 661 '' is not marked as EOG load: control token: 459 '' is not marked as EOG load: control token: 134 '' is not marked as EOG load: control token: 708 '' is not marked as EOG load: control token: 426 '' is not marked as EOG load: control token: 727 '' is not marked as EOG load: control token: 506 '' is not marked as EOG load: control token: 265 '' is not marked as EOG load: control token: 861 '' is not marked as EOG load: control token: 988 '' is not marked as EOG load: control token: 189 '' is not marked as EOG load: control token: 367 '' is not marked as EOG load: control token: 118 '' is not marked as EOG load: control token: 526 '' is not marked as EOG load: control token: 527 '' is not marked as EOG load: control token: 448 '' is not marked as EOG load: control token: 86 '' is not marked as EOG load: control token: 554 '' is not marked as EOG load: control token: 895 '' is not marked as EOG load: control token: 608 '' is not marked as EOG load: control token: 681 '' is not marked as EOG load: control token: 403 '' is not marked as EOG load: control token: 503 '' is not marked as EOG load: control token: 20 '' is not marked as EOG load: control token: 797 '' is not marked as EOG load: control token: 476 '' is not marked as EOG load: control token: 359 '' is not marked as EOG load: control token: 909 '' is not marked as EOG load: control token: 323 '' is not marked as EOG load: control token: 435 '' is not marked as EOG load: control token: 43 '' is not marked as EOG load: control token: 970 '' is not marked as EOG load: control token: 175 '' is not marked as EOG load: control token: 300 '' is not marked as EOG load: control token: 505 '' is not marked as EOG load: control token: 158 '' is not marked as EOG load: control token: 890 '' is not marked as EOG load: control token: 855 '' is not marked as EOG load: control token: 182 '' is not marked as EOG load: control token: 312 '' is not marked as EOG load: control token: 688 '' is not marked as EOG load: control token: 795 '' is not marked as EOG load: control token: 883 '' is not marked as EOG load: control token: 817 '' is not marked as EOG load: control token: 886 '' is not marked as EOG load: control token: 50 '' is not marked as EOG load: control token: 10 '<|im_start|>' is not marked as EOG load: control token: 314 '' is not marked as EOG load: control token: 519 '' is not marked as EOG load: control token: 989 '' is not marked as EOG load: control token: 653 '' is not marked as EOG load: control token: 779 '' is not marked as EOG load: control token: 377 '' is not marked as EOG load: control token: 473 '' is not marked as EOG load: control token: 61 '' is not marked as EOG load: control token: 165 '' is not marked as EOG load: control token: 745 '' is not marked as EOG load: control token: 963 '' is not marked as EOG load: control token: 775 '' is not marked as EOG load: control token: 450 '' is not marked as EOG load: control token: 396 '' is not marked as EOG load: control token: 587 '' is not marked as EOG load: control token: 418 '' is not marked as EOG load: control token: 414 '' is not marked as EOG load: control token: 859 '' is not marked as EOG load: control token: 218 '' is not marked as EOG load: control token: 252 '' is not marked as EOG load: control token: 55 '' is not marked as EOG load: control token: 802 '' is not marked as EOG load: control token: 249 '' is not marked as EOG load: control token: 130 '' is not marked as EOG load: control token: 34 '' is not marked as EOG load: control token: 834 '' is not marked as EOG load: control token: 660 '' is not marked as EOG load: control token: 638 '' is not marked as EOG load: control token: 380 '' is not marked as EOG load: control token: 411 '' is not marked as EOG load: control token: 62 '' is not marked as EOG load: control token: 719 '' is not marked as EOG load: control token: 340 '' is not marked as EOG load: control token: 939 '' is not marked as EOG load: control token: 891 '' is not marked as EOG load: control token: 184 '' is not marked as EOG load: control token: 714 '' is not marked as EOG load: control token: 19 '' is not marked as EOG load: control token: 794 '' is not marked as EOG load: control token: 736 '' is not marked as EOG load: control token: 509 '' is not marked as EOG load: control token: 983 '' is not marked as EOG load: control token: 720 '' is not marked as EOG load: control token: 862 '' is not marked as EOG load: control token: 829 '' is not marked as EOG load: control token: 925 '' is not marked as EOG load: control token: 698 '' is not marked as EOG load: control token: 451 '' is not marked as EOG load: control token: 570 '' is not marked as EOG load: control token: 214 '' is not marked as EOG load: control token: 144 '' is not marked as EOG load: control token: 561 '' is not marked as EOG load: control token: 582 '' is not marked as EOG load: control token: 523 '' is not marked as EOG load: control token: 65 '' is not marked as EOG load: control token: 311 '' is not marked as EOG load: control token: 693 '' is not marked as EOG load: control token: 857 '' is not marked as EOG load: control token: 513 '' is not marked as EOG load: control token: 271 '' is not marked as EOG load: control token: 919 '' is not marked as EOG load: control token: 542 '' is not marked as EOG load: control token: 594 '' is not marked as EOG load: control token: 338 '' is not marked as EOG load: control token: 32 '' is not marked as EOG load: control token: 955 '' is not marked as EOG load: control token: 969 '' is not marked as EOG load: control token: 460 '' is not marked as EOG load: control token: 518 '' is not marked as EOG load: control token: 588 '' is not marked as EOG load: control token: 701 '' is not marked as EOG load: control token: 319 '' is not marked as EOG load: control token: 320 '' is not marked as EOG load: control token: 233 '' is not marked as EOG load: control token: 277 '' is not marked as EOG load: control token: 585 '' is not marked as EOG load: control token: 213 '' is not marked as EOG load: control token: 599 '' is not marked as EOG load: control token: 291 '' is not marked as EOG load: control token: 583 '' is not marked as EOG load: control token: 160 '' is not marked as EOG load: control token: 109 '' is not marked as EOG load: control token: 164 '' is not marked as EOG load: control token: 436 '' is not marked as EOG load: control token: 461 '' is not marked as EOG load: control token: 982 '' is not marked as EOG load: control token: 621 '' is not marked as EOG load: control token: 303 '' is not marked as EOG load: control token: 207 '' is not marked as EOG load: control token: 654 '' is not marked as EOG load: control token: 868 '' is not marked as EOG load: control token: 437 '' is not marked as EOG load: control token: 641 '' is not marked as EOG load: control token: 200 '' is not marked as EOG load: control token: 741 '' is not marked as EOG load: control token: 738 '' is not marked as EOG load: control token: 786 '' is not marked as EOG load: control token: 82 '' is not marked as EOG load: control token: 234 '' is not marked as EOG load: control token: 563 '' is not marked as EOG load: control token: 358 '' is not marked as EOG load: control token: 497 '' is not marked as EOG load: control token: 296 '' is not marked as EOG load: control token: 652 '' is not marked as EOG load: control token: 756 '' is not marked as EOG load: control token: 170 '' is not marked as EOG load: control token: 495 '' is not marked as EOG load: control token: 176 '' is not marked as EOG load: control token: 89 '' is not marked as EOG load: control token: 724 '' is not marked as EOG load: control token: 251 '' is not marked as EOG load: control token: 238 '' is not marked as EOG load: control token: 913 '' is not marked as EOG load: control token: 766 '' is not marked as EOG load: control token: 826 '' is not marked as EOG load: control token: 547 '' is not marked as EOG load: control token: 374 '' is not marked as EOG load: control token: 514 '' is not marked as EOG load: control token: 950 '' is not marked as EOG load: control token: 598 '' is not marked as EOG load: control token: 850 '' is not marked as EOG load: control token: 908 '' is not marked as EOG load: control token: 517 '' is not marked as EOG load: control token: 679 '' is not marked as EOG load: control token: 578 '' is not marked as EOG load: control token: 75 '' is not marked as EOG load: control token: 462 '' is not marked as EOG load: control token: 456 '' is not marked as EOG load: control token: 979 '' is not marked as EOG load: control token: 446 '' is not marked as EOG load: control token: 917 '' is not marked as EOG load: control token: 549 '' is not marked as EOG load: control token: 183 '' is not marked as EOG load: control token: 424 '' is not marked as EOG load: control token: 419 '' is not marked as EOG load: control token: 413 '' is not marked as EOG load: control token: 119 '' is not marked as EOG load: control token: 66 '' is not marked as EOG load: control token: 478 '' is not marked as EOG load: control token: 398 '' is not marked as EOG load: control token: 386 '' is not marked as EOG load: control token: 440 '' is not marked as EOG load: control token: 202 '' is not marked as EOG load: control token: 747 '' is not marked as EOG load: control token: 825 '' is not marked as EOG load: control token: 997 '' is not marked as EOG load: control token: 369 '' is not marked as EOG load: control token: 696 '' is not marked as EOG load: control token: 512 '' is not marked as EOG load: control token: 349 '' is not marked as EOG load: control token: 985 '' is not marked as EOG load: control token: 337 '' is not marked as EOG load: control token: 135 '' is not marked as EOG load: control token: 283 '' is not marked as EOG load: control token: 776 '' is not marked as EOG load: control token: 327 '' is not marked as EOG load: control token: 610 '' is not marked as EOG load: control token: 752 '' is not marked as EOG load: control token: 510 '' is not marked as EOG load: control token: 92 '' is not marked as EOG load: control token: 69 '' is not marked as EOG load: control token: 703 '' is not marked as EOG load: control token: 284 '' is not marked as EOG load: control token: 325 '' is not marked as EOG load: control token: 228 '' is not marked as EOG load: control token: 355 '' is not marked as EOG load: control token: 389 '' is not marked as EOG load: control token: 458 '' is not marked as EOG load: control token: 264 '' is not marked as EOG load: control token: 335 '' is not marked as EOG load: control token: 181 '' is not marked as EOG load: control token: 263 '' is not marked as EOG load: control token: 671 '' is not marked as EOG load: control token: 63 '' is not marked as EOG load: control token: 261 '' is not marked as EOG load: control token: 760 '' is not marked as EOG load: control token: 399 '' is not marked as EOG load: control token: 692 '' is not marked as EOG load: control token: 956 '' is not marked as EOG load: control token: 400 '' is not marked as EOG load: control token: 83 '' is not marked as EOG load: control token: 79 '' is not marked as EOG load: control token: 564 '' is not marked as EOG load: control token: 537 '' is not marked as EOG load: control token: 959 '' is not marked as EOG load: control token: 910 '' is not marked as EOG load: control token: 285 '' is not marked as EOG load: control token: 524 '' is not marked as EOG load: control token: 565 '' is not marked as EOG load: control token: 744 '' is not marked as EOG load: control token: 255 '' is not marked as EOG load: control token: 866 '' is not marked as EOG load: control token: 538 '' is not marked as EOG load: control token: 280 '' is not marked as EOG load: control token: 815 '' is not marked as EOG load: control token: 204 '' is not marked as EOG load: control token: 475 '' is not marked as EOG load: control token: 480 '' is not marked as EOG load: control token: 166 '' is not marked as EOG load: control token: 562 '' is not marked as EOG load: control token: 47 '' is not marked as EOG load: control token: 471 '' is not marked as EOG load: control token: 486 '' is not marked as EOG load: control token: 244 '' is not marked as EOG load: control token: 301 '' is not marked as EOG load: control token: 2 '' is not marked as EOG load: control token: 580 '' is not marked as EOG load: control token: 246 '' is not marked as EOG load: control token: 52 '' is not marked as EOG load: control token: 295 '' is not marked as EOG load: control token: 994 '' is not marked as EOG load: control token: 90 '' is not marked as EOG load: control token: 122 '' is not marked as EOG load: control token: 370 '' is not marked as EOG load: control token: 124 '' is not marked as EOG load: control token: 644 '' is not marked as EOG load: control token: 357 '' is not marked as EOG load: control token: 275 '' is not marked as EOG load: control token: 488 '' is not marked as EOG load: control token: 492 '' is not marked as EOG load: control token: 944 '' is not marked as EOG load: control token: 45 '' is not marked as EOG load: control token: 262 '' is not marked as EOG load: control token: 276 '' is not marked as EOG load: control token: 757 '' is not marked as EOG load: control token: 305 '' is not marked as EOG load: control token: 683 '' is not marked as EOG load: control token: 382 '' is not marked as EOG load: control token: 432 '' is not marked as EOG load: control token: 831 '' is not marked as EOG load: control token: 824 '\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n\' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- \'<|im_end|>\\n\' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \'<|im_end|>\\n\' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == "assistant" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = "\\n" ~ message.reasoning_content ~ "\\n\\n" ~ (message.content | default(\'\', true)) %}\n {%- else %}\n {%- set content = message.content | default(\'\', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if \'\' not in content and \'\' not in content -%}\n {%- set content = "" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- \'<|im_start|>assistant\\n\' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ \'\\n\' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if \'\' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split(\'\')[-1] %}\n {%- elif \'\' in c %}\n {# If was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split(\'\')[0] %}\n {%- endif %}\n {%- set c = "" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ \'\\n\' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- "" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- \'\\n\\n\' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- \'\\n\' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ \'\\n\\n\' -}}\n {%- endfor %}\n {%- endif %}\n {{- \'\\n\\n\' -}}\n {%- endfor %}\n {{- \'<|im_end|>\\n\' }}\n {%- else %}\n {# Assistant message doesn\'t have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- \'<|im_start|>assistant\\n\' ~ (content | default(\'\', true) | string | trim) ~ \'<|im_end|>\\n\' }}\n {%- else %}\n {%- set c = (content | default(\'\', true) | string) %}\n {%- if \'\' in c and \'\' in c %}\n {%- set c = "" ~ c.split(\'\')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- \'<|im_start|>assistant\\n\' ~ c ~ \'<|im_end|>\\n\' }}\n {%- else %}\n {{- \'<|im_start|>assistant\\n<|im_end|>\\n\' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == "user" or message.role == "system" %}\n {{- \'<|im_start|>\' + message.role + \'\\n\' }}\n {%- set content = message.content | string %}\n {{- content }}\n {{- \'<|im_end|>\\n\' }}\n {%- elif message.role == "tool" %}\n {%- if loop.previtem and loop.previtem.role != "tool" %}\n {{- \'<|im_start|>user\\n\' }}\n {%- endif %}\n {{- \'\\n\' }}\n {{- message.content }}\n {{- \'\\n\\n\' }}\n {%- if not loop.last and loop.nextitem.role != "tool" %}\n {{- \'<|im_end|>\\n\' }}\n {%- elif loop.last %}\n {{- \'<|im_end|>\\n\' }}\n {%- endif %}\n {%- else %}\n {{- \'<|im_start|>\' + message.role + \'\\n\' + message.content + \'<|im_end|>\\n\' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- \'<|im_start|>assistant\\n\\n\' }}\n {%- else %}\n {{- \'<|im_start|>assistant\\n\' }}\n {%- endif %}\n{%- endif %}\n', 'nemotron_h_moe.ssm.group_count': '8', 'nemotron_h_moe.rope.scaling.finetuned': 'false', 'nemotron_h_moe.attention.key_length': '128', 'tokenizer.ggml.pre': 'pixtral', 'nemotron_h_moe.expert_feed_forward_length': '1856', 'nemotron_h_moe.expert_weights_norm': 'true', 'nemotron_h_moe.embedding_length': '2688', 'nemotron_h_moe.expert_shared_feed_forward_length': '3712'} Available chat formats from metadata: chat_template.default Using gguf chat template: {% macro render_extra_keys(json_dict, handled_keys) %} {%- if json_dict is mapping %} {%- for json_key in json_dict if json_key not in handled_keys %} {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %} {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '' }} {%- else %} {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '' }} {%- endif %} {%- endfor %} {%- endif %} {% endmacro %} {%- set enable_thinking = enable_thinking if enable_thinking is defined else True %} {%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %} {%- set ns = namespace(last_user_idx = -1) %} {%- set loop_messages = messages %} {%- for m in loop_messages %} {%- if m["role"] == "user" %} {%- set ns.last_user_idx = loop.index0 %} {%- endif %} {%- endfor %} {%- if messages[0]["role"] == "system" %} {%- set system_message = messages[0]["content"] %} {%- set loop_messages = messages[1:] %} {%- else %} {%- set system_message = "" %} {%- set loop_messages = messages %} {%- endif %} {%- if not tools is defined %} {%- set tools = [] %} {%- endif %} {# Recompute last_user_idx relative to loop_messages after handling system #} {%- set ns = namespace(last_user_idx = -1) %} {%- for m in loop_messages %} {%- if m["role"] == "user" %} {%- set ns.last_user_idx = loop.index0 %} {%- endif %} {%- endfor %} {%- if system_message is defined %} {{- "<|im_start|>system\n" + system_message }} {%- else %} {%- if tools is iterable and tools | length > 0 %} {{- "<|im_start|>system\n" }} {%- endif %} {%- endif %} {%- if tools is iterable and tools | length > 0 %} {%- if system_message is defined and system_message | length > 0 %} {{- "\n\n" }} {%- endif %} {{- "# Tools\n\nYou have access to the following functions:\n\n" }} {{- "" }} {%- for tool in tools %} {%- if tool.function is defined %} {%- set tool = tool.function %} {%- endif %} {{- "\n\n" ~ tool.name ~ "" }} {%- if tool.description is defined %} {{- '\n' ~ (tool.description | trim) ~ '' }} {%- endif %} {{- '\n' }} {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %} {%- for param_name, param_fields in tool.parameters.properties|items %} {{- '\n' }} {{- '\n' ~ param_name ~ '' }} {%- if param_fields.type is defined %} {{- '\n' ~ (param_fields.type | string) ~ '' }} {%- endif %} {%- if param_fields.description is defined %} {{- '\n' ~ (param_fields.description | trim) ~ '' }} {%- endif %} {%- if param_fields.enum is defined %} {{- '\n' ~ (param_fields.enum | tojson | safe) ~ '' }} {%- endif %} {%- set handled_keys = ['name', 'type', 'description', 'enum'] %} {{- render_extra_keys(param_fields, handled_keys) }} {{- '\n' }} {%- endfor %} {%- endif %} {% set handled_keys = ['type', 'properties', 'required'] %} {{- render_extra_keys(tool.parameters, handled_keys) }} {%- if tool.parameters is defined and tool.parameters.required is defined %} {{- '\n' ~ (tool.parameters.required | tojson | safe) ~ '' }} {%- endif %} {{- '\n' }} {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %} {{- render_extra_keys(tool, handled_keys) }} {{- '\n' }} {%- endfor %} {{- "\n" }} {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} {%- endif %} {%- if system_message is defined %} {{- '<|im_end|>\n' }} {%- else %} {%- if tools is iterable and tools | length > 0 %} {{- '<|im_end|>\n' }} {%- endif %} {%- endif %} {%- for message in loop_messages %} {%- if message.role == "assistant" %} {# Add reasoning content in to content field for unified processing below. #} {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %} {%- set content = "\n" ~ message.reasoning_content ~ "\n\n" ~ (message.content | default('', true)) %} {%- else %} {%- set content = message.content | default('', true) %} {%- if content is string -%} {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #} {%- if '' not in content and '' not in content -%} {%- set content = "" ~ content -%} {%- endif -%} {%- else -%} {%- set content = content -%} {%- endif -%} {%- endif %} {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %} {# Assistant message has tool calls. #} {{- '<|im_start|>assistant\n' }} {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} {%- if content is string and content | trim | length > 0 %} {%- if include_content %} {{- (content | trim) ~ '\n' -}} {%- else %} {%- set c = (content | string) %} {%- if '' in c %} {# Keep only content after the last closing think. Also generation prompt causes this. #} {%- set c = c.split('')[-1] %} {%- elif '' in c %} {# If was opened but never closed, drop the trailing think segment #} {%- set c = c.split('')[0] %} {%- endif %} {%- set c = "" ~ c | trim %} {%- if c | length > 0 %} {{- c ~ '\n' -}} {%- endif %} {%- endif %} {%- else %} {{- "" -}} {%- endif %} {%- for tool_call in message.tool_calls %} {%- if tool_call.function is defined %} {%- set tool_call = tool_call.function %} {%- endif %} {{- '\n\n' -}} {%- if tool_call.arguments is defined %} {%- for args_name, args_value in tool_call.arguments|items %} {{- '\n' -}} {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} {{- args_value ~ '\n\n' -}} {%- endfor %} {%- endif %} {{- '\n\n' -}} {%- endfor %} {{- '<|im_end|>\n' }} {%- else %} {# Assistant message doesn't have tool calls. #} {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} {{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }} {%- else %} {%- set c = (content | default('', true) | string) %} {%- if '' in c and '' in c %} {%- set c = "" ~ c.split('')[-1] %} {%- endif %} {%- set c = c | trim %} {%- if c | length > 0 %} {{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }} {%- else %} {{- '<|im_start|>assistant\n<|im_end|>\n' }} {%- endif %} {%- endif %} {%- endif %} {%- elif message.role == "user" or message.role == "system" %} {{- '<|im_start|>' + message.role + '\n' }} {%- set content = message.content | string %} {{- content }} {{- '<|im_end|>\n' }} {%- elif message.role == "tool" %} {%- if loop.previtem and loop.previtem.role != "tool" %} {{- '<|im_start|>user\n' }} {%- endif %} {{- '\n' }} {{- message.content }} {{- '\n\n' }} {%- if not loop.last and loop.nextitem.role != "tool" %} {{- '<|im_end|>\n' }} {%- elif loop.last %} {{- '<|im_end|>\n' }} {%- endif %} {%- else %} {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }} {%- endif %} {%- endfor %} {%- if add_generation_prompt %} {%- if enable_thinking %} {{- '<|im_start|>assistant\n\n' }} {%- else %} {{- '<|im_start|>assistant\n' }} {%- endif %} {%- endif %} Using chat eos_token: <|im_end|> Using chat bos_token: