#define LLAMA_API_INTERNAL

#include "llama.h"

#include "unicode.h"

#include "ggml.h"

#include "ggml-alloc.h"

#ifdef GGML_USE_CUBLAS

# include "ggml-cuda.h"

#elif defined(GGML_USE_CLBLAST)

# include "ggml-opencl.h"

#endif

#ifdef GGML_USE_METAL

# include "ggml-metal.h"

#endif

#ifdef GGML_USE_MPI

# include "ggml-mpi.h"

#endif

#ifdef GGML_USE_K_QUANTS

# ifndef QK_K

# ifdef GGML_QKK_64

# define QK_K 64

# else

# define QK_K 256

# endif

# endif

#endif

#ifdef __has_include

#if __has_include(<unistd.h>)

#include <unistd.h>

#if defined(_POSIX_MAPPED_FILES)

#include <sys/mman.h>

#endif

#if defined(_POSIX_MEMLOCK_RANGE)

#include <sys/resource.h>

#endif

#endif

#endif

#if defined(_WIN32)

#define WIN32_LEAN_AND_MEAN

#ifndef NOMINMAX

#define NOMINMAX

#endif

#include <windows.h>

#include <io.h>

#include <stdio.h> // for _fseeki64

#endif

#include <algorithm>

#include <array>

#include <cassert>

#include <cinttypes>

#include <climits>

#include <cstdarg>

#include <cstddef>

#include <cstdint>

#include <cstdio>

#include <cstring>

#include <ctime>

#include <fstream>

#include <initializer_list>

#include <map>

#include <memory>

#include <mutex>

#include <numeric>

#include <queue>

#include <random>

#include <regex>

#include <sstream>

#include <thread>

#include <unordered_map>

#include <set>

#include <forward_list>

#if defined(_MSC_VER)

#pragma warning(disable: 4244 4267) // possible loss of data

#endif

#ifdef __GNUC__

#ifdef __MINGW32__

#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))

#else

#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))

#endif

#else

#define LLAMA_ATTRIBUTE_FORMAT(...)

#endif

//

// logging

//

LLAMA_ATTRIBUTE_FORMAT(2, 3)

static void llama_log_internal (ggml_log_level level, const char* format, ...);

static void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);

#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)

#define LLAMA_LOG_WARN(...) llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)

#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)

//

// helpers

//

static size_t utf8_len(char src) {

const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };

uint8_t highbits = static_cast<uint8_t>(src) >> 4;

return lookup[highbits];

}

static void replace_all(std::string & s, const std::string & search, const std::string & replace) {

std::string result;

for (size_t pos = 0; ; pos += search.length()) {

auto new_pos = s.find(search, pos);

if (new_pos == std::string::npos) {

result += s.substr(pos, s.size() - pos);

break;

}

result += s.substr(pos, new_pos - pos) + replace;

pos = new_pos;

}

s = std::move(result);

}

static bool is_float_close(float a, float b, float abs_tol) {

// Check for non-negative tolerance

if (abs_tol < 0.0) {

throw std::invalid_argument("Tolerance must be non-negative");

}

// Exact equality check

if (a == b) {

return true;

}

// Check for infinities

if (std::isinf(a) || std::isinf(b)) {

return false;

}

// Regular comparison using the provided absolute tolerance

return std::fabs(b - a) <= abs_tol;

}

#ifdef GGML_USE_CPU_HBM

#include <hbwmalloc.h>

#endif

static void zeros(std::ofstream & file, size_t n) {

char zero = 0;

for (size_t i = 0; i < n; ++i) {

file.write(&zero, 1);

}

}

LLAMA_ATTRIBUTE_FORMAT(1, 2)

static std::string format(const char * fmt, ...) {

va_list ap;

va_list ap2;

va_start(ap, fmt);

va_copy(ap2, ap);

int size = vsnprintf(NULL, 0, fmt, ap);

GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT

std::vector<char> buf(size + 1);

int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);

GGML_ASSERT(size2 == size);

va_end(ap2);

va_end(ap);

return std::string(buf.data(), size);

}

//

// gguf constants (sync with gguf.py)

//

enum llm_arch {

LLM_ARCH_LLAMA,

LLM_ARCH_FALCON,

LLM_ARCH_BAICHUAN,

LLM_ARCH_GPT2,

LLM_ARCH_GPTJ,

LLM_ARCH_GPTNEOX,

LLM_ARCH_MPT,

LLM_ARCH_STARCODER,

LLM_ARCH_PERSIMMON,

LLM_ARCH_REFACT,

LLM_ARCH_BLOOM,

LLM_ARCH_CODESHELL,

LLM_ARCH_UNKNOWN,

};

static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {

{ LLM_ARCH_LLAMA, "llama" },

{ LLM_ARCH_FALCON, "falcon" },

{ LLM_ARCH_GPT2, "gpt2" },

{ LLM_ARCH_GPTJ, "gptj" },

{ LLM_ARCH_GPTNEOX, "gptneox" },

{ LLM_ARCH_MPT, "mpt" },

{ LLM_ARCH_BAICHUAN, "baichuan" },

{ LLM_ARCH_STARCODER, "starcoder" },

{ LLM_ARCH_PERSIMMON, "persimmon" },

{ LLM_ARCH_REFACT, "refact" },

{ LLM_ARCH_BLOOM, "bloom" },

{ LLM_ARCH_CODESHELL, "codeshell" },

};

enum llm_kv {

LLM_KV_GENERAL_ARCHITECTURE,

LLM_KV_GENERAL_QUANTIZATION_VERSION,

LLM_KV_GENERAL_ALIGNMENT,

LLM_KV_GENERAL_NAME,

LLM_KV_GENERAL_AUTHOR,

LLM_KV_GENERAL_URL,

LLM_KV_GENERAL_DESCRIPTION,

LLM_KV_GENERAL_LICENSE,

LLM_KV_GENERAL_SOURCE_URL,

LLM_KV_GENERAL_SOURCE_HF_REPO,

LLM_KV_CONTEXT_LENGTH,

LLM_KV_EMBEDDING_LENGTH,

LLM_KV_BLOCK_COUNT,

LLM_KV_FEED_FORWARD_LENGTH,

LLM_KV_USE_PARALLEL_RESIDUAL,

LLM_KV_TENSOR_DATA_LAYOUT,

LLM_KV_ATTENTION_HEAD_COUNT,

LLM_KV_ATTENTION_HEAD_COUNT_KV,

LLM_KV_ATTENTION_MAX_ALIBI_BIAS,

LLM_KV_ATTENTION_CLAMP_KQV,

LLM_KV_ATTENTION_LAYERNORM_EPS,

LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,

LLM_KV_ROPE_DIMENSION_COUNT,

LLM_KV_ROPE_FREQ_BASE,

LLM_KV_ROPE_SCALE_LINEAR,

LLM_KV_TOKENIZER_MODEL,

LLM_KV_TOKENIZER_LIST,

LLM_KV_TOKENIZER_TOKEN_TYPE,

LLM_KV_TOKENIZER_SCORES,

LLM_KV_TOKENIZER_MERGES,

LLM_KV_TOKENIZER_BOS_ID,

LLM_KV_TOKENIZER_EOS_ID,

LLM_KV_TOKENIZER_UNK_ID,

LLM_KV_TOKENIZER_SEP_ID,

LLM_KV_TOKENIZER_PAD_ID,

LLM_KV_TOKENIZER_HF_JSON,

LLM_KV_TOKENIZER_RWKV,

};

static std::map<llm_kv, std::string> LLM_KV_NAMES = {

{ LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" },

{ LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" },

{ LLM_KV_GENERAL_ALIGNMENT, "general.alignment" },

{ LLM_KV_GENERAL_NAME, "general.name" },

{ LLM_KV_GENERAL_AUTHOR, "general.author" },

{ LLM_KV_GENERAL_URL, "general.url" },

{ LLM_KV_GENERAL_DESCRIPTION, "general.description" },

{ LLM_KV_GENERAL_LICENSE, "general.license" },

{ LLM_KV_GENERAL_SOURCE_URL, "general.source.url" },

{ LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" },

{ LLM_KV_CONTEXT_LENGTH, "%s.context_length" },

{ LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },

{ LLM_KV_BLOCK_COUNT, "%s.block_count" },

{ LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },

{ LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" },

{ LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },

{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },

{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },

{ LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },

{ LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },

{ LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },

{ LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },

{ LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },

{ LLM_KV_ROPE_FREQ_BASE, "%s.rope.freq_base" },

{ LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },

{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },

{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },

{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },

{ LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },

{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },

{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },

{ LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },

{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },

{ LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },

{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },

{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },

{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },

};

struct LLM_KV {

LLM_KV(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_kv kv) const {

return ::format(LLM_KV_NAMES[kv].c_str(), LLM_ARCH_NAMES[arch].c_str());

}

};

enum llm_tensor {

LLM_TENSOR_TOKEN_EMBD,

LLM_TENSOR_TOKEN_EMBD_NORM,

LLM_TENSOR_POS_EMBD,

LLM_TENSOR_OUTPUT,

LLM_TENSOR_OUTPUT_NORM,

LLM_TENSOR_ROPE_FREQS,

LLM_TENSOR_ATTN_Q,

LLM_TENSOR_ATTN_K,

LLM_TENSOR_ATTN_V,

LLM_TENSOR_ATTN_QKV,

LLM_TENSOR_ATTN_OUT,

LLM_TENSOR_ATTN_NORM,

LLM_TENSOR_ATTN_NORM_2,

LLM_TENSOR_ATTN_ROT_EMBD,

LLM_TENSOR_FFN_GATE,

LLM_TENSOR_FFN_DOWN,

LLM_TENSOR_FFN_UP,

LLM_TENSOR_FFN_NORM,

LLM_TENSOR_ATTN_Q_NORM,

LLM_TENSOR_ATTN_K_NORM,

};

static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {

{

LLM_ARCH_LLAMA,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ROPE_FREQS, "rope_freqs" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },

{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },

{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_BAICHUAN,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ROPE_FREQS, "rope_freqs" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },

{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },

{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_FALCON,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_GPT2,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

},

},

{

LLM_ARCH_GPTJ,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

},

},

{

LLM_ARCH_GPTNEOX,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_PERSIMMON,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd"},

{ LLM_TENSOR_OUTPUT_NORM, "output_norm"},

{ LLM_TENSOR_OUTPUT, "output"},

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm"},

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv"},

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output"},

{ LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm"},

{ LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm"},

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm"},

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down"},

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up"},

{ LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd"},

},

},

{

LLM_ARCH_MPT,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_STARCODER,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_POS_EMBD, "position_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

},

},

{

LLM_ARCH_REFACT,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },

{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },

{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_BLOOM,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

},

},

{

LLM_ARCH_CODESHELL,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },

{ LLM_TENSOR_OUTPUT, "output" },

{ LLM_TENSOR_ROPE_FREQS, "rope_freqs" },

{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },

{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },

{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },

{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },

{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },

{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },

{ LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },

{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },

{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },

{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },

{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

},

},

{

LLM_ARCH_UNKNOWN,

{

{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },

},

},

};

static llm_arch llm_arch_from_string(const std::string & name) {

for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT

if (kv.second == name) {

return kv.first;

}

}

return LLM_ARCH_UNKNOWN;

}

// helper to handle gguf constants

// usage:

//

// const auto tn = LLM_TN(LLM_ARCH_LLAMA);

//

// std::string name = tn(LLM_TENSOR_OUTPUT); -> "output"

// std::string name = tn(LLM_TENSOR_TOKEN_EMBD, "bias"); -> "token_embd.bias"

// std::string name = tn(LLM_TENSOR_ATTN_NORM, "weight", 3); -> "blk.3.attn_norm.weight"

//

struct LLM_TN {

LLM_TN(llm_arch arch) : arch(arch) {}

llm_arch arch;

std::string operator()(llm_tensor tensor) const {

return LLM_TENSOR_NAMES[arch].at(tensor);

}

std::string operator()(llm_tensor tensor, const std::string & suffix) const {

return LLM_TENSOR_NAMES[arch].at(tensor) + "." + suffix;

}

std::string operator()(llm_tensor tensor, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid);

}

std::string operator()(llm_tensor tensor, const std::string & suffix, int bid) const {

return ::format(LLM_TENSOR_NAMES[arch].at(tensor).c_str(), bid) + "." + suffix;

}

};

//

// gguf helpers

//

#define GGUF_GET_KEY(ctx, dst, func, type, req, key) \

do { \

const std::string skey(key); \

const int kid = gguf_find_key(ctx, skey.c_str()); \

if (kid >= 0) { \

enum gguf_type ktype = gguf_get_kv_type(ctx, kid); \

if (ktype != (type)) { \

throw std::runtime_error(format("key %s has wrong type: %s", skey.c_str(), gguf_type_name(ktype))); \

} \

(dst) = func(ctx, kid); \

} else if (req) { \

throw std::runtime_error(format("key not found in model: %s", skey.c_str())); \

} \

} while (0)

//

// ggml helpers

//

static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph * graph, int n_threads) {

struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);

if (plan.work_size > 0) {

buf.resize(plan.work_size);

plan.work_data = buf.data();

}

ggml_graph_compute(graph, &plan);

}

//

// llama helpers

//

#ifdef GGML_USE_CUBLAS

# define llama_host_malloc(n) ggml_cuda_host_malloc(n)

# define llama_host_free(data) ggml_cuda_host_free(data)

#elif GGML_USE_METAL

# define llama_host_malloc(n) ggml_metal_host_malloc(n)

# define llama_host_free(data) ggml_metal_host_free(data)

#elif GGML_USE_CPU_HBM

# define llama_host_malloc(n) hbw_malloc(n)

# define llama_host_free(data) if (data != NULL) hbw_free(data)

#else

# define llama_host_malloc(n) malloc(n)

# define llama_host_free(data) free(data)

#endif

#if defined(_WIN32)

static std::string llama_format_win_err(DWORD err) {

LPSTR buf;

size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,

NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);

if (!size) {

return "FormatMessageA failed";

}

std::string ret(buf, size);

LocalFree(buf);

return ret;

}

#endif

struct llama_buffer {

void * data = NULL;

size_t size = 0;

// fallback to malloc / free

// useful in cases where CUDA can try to allocate PINNED memory

bool fallback = false;

void resize(size_t n) {

llama_host_free(data);

data = llama_host_malloc(n);

if (!data) {

fallback = true;

data = malloc(n);

} else {

fallback = false;

}

GGML_ASSERT(data);

size = n;

}

~llama_buffer() {

if (data) {

if (fallback) { // NOLINT

free(data);

} else {

llama_host_free(data);

}

}

data = NULL;

}

};

struct llama_file {

// use FILE * so we don't have to re-open the file to mmap

FILE * fp;

size_t size;

llama_file(const char * fname, const char * mode) {

fp = std::fopen(fname, mode);

if (fp == NULL) {

throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));

}

seek(0, SEEK_END);

size = tell();

seek(0, SEEK_SET);

}

size_t tell() const {

#ifdef _WIN32

__int64 ret = _ftelli64(fp);

#else

long ret = std::ftell(fp);

#endif

GGML_ASSERT(ret != -1); // this really shouldn't fail

return (size_t) ret;

}

void seek(size_t offset, int whence) const {

#ifdef _WIN32

int ret = _fseeki64(fp, (__int64) offset, whence);

#else

int ret = std::fseek(fp, (long) offset, whence);

#endif

GGML_ASSERT(ret == 0); // same

}

void read_raw(void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

std::size_t ret = std::fread(ptr, len, 1, fp);

if (ferror(fp)) {

throw std::runtime_error(format("read error: %s", strerror(errno)));

}

if (ret != 1) {

throw std::runtime_error(std::string("unexpectedly reached end of file"));

}

}

uint32_t read_u32() const {

uint32_t ret;

read_raw(&ret, sizeof(ret));

return ret;

}

void write_raw(const void * ptr, size_t len) const {

if (len == 0) {

return;

}

errno = 0;

size_t ret = std::fwrite(ptr, len, 1, fp);

if (ret != 1) {

throw std::runtime_error(format("write error: %s", strerror(errno)));

}

}

void write_u32(std::uint32_t val) const {

write_raw(&val, sizeof(val));

}

~llama_file() {

if (fp) {

std::fclose(fp);

}

}

};

struct llama_mmap {

void * addr;

size_t size;

llama_mmap(const llama_mmap &) = delete;

#ifdef _POSIX_MAPPED_FILES

static constexpr bool SUPPORTED = true;

llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {

size = file->size;

int fd = fileno(file->fp);

int flags = MAP_SHARED;

// prefetch/readahead impairs performance on NUMA systems

if (numa) { prefetch = 0; }

#ifdef __linux__

if (prefetch) { flags |= MAP_POPULATE; }

#endif

addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);

if (addr == MAP_FAILED) {

throw std::runtime_error(format("mmap failed: %s", strerror(errno)));

}

if (prefetch > 0) {

// Advise the kernel to preload the mapped memory

if (posix_madvise(addr, std::min(file->size, prefetch), POSIX_MADV_WILLNEED)) {

fprintf(stderr, "warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",

strerror(errno));

}

}

if (numa) {

// advise the kernel not to use readahead

// (because the next page might not belong on the same node)

if (posix_madvise(addr, file->size, POSIX_MADV_RANDOM)) {

fprintf(stderr, "warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",

strerror(errno));

}

}

}

~llama_mmap() {

munmap(addr, size);

}

#elif defined(_WIN32)

static constexpr bool SUPPORTED = true;

llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {

(void) numa;

size = file->size;

HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));

HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

DWORD error = GetLastError();

if (hMapping == NULL) {

throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));

}

addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);

error = GetLastError();

CloseHandle(hMapping);

if (addr == NULL) {

throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));

}

if (prefetch) {

// PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it

BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);

HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");

// may fail on pre-Windows 8 systems

pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));

if (pPrefetchVirtualMemory) {

// advise the kernel to preload the mapped memory

WIN32_MEMORY_RANGE_ENTRY range;

range.VirtualAddress = addr;

range.NumberOfBytes = (SIZE_T)size;

if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {

fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

}

}

~llama_mmap() {

if (!UnmapViewOfFile(addr)) {

fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

#else

static constexpr bool SUPPORTED = false;

llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {

(void) file;

(void) prefetch;

(void) numa;

throw std::runtime_error(std::string("mmap not supported"));

}

#endif

};

// Represents some region of memory being locked using mlock or VirtualLock;

// will automatically unlock on destruction.

struct llama_mlock {

void * addr = NULL;

size_t size = 0;

bool failed_already = false;

llama_mlock() {}

llama_mlock(const llama_mlock &) = delete;

~llama_mlock() {

if (size) {

raw_unlock(addr, size);

}

}

void init(void * ptr) {

GGML_ASSERT(addr == NULL && size == 0); // NOLINT

addr = ptr;

}

void grow_to(size_t target_size) {

GGML_ASSERT(addr);

if (failed_already) {

return;

}

size_t granularity = lock_granularity();

target_size = (target_size + granularity - 1) & ~(granularity - 1);

if (target_size > size) {

if (raw_lock((uint8_t *) addr + size, target_size - size)) {

size = target_size;

} else {

failed_already = true;

}

}

}

#ifdef _POSIX_MEMLOCK_RANGE

static constexpr bool SUPPORTED = true;

static size_t lock_granularity() {

return (size_t) sysconf(_SC_PAGESIZE);

}

#ifdef __APPLE__

#define MLOCK_SUGGESTION \

"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \

"decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n"

#else

#define MLOCK_SUGGESTION \

"Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"

#endif

bool raw_lock(const void * addr, size_t size) const {

if (!mlock(addr, size)) {

return true;

}

char* errmsg = std::strerror(errno);

bool suggest = (errno == ENOMEM);

// Check if the resource limit is fine after all

struct rlimit lock_limit;

if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {

suggest = false;

}

if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) {

suggest = false;

}

fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",

size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");

return false;

}

#undef MLOCK_SUGGESTION

static void raw_unlock(void * addr, size_t size) {

if (munlock(addr, size)) {

fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));

}

}

#elif defined(_WIN32)

static constexpr bool SUPPORTED = true;

static size_t lock_granularity() {

SYSTEM_INFO si;

GetSystemInfo(&si);

return (size_t) si.dwPageSize;

}

bool raw_lock(void * ptr, size_t len) const {

for (int tries = 1; ; tries++) {

if (VirtualLock(ptr, len)) {

return true;

}

if (tries == 2) {

fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",

len, size, llama_format_win_err(GetLastError()).c_str());

return false;

}

// It failed but this was only the first try; increase the working

// set size and try again.

SIZE_T min_ws_size, max_ws_size;

if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {

fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

return false;

}

// Per MSDN: "The maximum number of pages that a process can lock

// is equal to the number of pages in its minimum working set minus

// a small overhead."

// Hopefully a megabyte is enough overhead:

size_t increment = len + 1048576;

// The minimum must be <= the maximum, so we need to increase both:

min_ws_size += increment;

max_ws_size += increment;

if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {

fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",

llama_format_win_err(GetLastError()).c_str());

return false;

}

}

}

static void raw_unlock(void * ptr, size_t len) {

if (!VirtualUnlock(ptr, len)) {

fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",

llama_format_win_err(GetLastError()).c_str());

}

}

#else

static constexpr bool SUPPORTED = false;

static size_t lock_granularity() {

return (size_t) 65536;

}

bool raw_lock(const void * addr, size_t len) const {

fprintf(stderr, "warning: mlock not supported on this system\n");

return false;

}

static void raw_unlock(const void * addr, size_t len) {}

#endif

};

typedef void (*offload_func_t)(struct ggml_tensor * tensor);

static void llama_nop(struct ggml_tensor * tensor) { // don't offload by default

(void) tensor;

}

static std::string llama_token_to_str(const struct llama_context * ctx, llama_token token) {