#include "utils.hpp"

#include "json-schema-to-grammar.h"

#include "sampling.h"

#include "speculative.h"

#include <atomic>

#include <chrono>

#include <cinttypes>

#include <condition_variable>

#include <cstddef>

#include <deque>

#include <memory>

#include <mutex>

#include <signal.h>

#include <thread>

#include <unordered_map>

#include <unordered_set>

using json = nlohmann::ordered_json;

constexpr int HTTP_POLLING_SECONDS = 1;

enum stop_type {

STOP_TYPE_NONE,

STOP_TYPE_EOS,

STOP_TYPE_WORD,

STOP_TYPE_LIMIT,

};

// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283

enum slot_state {

SLOT_STATE_IDLE,

SLOT_STATE_STARTED, // TODO: this state is only used for setting up the initial prompt processing; maybe merge it

// with launch_slot_with_task in the future

SLOT_STATE_PROCESSING_PROMPT,

SLOT_STATE_DONE_PROMPT,

SLOT_STATE_GENERATING,

};

enum server_state {

SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet

SERVER_STATE_READY, // Server is ready and model is loaded

};

enum server_task_type {

SERVER_TASK_TYPE_COMPLETION,

SERVER_TASK_TYPE_EMBEDDING,

SERVER_TASK_TYPE_RERANK,

SERVER_TASK_TYPE_INFILL,

SERVER_TASK_TYPE_CANCEL,

SERVER_TASK_TYPE_NEXT_RESPONSE,

SERVER_TASK_TYPE_METRICS,

SERVER_TASK_TYPE_SLOT_SAVE,

SERVER_TASK_TYPE_SLOT_RESTORE,

SERVER_TASK_TYPE_SLOT_ERASE,

SERVER_TASK_TYPE_SET_LORA,

};

enum oaicompat_type {

OAICOMPAT_TYPE_NONE,

OAICOMPAT_TYPE_CHAT,

OAICOMPAT_TYPE_COMPLETION,

OAICOMPAT_TYPE_EMBEDDING,

};

// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/11

enum error_type {

ERROR_TYPE_INVALID_REQUEST,

ERROR_TYPE_AUTHENTICATION,

ERROR_TYPE_SERVER,

ERROR_TYPE_NOT_FOUND,

ERROR_TYPE_PERMISSION,

ERROR_TYPE_UNAVAILABLE, // custom error

ERROR_TYPE_NOT_SUPPORTED, // custom error

};

struct slot_params {

bool stream = true;

bool cache_prompt = true; // remember the prompt to avoid reprocessing all prompt

bool return_tokens = false;

int32_t n_keep = 0; // number of tokens to keep from initial prompt

int32_t n_discard =

0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half

int32_t n_predict = -1; // new tokens to predict

int32_t n_indent = 0; // mininum line indentation for the generated text in number of whitespace characters

int64_t t_max_prompt_ms = -1; // TODO: implement

int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit

std::vector<common_adapter_lora_info> lora;

std::vector<std::string> antiprompt;

std::vector<std::string> response_fields;

bool timings_per_token = false;

bool post_sampling_probs = false;

bool ignore_eos = false;

struct common_params_sampling sampling;

struct common_params_speculative speculative;

// OAI-compat fields

bool verbose = false;

oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;

std::string oaicompat_model;

std::string oaicompat_cmpl_id;

common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;

json to_json() const {

std::vector<std::string> samplers;

samplers.reserve(sampling.samplers.size());

for (const auto &sampler : sampling.samplers) {

samplers.emplace_back(common_sampler_type_to_str(sampler));

}

json lora = json::array();

for (size_t i = 0; i < this->lora.size(); ++i) {

lora.push_back({{"id", i}, {"scale", this->lora[i].scale}});

}

auto grammar_triggers = json::array();

for (const auto &trigger : sampling.grammar_triggers) {

grammar_triggers.push_back(trigger.to_json<json>());

}

return json{

{"n_predict", n_predict}, // Server configured n_predict

{"seed", sampling.seed},

{"temperature", sampling.temp},

{"dynatemp_range", sampling.dynatemp_range},

{"dynatemp_exponent", sampling.dynatemp_exponent},

{"top_k", sampling.top_k},

{"top_p", sampling.top_p},

{"min_p", sampling.min_p},

{"xtc_probability", sampling.xtc_probability},

{"xtc_threshold", sampling.xtc_threshold},

{"typical_p", sampling.typ_p},

{"repeat_last_n", sampling.penalty_last_n},

{"repeat_penalty", sampling.penalty_repeat},

{"presence_penalty", sampling.penalty_present},

{"frequency_penalty", sampling.penalty_freq},

{"dry_multiplier", sampling.dry_multiplier},

{"dry_base", sampling.dry_base},

{"dry_allowed_length", sampling.dry_allowed_length},

{"dry_penalty_last_n", sampling.dry_penalty_last_n},

{"dry_sequence_breakers", sampling.dry_sequence_breakers},

{"mirostat", sampling.mirostat},

{"mirostat_tau", sampling.mirostat_tau},

{"mirostat_eta", sampling.mirostat_eta},

{"stop", antiprompt},

{"max_tokens", n_predict}, // User configured n_predict

{"n_keep", n_keep},

{"n_discard", n_discard},

{"ignore_eos", sampling.ignore_eos},

{"stream", stream},

{"logit_bias", format_logit_bias(sampling.logit_bias)},

{"n_probs", sampling.n_probs},

{"min_keep", sampling.min_keep},

{"grammar", sampling.grammar},

{"grammar_lazy", sampling.grammar_lazy},

{"grammar_triggers", grammar_triggers},

{"preserved_tokens", sampling.preserved_tokens},

{"chat_format", common_chat_format_name(oaicompat_chat_format)},

{"samplers", samplers},

{"speculative.n_max", speculative.n_max},

{"speculative.n_min", speculative.n_min},

{"speculative.p_min", speculative.p_min},

{"timings_per_token", timings_per_token},

{"post_sampling_probs", post_sampling_probs},

{"lora", lora},

};

}

};

struct server_task {

int id = -1; // to be filled by server_queue

int index = -1; // used when there are multiple prompts (batch request)

server_task_type type;

// used by SERVER_TASK_TYPE_CANCEL

int id_target = -1;

// used by SERVER_TASK_TYPE_INFERENCE

slot_params params;

llama_tokens prompt_tokens;

int id_selected_slot = -1;

// used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE

struct slot_action {

int slot_id;

std::string filename;

std::string filepath;

};

slot_action slot_action;

// used by SERVER_TASK_TYPE_METRICS

bool metrics_reset_bucket = false;

// used by SERVER_TASK_TYPE_SET_LORA

std::vector<common_adapter_lora_info> set_lora;

server_task(server_task_type type) : type(type) {}

static slot_params params_from_json_cmpl(const llama_context *ctx, const common_params &params_base,

const json &data) {

const llama_model *model = llama_get_model(ctx);

const llama_vocab *vocab = llama_model_get_vocab(model);

slot_params params;

// Sampling parameter defaults are loaded from the global server context (but individual requests can still

// override them)

slot_params defaults;

defaults.sampling = params_base.sampling;

defaults.speculative = params_base.speculative;

// enabling this will output extra debug information in the HTTP responses from the server

params.verbose = params_base.verbosity > 9;

params.timings_per_token = json_value(data, "timings_per_token", false);

params.stream = json_value(data, "stream", false);

params.cache_prompt = json_value(data, "cache_prompt", true);

params.return_tokens = json_value(data, "return_tokens", false);

params.n_predict = json_value(data, "n_predict", json_value(data, "max_tokens", defaults.n_predict));

params.n_indent = json_value(data, "n_indent", defaults.n_indent);

params.n_keep = json_value(data, "n_keep", defaults.n_keep);

params.n_discard = json_value(data, "n_discard", defaults.n_discard);

// params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO:

// implement

params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms);

params.response_fields = json_value(data, "response_fields", std::vector<std::string>());

params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k);

params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p);

params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p);

params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability);

params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold);

params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p);

params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp);

params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range);

params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent);

params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n);

params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat);

params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq);

params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present);

params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier);

params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base);

params.sampling.dry_allowed_length =

json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length);

params.sampling.dry_penalty_last_n =

json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n);

params.sampling.mirostat = json_value(data, "mirostat", defaults.sampling.mirostat);

params.sampling.mirostat_tau = json_value(data, "mirostat_tau", defaults.sampling.mirostat_tau);

params.sampling.mirostat_eta = json_value(data, "mirostat_eta", defaults.sampling.mirostat_eta);

params.sampling.seed = json_value(data, "seed", defaults.sampling.seed);

params.sampling.n_probs = json_value(data, "n_probs", defaults.sampling.n_probs);

params.sampling.min_keep = json_value(data, "min_keep", defaults.sampling.min_keep);

params.post_sampling_probs = json_value(data, "post_sampling_probs", defaults.post_sampling_probs);

params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min);

params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max);

params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min);

params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min);

params.speculative.n_min = std::max(params.speculative.n_min, 0);

params.speculative.n_max = std::max(params.speculative.n_max, 0);

// Use OpenAI API logprobs only if n_probs wasn't provided

if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs) {

params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs);

}

if (data.contains("lora")) {

if (data.at("lora").is_array()) {

params.lora = parse_lora_request(params_base.lora_adapters, data.at("lora"));

} else {

throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");

}

} else {

params.lora = params_base.lora_adapters;

}

// TODO: add more sanity checks for the input parameters

if (params.sampling.penalty_last_n < -1) {

throw std::runtime_error("Error: repeat_last_n must be >= -1");

}

if (params.sampling.dry_penalty_last_n < -1) {

throw std::runtime_error("Error: dry_penalty_last_n must be >= -1");

}

if (params.sampling.penalty_last_n == -1) {

// note: should be the slot's context and not the full context, but it's ok

params.sampling.penalty_last_n = llama_n_ctx(ctx);

}

if (params.sampling.dry_penalty_last_n == -1) {

params.sampling.dry_penalty_last_n = llama_n_ctx(ctx);

}

if (params.sampling.dry_base < 1.0f) {

params.sampling.dry_base = defaults.sampling.dry_base;

}

// sequence breakers for DRY

{

// Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format

// Ref:

// https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39

if (data.contains("dry_sequence_breakers")) {

params.sampling.dry_sequence_breakers =

json_value(data, "dry_sequence_breakers", std::vector<std::string>());

if (params.sampling.dry_sequence_breakers.empty()) {

throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");

}

}

}

// process "json_schema" and "grammar"

if (data.contains("json_schema") && !data.contains("grammar")) {

try {

auto schema = json_value(data, "json_schema", json::object());

SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());

params.sampling.grammar = json_schema_to_grammar(schema);

SRV_DBG("Converted grammar: %s\n", params.sampling.grammar.c_str());

} catch (const std::exception &e) {

throw std::runtime_error(std::string("\"json_schema\": ") + e.what());

}

} else {

params.sampling.grammar = json_value(data, "grammar", defaults.sampling.grammar);

SRV_DBG("Grammar: %s\n", params.sampling.grammar.c_str());

params.sampling.grammar_lazy = json_value(data, "grammar_lazy", defaults.sampling.grammar_lazy);

SRV_DBG("Grammar lazy: %s\n", params.sampling.grammar_lazy ? "true" : "false");

}

{

auto it = data.find("chat_format");

if (it != data.end()) {

params.oaicompat_chat_format = static_cast<common_chat_format>(it->get<int>());

SRV_INF("Chat format: %s\n", common_chat_format_name(params.oaicompat_chat_format).c_str());

} else {

params.oaicompat_chat_format = defaults.oaicompat_chat_format;

}

}

{

const auto preserved_tokens = data.find("preserved_tokens");

if (preserved_tokens != data.end()) {

for (const auto &t : *preserved_tokens) {

auto ids = common_tokenize(vocab, t.get<std::string>(), /* add_special= */ false,

/* parse_special= */ true);

if (ids.size() == 1) {

SRV_DBG("Preserved token: %d\n", ids[0]);

params.sampling.preserved_tokens.insert(ids[0]);

} else {

// This may happen when using a tool call style meant for a model with special tokens to

// preserve on a model without said tokens.

SRV_DBG("Not preserved because more than 1 token: %s\n", t.get<std::string>().c_str());

}

}

}

const auto grammar_triggers = data.find("grammar_triggers");

if (grammar_triggers != data.end()) {

for (const auto &t : *grammar_triggers) {

auto ct = common_grammar_trigger::from_json(t);

if (ct.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {

const auto &word = ct.value;

auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true);

if (ids.size() == 1) {

auto token = ids[0];

if (std::find(params.sampling.preserved_tokens.begin(),

params.sampling.preserved_tokens.end(),

(llama_token)token) == params.sampling.preserved_tokens.end()) {

throw std::runtime_error("Grammar trigger word should be marked as preserved token: " +

word);

}

SRV_DBG("Grammar trigger token: %d (`%s`)\n", token, word.c_str());

common_grammar_trigger trigger;

trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;

trigger.value = (llama_token)token;

params.sampling.grammar_triggers.push_back(trigger);

} else {

SRV_DBG("Grammar trigger word: `%s`\n", word.c_str());

params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});

}

} else {

params.sampling.grammar_triggers.push_back(ct);

}

}

}

if (params.sampling.grammar_lazy && params.sampling.grammar_triggers.empty()) {

throw std::runtime_error("Error: no triggers set for lazy grammar!");

}

}

{

params.sampling.logit_bias.clear();

params.ignore_eos = json_value(data, "ignore_eos", false);

const auto &logit_bias = data.find("logit_bias");

if (logit_bias != data.end() && logit_bias->is_array()) {

const int n_vocab = llama_vocab_n_tokens(vocab);

for (const auto &el : *logit_bias) {

// TODO: we may want to throw errors here, in case "el" is incorrect

if (el.is_array() && el.size() == 2) {

float bias;

if (el[1].is_number()) {

bias = el[1].get<float>();

} else if (el[1].is_boolean() && !el[1].get<bool>()) {

bias = -INFINITY;

} else {

continue;

}

if (el[0].is_number_integer()) {

llama_token tok = el[0].get<llama_token>();

if (tok >= 0 && tok < n_vocab) {

params.sampling.logit_bias.push_back({tok, bias});

}

} else if (el[0].is_string()) {

auto toks = common_tokenize(vocab, el[0].get<std::string>(), false);

for (auto tok : toks) {

params.sampling.logit_bias.push_back({tok, bias});

}

}

}

}

}

}

{

params.antiprompt.clear();

const auto &stop = data.find("stop");

if (stop != data.end() && stop->is_array()) {

for (const auto &word : *stop) {

if (!word.empty()) {

params.antiprompt.push_back(word);

}

}

}

}

{

const auto samplers = data.find("samplers");

if (samplers != data.end()) {

if (samplers->is_array()) {

params.sampling.samplers = common_sampler_types_from_names(*samplers, false);

} else if (samplers->is_string()) {

params.sampling.samplers = common_sampler_types_from_chars(samplers->get<std::string>());

}

} else {

params.sampling.samplers = defaults.sampling.samplers;

}

}

std::string model_name = params_base.model_alias.empty() ? DEFAULT_OAICOMPAT_MODEL : params_base.model_alias;

params.oaicompat_model = json_value(data, "model", model_name);

return params;

}

// utility function

static std::unordered_set<int> get_list_id(const std::vector<server_task> &tasks) {

std::unordered_set<int> ids(tasks.size());

for (size_t i = 0; i < tasks.size(); i++) {

ids.insert(tasks[i].id);

}

return ids;

}

};

struct result_timings {

int32_t prompt_n = -1;

double prompt_ms;

double prompt_per_token_ms;

double prompt_per_second;

int32_t predicted_n = -1;

double predicted_ms;

double predicted_per_token_ms;

double predicted_per_second;

json to_json() const {

return {

{"prompt_n", prompt_n},

{"prompt_ms", prompt_ms},

{"prompt_per_token_ms", prompt_per_token_ms},

{"prompt_per_second", prompt_per_second},

{"predicted_n", predicted_n},

{"predicted_ms", predicted_ms},

{"predicted_per_token_ms", predicted_per_token_ms},

{"predicted_per_second", predicted_per_second},

};

}

};

struct server_task_result {

int id = -1;

int id_slot = -1;

virtual bool is_error() {

// only used by server_task_result_error

return false;

}

virtual bool is_stop() {

// only used by server_task_result_cmpl_*

return false;

}

virtual int get_index() { return -1; }

virtual json to_json() = 0;

virtual ~server_task_result() = default;

};

// using shared_ptr for polymorphism of server_task_result

using server_task_result_ptr = std::unique_ptr<server_task_result>;

inline std::string stop_type_to_str(stop_type type) {

switch (type) {

case STOP_TYPE_EOS:

return "eos";

case STOP_TYPE_WORD:

return "word";

case STOP_TYPE_LIMIT:

return "limit";

default:

return "none";

}

}

struct completion_token_output {

llama_token tok;

float prob;

std::string text_to_send;

struct prob_info {

llama_token tok;

std::string txt;

float prob;

};

std::vector<prob_info> probs;

json to_json(bool post_sampling_probs) const {

json probs_for_token = json::array();

for (const auto &p : probs) {

std::string txt(p.txt);

txt.resize(validate_utf8(txt));

probs_for_token.push_back(json{

{"id", p.tok},

{"token", txt},

{"bytes", str_to_bytes(p.txt)},

{post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob)},

});

}

return probs_for_token;

}

static json probs_vector_to_json(const std::vector<completion_token_output> &probs, bool post_sampling_probs) {

json out = json::array();

for (const auto &p : probs) {

std::string txt(p.text_to_send);

txt.resize(validate_utf8(txt));

out.push_back(json{

{"id", p.tok},

{"token", txt},

{"bytes", str_to_bytes(p.text_to_send)},

{post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob)},

{post_sampling_probs ? "top_probs" : "top_logprobs", p.to_json(post_sampling_probs)},

});

}

return out;

}

static float logarithm(float x) {

// nlohmann::json converts -inf to null, so we need to prevent that

return x == 0.0f ? std::numeric_limits<float>::lowest() : std::log(x);

}

static std::vector<unsigned char> str_to_bytes(const std::string &str) {

std::vector<unsigned char> bytes;

for (unsigned char c : str) {

bytes.push_back(c);

}

return bytes;

}

};

struct server_task_result_cmpl_final : server_task_result {

int index = 0;

std::string content;

llama_tokens tokens;

bool stream;

result_timings timings;

std::string prompt;

bool truncated;

int32_t n_decoded;

int32_t n_prompt_tokens;

int32_t n_tokens_cached;

bool has_new_line;

std::string stopping_word;

stop_type stop = STOP_TYPE_NONE;

bool post_sampling_probs;

std::vector<completion_token_output> probs_output;

std::vector<std::string> response_fields;

slot_params generation_params;

// OAI-compat fields

bool verbose = false;

oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;

std::string oaicompat_model;

std::string oaicompat_cmpl_id;

common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;

virtual int get_index() override { return index; }

virtual bool is_stop() override {

return true; // in stream mode, final responses are considered stop

}

virtual json to_json() override {

switch (oaicompat) {

case OAICOMPAT_TYPE_NONE:

return to_json_non_oaicompat();

case OAICOMPAT_TYPE_COMPLETION:

return to_json_oaicompat();

case OAICOMPAT_TYPE_CHAT:

return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat();

default:

GGML_ASSERT(false && "Invalid oaicompat_type");

}

}

json to_json_non_oaicompat() {

json res = json{

{"index", index},

{"content", stream ? "" : content}, // in stream mode, content is already in last partial chunk

{"tokens", stream ? llama_tokens{} : tokens},

{"id_slot", id_slot},

{"stop", true},

{"model", oaicompat_model},

{"tokens_predicted", n_decoded},

{"tokens_evaluated", n_prompt_tokens},

{"generation_settings", generation_params.to_json()},

{"prompt", prompt},

{"has_new_line", has_new_line},

{"truncated", truncated},

{"stop_type", stop_type_to_str(stop)},

{"stopping_word", stopping_word},

{"tokens_cached", n_tokens_cached},

{"timings", timings.to_json()},

};

if (!stream && !probs_output.empty()) {

res["completion_probabilities"] =

completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);

}

return response_fields.empty() ? res : json_get_nested_values(response_fields, res);

}

json to_json_oaicompat() {

std::time_t t = std::time(0);

json logprobs = json(nullptr); // OAI default to null

if (!stream && probs_output.size() > 0) {

logprobs = json{

{"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},

};

}

json finish_reason = "length";

if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {

finish_reason = "stop";

}

json res = json{

{"choices", json::array({json{

{"text", stream ? "" : content}, // in stream mode, content is already in last partial chunk

{"index", index},

{"logprobs", logprobs},

{"finish_reason", finish_reason},

}})},

{"created", t},

{"model", oaicompat_model},

{"system_fingerprint", build_info},

{"object", "text_completion"},

{"usage", json{{"completion_tokens", n_decoded},

{"prompt_tokens", n_prompt_tokens},

{"total_tokens", n_decoded + n_prompt_tokens}}},

{"id", oaicompat_cmpl_id}};

// extra fields for debugging purposes

if (verbose) {

res["__verbose"] = to_json_non_oaicompat();

}

if (timings.prompt_n >= 0) {

res.push_back({"timings", timings.to_json()});

}

return res;

}

json to_json_oaicompat_chat() {

std::string finish_reason = "length";

common_chat_msg msg;

if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {

SRV_DBG("Parsing chat message: %s\n", content.c_str());

msg = common_chat_parse(content, oaicompat_chat_format);

finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";

} else {

msg.content = content;

}

json message{

{"role", "assistant"},

};

if (!msg.reasoning_content.empty()) {

message["reasoning_content"] = msg.reasoning_content;

}

if (msg.content.empty() && !msg.tool_calls.empty()) {

message["content"] = json();

} else {

message["content"] = msg.content;

}

if (!msg.tool_calls.empty()) {

auto tool_calls = json::array();

for (const auto &tc : msg.tool_calls) {

tool_calls.push_back({

{"type", "function"},

{"function",

{

{"name", tc.name},

{"arguments", tc.arguments},

}},

{"id", tc.id},

});

}

message["tool_calls"] = tool_calls;

}

json choice{

{"finish_reason", finish_reason},

{"index", 0},

{"message", message},

};

if (!stream && probs_output.size() > 0) {

choice["logprobs"] = json{

{"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},

};

}

std::time_t t = std::time(0);

json res = json{{"choices", json::array({choice})},

{"created", t},

{"model", oaicompat_model},

{"system_fingerprint", build_info},

{"object", "chat.completion"},

{"usage", json{{"completion_tokens", n_decoded},

{"prompt_tokens", n_prompt_tokens},

{"total_tokens", n_decoded + n_prompt_tokens}}},

{"id", oaicompat_cmpl_id}};

// extra fields for debugging purposes

if (verbose) {

res["__verbose"] = to_json_non_oaicompat();

}

if (timings.prompt_n >= 0) {

res.push_back({"timings", timings.to_json()});

}

return res;

}

json to_json_oaicompat_chat_stream() {

std::time_t t = std::time(0);

std::string finish_reason = "length";

if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {

finish_reason = "stop";

}

json choice = json{{"finish_reason", finish_reason}, {"index", 0}, {"delta", json::object()}};

json ret = json{

{"choices", json::array({choice})},

{"created", t},

{"id", oaicompat_cmpl_id},

{"model", oaicompat_model},

{"system_fingerprint", build_info},

{"object", "chat.completion.chunk"},

{"usage",

json{

{"completion_tokens", n_decoded},

{"prompt_tokens", n_prompt_tokens},

{"total_tokens", n_decoded + n_prompt_tokens},

}},

};

if (timings.prompt_n >= 0) {

ret.push_back({"timings", timings.to_json()});

}

return ret;

}

};

struct server_task_result_cmpl_partial : server_task_result {

int index = 0;

std::string content;

llama_tokens tokens;

int32_t n_decoded;

int32_t n_prompt_tokens;

bool post_sampling_probs;

completion_token_output prob_output;

result_timings timings;

// OAI-compat fields

bool verbose = false;

oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;

std::string oaicompat_model;

std::string oaicompat_cmpl_id;

virtual int get_index() override { return index; }

virtual bool is_stop() override {

return false; // in stream mode, partial responses are not considered stop

}

virtual json to_json() override {

switch (oaicompat) {

case OAICOMPAT_TYPE_NONE:

return to_json_non_oaicompat();

case OAICOMPAT_TYPE_COMPLETION:

return to_json_oaicompat();

case OAICOMPAT_TYPE_CHAT:

return to_json_oaicompat_chat();

default:

GGML_ASSERT(false && "Invalid oaicompat_type");

}

}

json to_json_non_oaicompat() {

// non-OAI-compat JSON

json res = json{

{"index", index},

{"content", content},

{"tokens", tokens},

{"stop", false},

{"id_slot", id_slot},

{"tokens_predicted", n_decoded},

{"tokens_evaluated", n_prompt_tokens},

};

// populate the timings object when needed (usually for the last response or with timings_per_token enabled)

if (timings.prompt_n > 0) {

res.push_back({"timings", timings.to_json()});

}

if (!prob_output.probs.empty()) {

res["completion_probabilities"] =

completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs);

}

return res;

}

json to_json_oaicompat() {

std::time_t t = std::time(0);

json logprobs = json(nullptr); // OAI default to null

if (prob_output.probs.size() > 0) {

logprobs = json{

{"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},

};

}

json res = json{{"choices", json::array({json{

{"text", content},

{"index", index},

{"logprobs", logprobs},

{"finish_reason", nullptr},

}})},

{"created", t},

{"model", oaicompat_model},

{"system_fingerprint", build_info},

{"object", "text_completion"},

{"id", oaicompat_cmpl_id}};

// extra fields for debugging purposes

if (verbose) {

res["__verbose"] = to_json_non_oaicompat();

}

if (timings.prompt_n >= 0) {

res.push_back({"timings", timings.to_json()});

}

return res;

}

json to_json_oaicompat_chat() {

bool first = n_decoded == 0;

std::time_t t = std::time(0);

json choices;

if (first) {

if (content.empty()) {

choices = json::array(

{json{{"finish_reason", nullptr}, {"index", 0}, {"delta", json{{"role", "assistant"}}}}});

} else {

// We have to send this as two updates to conform to openai behavior

json initial_ret = json{{"choices", json::array({json{{"finish_reason", nullptr},

{"index", 0},

{"delta", json{{"role", "assistant"}}}}})},

{"created", t},

{"id", oaicompat_cmpl_id},

{"model", oaicompat_model},

{"object", "chat.completion.chunk"}};

json second_ret =

json{{"choices",

json::array(

{json{{"finish_reason", nullptr}, {"index", 0}, {"delta", json{{"content", content}}}}})},

{"created", t},

{"id", oaicompat_cmpl_id},

{"model", oaicompat_model},

{"object", "chat.completion.chunk"}};

return std::vector<json>({initial_ret, second_ret});

}

} else {

choices = json::array({json{

{"finish_reason", nullptr},

{"index", 0},

{"delta",

json{

{"content", content},

}},

}});

}

GGML_ASSERT(choices.size() >= 1);

if (prob_output.probs.size() > 0) {

choices[0]["logprobs"] = json{

{"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},

};

}

json ret = json{{"choices", choices},

{"created", t},

{"id", oaicompat_cmpl_id},

{"model", oaicompat_model},

{"system_fingerprint", build_info},

{"object", "chat.completion.chunk"}};

if (timings.prompt_n >= 0) {

ret.push_back({"timings", timings.to_json()});

}

return std::vector<json>({ret});

}

};

struct server_task_result_embd : server_task_result {

int index = 0;

std::vector<std::vector<float>> embedding;

int32_t n_tokens;

// OAI-compat fields

oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;

virtual int get_index() override { return index; }

virtual json to_json() override {

return oaicompat == OAICOMPAT_TYPE_EMBEDDING ? to_json_oaicompat() : to_json_non_oaicompat();

}

json to_json_non_oaicompat() {

return json{

{"index", index},

{"embedding", embedding},

};

}

json to_json_oaicompat() {

return json{

{"index", index},

{"embedding", embedding[0]},

{"tokens_evaluated", n_tokens},

};

}

};

struct server_task_result_rerank : server_task_result {

int index = 0;

float score = -1e6;