// SPDX-License-Identifier: GPL-3.0-or-later

#include "sqlite_metadata.h"

#include "sqlite3recover.h"

//#include "sqlite_db_migration.h"

#define DB_METADATA_VERSION 18

const char *database_config[] = {

"CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "

"registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "

"os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default '',"

"hops INT NOT NULL DEFAULT 0,"

"memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"

"program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "

"entries INT NOT NULL DEFAULT 0,"

"health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)",

"CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "

"family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "

"chart_type int, memory_mode int, history_entries)",

"CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "

"multiplier int, divisor int , algorithm int, options text)",

"CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)",

"CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "

"label_value text, date_created int, PRIMARY KEY (chart_id, label_key))",

"CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)",

"CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "

"on_key text, class text, component text, type text, os text, hosts text, lookup text, "

"every text, units text, calc text, families text, plugin text, module text, charts text, green text, "

"red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "

"repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "

"p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, "

"summary text, time_group_condition INT, time_group_value DOUBLE, dims_group INT, data_source INT)",

"CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "

"date_created INT, PRIMARY KEY(host_id, system_key))",

"CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "

"label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))",

"CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)"

" SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END",

"CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, "

"config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "

"chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))",

"CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "

"updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "

"flags int, exec_run_timestamp int, delay_up_to_timestamp int, "

"info text, exec_code int, new_status real, old_status real, delay int, "

"new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)",

"CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)",

"CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)",

"CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",

"CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",

NULL

};

const char *database_cleanup[] = {

"DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)",

"DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)",

"DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)",

"DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)",

"DROP TRIGGER IF EXISTS tr_dim_del",

"DROP INDEX IF EXISTS ind_d1",

"DROP INDEX IF EXISTS ind_c1",

"DROP INDEX IF EXISTS ind_c2",

"DROP INDEX IF EXISTS alert_hash_index",

"DROP INDEX IF EXISTS health_log_d_ind_4",

"DROP INDEX IF EXISTS health_log_d_ind_1",

"DROP INDEX IF EXISTS health_log_d_ind_5",

NULL

};

sqlite3 *db_meta = NULL;

// SQL statements

#define SQL_STORE_CLAIM_ID \

"INSERT INTO node_instance " \

"(host_id, claim_id, date_created) VALUES (@host_id, @claim_id, UNIXEPOCH()) " \

"ON CONFLICT(host_id) DO UPDATE SET claim_id = excluded.claim_id"

#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid"

#define STORE_HOST_LABEL \

"INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "

#define STORE_CHART_LABEL \

"INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "

#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"

#define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid"

#define SQL_STORE_HOST_INFO \

"INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, " \

"memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \

"VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, " \

"@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected)"

#define SQL_STORE_CHART \

"INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, " \

"update_every, chart_type, memory_mode, history_entries) " \

"values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, " \

"@update_every, @chart_type, @memory_mode, @history_entries) " \

"ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, " \

"family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, " \

"plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \

"chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries"

#define SQL_STORE_DIMENSION \

"INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \

"VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) " \

"ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, " \

"divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options"

#define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"

#define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id"

#define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id"

#define SQL_STORE_HOST_SYSTEM_INFO_VALUES \

"INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \

"(@uuid, @name, @value, UNIXEPOCH())"

#define CONVERT_EXISTING_LOCALHOST "UPDATE host SET hops = 1 WHERE hops = 0 AND host_id <> @host_id"

#define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)"

#define METADATA_MAINTENANCE_FIRST_CHECK (1800) // Maintenance first run after agent startup in seconds

#define METADATA_MAINTENANCE_REPEAT (60) // Repeat if last run for dimensions, charts, labels needs more work

#define METADATA_HEALTH_LOG_INTERVAL (3600) // Repeat maintenance for health

#define METADATA_DIM_CHECK_INTERVAL (3600) // Repeat maintenance for dimensions

#define METADATA_CHART_CHECK_INTERVAL (3600) // Repeat maintenance for charts

#define METADATA_LABEL_CHECK_INTERVAL (3600) // Repeat maintenance for labels

#define METADATA_RUNTIME_THRESHOLD (5) // Run time threshold for cleanup task

#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata

#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata

#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write

#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)

#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop

#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum

#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum

enum metadata_opcode {

METADATA_DATABASE_NOOP = 0,

METADATA_DATABASE_TIMER,

METADATA_DEL_DIMENSION,

METADATA_STORE_CLAIM_ID,

METADATA_ADD_HOST_INFO,

METADATA_SCAN_HOSTS,

METADATA_LOAD_HOST_CONTEXT,

METADATA_DELETE_HOST_CHART_LABELS,

METADATA_MAINTENANCE,

METADATA_SYNC_SHUTDOWN,

METADATA_UNITTEST,

// leave this last

// we need it to check for worker utilization

METADATA_MAX_ENUMERATIONS_DEFINED

};

#define MAX_PARAM_LIST (2)

struct metadata_cmd {

enum metadata_opcode opcode;

struct completion *completion;

const void *param[MAX_PARAM_LIST];

struct metadata_cmd *prev, *next;

};

typedef enum {

METADATA_FLAG_PROCESSING = (1 << 0), // store or cleanup

METADATA_FLAG_SHUTDOWN = (1 << 1), // Shutting down

} METADATA_FLAG;

struct metadata_wc {

uv_thread_t thread;

uv_loop_t *loop;

uv_async_t async;

uv_timer_t timer_req;

time_t metadata_check_after;

METADATA_FLAG flags;

struct completion start_stop_complete;

struct completion *scan_complete;

/* FIFO command queue */

SPINLOCK cmd_queue_lock;

struct metadata_cmd *cmd_base;

};

#define metadata_flag_check(target_flags, flag) (__atomic_load_n(&((target_flags)->flags), __ATOMIC_SEQ_CST) & (flag))

#define metadata_flag_set(target_flags, flag) __atomic_or_fetch(&((target_flags)->flags), (flag), __ATOMIC_SEQ_CST)

#define metadata_flag_clear(target_flags, flag) __atomic_and_fetch(&((target_flags)->flags), ~(flag), __ATOMIC_SEQ_CST)

struct metadata_wc metasync_worker = {.loop = NULL};

//

// For unittest

//

struct thread_unittest {

int join;

unsigned added;

unsigned processed;

unsigned *done;

};

int sql_metadata_cache_stats(int op)

{

int count, dummy;

if (!REQUIRE_DB(db_meta))

return 0;

sqlite3_db_status(db_meta, op, &count, &dummy, 0);

return count;

}

static inline void set_host_node_id(RRDHOST *host, nd_uuid_t *node_id)

{

if (unlikely(!host))

return;

if (unlikely(!node_id)) {

freez(host->node_id);

__atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED);

return;

}

struct aclk_sync_cfg_t *wc = host->aclk_config;

if (unlikely(!host->node_id)) {

nd_uuid_t *t = mallocz(sizeof(*host->node_id));

uuid_copy(*t, *node_id);

__atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED);

}

else {

uuid_copy(*(host->node_id), *node_id);

}

if (unlikely(!wc))

sql_create_aclk_table(host, &host->host_uuid, node_id);

else

uuid_unparse_lower(*node_id, wc->node_id);

}

#define SQL_SET_HOST_LABEL \

"INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) " \

"VALUES (@host_id, @source_type, @label_key, @label_value, UNIXEPOCH()) ON CONFLICT (host_id, label_key) " \

" DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()"

bool sql_set_host_label(nd_uuid_t *host_id, const char *label_key, const char *label_value)

{

sqlite3_stmt *res = NULL;

bool status = false;

if (!label_key || !label_value || !host_id)

return false;

if (!PREPARE_STATEMENT(db_meta, SQL_SET_HOST_LABEL, &res))

return 1;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, RRDLABEL_SRC_AUTO));

SQLITE_BIND_FAIL(done, sqlite3_bind_text(res, ++param, label_key, -1, SQLITE_STATIC));

SQLITE_BIND_FAIL(done, sqlite3_bind_text(res, ++param, label_value, -1, SQLITE_STATIC));

param = 0;

int rc = execute_insert(res);

status = (rc == SQLITE_DONE);

if (false == status)

error_report("Failed to store node instance information, rc = %d", rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return status;

}

#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id"

int update_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id)

{

sqlite3_stmt *res = NULL;

RRDHOST *host = NULL;

int rc = 2;

char host_guid[GUID_LEN + 1];

uuid_unparse_lower(*host_id, host_guid);

rrd_wrlock();

host = rrdhost_find_by_guid(host_guid);

if (likely(host))

set_host_node_id(host, node_id);

rrd_wrunlock();

if (!REQUIRE_DB(db_meta))

return 1;

if (!PREPARE_STATEMENT(db_meta, SQL_UPDATE_NODE_ID, &res))

return 1;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, node_id, sizeof(*node_id), SQLITE_STATIC));

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

param = 0;

rc = execute_insert(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to store node instance information, rc = %d", rc);

rc = sqlite3_changes(db_meta);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return rc - 1;

}

#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL"

int get_node_id(nd_uuid_t *host_id, nd_uuid_t *node_id)

{

sqlite3_stmt *res = NULL;

if (!REQUIRE_DB(db_meta))

return 1;

if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_NODE_ID, &res))

return 1;

int param = 0, rc = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

param = 0;

rc = sqlite3_step_monitored(res);

if (likely(rc == SQLITE_ROW && node_id))

uuid_copy(*node_id, *((nd_uuid_t *) sqlite3_column_blob(res, 0)));

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return (rc == SQLITE_ROW) ? 0 : -1;

}

#define SQL_INVALIDATE_NODE_INSTANCES \

"UPDATE node_instance SET node_id = NULL WHERE EXISTS " \

"(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))"

void invalidate_node_instances(nd_uuid_t *host_id, nd_uuid_t *claim_id)

{

sqlite3_stmt *res = NULL;

if (!REQUIRE_DB(db_meta))

return;

if (!PREPARE_STATEMENT(db_meta, SQL_INVALIDATE_NODE_INSTANCES, &res))

return;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

if (claim_id)

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, claim_id, sizeof(*claim_id), SQLITE_STATIC));

else

SQLITE_BIND_FAIL(done, sqlite3_bind_null(res, ++param));

param = 0;

int rc = execute_insert(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to invalidate node instance information, rc = %d", rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

}

#define SQL_GET_NODE_INSTANCE_LIST \

"SELECT ni.node_id, ni.host_id, h.hostname " \

"FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0"

struct node_instance_list *get_node_list(void)

{

struct node_instance_list *node_list = NULL;

sqlite3_stmt *res = NULL;

if (!REQUIRE_DB(db_meta))

return NULL;

if (!PREPARE_STATEMENT(db_meta, SQL_GET_NODE_INSTANCE_LIST, &res))

return NULL;

int row = 0;

char host_guid[UUID_STR_LEN];

while (sqlite3_step_monitored(res) == SQLITE_ROW)

row++;

if (sqlite3_reset(res) != SQLITE_OK) {

error_report("Failed to reset the prepared statement while fetching node instance information");

goto failed;

}

node_list = callocz(row + 1, sizeof(*node_list));

int max_rows = row;

row = 0;

// TODO: Check to remove lock

rrd_rdlock();

while (sqlite3_step_monitored(res) == SQLITE_ROW) {

if (sqlite3_column_bytes(res, 0) == sizeof(nd_uuid_t))

uuid_copy(node_list[row].node_id, *((nd_uuid_t *)sqlite3_column_blob(res, 0)));

if (sqlite3_column_bytes(res, 1) == sizeof(nd_uuid_t)) {

nd_uuid_t *host_id = (nd_uuid_t *)sqlite3_column_blob(res, 1);

uuid_unparse_lower(*host_id, host_guid);

RRDHOST *host = rrdhost_find_by_guid(host_guid);

if (!host)

continue;

if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {

netdata_log_info(

"ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));

continue;

}

uuid_copy(node_list[row].host_id, *host_id);

node_list[row].queryable = 1;

node_list[row].live =

(host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;

node_list[row].hops = host->system_info ? host->system_info->hops :

uuid_eq(*host_id, localhost->host_uuid) ? 0 : 1;

node_list[row].hostname =

sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;

}

row++;

if (row == max_rows)

break;

}

rrd_rdunlock();

failed:

SQLITE_FINALIZE(res);

return node_list;

}

#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id"

void sql_load_node_id(RRDHOST *host)

{

sqlite3_stmt *res = NULL;

if (!REQUIRE_DB(db_meta))

return;

if (!PREPARE_STATEMENT(db_meta, SQL_GET_HOST_NODE_ID, &res))

return;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

param = 0;

int rc = sqlite3_step_monitored(res);

if (likely(rc == SQLITE_ROW)) {

if (likely(sqlite3_column_bytes(res, 0) == sizeof(nd_uuid_t)))

set_host_node_id(host, (nd_uuid_t *)sqlite3_column_blob(res, 0));

else

set_host_node_id(host, NULL);

}

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

}

#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id"

void sql_build_host_system_info(nd_uuid_t *host_id, struct rrdhost_system_info *system_info)

{

sqlite3_stmt *res = NULL;

if (!PREPARE_STATEMENT(db_meta, SELECT_HOST_INFO, &res))

return;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

param = 0;

while (sqlite3_step_monitored(res) == SQLITE_ROW) {

rrdhost_set_system_info_variable(

system_info, (char *)sqlite3_column_text(res, 0), (char *)sqlite3_column_text(res, 1));

}

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

}

#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \

"AND label_key IS NOT NULL AND label_value IS NOT NULL"

RRDLABELS *sql_load_host_labels(nd_uuid_t *host_id)

{

RRDLABELS *labels = NULL;

sqlite3_stmt *res = NULL;

if (!PREPARE_STATEMENT(db_meta, SELECT_HOST_LABELS, &res))

return NULL;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

param = 0;

labels = rrdlabels_create();

while (sqlite3_step_monitored(res) == SQLITE_ROW) {

rrdlabels_add(

labels,

(const char *)sqlite3_column_text(res, 0),

(const char *)sqlite3_column_text(res, 1),

sqlite3_column_int(res, 2));

}

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return labels;

}

static int exec_statement_with_uuid(const char *sql, nd_uuid_t *uuid)

{

int result = 1;

sqlite3_stmt *res = NULL;

if (!PREPARE_STATEMENT(db_meta, sql, &res)) {

error_report("Failed to prepare statement %s", sql);

return 1;

}

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, uuid, sizeof(*uuid), SQLITE_STATIC));

param = 0;

int rc = execute_insert(res);

if (likely(rc == SQLITE_DONE))

result = SQLITE_OK;

else

error_report("Failed to execute %s, rc = %d", sql, rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return result;

}

static void recover_database(const char *sqlite_database, const char *new_sqlite_database)

{

sqlite3 *database;

int rc = sqlite3_open(sqlite_database, &database);

if (rc != SQLITE_OK)

return;

netdata_log_info("Recover %s", sqlite_database);

netdata_log_info(" to %s", new_sqlite_database);

// This will remove the -shm and -wal files when we close the database

(void) db_execute(database, "select count(*) from sqlite_master limit 0");

sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);

if (recover) {

rc = sqlite3_recover_run(recover);

if (rc == SQLITE_OK)

netdata_log_info("Recover complete");

else

netdata_log_error("Recover encountered an error but the database may be usable");

rc = sqlite3_recover_finish(recover);

(void) sqlite3_close(database);

if (rc == SQLITE_OK) {

rc = rename(new_sqlite_database, sqlite_database);

if (rc == 0) {

netdata_log_info("Renamed %s", new_sqlite_database);

netdata_log_info(" to %s", sqlite_database);

}

}

else

netdata_log_error("Recover failed to free resources");

}

else

(void) sqlite3_close(database);

}

static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)

{

nd_uuid_t uuid;

if ( argc != 1 ){

sqlite3_result_null(context);

return ;

}

int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);

if (rc == -1) {

sqlite3_result_null(context);

return ;

}

sqlite3_result_blob(context, &uuid, sizeof(nd_uuid_t), SQLITE_TRANSIENT);

}

void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv)

{

if (argc != 1 ){

sqlite3_result_null(context);

return ;

}

if (sqlite3_value_int(argv[0]) != 0) {

struct timespec req = {.tv_sec = 0, .tv_nsec = 1};

nanosleep(&req, NULL);

}

sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec());

}

void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv)

{

(void)argc;

(void)argv;

nd_uuid_t uuid;

uuid_generate_random(uuid);

sqlite3_result_blob(context, &uuid, sizeof(nd_uuid_t), SQLITE_TRANSIENT);

}

// Init

/*

* Initialize the SQLite database

* Return 0 on success

*/

int sql_init_meta_database(db_check_action_type_t rebuild, int memory)

{

char *err_msg = NULL;

char sqlite_database[FILENAME_MAX + 1];

int rc;

if (likely(!memory)) {

snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);

rc = unlink(sqlite_database);

snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);

if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {

char new_sqlite_database[FILENAME_MAX + 1];

snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);

recover_database(sqlite_database, new_sqlite_database);

if (rebuild & DB_CHECK_RECOVER)

return 0;

}

}

else

strncpyz(sqlite_database, ":memory:", sizeof(sqlite_database) - 1);

rc = sqlite3_open(sqlite_database, &db_meta);

if (rc != SQLITE_OK) {

error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));

char *error_str = get_database_extented_error(db_meta, 0, "meta_open");

if (error_str)

analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);

freez(error_str);

sqlite3_close(db_meta);

db_meta = NULL;

return 1;

}

if (rebuild & DB_CHECK_RECLAIM_SPACE) {

netdata_log_info("Reclaiming space of %s", sqlite_database);

rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg);

if (rc != SQLITE_OK) {

error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);

sqlite3_free(err_msg);

}

else {

(void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

(void) sqlite3_close(db_meta);

}

return 1;

}

if (rebuild & DB_CHECK_ANALYZE) {

errno = 0;

netdata_log_info("Running ANALYZE on %s", sqlite_database);

rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);

if (rc != SQLITE_OK) {

error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg);

sqlite3_free(err_msg);

}

else {

(void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");

(void) sqlite3_close(db_meta);

}

return 1;

}

errno = 0;

netdata_log_info("SQLite database %s initialization", sqlite_database);

rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to register internal u2h function");

rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to register internal now_usec function");

rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to register internal uuid_random function");

int target_version = DB_METADATA_VERSION;

if (likely(!memory))

target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);

if (configure_sqlite_database(db_meta, target_version, "meta_config"))

return 1;

if (init_database_batch(db_meta, &database_config[0], "meta_init"))

return 1;

if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup"))

return 1;

netdata_log_info("SQLite database initialization completed");

return 0;

}

// Metadata functions

struct query_build {

BUFFER *sql;

int count;

char uuid_str[UUID_STR_LEN];

};

#define SQL_DELETE_CHART_LABELS_BY_HOST \

"DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)"

static void delete_host_chart_labels(nd_uuid_t *host_uuid)

{

sqlite3_stmt *res = NULL;

if (!PREPARE_STATEMENT(db_meta, SQL_DELETE_CHART_LABELS_BY_HOST, &res))

return;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));

param = 0;

int rc = sqlite3_step_monitored(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to execute command to remove chart labels, rc = %d", rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

}

static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

struct query_build *lb = data;

if (unlikely(!lb->count))

buffer_sprintf(lb->sql, STORE_HOST_LABEL);

else

buffer_strcat(lb->sql, ", ");

buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

lb->count++;

return 1;

}

static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

struct query_build *lb = data;

if (unlikely(!lb->count))

buffer_sprintf(lb->sql, STORE_CHART_LABEL);

else

buffer_strcat(lb->sql, ", ");

buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

lb->count++;

return 1;

}

#define SQL_DELETE_CHART_LABEL "DELETE FROM chart_label WHERE chart_id = @chart_id"

#define SQL_DELETE_CHART_LABEL_HISTORY "DELETE FROM chart_label WHERE date_created < %ld AND chart_id = @chart_id"

static void clean_old_chart_labels(RRDSET *st)

{

char sql[512];

time_t first_time_s = rrdset_first_entry_s(st);

if (unlikely(!first_time_s))

snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL);

else

snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL_HISTORY, first_time_s);

int rc = exec_statement_with_uuid(sql, &st->chart_uuid);

if (unlikely(rc))

error_report("METADATA: 'host:%s' Failed to clean old labels for chart %s", rrdhost_hostname(st->rrdhost), rrdset_name(st));

}

static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter)

{

size_t old_version = st->rrdlabels_last_saved_version;

size_t new_version = rrdlabels_version(st->rrdlabels);

if (new_version == old_version)

return 0;

struct query_build tmp = {.sql = work_buffer, .count = 0};

uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);

rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);

buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");

int rc = db_execute(db_meta, buffer_tostring(work_buffer));

if (likely(!rc)) {

st->rrdlabels_last_saved_version = new_version;

(*query_counter)++;

}

clean_old_chart_labels(st);

return rc;

}

// If the machine guid has changed, then existing one with hops 0 will be marked as hops 1 (child)

void detect_machine_guid_change(nd_uuid_t *host_uuid)

{

int rc;

rc = exec_statement_with_uuid(CONVERT_EXISTING_LOCALHOST, host_uuid);

if (!rc) {

if (unlikely(db_execute(db_meta, DELETE_MISSING_NODE_INSTANCES)))

error_report("Failed to remove deleted hosts from node instances");

}

}

static int store_claim_id(nd_uuid_t *host_id, nd_uuid_t *claim_id)

{

sqlite3_stmt *res = NULL;

int rc = 0;

if (!REQUIRE_DB(db_meta))

return 1;

if (!PREPARE_STATEMENT(db_meta, SQL_STORE_CLAIM_ID, &res))

return 1;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

if (claim_id)

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param,claim_id, sizeof(*claim_id), SQLITE_STATIC));

else

SQLITE_BIND_FAIL(done, sqlite3_bind_null(res, ++param));

param = 0;

rc = execute_insert(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to store host claim id rc = %d", rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_FINALIZE(res);

return rc != SQLITE_DONE;

}

static void delete_dimension_uuid(nd_uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)

{

static __thread sqlite3_stmt *res = NULL;

int rc;

if (!PREPARE_COMPILED_STATEMENT(db_meta, DELETE_DIMENSION_UUID, &res))

return;

int param = 0;

SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC));

param = 0;

rc = sqlite3_step_monitored(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to delete dimension uuid, rc = %d", rc);

done:

REPORT_BIND_FAIL(res, param);

SQLITE_RESET(res);

}

//

// Store host and host system info information in the database

static int store_host_metadata(RRDHOST *host)

{

static __thread sqlite3_stmt *res = NULL;

if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_STORE_HOST_INFO, &res))

return false;

int param = 0;

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_hostname(host), 0));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_registry_hostname(host), 1));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, host->rrd_update_every));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_os(host), 1));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_timezone(host), 1));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, "", 1));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, host->system_info ? host->system_info->hops : 0));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, host->rrd_memory_mode));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_abbrev_timezone(host), 1));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, host->utc_offset));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_program_name(host), 1));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, rrdhost_program_version(host), 1));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int64(res, ++param, host->rrd_history_entries));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int(res, ++param, (int ) host->health.health_enabled));

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_int64(res, ++param, (sqlite3_int64) host->last_connected));

int store_rc = sqlite3_step_monitored(res);

if (unlikely(store_rc != SQLITE_DONE))

error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), store_rc);

SQLITE_RESET(res);

return store_rc != SQLITE_DONE;

bind_fail:

REPORT_BIND_FAIL(res, param);

SQLITE_RESET(res);

return 1;

}

static int add_host_sysinfo_key_value(const char *name, const char *value, nd_uuid_t *uuid)

{

static __thread sqlite3_stmt *res = NULL;

if (!REQUIRE_DB(db_meta))

return 0;

if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_STORE_HOST_SYSTEM_INFO_VALUES, &res))

return 0;

int param = 0;

SQLITE_BIND_FAIL(bind_fail, sqlite3_bind_blob(res, ++param, uuid, sizeof(*uuid), SQLITE_STATIC));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, name, 0));

SQLITE_BIND_FAIL(bind_fail, bind_text_null(res, ++param, value ? value : "unknown", 0));

int store_rc = sqlite3_step_monitored(res);

if (unlikely(store_rc != SQLITE_DONE))

error_report("Failed to store host info value %s, rc = %d", name, store_rc);

SQLITE_RESET(res);

return store_rc == SQLITE_DONE;

bind_fail:

REPORT_BIND_FAIL(res, param);

SQLITE_RESET(res);

return 0;

}

static bool store_host_systeminfo(RRDHOST *host)

{

struct rrdhost_system_info *system_info = host->system_info;

if (unlikely(!system_info))

return false;

int ret = 0;

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &host->host_uuid);