// SPDX-License-Identifier: GPL-3.0-or-later

#include "sqlite_metadata.h"

// SQL statements

#define SQL_STORE_CLAIM_ID \

"INSERT INTO node_instance " \

"(host_id, claim_id, date_created) VALUES (@host_id, @claim_id, UNIXEPOCH()) " \

"ON CONFLICT(host_id) DO UPDATE SET claim_id = excluded.claim_id"

#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid"

#define STORE_HOST_LABEL \

"INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "

#define STORE_CHART_LABEL \

"INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "

#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"

#define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid"

#define SQL_STORE_HOST_INFO \

"INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, " \

"memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \

"VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, " \

"@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected)"

#define SQL_STORE_CHART \

"INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, " \

"update_every, chart_type, memory_mode, history_entries) " \

"values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, " \

"@update_every, @chart_type, @memory_mode, @history_entries) " \

"ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, " \

"family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, " \

"plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \

"chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries"

#define SQL_STORE_DIMENSION \

"INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \

"VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) " \

"ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, " \

"divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options"

#define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"

#define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id"

#define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id"

#define SQL_STORE_HOST_SYSTEM_INFO_VALUES \

"INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \

"(@uuid, @name, @value, UNIXEPOCH())"

#define MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID \

"UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0)"

#define DELETE_NON_EXISTING_LOCALHOST "DELETE FROM host WHERE hops = 0 AND host_id <> @host_id"

#define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)"

#define METADATA_MAINTENANCE_FIRST_CHECK (1800) // Maintenance first run after agent startup in seconds

#define METADATA_MAINTENANCE_REPEAT (60) // Repeat if last run for dimensions, charts, labels needs more work

#define METADATA_HEALTH_LOG_INTERVAL (3600) // Repeat maintenance for health

#define METADATA_DIM_CHECK_INTERVAL (3600) // Repeat maintenance for dimensions

#define METADATA_CHART_CHECK_INTERVAL (3600) // Repeat maintenance for charts

#define METADATA_LABEL_CHECK_INTERVAL (3600) // Repeat maintenance for labels

#define METADATA_RUNTIME_THRESHOLD (5) // Run time threshold for cleanup task

#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata

#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata

#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write

#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)

#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop

#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum

#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum

enum metadata_opcode {

METADATA_DATABASE_NOOP = 0,

METADATA_DATABASE_TIMER,

METADATA_DEL_DIMENSION,

METADATA_STORE_CLAIM_ID,

METADATA_ADD_HOST_INFO,

METADATA_SCAN_HOSTS,

METADATA_LOAD_HOST_CONTEXT,

METADATA_DELETE_HOST_CHART_LABELS,

METADATA_MAINTENANCE,

METADATA_SYNC_SHUTDOWN,

METADATA_UNITTEST,

// leave this last

// we need it to check for worker utilization

METADATA_MAX_ENUMERATIONS_DEFINED

};

#define MAX_PARAM_LIST (2)

struct metadata_cmd {

enum metadata_opcode opcode;

struct completion *completion;

const void *param[MAX_PARAM_LIST];

struct metadata_cmd *prev, *next;

};

typedef enum {

METADATA_FLAG_PROCESSING = (1 << 0), // store or cleanup

METADATA_FLAG_SHUTDOWN = (1 << 1), // Shutting down

} METADATA_FLAG;

struct metadata_wc {

uv_thread_t thread;

uv_loop_t *loop;

uv_async_t async;

uv_timer_t timer_req;

time_t metadata_check_after;

METADATA_FLAG flags;

struct completion start_stop_complete;

struct completion *scan_complete;

/* FIFO command queue */

SPINLOCK cmd_queue_lock;

struct metadata_cmd *cmd_base;

};

#define metadata_flag_check(target_flags, flag) (__atomic_load_n(&((target_flags)->flags), __ATOMIC_SEQ_CST) & (flag))

#define metadata_flag_set(target_flags, flag) __atomic_or_fetch(&((target_flags)->flags), (flag), __ATOMIC_SEQ_CST)

#define metadata_flag_clear(target_flags, flag) __atomic_and_fetch(&((target_flags)->flags), ~(flag), __ATOMIC_SEQ_CST)

struct metadata_wc metasync_worker = {.loop = NULL};

//

// For unittest

//

struct thread_unittest {

int join;

unsigned added;

unsigned processed;

unsigned *done;

};

// Metadata functions

struct query_build {

BUFFER *sql;

int count;

char uuid_str[UUID_STR_LEN];

};

#define SQL_DELETE_CHART_LABELS_BY_HOST \

"DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)"

static void delete_host_chart_labels(uuid_t *host_uuid)

{

sqlite3_stmt *res = NULL;

int rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABELS_BY_HOST, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to delete chart labels by host");

return;

}

rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to bind host_id parameter to host chart labels");

goto failed;

}

rc = sqlite3_step_monitored(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to execute command to remove host chart labels");

failed:

if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

error_report("Failed to finalize statement to remove host chart labels");

}

static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

struct query_build *lb = data;

if (unlikely(!lb->count))

buffer_sprintf(lb->sql, STORE_HOST_LABEL);

else

buffer_strcat(lb->sql, ", ");

buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

lb->count++;

return 1;

}

static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {

struct query_build *lb = data;

if (unlikely(!lb->count))

buffer_sprintf(lb->sql, STORE_CHART_LABEL);

else

buffer_strcat(lb->sql, ", ");

buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);

lb->count++;

return 1;

}

#define SQL_DELETE_CHART_LABEL "DELETE FROM chart_label WHERE chart_id = @chart_id"

#define SQL_DELETE_CHART_LABEL_HISTORY "DELETE FROM chart_label WHERE date_created < %ld AND chart_id = @chart_id"

static void clean_old_chart_labels(RRDSET *st)

{

char sql[512];

time_t first_time_s = rrdset_first_entry_s(st);

if (unlikely(!first_time_s))

snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL);

else

snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL_HISTORY, first_time_s);

int rc = exec_statement_with_uuid(sql, &st->chart_uuid);

if (unlikely(rc))

error_report("METADATA: 'host:%s' Failed to clean old labels for chart %s", rrdhost_hostname(st->rrdhost), rrdset_name(st));

}

static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter)

{

size_t old_version = st->rrdlabels_last_saved_version;

size_t new_version = rrdlabels_version(st->rrdlabels);

if (new_version == old_version)

return 0;

struct query_build tmp = {.sql = work_buffer, .count = 0};

uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);

rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);

buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");

int rc = db_execute(db_meta, buffer_tostring(work_buffer));

if (likely(!rc)) {

st->rrdlabels_last_saved_version = new_version;

(*query_counter)++;

}

clean_old_chart_labels(st);

return rc;

}

// Migrate all hosts with hops zero to this host_uuid

void migrate_localhost(uuid_t *host_uuid)

{

int rc;

rc = exec_statement_with_uuid(MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID, host_uuid);

if (!rc)

rc = exec_statement_with_uuid(DELETE_NON_EXISTING_LOCALHOST, host_uuid);

if (!rc) {

if (unlikely(db_execute(db_meta, DELETE_MISSING_NODE_INSTANCES)))

error_report("Failed to remove deleted hosts from node instances");

}

}

static int store_claim_id(uuid_t *host_id, uuid_t *claim_id)

{

sqlite3_stmt *res = NULL;

int rc;

if (unlikely(!db_meta)) {

if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

error_report("Database has not been initialized");

return 1;

}

rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CLAIM_ID, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to store host claim id");

return 1;

}

rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to bind host_id parameter to store claim id");

goto failed;

}

if (claim_id)

rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);

else

rc = sqlite3_bind_null(res, 2);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to bind claim_id parameter to host claim id");

goto failed;

}

rc = execute_insert(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to store host claim id rc = %d", rc);

failed:

if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

error_report("Failed to finalize the prepared statement when storing a host claim id");

return rc != SQLITE_DONE;

}

static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)

{

static __thread sqlite3_stmt *res = NULL;

int rc;

if (unlikely(!res)) {

rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res);

if (rc != SQLITE_OK) {

error_report("Failed to prepare statement to delete a dimension uuid");

return;

}

}

rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto skip_execution;

rc = sqlite3_step_monitored(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to delete dimension uuid, rc = %d", rc);

skip_execution:

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement when deleting dimension UUID, rc = %d", rc);

}

//

// Store host and host system info information in the database

static int store_host_metadata(RRDHOST *host)

{

static __thread sqlite3_stmt *res = NULL;

int rc, param = 0;

if (unlikely((!res))) {

rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to store host, rc = %d", rc);

return 1;

}

}

rc = sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_hostname(host), 0);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_registry_hostname(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, host->rrd_update_every);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_os(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_timezone(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_tags(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, host->system_info ? host->system_info->hops : 0);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_abbrev_timezone(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, host->utc_offset);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_program_name(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, rrdhost_program_version(host), 1);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int64(res, ++param, host->rrd_history_entries);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, (int ) host->health.health_enabled);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int64(res, ++param, (sqlite3_int64) host->last_connected);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

int store_rc = sqlite3_step_monitored(res);

if (unlikely(store_rc != SQLITE_DONE))

error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);

return store_rc != SQLITE_DONE;

bind_fail:

error_report("Failed to bind %d parameter to store host %s, rc = %d", param, rrdhost_hostname(host), rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);

return 1;

}

static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_t *uuid)

{

static __thread sqlite3_stmt *res = NULL;

int rc, param = 0;

if (unlikely(!db_meta)) {

if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)

return 0;

error_report("Database has not been initialized");

return 0;

}

if (unlikely((!res))) {

rc = prepare_statement(db_meta, SQL_STORE_HOST_SYSTEM_INFO_VALUES, &res);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to store host info values, rc = %d", rc);

return 0;

}

}

rc = sqlite3_bind_blob(res, ++param, uuid, sizeof(*uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, name, 0);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = bind_text_null(res, ++param, value ? value : "unknown", 0);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

int store_rc = sqlite3_step_monitored(res);

if (unlikely(store_rc != SQLITE_DONE))

error_report("Failed to store host info value %s, rc = %d", name, rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement to store host info value %s, rc = %d", name, rc);

return store_rc == SQLITE_DONE;

bind_fail:

error_report("Failed to bind %d parameter to store host info values %s, rc = %d", param, name, rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement to store host info values %s, rc = %d", name, rc);

return 0;

}

static bool store_host_systeminfo(RRDHOST *host)

{

struct rrdhost_system_info *system_info = host->system_info;

if (unlikely(!system_info))

return false;

int ret = 0;

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &host->host_uuid);

ret += add_host_sysinfo_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &host->host_uuid);

return !(24 == ret);

}

/*

* Store a chart in the database

*/

static int store_chart_metadata(RRDSET *st)

{

static __thread sqlite3_stmt *res = NULL;

int rc, param = 0, store_rc = 0;

if (unlikely(!res)) {

rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to store chart, rc = %d", rc);

return 1;

}

}

rc = sqlite3_bind_blob(res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_blob(res, ++param, &st->rrdhost->host_uuid, sizeof(st->rrdhost->host_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

const char *name = string2str(st->parts.name);

if (name && *name)

rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC);

else

rc = sqlite3_bind_null(res, ++param);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_family(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_context(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_title(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_units(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, (int) st->priority);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, st->update_every);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, st->chart_type);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, (int) st->db.entries);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

store_rc = execute_insert(res);

if (unlikely(store_rc != SQLITE_DONE))

error_report("Failed to store chart, rc = %d", store_rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement in chart store function, rc = %d", rc);

return store_rc != SQLITE_DONE;

bind_fail:

error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement in chart store function, rc = %d", rc);

return 1;

}

/*

* Store a dimension

*/

static int store_dimension_metadata(RRDDIM *rd)

{

static __thread sqlite3_stmt *res = NULL;

int rc, param = 0;

if (unlikely(!res)) {

rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to store dimension, rc = %d", rc);

return 1;

}

}

rc = sqlite3_bind_blob(res, ++param, &rd->metric_uuid, sizeof(rd->metric_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_blob(res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, string2str(rd->id), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_text(res, ++param, string2str(rd->name), -1, SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, (int) rd->multiplier);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, (int ) rd->divisor);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = sqlite3_bind_int(res, ++param, rd->algorithm);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))

rc = sqlite3_bind_text(res, ++param, "hidden", -1, SQLITE_STATIC);

else

rc = sqlite3_bind_null(res, ++param);

if (unlikely(rc != SQLITE_OK))

goto bind_fail;

rc = execute_insert(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to store dimension, rc = %d", rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement in store dimension, rc = %d", rc);

return 0;

bind_fail:

error_report("Failed to bind parameter %d to store dimension, rc = %d", param, rc);

rc = sqlite3_reset(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to reset statement in store dimension, rc = %d", rc);

return 1;

}

static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused)

{

#ifdef ENABLE_DBENGINE

if(dbengine_enabled) {

bool no_retention = true;

for (size_t tier = 0; tier < storage_tiers; tier++) {

if (!multidb_ctx[tier])

continue;

time_t first_time_t = 0, last_time_t = 0;

if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {

if (first_time_t > 0) {

no_retention = false;

break;

}

}

}

return no_retention;

}

else

return false;

#else

return false;

#endif

}

int get_pragma_value(sqlite3 *database, const char *sql)

{

sqlite3_stmt *res = NULL;

int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);

if (unlikely(rc != SQLITE_OK))

return -1;

int result = -1;

rc = sqlite3_step_monitored(res);

if (likely(rc == SQLITE_ROW))

result = sqlite3_column_int(res, 0);

rc = sqlite3_finalize(res);

(void) rc;

return result;

}

int get_free_page_count(sqlite3 *database)

{

return get_pragma_value(database, "PRAGMA freelist_count");

}

int get_database_page_count(sqlite3 *database)

{

return get_pragma_value(database, "PRAGMA page_count");

}

static bool run_cleanup_loop(

sqlite3_stmt *res,

struct metadata_wc *wc,

bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool),

void (*action_cb)(uuid_t *, sqlite3_stmt **, bool),

uint32_t *total_checked,

uint32_t *total_deleted,

uint64_t *row_id,

sqlite3_stmt **check_stmt,

sqlite3_stmt **action_stmt,

bool check_flag,

bool action_flag)

{

if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))

return true;

int rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) *row_id);

if (unlikely(rc != SQLITE_OK))

return true;

time_t start_running = now_monotonic_sec();

bool time_expired = false;

while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW &&

(*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) {

if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))

break;

*row_id = sqlite3_column_int64(res, 1);

rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag);

if (rc == true) {

action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag);

(*total_deleted)++;

}

(*total_checked)++;

time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD);

}

return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP);

}

#define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id"

#define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id"

static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension)

{

int rc, result = 1;

sqlite3_stmt *res = check_res ? *check_res : NULL;

if (!res) {

if (check_in_dimension)

rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0);

else

rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);

return 0;

}

if (check_res)

*check_res = res;

}

rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to bind chart uuid parameter, rc = %d", rc);

goto skip;

}

rc = sqlite3_step_monitored(res);

if (likely(rc == SQLITE_ROW))

result = sqlite3_column_int(res, 0);

skip:

if (check_res)

rc = sqlite3_reset(res);

else

rc = sqlite3_finalize(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to %s statement that checks chart uuid existence rc = %d", check_res ? "reset" : "finalize", rc);

return result == 0;

}

#define SQL_DELETE_CHART_BY_UUID "DELETE FROM chart WHERE chart_id = @chart_id"

#define SQL_DELETE_CHART_LABEL_BY_UUID "DELETE FROM chart_label WHERE chart_id = @chart_id"

static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only)

{

int rc;

sqlite3_stmt *res = action_res ? *action_res : NULL;

if (!res) {

if (label_only)

rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABEL_BY_UUID, -1, &res, 0);

else

rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_BY_UUID, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);

return;

}

if (action_res)

*action_res = res;

}

rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to bind chart uuid parameter, rc = %d", rc);

goto skip;

}

rc = sqlite3_step_monitored(res);

if (unlikely(rc != SQLITE_DONE))

error_report("Failed to delete a chart uuid from the %s table, rc = %d", label_only ? "labels" : "chart", rc);

skip:

if (action_res)

rc = sqlite3_reset(res);

else

rc = sqlite3_finalize(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to %s statement that deletes a chart uuid rc = %d", action_res ? "reset" : "finalize", rc);

}

static void check_dimension_metadata(struct metadata_wc *wc)

{

static time_t next_execution_t = 0;

static uint64_t last_row_id = 0;

time_t now = now_realtime_sec();

if (!next_execution_t)

next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

if (next_execution_t && next_execution_t > now)

return;

int rc;

sqlite3_stmt *res = NULL;

rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION_LIST, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to fetch host dimensions");

return;

}

uint32_t total_checked = 0;

uint32_t total_deleted = 0;

nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id);

bool more_to_do = run_cleanup_loop(

res,

wc,

dimension_can_be_deleted,

delete_dimension_uuid,

&total_checked,

&total_deleted,

&last_row_id,

NULL,

NULL,

false,

false);

now = now_realtime_sec();

if (more_to_do)

next_execution_t = now + METADATA_MAINTENANCE_REPEAT;

else {

last_row_id = 0;

next_execution_t = now + METADATA_DIM_CHECK_INTERVAL;

}

nd_log(

NDLS_DAEMON,

NDLP_DEBUG,

"Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",

total_checked,

total_deleted,

last_row_id ? "resume" : "restart",

(long long)(next_execution_t - now));

rc = sqlite3_finalize(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to finalize the prepared statement to check dimensions");

}

static void check_chart_metadata(struct metadata_wc *wc)

{

static time_t next_execution_t = 0;

static uint64_t last_row_id = 0;

time_t now = now_realtime_sec();

if (!next_execution_t)

next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

if (next_execution_t && next_execution_t > now)

return;

sqlite3_stmt *res = NULL;

int rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LIST, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to fetch charts");

return;

}

uint32_t total_checked = 0;

uint32_t total_deleted = 0;

nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id);

sqlite3_stmt *check_res = NULL;

sqlite3_stmt *action_res = NULL;

bool more_to_do = run_cleanup_loop(

res,

wc,

chart_can_be_deleted,

delete_chart_uuid,

&total_checked,

&total_deleted,

&last_row_id,

&check_res,

&action_res,

true,

false);

if (check_res)

sqlite3_finalize(check_res);

if (action_res)

sqlite3_finalize(action_res);

now = now_realtime_sec();

if (more_to_do)

next_execution_t = now + METADATA_MAINTENANCE_REPEAT;

else {

last_row_id = 0;

next_execution_t = now + METADATA_CHART_CHECK_INTERVAL;

}

nd_log(

NDLS_DAEMON,

NDLP_DEBUG,

"Charts checked %u, deleted %u. Checks will %s in %lld seconds",

total_checked,

total_deleted,

last_row_id ? "resume" : "restart",

(long long)(next_execution_t - now));

rc = sqlite3_finalize(res);

if (unlikely(rc != SQLITE_OK))

error_report("Failed to finalize the prepared statement when reading charts");

}

static void check_label_metadata(struct metadata_wc *wc)

{

static time_t next_execution_t = 0;

static uint64_t last_row_id = 0;

time_t now = now_realtime_sec();

if (!next_execution_t)

next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;

if (next_execution_t && next_execution_t > now)

return;

int rc;

sqlite3_stmt *res = NULL;

rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LABEL_LIST, -1, &res, 0);

if (unlikely(rc != SQLITE_OK)) {

error_report("Failed to prepare statement to fetch charts");