// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements. See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership. The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License. You may obtain a copy of the License at

//

// http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied. See the License for the

// specific language governing permissions and limitations

// under the License.

#include "exprs/vsearch.h"

#include <fmt/format.h>

#include <memory>

#include <roaring/roaring.hh>

#include "common/logging.h"

#include "common/status.h"

#include "core/column/column_const.h"

#include "exprs/function/function_search.h"

#include "exprs/vexpr_context.h"

#include "exprs/vliteral.h"

#include "exprs/vslot_ref.h"

#include "glog/logging.h"

#include "runtime/runtime_state.h"

#include "storage/index/inverted/inverted_index_reader.h"

#include "storage/olap_common.h"

#include "storage/segment/segment.h"

namespace doris {

using namespace segment_v2;

namespace {

struct SearchInputBundle {

std::unordered_map<std::string, IndexIterator*> iterators;

std::unordered_map<std::string, IndexFieldNameAndTypePair> field_types;

std::unordered_map<std::string, int> field_name_to_column_id;

std::vector<int> column_ids;

ColumnsWithTypeAndName literal_args;

};

void add_search_binding_diagnostic(const IndexExecContext* index_context,

const std::string& diagnostic) {

VLOG_DEBUG << diagnostic;

if (index_context == nullptr) {

return;

}

const auto& index_query_context = index_context->get_index_query_context();

if (index_query_context != nullptr && index_query_context->stats != nullptr) {

index_query_context->stats->inverted_index_stats.add_binding_diagnostic(diagnostic);

}

}

Status collect_search_inputs(const VSearchExpr& expr, VExprContext* context,

SearchInputBundle* bundle) {

DCHECK(bundle != nullptr);

auto index_context = context->get_index_context();

if (index_context == nullptr) {

LOG(WARNING) << "collect_search_inputs: No inverted index context available";

return Status::InternalError("No inverted index context available");

}

// Get field bindings for variant subcolumn support

const auto& search_param = expr.get_search_param();

const auto& field_bindings = search_param.field_bindings;

std::unordered_map<std::string, ColumnId> parent_to_base_column_id;

std::unordered_map<std::string, std::string> parent_to_storage_field_prefix;

// Resolve and cache the base (parent) column id for a variant field binding.

// This avoids repeated schema lookups when multiple subcolumns share the same parent column.

auto resolve_parent_column_id = [&](const std::string& parent_field, ColumnId* column_id) {

// Guard against invalid inputs: variant bindings may miss parent_field, and callers must

// provide a valid output pointer to receive the resolved id.

if (parent_field.empty() || column_id == nullptr) {

return false;

}

auto it = parent_to_base_column_id.find(parent_field);

if (it != parent_to_base_column_id.end()) {

*column_id = it->second;

return true;

}

if (index_context == nullptr || index_context->segment() == nullptr) {

return false;

}

const int32_t ordinal =

index_context->segment()->tablet_schema()->field_index(parent_field);

if (ordinal < 0) {

return false;

}

ColumnId resolved_id = static_cast<ColumnId>(ordinal);

parent_to_base_column_id.emplace(parent_field, resolved_id);

if (auto* storage_name_type = index_context->get_storage_name_and_type_by_id(resolved_id);

storage_name_type != nullptr) {

parent_to_storage_field_prefix[parent_field] = storage_name_type->first;

}

*column_id = resolved_id;

return true;

};

int child_index = 0; // Index for iterating through children

for (const auto& child : expr.children()) {

if (child->is_slot_ref()) {

auto* column_slot_ref = assert_cast<VSlotRef*>(child.get());

int column_id = column_slot_ref->column_id();

// Determine the field_name from field_bindings (for variant subcolumns)

// field_bindings and children should have the same order

std::string field_name;

const TSearchFieldBinding* binding = nullptr;

if (child_index < field_bindings.size()) {

// Use field_name from binding (may include "parent.subcolumn" for variant)

binding = &field_bindings[child_index];

field_name = binding->field_name;

} else {

// Fallback to column_name if binding not found

field_name = column_slot_ref->column_name();

}

bundle->field_name_to_column_id[field_name] = column_id;

auto* iterator = index_context->get_inverted_index_iterator_by_column_id(column_id);

const auto* storage_name_type =

index_context->get_storage_name_and_type_by_column_id(column_id);

bool field_added = false;

// For variant subcolumns, slot_ref might not map to a real indexed column in the scan schema.

// Fall back to the parent variant column's iterator and synthesize lucene field name.

if (iterator == nullptr && binding != nullptr &&

binding->__isset.is_variant_subcolumn && binding->is_variant_subcolumn &&

binding->__isset.parent_field_name && !binding->parent_field_name.empty()) {

ColumnId base_column_id = 0;

if (resolve_parent_column_id(binding->parent_field_name, &base_column_id)) {

iterator = index_context->get_inverted_index_iterator_by_id(base_column_id);

const auto* base_storage_name_type =

index_context->get_storage_name_and_type_by_id(base_column_id);

if (iterator != nullptr && base_storage_name_type != nullptr) {

std::string prefix = base_storage_name_type->first;

if (auto pit =

parent_to_storage_field_prefix.find(binding->parent_field_name);

pit != parent_to_storage_field_prefix.end() && !pit->second.empty()) {

prefix = pit->second;

} else {

parent_to_storage_field_prefix[binding->parent_field_name] = prefix;

}

std::string sub_path;

if (binding->__isset.subcolumn_path) {

sub_path = binding->subcolumn_path;

}

if (sub_path.empty()) {

// Fallback: strip "parent." prefix from logical field name

std::string pfx = binding->parent_field_name + ".";

if (field_name.starts_with(pfx)) {

sub_path = field_name.substr(pfx.size());

}

}

if (!sub_path.empty()) {

bundle->iterators[field_name] = iterator;

bundle->field_types[field_name] =

std::make_pair(prefix + "." + sub_path, nullptr);

int base_column_index =

index_context->column_index_by_id(base_column_id);

if (base_column_index >= 0) {

bundle->column_ids.emplace_back(base_column_index);

}

add_search_binding_diagnostic(

index_context.get(),

fmt::format("[VariantSearchBinding] phase=collect_inputs "

"result=parent_fallback logical_field={} "

"parent_field={} sub_path={} base_column_id={} "

"stored_field={} reason=slot_iterator_missing",

field_name, binding->parent_field_name, sub_path,

base_column_id, prefix + "." + sub_path));

field_added = true;

}

}

} else {

add_search_binding_diagnostic(

index_context.get(),

fmt::format("[VariantSearchBinding] phase=collect_inputs "

"result=reject logical_field={} parent_field={} "

"reason=parent_column_not_found",

field_name, binding->parent_field_name));

}

}

// Only collect fields that have iterators (materialized columns with indexes)

if (!field_added && iterator != nullptr) {

if (storage_name_type == nullptr) {

return Status::InternalError("storage_name_type not found for column {} in {}",

column_id, expr.expr_name());

}

bundle->iterators.emplace(field_name, iterator);

bundle->field_types.emplace(field_name, *storage_name_type);

bundle->column_ids.emplace_back(column_id);

if (binding != nullptr && binding->__isset.is_variant_subcolumn &&

binding->is_variant_subcolumn) {

add_search_binding_diagnostic(

index_context.get(),

fmt::format("[VariantSearchBinding] phase=collect_inputs "

"result=direct_iterator logical_field={} column_id={} "

"stored_field={}",

field_name, column_id, storage_name_type->first));

}

}

child_index++;

} else if (child->is_literal()) {

auto* literal = assert_cast<VLiteral*>(child.get());

bundle->literal_args.emplace_back(literal->get_column_ptr(), literal->get_data_type(),

literal->expr_name());

} else {

// Check if this is ElementAt expression (for variant subcolumn access)

if (child->expr_name() == "element_at" && child_index < field_bindings.size() &&

field_bindings[child_index].__isset.is_variant_subcolumn &&

field_bindings[child_index].is_variant_subcolumn) {

// Variant subcolumn not materialized - skip, will create empty BitSetQuery in function_search

add_search_binding_diagnostic(

index_context.get(),

fmt::format("[VariantSearchBinding] phase=collect_inputs "

"result=unmaterialized_element_at logical_field={} "

"parent_field={} sub_path={} reason=no_slot_ref",

field_bindings[child_index].field_name,

field_bindings[child_index].__isset.parent_field_name

? field_bindings[child_index].parent_field_name

: "",

field_bindings[child_index].__isset.subcolumn_path

? field_bindings[child_index].subcolumn_path

: ""));

child_index++;

continue;

}

// Not a supported child type

return Status::InvalidArgument("Unsupported child node type: {}", child->expr_name());

}

}

return Status::OK();

}

} // namespace

VSearchExpr::VSearchExpr(const TExprNode& node) : VExpr(node) {

if (node.__isset.search_param) {

_search_param = node.search_param;

_original_dsl = _search_param.original_dsl;

}

}

Status VSearchExpr::prepare(RuntimeState* state, const RowDescriptor& row_desc,

VExprContext* context) {

RETURN_IF_ERROR(VExpr::prepare(state, row_desc, context));

const auto& query_options = state->query_options();

if (query_options.__isset.enable_inverted_index_query_cache) {

_enable_cache = query_options.enable_inverted_index_query_cache;

}

return Status::OK();

}

const std::string& VSearchExpr::expr_name() const {

static const std::string name = "VSearchExpr";

return name;

}

Status VSearchExpr::execute_column_impl(VExprContext* context, const Block* block,

const Selector* selector, size_t count,

ColumnPtr& result_column) const {

if (fast_execute(context, selector, count, result_column)) {

return Status::OK();

}

return Status::InternalError("SearchExpr should not be executed without inverted index");

}

Status VSearchExpr::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) {

if (_search_param.original_dsl.empty()) {

return Status::InvalidArgument("search DSL is empty");

}

auto index_context = context->get_index_context();

if (!index_context) {

LOG(WARNING) << "VSearchExpr: No inverted index context available";

return Status::OK();

}

SearchInputBundle bundle;

RETURN_IF_ERROR(collect_search_inputs(*this, context, &bundle));

VLOG_DEBUG << "VSearchExpr: bundle.iterators.size()=" << bundle.iterators.size();

const bool is_nested_query = _search_param.root.clause_type == "NESTED";

if (bundle.iterators.empty() && !is_nested_query) {

LOG(WARNING) << "VSearchExpr: No indexed columns available for evaluation, DSL: "

<< _original_dsl;

add_search_binding_diagnostic(

index_context.get(),

fmt::format("[VariantSearchBinding] phase=evaluate_search result=no_iterator "

"dsl={} reason=no_indexed_columns",

_original_dsl));

auto empty_bitmap = InvertedIndexResultBitmap(std::make_shared<roaring::Roaring>(),

std::make_shared<roaring::Roaring>());

index_context->set_index_result_for_expr(this, std::move(empty_bitmap));

return Status::OK();

}

auto index_query_context = index_context->get_index_query_context();

auto function = std::make_shared<FunctionSearch>();

auto result_bitmap = InvertedIndexResultBitmap();

auto status = function->evaluate_inverted_index_with_search_param(

_search_param, bundle.field_types, bundle.iterators, segment_num_rows, result_bitmap,

_enable_cache, index_context.get(), bundle.field_name_to_column_id,

index_query_context);

if (!status.ok()) {

LOG(WARNING) << "VSearchExpr: Function evaluation failed: " << status.to_string();

return status;

}

index_context->set_index_result_for_expr(this, result_bitmap);

for (int column_id : bundle.column_ids) {

index_context->set_true_for_index_status(this, column_id);

}

return Status::OK();

}

} // namespace doris