/**

* Copyright (c) 2021 OceanBase

* OceanBase CE is licensed under Mulan PubL v2.

* You can use this software according to the terms and conditions of the Mulan PubL v2.

* You may obtain a copy of Mulan PubL v2 at:

* http://license.coscl.org.cn/MulanPubL-2.0

* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,

* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,

* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.

* See the Mulan PubL v2 for more details.

*/

#define USING_LOG_PREFIX SQL_OPT

#include "sql/optimizer/ob_log_table_scan.h"

#include "lib/container/ob_se_array_iterator.h"

#include "sql/optimizer/ob_table_partition_info.h"

#include "sql/resolver/expr/ob_raw_expr_util.h"

#include "sql/resolver/dml/ob_select_stmt.h"

#include "sql/ob_sql_utils.h"

#include "sql/optimizer/ob_optimizer_util.h"

#include "sql/code_generator/ob_expr_generator_impl.h"

#include "sql/plan_cache/ob_plan_set.h"

#include "sql/optimizer/ob_log_operator_factory.h"

#include "share/schema/ob_schema_utils.h"

#include "sql/rewrite/ob_transform_utils.h"

#include "sql/optimizer/ob_join_order.h"

#include "sql/optimizer/ob_log_join.h"

#include "sql/dblink/ob_dblink_utils.h"

using namespace oceanbase::sql;

using namespace oceanbase::common;

using namespace oceanbase::share;

using namespace oceanbase::storage;

using oceanbase::share::schema::ObTableSchema;

using oceanbase::share::schema::ObSchemaGetterGuard;

const char *ObLogTableScan::get_name() const

{

bool is_get = false;

bool is_range = false;

const char *name = NULL;

int ret = OB_SUCCESS;

SampleInfo::SampleMethod sample_method = get_sample_info().method_;

if (NULL != pre_query_range_) {

if (OB_FAIL(get_pre_query_range()->is_get(is_get))) {

// is_get always return true

LOG_WARN("failed to get is_get", K(ret));

} else if (range_conds_.count() > 0) {

is_range = true;

}

}

if (sample_method != SampleInfo::NO_SAMPLE) {

name = (sample_method == SampleInfo::ROW_SAMPLE) ? "TABLE ROW SAMPLE SCAN" : "TABLE BLOCK SAMPLE SCAN";

} else if (is_skip_scan()) {

name = use_das() ? "DISTRIBUTED TABLE SKIP SCAN" : "TABLE SKIP SCAN";

} else if (EXTERNAL_TABLE == get_table_type()) {

name = "EXTERNAL TABLE SCAN";

} else if (use_das()) {

if (is_get) {

name = "DISTRIBUTED TABLE GET";

} else if (is_range) {

name = "DISTRIBUTED TABLE RANGE SCAN";

} else {

name = "DISTRIBUTED TABLE FULL SCAN";

}

} else if (use_column_store()) {

if (is_get) {

name = "COLUMN TABLE GET";

} else if (is_range) {

name = "COLUMN TABLE RANGE SCAN";

} else {

name = "COLUMN TABLE FULL SCAN";

}

} else {

if (is_get) {

name = "TABLE GET";

} else if (is_range) {

name = "TABLE RANGE SCAN";

} else {

name = "TABLE FULL SCAN";

}

}

return name;

}

void ObLogTableScan::set_ref_table_id(uint64_t ref_table_id)

{

ref_table_id_ = ref_table_id;

}

int ObLogTableScan::set_range_columns(const ObIArray<ColumnItem> &range_columns)

{

int ret = OB_SUCCESS;

range_columns_.reset();

if (OB_FAIL(range_columns_.assign(range_columns))) {

LOG_WARN("failed to assign range columns", K(ret));

} else if (!is_virtual_table(ref_table_id_)) {

// banliu.zyd: 虚拟表不保序，仅对非虚拟表做处理

OrderItem item;

int64_t N = range_columns.count();

reset_op_ordering();

common::ObIArray<OrderItem> &op_ordering = get_op_ordering();

for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) {

item.expr_ = range_columns.at(i).expr_;

item.order_type_ = scan_direction_;

if (OB_FAIL(op_ordering.push_back(item))) {

LOG_WARN("failed to push back item", K(ret));

}

}

} else { /*do nothing*/ }

return ret;

}

int ObLogTableScan::do_re_est_cost(EstimateCostInfo &param, double &card, double &op_cost, double &cost)

{

int ret = OB_SUCCESS;

double limit_percent = -1.0;

int64_t limit_count = -1;

int64_t offset_count = 0;

ObOptimizerContext *opt_ctx = NULL;

if (OB_ISNULL(access_path_)) { // table scan create from CteTablePath

card = get_card();

op_cost = get_op_cost();

cost = get_cost();

} else if (OB_ISNULL(get_plan()) || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context())

|| OB_ISNULL(est_cost_info_) || OB_UNLIKELY(1 > param.need_parallel_)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected params", K(ret), K(opt_ctx), K(est_cost_info_), K(param));

} else if (OB_FAIL(get_limit_offset_value(NULL, limit_count_expr_, limit_offset_expr_,

limit_percent, limit_count, offset_count))) {

LOG_WARN("failed to get limit offset value", K(ret));

} else {

card = get_card();

double limit_count_double = static_cast<double>(limit_count);

double offset_count_double = static_cast<double>(offset_count);

param.need_row_count_ = 0 > param.need_row_count_ ? card : param.need_row_count_;

if (0 <= limit_count) {

param.need_row_count_ = std::min(param.need_row_count_, limit_count_double * param.need_parallel_);

}

param.need_row_count_ = std::min(param.need_row_count_, card);

param.need_row_count_ += offset_count_double;

if (OB_FAIL(AccessPath::re_estimate_cost(param,

*est_cost_info_,

sample_info_,

*opt_ctx,

card,

op_cost))) {

LOG_WARN("failed to re estimate cost", K(ret));

} else {

cost = op_cost;

card = std::min(param.need_row_count_ - offset_count_double, card);

}

}

return ret;

}

int ObLogTableScan::get_op_exprs(ObIArray<ObRawExpr*> &all_exprs)

{

int ret = OB_SUCCESS;

if (OB_ISNULL(get_plan())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

} else if (OB_FAIL(generate_access_exprs())) {

LOG_WARN("failed to generate access exprs", K(ret));

} else if (NULL != limit_count_expr_ &&

OB_FAIL(all_exprs.push_back(limit_count_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (NULL != limit_offset_expr_ &&

OB_FAIL(all_exprs.push_back(limit_offset_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (NULL != fq_expr_ && OB_FAIL(all_exprs.push_back(fq_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (NULL != tablet_id_expr_ && OB_FAIL(all_exprs.push_back(tablet_id_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (NULL != calc_part_id_expr_ && OB_FAIL(all_exprs.push_back(calc_part_id_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (OB_FAIL(allocate_lookup_trans_info_expr())) {

LOG_WARN("failed to add lookup trans expr", K(ret));

} else if (NULL != trans_info_expr_ && OB_FAIL(all_exprs.push_back(trans_info_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (OB_FAIL(append(all_exprs, access_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append(all_exprs, pushdown_aggr_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(ObLogicalOperator::get_op_exprs(all_exprs))) {

LOG_WARN("failed to get exprs", K(ret));

} else { /*do nothing*/ }

return ret;

}

int ObLogTableScan::allocate_expr_post(ObAllocExprContext &ctx)

{

int ret = OB_SUCCESS;

for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs_.count(); i++) {

ObRawExpr *expr = access_exprs_.at(i);

if (OB_ISNULL(expr)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("null expr", K(ret));

} else if (OB_FAIL(mark_expr_produced(expr, branch_id_, id_, ctx))) {

LOG_WARN("failed to mark expr as produced", K(*expr), K(branch_id_), K(id_), K(ret));

} else { /*do nothing*/ }

}

for (int64_t i = 0; OB_SUCC(ret) && i < pushdown_aggr_exprs_.count(); i++) {

ObRawExpr *expr = NULL;

if (OB_ISNULL(expr = pushdown_aggr_exprs_.at(i))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get null expr", K(ret));

} else if (OB_FAIL(mark_expr_produced(expr, branch_id_, id_, ctx))) {

LOG_WARN("failed to mark expr as produced", K(*expr), K(branch_id_), K(id_), K(ret));

} else { /*do nothing*/ }

}

// check if we can produce some more exprs, such as 1 + 'c1' after we have produced 'c1'

if (OB_SUCC(ret)) {

if (!is_plan_root() && OB_FAIL(append(output_exprs_, access_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append(output_exprs_, pushdown_aggr_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(ObLogicalOperator::allocate_expr_post(ctx))) {

LOG_WARN("failed to allocate expr post", K(ret));

} else { /*do nothing*/ }

}

// add special exprs to all exprs

if (OB_SUCC(ret)) {

ObRawExprUniqueSet &all_exprs = get_plan()->get_optimizer_context().get_all_exprs();

if (NULL != part_expr_ && OB_FAIL(all_exprs.append(part_expr_))) {

LOG_WARN("failed to get part expr", K(ret));

} else if (NULL != subpart_expr_ && OB_FAIL(all_exprs.append(subpart_expr_))) {

LOG_WARN("failed to get subpart expr", K(ret));

} else if (OB_FAIL(all_exprs.append(range_conds_))) {

LOG_WARN("failed to append range exprs", K(ret));

} else {

for (int64_t i = 0; OB_SUCC(ret) && i < real_expr_map_.count(); ++i) {

if (OB_FAIL(all_exprs.append(real_expr_map_.at(i).second))) {

LOG_WARN("failed to append real expr", K(ret));

}

}

}

}

return ret;

}

int ObLogTableScan::check_output_dependance(common::ObIArray<ObRawExpr *> &child_output, PPDeps &deps)

{

int ret = OB_SUCCESS;

ObSEArray<ObRawExpr*, 8> exprs;

LOG_TRACE("start to check output exprs", K(type_), K(child_output), K(deps));

ObRawExprCheckDep dep_checker(child_output, deps, true);

if (OB_FAIL(append(exprs, filter_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(exprs, output_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(exprs, rowkey_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(exprs, part_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(exprs, spatial_exprs_))) {

LOG_WARN("failed to append exprs", K(ret));

} else if (use_batch() && nullptr != group_id_expr_

&& OB_FAIL(add_var_to_array_no_dup(exprs, group_id_expr_))) {

LOG_WARN("failed to push back group id expr", K(ret));

} else if (index_back_ &&

nullptr != trans_info_expr_ &&

OB_FAIL(add_var_to_array_no_dup(exprs, trans_info_expr_))) {

LOG_WARN("fail to add lookup trans info expr", K(ret));

} else if (OB_FAIL(dep_checker.check(exprs))) {

LOG_WARN("failed to check op_exprs", K(ret));

} else {

LOG_TRACE("succeed to check output exprs", K(exprs), K(type_), K(deps));

}

return ret;

}

int ObLogTableScan::extract_file_column_exprs_recursively(ObRawExpr *expr)

{

int ret = OB_SUCCESS;

if (OB_ISNULL(expr)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("expr is null", K(ret));

} else if (T_PSEUDO_EXTERNAL_FILE_COL == expr->get_expr_type()) {

auto pseudo_col_expr = static_cast<ObPseudoColumnRawExpr*>(expr);

if (pseudo_col_expr->get_table_id() != table_id_) {

//table id may be changed because of rewrite

pseudo_col_expr->set_table_id(table_id_);

}

if (OB_FAIL(add_var_to_array_no_dup(ext_file_column_exprs_, expr))) {

LOG_WARN("failed to add var to array", K(ret));

}

} else {

for (int i = 0; OB_SUCC(ret) && i < expr->get_children_count(); ++i) {

if (OB_FAIL(extract_file_column_exprs_recursively(expr->get_param_expr(i)))) {

LOG_WARN("fail to extract file column expr", K(ret));

}

}

}

return ret;

}

// If the filter is indexed before returning to the table,

// and there are generated columns, deep copy is required

int ObLogTableScan::copy_filter_before_index_back()

{

int ret = OB_SUCCESS;

ObIArray<ObRawExpr*> &filters = get_filter_exprs();

const auto &flags = get_filter_before_index_flags();

if (OB_FAIL(filter_before_index_back_set())) {

LOG_WARN("Failed to filter_before_index_back_set", K(ret));

} else if (get_index_back() && !flags.empty()) {

ObRawExprCopier copier(get_plan()->get_optimizer_context().get_expr_factory());

const ObDMLStmt *stmt = NULL;

ObArray<ObRawExpr *> column_exprs;

if (OB_ISNULL(stmt = get_stmt())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(get_stmt()), K(ret));

} else if (OB_FAIL(stmt->get_column_exprs(column_exprs))) {

LOG_WARN("failed to get column exprs", K(ret));

} else if (OB_FAIL(copier.add_skipped_expr(column_exprs))) {

LOG_WARN("failed to add skipped exprs", K(ret));

}

for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) {

if (filters.at(i)->has_flag(CNT_PL_UDF)) {

// do nothing.

} else if (flags.at(i)) {

if (get_index_back() && get_is_index_global() && filters.at(i)->has_flag(CNT_SUB_QUERY)) {

// do nothing.

} else {

bool is_contain_vir_gen_column = false;

// ObArray<ObRawExpr *> column_exprs;

// scan_pushdown before index back conclude virtual generated column

// need copy for avoiding shared expression.

if (OB_FAIL(ObRawExprUtils::contain_virtual_generated_column(filters.at(i), is_contain_vir_gen_column))) {

LOG_WARN("fail to contain virtual gen column", K(ret));

} else if (is_contain_vir_gen_column) {

if (OB_FAIL(copier.copy(filters.at(i), filters.at(i)))) {

LOG_WARN("failed to copy exprs", K(ret));

}

}

}

}

}

}

return ret;

}

int ObLogTableScan::generate_access_exprs()

{

int ret = OB_SUCCESS;

const ObDMLStmt *stmt = NULL;

ObSEArray<ObRawExpr*, 8> temp_exprs;

if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_stmt())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(get_plan()), K(get_stmt()), K(ret));

} else if (OB_FAIL(copy_filter_before_index_back())) {

LOG_WARN("failed to copy filter before index back", K(ret));

} else if (OB_FAIL(generate_necessary_rowkey_and_partkey_exprs())) {

LOG_WARN("failed to generate rowkey and part exprs", K(ret));

} else if (use_batch()

&& OB_FAIL(ObOptimizerUtil::allocate_group_id_expr(get_plan(), group_id_expr_))) {

LOG_WARN("allocate group id expr failed", K(ret));

} else if (nullptr != group_id_expr_ && OB_FAIL(access_exprs_.push_back(group_id_expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else if (OB_FAIL(append_array_no_dup(access_exprs_, rowkey_exprs_))) {

LOG_WARN("failed to push back exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(access_exprs_, part_exprs_))) {

LOG_WARN("failed to push back exprs", K(ret));

} else if (is_spatial_index_ && OB_FAIL(append_array_no_dup(access_exprs_, spatial_exprs_))) {

LOG_WARN("failed to push back exprs", K(ret));

} else if (is_index_global_ && index_back_) {

if (OB_FAIL(ObRawExprUtils::extract_column_exprs(filter_exprs_, temp_exprs))) {

LOG_WARN("failed to extract column exprs", K(ret));

} else if (OB_FAIL(append_array_no_dup(access_exprs_, temp_exprs))) {

LOG_WARN("failed to append array no dup", K(ret));

} else { /*do nothing*/}

}

if (OB_SUCC(ret)) {

for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_column_size(); i++) {

const ColumnItem *col_item = stmt->get_column_item(i);

if (OB_ISNULL(col_item) || OB_ISNULL(col_item->expr_)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(col_item), K(ret));

} else if (col_item->table_id_ != table_id_ || !col_item->expr_->is_explicited_reference()) {

//do nothing

} else if (col_item->expr_->is_only_referred_by_stored_gen_col()) {

//skip if is only referred by stored generated columns which don't need to be recalculated

} else if (is_index_scan() && !get_index_back() && !col_item->expr_->is_referred_by_normal()) {

//skip the dependant columns of partkeys and generated columns if index_back is false in index scan

} else if (OB_FAIL(temp_exprs.push_back(col_item->expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else { /*do nothing*/}

}

if (OB_FAIL(ret)) {

/*do nothing*/

} else if (OB_FAIL(append_array_no_dup(access_exprs_, temp_exprs))) {

LOG_WARN("failed to append exprs", K(ret));

} else { /*do nothing*/ }

for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_pseudo_column_like_exprs().count(); i++) {

ObRawExpr *expr = stmt->get_pseudo_column_like_exprs().at(i);

if (OB_ISNULL(expr)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

} else if ((T_ORA_ROWSCN == expr->get_expr_type())

&& static_cast<ObPseudoColumnRawExpr*>(expr)->get_table_id() == table_id_) {

if (OB_FAIL(access_exprs_.push_back(expr))) {

LOG_WARN("fail to push back expr", K(ret));

}

}

}

if (OB_SUCC(ret)) {

if (OB_FAIL(add_mapping_columns_for_vt(access_exprs_))) {

LOG_WARN("failed to add mapping columns for vt", K(ret));

} else {

LOG_TRACE("succeed to generate access exprs", K(access_exprs_));

}

}

}

return ret;

}

int ObLogTableScan::replace_gen_col_op_exprs(ObRawExprReplacer &replacer)

{

int ret = OB_SUCCESS;

if (is_index_scan() && !(get_index_back())) {

// do nothing.

} else if (!replacer.empty()) {

FOREACH_CNT_X(it, get_op_ordering(), OB_SUCC(ret)) {

if (OB_FAIL(replace_expr_action(replacer, it->expr_))) {

LOG_WARN("replace agg expr failed", K(ret));

}

}

if (OB_FAIL(ret)) {

} else if (OB_FAIL(replace_exprs_action(replacer, get_output_exprs()))) {

LOG_WARN("failed to replace agg expr", K(ret));

} else if (NULL != limit_offset_expr_ &&

OB_FAIL(replace_expr_action(replacer, limit_count_expr_))) {

LOG_WARN("failed to replace limit count expr", K(ret));

} else if (NULL != limit_offset_expr_ &&

OB_FAIL(replace_expr_action(replacer, limit_offset_expr_))) {

LOG_WARN("failed to replace limit offset expr ", K(ret));

} else {

for (int64_t i = 0; OB_SUCC(ret) && i < pushdown_aggr_exprs_.count(); ++i) {

ObAggFunRawExpr *pushdown_aggr_expr = pushdown_aggr_exprs_.at(i);

for (int64_t j = 0; OB_SUCC(ret) && j < pushdown_aggr_expr->get_param_count(); j++) {

if (OB_ISNULL(pushdown_aggr_expr->get_param_expr(j))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("param_expr is NULL", K(j), K(ret));

} else if (OB_FAIL(replace_expr_action(replacer,

pushdown_aggr_expr->get_param_expr(j)))) {

LOG_WARN("Fail to replace push down aggr expr", K(i), K(j), K(ret));

}

}

}

}

// Scenario processing without index table.

if (OB_SUCC(ret) && !get_index_back()) {

if (NULL != part_expr_ &&

OB_FAIL(replace_expr_action(replacer, part_expr_))) {

LOG_WARN("failed to replace part expr ", K(ret));

} else if (NULL != subpart_expr_ && OB_FAIL(replace_expr_action(replacer,

subpart_expr_))) {

LOG_WARN("failed to replace subpart expr ", K(ret));

} else if (OB_FAIL(replace_exprs_action(replacer, get_filter_exprs()))) {

LOG_WARN("failed to replace agg expr", K(ret));

}

}

// Index back to table scene processing

if (OB_SUCC(ret) && get_index_back()) {

if (OB_FAIL(replace_index_back_pushdown_filters(replacer))) {

LOG_WARN("failed to replace pushdown exprs", K(ret));

}

}

} else { /* Do nothing */ }

return ret;

}

int ObLogTableScan::replace_index_back_pushdown_filters(ObRawExprReplacer &replacer)

{

int ret = OB_SUCCESS;

ObIArray<ObRawExpr*> &filters = get_filter_exprs();

const auto &flags = get_filter_before_index_flags();

if (get_contains_fake_cte() || is_virtual_table(get_ref_table_id())) {

// nonpushdown need replace.

if (OB_FAIL(replace_exprs_action(replacer, filters))) {

LOG_WARN("failed to replace agg expr", K(ret));

}

} else {

for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) {

if (filters.at(i)->has_flag(CNT_PL_UDF)) {

// nonpushdown need replace.

if (OB_FAIL(replace_expr_action(replacer, filters.at(i)))) {

LOG_WARN("failed to replace agg expr", K(ret));

}

} else if (!get_index_back()) {

// scan_pushdown no need replace.

} else if (flags.empty() || i >= flags.count()) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("filter before index flag is invalid", K(ret), K(i), K(flags), K(filters));

} else if (flags.at(i)) {

if (get_index_back() && get_is_index_global() && filters.at(i)->has_flag(CNT_SUB_QUERY)) {

// lookup_pushdown need replace.

if (OB_FAIL(replace_expr_action(replacer, filters.at(i)))) {

LOG_WARN("failed to replace agg expr", K(ret));

}

} else {

// scan_pushdown no need replace.

}

} else if (OB_FAIL(replace_expr_action(replacer, filters.at(i)))) {

LOG_WARN("failed to replace agg expr", K(ret));

}

}

}

return ret;

}

int ObLogTableScan::has_nonpushdown_filter(bool &has_npd_filter)

{

int ret = OB_SUCCESS;

has_npd_filter = false;

ObArray<ObRawExpr*> nonpushdown_filters;

ObArray<ObRawExpr*> scan_pushdown_filters;

ObArray<ObRawExpr*> lookup_pushdown_filters;

if (OB_FAIL(extract_pushdown_filters(nonpushdown_filters,

scan_pushdown_filters,

lookup_pushdown_filters,

true /*ignore pushdown filters*/))) {

LOG_WARN("extract pushdnow filters failed", K(ret));

} else if (!nonpushdown_filters.empty()) {

has_npd_filter = true;

}

return ret;

}

int ObLogTableScan::extract_pushdown_filters(ObIArray<ObRawExpr*> &nonpushdown_filters,

ObIArray<ObRawExpr*> &scan_pushdown_filters,

ObIArray<ObRawExpr*> &lookup_pushdown_filters,

bool ignore_pd_filter /*= false */)

{

int ret = OB_SUCCESS;

const ObIArray<ObRawExpr*> &filters = get_filter_exprs();

const auto &flags = get_filter_before_index_flags();

if (get_contains_fake_cte() ||

is_virtual_table(get_ref_table_id()) ||

EXTERNAL_TABLE == get_table_type()) {

//all filters can not push down to storage

if (OB_FAIL(nonpushdown_filters.assign(filters))) {

LOG_WARN("store non-pushdown filters failed", K(ret));

}

} else {

//part of filters can push down to storage

//scan_pushdown_filters means that:

//1. index scan filter when TSC use index scan directly or

//(TSC use index scan and lookup the data table)

//2. data table scan filter when TSC use the data table scan directly

//lookup_pushdown_filters means that the data table filter when

//TSC use index scan and lookup the data table

for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) {

if (use_batch() && filters.at(i)->has_flag(CNT_DYNAMIC_PARAM)) {

//In Batch table scan the dynamic param filter do not push down to storage

if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("push dynamic filter to store non-pushdown filter failed", K(ret), K(i));

}

} else if (filters.at(i)->has_flag(CNT_PL_UDF) ||

filters.at(i)->has_flag(CNT_OBJ_ACCESS_EXPR)) {

//User Define Function/obj access expr filter do not push down to storage

if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("push UDF/obj access filter store non-pushdown filter failed", K(ret), K(i));

}

} else if (filters.at(i)->has_flag(CNT_DYNAMIC_USER_VARIABLE)

|| filters.at(i)->has_flag(CNT_ASSIGN_EXPR)) {

if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("push variable assign filter store non-pushdown filter failed", K(ret), K(i));

}

} else if (ignore_pd_filter) {

//ignore_pd_filter: only extract non-pushdown filters, ignore others

} else if (!get_index_back()) {

if (OB_FAIL(scan_pushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("store pushdown filter failed", K(ret));

}

} else if (flags.empty() || i >= flags.count()) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("filter before index flag is invalid", K(ret), K(i), K(flags), K(filters));

} else if (flags.at(i)) {

if (get_index_back() && get_is_index_global() && filters.at(i)->has_flag(CNT_SUB_QUERY)) {

if (OB_FAIL(lookup_pushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("store lookup pushdown filter failed", K(ret), K(i));

}

} else if (OB_FAIL(scan_pushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("store scan pushdown filter failed", K(ret), K(i));

}

} else if (OB_FAIL(lookup_pushdown_filters.push_back(filters.at(i)))) {

LOG_WARN("store lookup pushdown filter failed", K(ret), K(i));

}

}

}

return ret;

}

int ObLogTableScan::extract_virtual_gen_access_exprs(

ObIArray<ObRawExpr*> &access_exprs,

uint64_t scan_table_id)

{

int ret = OB_SUCCESS;

if (get_index_back() && scan_table_id == get_real_index_table_id()) {

//this das scan is index scan and will lookup the data table later

//index scan + lookup data table: the index scan only need access

//range condition columns + index filter columns + the data table rowkeys

const ObIArray<ObRawExpr*> &range_conditions = get_range_conditions();

if (OB_FAIL(ObRawExprUtils::extract_column_exprs(range_conditions, access_exprs))) {

LOG_WARN("extract column exprs failed", K(ret));

}

//store index filter columns

if (OB_SUCC(ret)) {

ObArray<ObRawExpr *> filter_columns; // the column in scan pushdown filters

ObArray<ObRawExpr *> nonpushdown_filters;

ObArray<ObRawExpr *> scan_pushdown_filters;

ObArray<ObRawExpr *> lookup_pushdown_filters;

if (OB_FAIL(extract_pushdown_filters(

nonpushdown_filters,

scan_pushdown_filters,

lookup_pushdown_filters))) {

LOG_WARN("extract pushdown filter failed", K(ret));

} else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(scan_pushdown_filters,

filter_columns))) {

LOG_WARN("extract column exprs failed", K(ret));

} else if (OB_FAIL(append_array_no_dup(access_exprs, filter_columns))) {

LOG_WARN("append filter column to access exprs failed", K(ret));

}

}

//store data table rowkeys

if (OB_SUCC(ret)) {

if (OB_FAIL(append_array_no_dup(access_exprs, get_rowkey_exprs()))) {

LOG_WARN("append the data table rowkey expr failed", K(ret), K(get_rowkey_exprs()));

} else if (OB_FAIL(append_array_no_dup(access_exprs, get_part_exprs()))) {

LOG_WARN("append the data table part expr failed", K(ret), K(get_part_exprs()));

} else if (NULL != get_group_id_expr()

&& OB_FAIL(add_var_to_array_no_dup(access_exprs,

const_cast<ObRawExpr *>(get_group_id_expr())))) {

LOG_WARN("fail to add group id", K(ret));

}

}

} else if (OB_FAIL(access_exprs.assign(get_access_exprs()))) {

LOG_WARN("assign access exprs failed", K(ret));

}

if (OB_SUCC(ret) && is_oracle_mapping_real_virtual_table(get_ref_table_id())) {

//the access exprs are the agent virtual table columns, but das need the real table columns

//now to replace the real table column

for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); ++i) {

ObRawExpr *expr = access_exprs.at(i);

ObRawExpr *mapping_expr = nullptr;

uint64_t column_id = UINT64_MAX;

if (OB_ISNULL(expr)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("expr is null", K(ret), K(expr));

} else if (T_ORA_ROWSCN == expr->get_expr_type()) {

ret = OB_NOT_SUPPORTED;

LOG_WARN("rowscan not supported", K(ret));

LOG_USER_ERROR(OB_NOT_SUPPORTED, "rowscan not supported");

} else if (OB_ISNULL(mapping_expr = get_real_expr(expr))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("mapping expr is null", K(ret), KPC(expr));

} else {

//replace the agent virtual table column expr

access_exprs.at(i) = mapping_expr;

}

}

}

ObArray<ObRawExpr*> tmp_access_exprs;

for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); ++i) {

ObRawExpr *expr = access_exprs.at(i);

if (expr->is_column_ref_expr() &&

static_cast<ObColumnRefRawExpr *>(expr)->is_virtual_generated_column()) {

if (OB_FAIL(add_var_to_array_no_dup(tmp_access_exprs, expr))) {

LOG_WARN("failed to add param expr", K(ret));

}

} else {

//do nothing.

}

}

if (OB_FAIL(ret)) {

} else if (OB_FAIL(access_exprs.assign(tmp_access_exprs))) {

LOG_WARN("failed to remove generated column exprs", K(ret));

}

return ret;

}

int ObLogTableScan::adjust_print_access_info(ObIArray<ObRawExpr*> &access)

{

int ret = OB_SUCCESS;

if (!is_index_scan() && !get_index_back()) {

ObArray<ObRawExpr *> main_table_virtual_gen_exprs;

ObArray<ObRawExpr *> tmp_access_exprs;

if (OB_FAIL(extract_virtual_gen_access_exprs(

main_table_virtual_gen_exprs, get_real_ref_table_id()))) {

LOG_WARN("failed to extract das access exprs", K(ret));

} else if (OB_FAIL(ObOptimizerUtil::except_exprs(access,

main_table_virtual_gen_exprs, tmp_access_exprs))) {

LOG_WARN("failed to except virtual generated column exprs", K(ret));

} else if (OB_FAIL(access.assign(tmp_access_exprs))) {

LOG_WARN("failed to assign exprs", K(ret));

}

} else if (!get_index_back()) {

// do nothing.

} else {

ObArray<ObRawExpr *> main_table_virtual_gen_exprs;

ObArray<ObRawExpr *> index_table_virtual_gen_exprs;

ObArray<ObRawExpr *> tmp_virtual_gen_exprs;

ObArray<ObRawExpr *> tmp_access_exprs;

if (OB_FAIL(extract_virtual_gen_access_exprs(

main_table_virtual_gen_exprs, get_real_ref_table_id()))) {

LOG_WARN("failed to extract das access exprs", K(ret));

} else if (OB_FAIL(extract_virtual_gen_access_exprs(

index_table_virtual_gen_exprs, get_real_index_table_id()))) {

LOG_WARN("failed to extract das access exprs", K(ret));

} else if (OB_FAIL(ObOptimizerUtil::except_exprs(main_table_virtual_gen_exprs,

index_table_virtual_gen_exprs, tmp_virtual_gen_exprs))) {

LOG_WARN("failed to except virtual generated column exprs", K(ret));

} else if (OB_FAIL(ObOptimizerUtil::except_exprs(access,

tmp_virtual_gen_exprs, tmp_access_exprs))) {

LOG_WARN("failed to except virtual generated column exprs", K(ret));

} else if (OB_FAIL(access.assign(tmp_access_exprs))) {

LOG_WARN("failed to assign exprs", K(ret));

}

}

return ret;

}

// for ddl scene.

int ObLogTableScan::generate_ddl_output_column_ids()

{

int ret = OB_SUCCESS;

if (OB_ISNULL(get_plan())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

} else {

ObOptimizerContext &opt_ctx = get_plan()->get_optimizer_context();

if (opt_ctx.is_online_ddl() &&

stmt::T_INSERT == opt_ctx.get_session_info()->get_stmt_type()) {

for (int64_t i = 0; OB_SUCC(ret) && i < get_output_exprs().count(); ++i) {

const ObRawExpr *output_expr = get_output_exprs().at(i);

if (OB_ISNULL(output_expr)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("output_expr is nullptr", K(ret));

} else if (!output_expr->is_column_ref_expr()) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("output expr is not column ref", K(ret), KPC(output_expr));

} else {

const ObColumnRefRawExpr *output_col = static_cast<const ObColumnRefRawExpr*>(

output_expr);

if (OB_FAIL(ddl_output_column_ids_.push_back(output_col->get_column_id()))) {

LOG_WARN("store ddl output column id failed", K(ret));

}

}

}

}

}

return ret;

}

int ObLogTableScan::get_mbr_column_exprs(const uint64_t table_id,

ObIArray<ObRawExpr *> &mbr_exprs)

{

int ret = OB_SUCCESS;

ObRawExpr *expr = NULL;

const ObDMLStmt *stmt = NULL;

ObSEArray<ObRawExpr*, 8> temp_exprs;

if (OB_ISNULL(stmt = get_stmt())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("stmt is null", K(ret));

} else {

for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_column_size(); i++) {

const ColumnItem *col_item = stmt->get_column_item(i);

if (OB_ISNULL(col_item) || OB_ISNULL(col_item->expr_)) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(col_item), K(ret));

} else if (table_id == col_item->table_id_ &&

OB_NOT_NULL(col_item->expr_->get_dependant_expr()) &&

col_item->expr_->get_dependant_expr()->get_expr_type() == T_FUN_SYS_SPATIAL_MBR &&

OB_FAIL(temp_exprs.push_back(col_item->expr_))) {

LOG_WARN("failed to push back expr", K(ret));

} else { /*do nothing*/}

}

}

if (OB_FAIL(ret)) {

} else if (OB_FAIL(append_array_no_dup(mbr_exprs, temp_exprs))) {

LOG_WARN("failed to append exprs", K(ret));

}

return ret;

}

int ObLogTableScan::allocate_lookup_trans_info_expr()

{

int ret = OB_SUCCESS;

// Is strict defensive check mode

// Is index_back (contain local lookup and global lookup)

// There is no trans_info_expr on the current table_scan operator

// Satisfy the three conditions, add trans_info_expr for lookup

// The result of Index_scan will contain the transaction information corresponding to each row

// The result of the lookup in the data table will also include the trans_info

// of the current row in the data table, But the trans_info will not be output to the upper operator

ObOptimizerContext *opt_ctx = nullptr;

ObOpPseudoColumnRawExpr *tmp_trans_info_expr = nullptr;

if (OB_ISNULL(get_plan())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("unexpected null", K(ret));

} else if (OB_ISNULL(opt_ctx = &(get_plan()->get_optimizer_context()))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("unexpected null", K(ret));

} else if (index_back_ &&

opt_ctx->is_strict_defensive_check() &&

GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_0_0 &&

nullptr == trans_info_expr_) {

if (OB_FAIL(OB_FAIL(ObOptimizerUtil::generate_pseudo_trans_info_expr(*opt_ctx,

index_name_,

tmp_trans_info_expr)))) {

LOG_WARN("fail to generate pseudo trans info expr", K(ret), K(index_name_));

} else {

trans_info_expr_ = tmp_trans_info_expr;

}

}

return ret;

}

int ObLogTableScan::generate_necessary_rowkey_and_partkey_exprs()

{

int ret = OB_SUCCESS;

bool has_lob_column = false;

ObSqlSchemaGuard *schema_guard = NULL;

const ObTableSchema *table_schema = NULL;

bool is_heap_table = false;

if (OB_ISNULL(get_stmt()) || OB_ISNULL(get_plan()) ||

OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

} else if (OB_FAIL(schema_guard->get_table_schema(ref_table_id_, table_schema))) {

LOG_WARN("failed to get table schema", K(ret));

} else if (table_schema != NULL && FALSE_IT(is_heap_table = table_schema->is_heap_table())) {

} else if (OB_FAIL(get_stmt()->has_lob_column(table_id_, has_lob_column))) {

LOG_WARN("failed to check whether stmt has lob column", K(ret));

} else if (OB_FAIL(get_mbr_column_exprs(table_id_, spatial_exprs_))) {

LOG_WARN("failed to check whether stmt has mbr column", K(ret));

} else if (is_heap_table && is_index_global_ && index_back_ &&

OB_FAIL(get_part_column_exprs(table_id_, ref_table_id_, part_exprs_))) {

LOG_WARN("failed to get part column exprs", K(ret));

} else if ((has_lob_column || index_back_) &&

OB_FAIL(get_plan()->get_rowkey_exprs(table_id_, ref_table_id_, rowkey_exprs_))) {

LOG_WARN("failed to generate rowkey exprs", K(ret));

} else { /*do nothing*/ }

return ret;

}

int ObLogTableScan::add_mapping_columns_for_vt(ObIArray<ObRawExpr*> &access_exprs)

{

int ret = OB_SUCCESS;

LOG_DEBUG("debug mapping columns for virtual table", K(ref_table_id_));

if (0 < access_exprs.count()) {

ObColumnRefRawExpr *first_col_expr = static_cast<ObColumnRefRawExpr*>(access_exprs.at(0));

LOG_DEBUG("debug mapping columns for virtual table", K(first_col_expr->get_table_id()), K(ref_table_id_));

if (share::is_oracle_mapping_real_virtual_table(ref_table_id_)) {

for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); ++i) {

ObRawExpr *real_expr = NULL;

if (OB_FAIL(add_mapping_column_for_vt(static_cast<ObColumnRefRawExpr *>(access_exprs.at(i)),

real_expr))) {

LOG_WARN("failed to add mapping column for vt", K(ret));

} else if (OB_FAIL(real_expr_map_.push_back(

std::pair<ObRawExpr *, ObRawExpr *>(access_exprs.at(i), real_expr)))) {

LOG_WARN("failed to push back real expr", K(ret));

}

}

}

}

return ret;

}

int ObLogTableScan::add_mapping_column_for_vt(ObColumnRefRawExpr *col_expr,

ObRawExpr *&real_expr)

{

int ret = OB_SUCCESS;

int64_t mapping_table_id = share::schema::ObSchemaUtils::get_real_table_mappings_tid(ref_table_id_);

if (OB_INVALID_ID == mapping_table_id) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("unexpected status: failed to get real table id", K(ret));

}

const share::schema::ObColumnSchemaV2 *col_schema = NULL;

const ObTableSchema *table_schema = NULL;

ObColumnRefRawExpr *mapping_col_expr = NULL;

ObOptimizerContext *opt_ctx = NULL;

ObSqlSchemaGuard *schema_guard = NULL;

if (OB_FAIL(ret)) {

} else if (OB_ISNULL(get_plan()) ||

OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) ||

OB_ISNULL(schema_guard = opt_ctx->get_sql_schema_guard())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("Get unexpected null", K(ret));

} else if (OB_FAIL(schema_guard->get_table_schema(table_id_, mapping_table_id, get_stmt(), table_schema))) {

ret = OB_TABLE_NOT_EXIST;

LOG_WARN("get table schema failed", K(mapping_table_id), K(ret));

} else if (OB_ISNULL(col_schema = table_schema->get_column_schema(col_expr->get_column_name()))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("failed to get column schema", K(col_expr->get_column_name()), K(ret));

} else if (OB_FAIL(ObRawExprUtils::build_column_expr(opt_ctx->get_expr_factory(), *col_schema, mapping_col_expr))) {

LOG_WARN("build column expr failed", K(ret));

} else {

mapping_col_expr->set_synonym_db_name(col_expr->get_synonym_db_name());

mapping_col_expr->set_synonym_name(col_expr->get_synonym_name());

mapping_col_expr->set_column_attr(table_schema->get_table_name_str(), col_schema->get_column_name_str());

mapping_col_expr->set_database_name(col_expr->get_database_name());

//column maybe from alias table, so must reset ref id by table id from table_item

mapping_col_expr->set_ref_id(mapping_table_id, col_schema->get_column_id());

mapping_col_expr->set_lob_column(col_expr->is_lob_column());

real_expr = mapping_col_expr;

}

return ret;

}

int ObLogTableScan::index_back_check()

{

int ret = OB_SUCCESS;

bool column_found = true;

if (!is_index_scan()) {

column_found = true;

} else {

for (int64_t i = 0; OB_SUCC(ret) && column_found && i < access_exprs_.count(); ++i) {

const ObColumnRefRawExpr *expr = NULL;

if (OB_ISNULL(expr = static_cast<const ObColumnRefRawExpr*>(access_exprs_.at(i)))) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected null", K(ret));

} else if (T_ORA_ROWSCN == expr->get_expr_type()) {

column_found = false;

} else if (ob_is_geometry_tc(expr->get_data_type())) { // 在此处先标记为需要index_back，具体是否需要需要结合谓词来判断。

column_found = false;

} else if (T_PSEUDO_GROUP_ID == expr->get_expr_type()) {

// do nothing

} else if (OB_UNLIKELY(!expr->is_column_ref_expr())) {

ret = OB_ERR_UNEXPECTED;

LOG_WARN("get unexpected expr type", K(expr->get_expr_type()), K(ret));

} else {

const uint64_t column_id = expr->get_column_id();

column_found = false;

if (OB_HIDDEN_LOGICAL_ROWID_COLUMN_ID == column_id) {

// rowid can be computed directly from index table.

column_found = true;

}

for (int64_t col_idx = 0;

OB_SUCC(ret) && !column_found && col_idx < idx_columns_.count();

++col_idx) {

if (column_id == idx_columns_.at(col_idx)) {

column_found = true;

} else { /* Do nothing */ }

}

column_found = ObOptimizerUtil::find_item(idx_columns_,

static_cast<const ObColumnRefRawExpr*>(expr)->get_column_id());

}

} //end for

}

if (OB_SUCC(ret)) {

index_back_ = !column_found;

if (OB_FAIL(filter_before_index_back_set())) {

LOG_WARN("Failed to filter_before_index_back_set", K(ret));

} else {/*Do nothing*/}

} else { /* Do nothing */ }

return ret;

}

int ObLogTableScan::filter_before_index_back_set()

{

int ret = OB_SUCCESS;

filter_before_index_back_.reset();

if (index_back_) {

if (OB_FAIL(ObOptimizerUtil::check_filter_before_indexback(filter_exprs_,

idx_columns_,

filter_before_index_back_))) {

LOG_WARN("failed to check filter before index back", K(ret));

} else { /*do nothing*/ }

} else {

for (int64_t i = 0; OB_SUCC(ret) && i < filter_exprs_.count(); ++i) {

if (OB_FAIL(filter_before_index_back_.push_back(false))) {

LOG_WARN("failed to add bool to filter_before_index_back_", K(ret));

} else { /* Do nothing */ }

}

}

LOG_TRACE("succeed to check filter before index back", K(filter_before_index_back_),

K(index_back_), K(ret));

return ret;

}

int ObLogTableScan::set_table_scan_filters(const common::ObIArray<ObRawExpr *> &filters)

{

int ret = OB_SUCCESS;

if (OB_FAIL(get_filter_exprs().assign(filters))) {

LOG_WARN("failed to get exprs", K(ret));

} else if (OB_FAIL(pick_out_query_range_exprs())) {

LOG_WARN("failed to pick out query range exprs", K(ret));