Skip to content

Commit 1ed503b

Browse files
asuhanasuhan
committed
Add table generation and start rowid
1 parent 1a369cb commit 1ed503b

14 files changed

Lines changed: 108 additions & 17 deletions

‎MapDServer.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,16 @@ StringDictionaryGenerations string_dictionary_generations_from_thrift(
7676
return string_dictionary_generations;
7777
}
7878

79+
TableGenerations table_generations_from_thrift(const std::vector<TTableGeneration>& thrift_table_generations) {
80+
TableGenerations table_generations;
81+
for (const auto& thrift_table_generation : thrift_table_generations) {
82+
table_generations.setGeneration(thrift_table_generation.table_id,
83+
TableGeneration{static_cast<size_t>(thrift_table_generation.tuple_count),
84+
static_cast<size_t>(thrift_table_generation.start_rowid)});
85+
}
86+
return table_generations;
87+
}
88+
7989
std::vector<LeafHostInfo> only_db_leaves(const std::vector<LeafHostInfo>& all_leaves) {
8090
std::vector<LeafHostInfo> data_leaves;
8191
std::copy_if(all_leaves.begin(), all_leaves.end(), std::back_inserter(data_leaves), [](const LeafHostInfo& leaf) {

‎MapDServer.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "gen-cpp/mapd_types.h"
1212
#include "QueryEngine/AggregatedColRange.h"
1313
#include "QueryEngine/StringDictionaryGenerations.h"
14+
#include "QueryEngine/TableGenerations.h"
1415
#include "QueryEngine/TargetMetaInfo.h"
1516

1617
#include <glog/logging.h>
@@ -175,4 +176,6 @@ AggregatedColRange column_ranges_from_thrift(const std::vector<TColumnRange>& th
175176
StringDictionaryGenerations string_dictionary_generations_from_thrift(
176177
const std::vector<TDictionaryGeneration>& thrift_string_dictionary_generations);
177178

179+
TableGenerations table_generations_from_thrift(const std::vector<TTableGeneration>& table_generations);
180+
178181
#endif // MAPDSERVER_H

‎QueryEngine/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ set(query_engine_source_files
4545
DynamicWatchdog.cpp
4646
SpeculativeTopN.cpp
4747
StringDictionaryGenerations.cpp
48+
TableGenerations.cpp
4849
StringFunctions.cpp
4950
RegexpFunctions.cpp
5051
JoinHashTable.cpp

‎QueryEngine/Execute.cpp‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -680,6 +680,7 @@ void Executor::clearMetaInfoCache() {
680680
input_table_info_cache_.clear();
681681
agg_col_range_cache_.clear();
682682
string_dictionary_generations_.clear();
683+
table_generations_.clear();
683684
}
684685

685686
std::vector<int8_t> Executor::serializeLiterals(const std::unordered_map<int, Executor::LiteralValues>& literals,
@@ -807,7 +808,7 @@ std::vector<llvm::Value*> Executor::codegen(const Analyzer::Expr* expr,
807808
}
808809
auto col_var = dynamic_cast<const Analyzer::ColumnVar*>(expr);
809810
if (col_var) {
810-
return codegen(col_var, fetch_columns, co.hoist_literals_);
811+
return codegen(col_var, fetch_columns, co);
811812
}
812813
auto constant = dynamic_cast<const Analyzer::Constant*>(expr);
813814
if (constant) {
@@ -1302,8 +1303,8 @@ size_t get_col_bit_width(const Analyzer::ColumnVar* col_var) {
13021303

13031304
std::vector<llvm::Value*> Executor::codegen(const Analyzer::ColumnVar* col_var,
13041305
const bool fetch_column,
1305-
const bool hoist_literals) {
1306-
const auto col_var_lvs = codegenColVar(col_var, fetch_column, hoist_literals);
1306+
const CompilationOptions& co) {
1307+
const auto col_var_lvs = codegenColVar(col_var, fetch_column, co);
13071308
if (!cgen_state_->outer_join_cond_lv_ || col_var->get_rte_idx() == 0) {
13081309
return col_var_lvs;
13091310
}
@@ -1312,7 +1313,8 @@ std::vector<llvm::Value*> Executor::codegen(const Analyzer::ColumnVar* col_var,
13121313

13131314
std::vector<llvm::Value*> Executor::codegenColVar(const Analyzer::ColumnVar* col_var,
13141315
const bool fetch_column,
1315-
const bool hoist_literals) {
1316+
const CompilationOptions& co) {
1317+
const bool hoist_literals = co.hoist_literals_;
13161318
auto col_id = col_var->get_column_id();
13171319
const auto rte_idx = col_var->get_rte_idx() == -1 ? int(0) : col_var->get_rte_idx();
13181320
#ifdef ENABLE_MULTIFRAG_JOIN
@@ -1331,7 +1333,18 @@ std::vector<llvm::Value*> Executor::codegenColVar(const Analyzer::ColumnVar* col
13311333
#endif
13321334
const auto offset = cgen_state_->frag_offsets_[rte_idx];
13331335
if (offset) {
1334-
return {cgen_state_->ir_builder_.CreateAdd(posArg(col_var), offset)};
1336+
const auto& table_generation = table_generations_.getGeneration(col_var->get_table_id());
1337+
if (table_generation.start_rowid > 0) {
1338+
Datum d;
1339+
d.bigintval = table_generation.start_rowid;
1340+
const auto start_rowid = makeExpr<Analyzer::Constant>(kBIGINT, false, d);
1341+
const auto start_rowid_lvs = codegen(start_rowid.get(), kENCODING_NONE, -1, co);
1342+
CHECK_EQ(size_t(1), start_rowid_lvs.size());
1343+
return {cgen_state_->ir_builder_.CreateAdd(cgen_state_->ir_builder_.CreateAdd(posArg(col_var), offset),
1344+
start_rowid_lvs.front())};
1345+
} else {
1346+
return {cgen_state_->ir_builder_.CreateAdd(posArg(col_var), offset)};
1347+
}
13351348
} else {
13361349
return {posArg(col_var)};
13371350
}
@@ -1353,7 +1366,7 @@ std::vector<llvm::Value*> Executor::codegenColVar(const Analyzer::ColumnVar* col
13531366
if (plan_state_->isLazyFetchColumn(col_var)) {
13541367
plan_state_->columns_to_fetch_.insert(std::make_pair(col_var->get_table_id(), col_var->get_column_id()));
13551368
}
1356-
return codegen(hash_join_lhs, fetch_column, hoist_literals);
1369+
return codegen(hash_join_lhs, fetch_column, co);
13571370
}
13581371
auto pos_arg = posArg(col_var);
13591372
auto col_byte_stream = colByteStream(col_var, fetch_column, hoist_literals);
@@ -1992,7 +2005,7 @@ std::string get_null_check_suffix(const SQLTypeInfo& lhs_ti, const SQLTypeInfo&
19922005
llvm::Value* Executor::codegenCmp(const Analyzer::BinOper* bin_oper, const CompilationOptions& co) {
19932006
for (const auto equi_join_tautology : plan_state_->join_info_.equi_join_tautologies_) {
19942007
if (*equi_join_tautology == *bin_oper) {
1995-
return plan_state_->join_info_.join_hash_table_->codegenSlot(co.hoist_literals_);
2008+
return plan_state_->join_info_.join_hash_table_->codegenSlot(co);
19962009
}
19972010
}
19982011
const auto optype = bin_oper->get_optype();
@@ -7421,11 +7434,14 @@ std::pair<bool, int64_t> Executor::skipFragment(const InputDescriptor& table_des
74217434
int64_t chunk_min{0};
74227435
int64_t chunk_max{0};
74237436
bool is_rowid{false};
7437+
size_t start_rowid{0};
74247438
if (chunk_meta_it == fragment.getChunkMetadataMap().end()) {
74257439
auto cd = get_column_descriptor(col_id, table_id, *catalog_);
74267440
CHECK(cd->isVirtualCol && cd->columnName == "rowid");
7427-
chunk_min = all_frag_row_offsets[frag_idx];
7428-
chunk_max = all_frag_row_offsets[frag_idx + 1] - 1;
7441+
const auto& table_generation = table_generations_.getGeneration(table_id);
7442+
start_rowid = table_generation.start_rowid;
7443+
chunk_min = all_frag_row_offsets[frag_idx] + start_rowid;
7444+
chunk_max = all_frag_row_offsets[frag_idx + 1] - 1 + start_rowid;
74297445
is_rowid = true;
74307446
} else {
74317447
const auto& chunk_type = lhs->get_type_info();
@@ -7458,7 +7474,7 @@ std::pair<bool, int64_t> Executor::skipFragment(const InputDescriptor& table_des
74587474
if (chunk_min > rhs_val || chunk_max < rhs_val) {
74597475
return {true, -1};
74607476
} else if (is_rowid) {
7461-
return {false, rhs_val};
7477+
return {false, rhs_val - start_rowid};
74627478
}
74637479
break;
74647480
default:

‎QueryEngine/Execute.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "NvidiaKernel.h"
1212
#include "RelAlgExecutionUnit.h"
1313
#include "StringDictionaryGenerations.h"
14+
#include "TableGenerations.h"
1415
#include "TargetMetaInfo.h"
1516

1617
#include "../Analyzer/Analyzer.h"
@@ -355,10 +356,10 @@ class Executor {
355356
std::vector<llvm::Value*> codegen(const Analyzer::Expr*, const bool fetch_columns, const CompilationOptions&);
356357
llvm::Value* codegen(const Analyzer::BinOper*, const CompilationOptions&);
357358
llvm::Value* codegen(const Analyzer::UOper*, const CompilationOptions&);
358-
std::vector<llvm::Value*> codegen(const Analyzer::ColumnVar*, const bool fetch_column, const bool hoist_literals);
359+
std::vector<llvm::Value*> codegen(const Analyzer::ColumnVar*, const bool fetch_column, const CompilationOptions&);
359360
std::vector<llvm::Value*> codegenColVar(const Analyzer::ColumnVar*,
360361
const bool fetch_column,
361-
const bool hoist_literals);
362+
const CompilationOptions&);
362363
llvm::Value* codgenAdjustFixedEncNull(llvm::Value*, const SQLTypeInfo&);
363364
std::vector<llvm::Value*> codegenOuterJoinNullPlaceholder(const std::vector<llvm::Value*>&,
364365
const Analyzer::ColumnVar*);
@@ -1195,6 +1196,7 @@ class Executor {
11951196
InputTableInfoCache input_table_info_cache_;
11961197
AggregatedColRange agg_col_range_cache_;
11971198
StringDictionaryGenerations string_dictionary_generations_;
1199+
TableGenerations table_generations_;
11981200

11991201
static std::map<std::pair<int, ::QueryRenderer::QueryRenderManager*>, std::shared_ptr<Executor>> executors_;
12001202
static std::mutex execute_mutex_;

‎QueryEngine/JoinHashTable.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -481,14 +481,14 @@ void JoinHashTable::putHashTableOnCpuToCache(
481481
join_hash_table_cache_.emplace_back(cache_key, cpu_hash_table_buff_);
482482
}
483483

484-
llvm::Value* JoinHashTable::codegenSlot(const bool hoist_literals) noexcept {
484+
llvm::Value* JoinHashTable::codegenSlot(const CompilationOptions& co) noexcept {
485485
CHECK(executor_->plan_state_->join_info_.join_impl_type_ == Executor::JoinImplType::HashOneToOne);
486486
const auto cols = get_cols(qual_bin_oper_, cat_, executor_->temporary_tables_);
487487
auto key_col = cols.second;
488488
CHECK(key_col);
489489
auto val_col = cols.first;
490490
CHECK(val_col);
491-
const auto key_lvs = executor_->codegen(key_col, true, hoist_literals);
491+
const auto key_lvs = executor_->codegen(key_col, true, co);
492492
CHECK_EQ(size_t(1), key_lvs.size());
493493
CHECK(executor_->plan_state_->join_info_.join_hash_table_);
494494
auto hash_ptr = get_arg_by_name(executor_->cgen_state_->row_func_, "join_hash_table");

‎QueryEngine/JoinHashTable.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class JoinHashTable {
108108
const int32_t hash_entry_count,
109109
const int32_t hash_join_invalid_val);
110110

111-
llvm::Value* codegenSlot(const bool hoist_literals) noexcept;
111+
llvm::Value* codegenSlot(const CompilationOptions&) noexcept;
112112

113113
const InputTableInfo& getInnerQueryInfo(const Analyzer::ColumnVar* inner_col);
114114

‎QueryEngine/QueryPhysicalInputsCollector.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,29 @@ PhysicalInputSet RelAlgPhysicalInputsVisitor::aggregateResult(const PhysicalInpu
111111
return result;
112112
}
113113

114+
class RelAlgPhysicalTableInputsVisitor : public RelAlgVisitor<std::unordered_set<int>> {
115+
public:
116+
std::unordered_set<int> visitScan(const RelScan* scan) const override {
117+
return {scan->getTableDescriptor()->tableId};
118+
}
119+
120+
protected:
121+
std::unordered_set<int> aggregateResult(const std::unordered_set<int>& aggregate,
122+
const std::unordered_set<int>& next_result) const override {
123+
auto result = aggregate;
124+
result.insert(next_result.begin(), next_result.end());
125+
return result;
126+
}
127+
};
128+
114129
} // namespace
115130

116131
std::unordered_set<PhysicalInput> get_physical_inputs(const RelAlgNode* ra) {
117132
RelAlgPhysicalInputsVisitor phys_inputs_visitor;
118133
return phys_inputs_visitor.visit(ra);
119134
}
135+
136+
std::unordered_set<int> get_physical_table_inputs(const RelAlgNode* ra) {
137+
RelAlgPhysicalTableInputsVisitor phys_table_inputs_visitor;
138+
return phys_table_inputs_visitor.visit(ra);
139+
}

‎QueryEngine/QueryPhysicalInputsCollector.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ struct hash<PhysicalInput> {
3030
} // std
3131

3232
std::unordered_set<PhysicalInput> get_physical_inputs(const RelAlgNode*);
33+
std::unordered_set<int> get_physical_table_inputs(const RelAlgNode*);
3334

3435
#endif // QUERYENGINE_QUERYPHYSICALINPUTSCOLLECTOR_H

‎QueryEngine/RelAlgExecutor.cpp‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ ExecutionResult RelAlgExecutor::executeRelAlgQuery(const std::string& query_ra,
3030
executor_->catalog_ = &cat_;
3131
executor_->agg_col_range_cache_ = computeColRangesCache(ra.get());
3232
executor_->string_dictionary_generations_ = computeStringDictionaryGenerations(ra.get());
33+
executor_->table_generations_ = computeTableGenerations(ra.get());
3334
ScopeGuard restore_metainfo_cache = [this] { executor_->clearMetaInfoCache(); };
3435
int64_t queue_time_ms = timer_stop(clock_begin);
3536
auto ed_list = get_execution_descriptors(ra.get());
@@ -102,6 +103,16 @@ StringDictionaryGenerations RelAlgExecutor::computeStringDictionaryGenerations(c
102103
return string_dictionary_generations;
103104
}
104105

106+
TableGenerations RelAlgExecutor::computeTableGenerations(const RelAlgNode* ra) {
107+
const auto phys_table_ids = get_physical_table_inputs(ra);
108+
TableGenerations table_generations;
109+
for (const int table_id : phys_table_ids) {
110+
const auto table_info = executor_->getTableInfo(table_id);
111+
table_generations.setGeneration(table_id, TableGeneration{table_info.numTuples, 0});
112+
}
113+
return table_generations;
114+
}
115+
105116
Executor* RelAlgExecutor::getExecutor() const {
106117
return executor_;
107118
}
@@ -141,7 +152,8 @@ FirstStepExecutionResult RelAlgExecutor::executeRelAlgQueryFirstStep(const RelAl
141152
}
142153

143154
void RelAlgExecutor::prepareLeafExecution(const AggregatedColRange& agg_col_range,
144-
const StringDictionaryGenerations& string_dictionary_generations) {
155+
const StringDictionaryGenerations& string_dictionary_generations,
156+
const TableGenerations& table_generations) {
145157
// capture the lock acquistion time
146158
auto clock_begin = timer_start();
147159
leaf_execution_cleanup_.reset(new ScopeGuard([this] {
@@ -155,6 +167,7 @@ void RelAlgExecutor::prepareLeafExecution(const AggregatedColRange& agg_col_rang
155167
}
156168
queue_time_ms_ = timer_stop(clock_begin);
157169
executor_->row_set_mem_owner_ = std::make_shared<RowSetMemoryOwner>();
170+
executor_->table_generations_ = table_generations;
158171
executor_->agg_col_range_cache_ = agg_col_range;
159172
executor_->string_dictionary_generations_ = string_dictionary_generations;
160173
}

0 commit comments

Comments
 (0)