Skip to content

Commit 936da4f

Browse files
authored
[feature](thread-pool) Support thread pool per disk for scanners (apache#7994)
Support thread pool per disk for scanners to prevent pool performance from some high ioutil disks happening key point: 1. each disk has a thread pool for scanners 2. whenever a thread pool of one disk runs out of local work, tasks can be retrieved from other threads(disks). This is done round-robin. performance testing: vec version: 25% faster than single thread pool in a high io util disk test case normal version: 8% faster than single thread pool in a high io util disk test case
1 parent a162f56 commit 936da4f

8 files changed

Lines changed: 205 additions & 15 deletions

File tree

be/src/common/config.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,10 @@ CONF_Int32(num_threads_per_core, "3");
147147
CONF_mBool(compress_rowbatches, "true");
148148
// interval between profile reports; in seconds
149149
CONF_mInt32(status_report_interval, "5");
150+
// if true, each disk will have a separate thread pool for scanner
151+
CONF_Bool(doris_enable_scanner_thread_pool_per_disk, "true");
152+
// the timeout of a work thread to wait the blocking priority queue to get a task
153+
CONF_mInt64(doris_blocking_priority_queue_wait_timeout_ms, "5");
150154
// number of olap scanner thread pool size
151155
CONF_Int32(doris_scanner_thread_pool_thread_num, "48");
152156
// number of olap scanner thread pool queue size

be/src/exec/olap_scan_node.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "runtime/string_value.h"
3737
#include "runtime/tuple_row.h"
3838
#include "util/priority_thread_pool.hpp"
39+
#include "util/priority_work_stealing_thread_pool.hpp"
3940
#include "util/runtime_profile.h"
4041

4142
namespace doris {
@@ -1436,6 +1437,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
14361437
PriorityThreadPool::Task task;
14371438
task.work_function = std::bind(&OlapScanNode::scanner_thread, this, *iter);
14381439
task.priority = _nice;
1440+
task.queue_id = state->exec_env()->store_path_to_index((*iter)->scan_disk());
14391441
(*iter)->start_wait_worker_timer();
14401442
if (thread_pool->offer(task)) {
14411443
olap_scanners.erase(iter++);

be/src/runtime/exec_env.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ class MemTracker;
4747
class StorageEngine;
4848
class PoolMemTrackerRegistry;
4949
class PriorityThreadPool;
50+
class PriorityWorkStealingThreadPool;
5051
class ReservationTracker;
5152
class ResultBufferMgr;
5253
class ResultQueueMgr;
@@ -145,6 +146,7 @@ class ExecEnv {
145146
SmallFileMgr* small_file_mgr() { return _small_file_mgr; }
146147

147148
const std::vector<StorePath>& store_paths() const { return _store_paths; }
149+
size_t store_path_to_index(const std::string& path) { return _store_path_map[path]; }
148150
void set_store_paths(const std::vector<StorePath>& paths) { _store_paths = paths; }
149151
StorageEngine* storage_engine() { return _storage_engine; }
150152
void set_storage_engine(StorageEngine* storage_engine) { _storage_engine = storage_engine; }
@@ -170,6 +172,8 @@ class ExecEnv {
170172
private:
171173
bool _is_init;
172174
std::vector<StorePath> _store_paths;
175+
// path => store index
176+
std::map<std::string, size_t> _store_path_map;
173177
// Leave protected so that subclasses can override
174178
ExternalScanContextMgr* _external_scan_context_mgr = nullptr;
175179
DataStreamMgr* _stream_mgr = nullptr;

be/src/runtime/exec_env_init.cpp

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
#include "util/parse_util.h"
6363
#include "util/pretty_printer.h"
6464
#include "util/priority_thread_pool.hpp"
65+
#include "util/priority_work_stealing_thread_pool.hpp"
6566
#include "vec/runtime/vdata_stream_mgr.h"
6667

6768
namespace doris {
@@ -83,6 +84,11 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
8384
return Status::OK();
8485
}
8586
_store_paths = store_paths;
87+
// path_name => path_index
88+
for (int i = 0; i < store_paths.size(); i++) {
89+
_store_path_map[store_paths[i].path] = i;
90+
}
91+
8692
_external_scan_context_mgr = new ExternalScanContextMgr(this);
8793
_stream_mgr = new DataStreamMgr();
8894
_vstream_mgr = new doris::vectorized::VDataStreamMgr();
@@ -95,8 +101,18 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
95101
new ExtDataSourceServiceClientCache(config::max_client_cache_size_per_host);
96102
_pool_mem_trackers = new PoolMemTrackerRegistry();
97103
_thread_mgr = new ThreadResourceMgr();
98-
_scan_thread_pool = new PriorityThreadPool(config::doris_scanner_thread_pool_thread_num,
104+
if (config::doris_enable_scanner_thread_pool_per_disk
105+
&& config::doris_scanner_thread_pool_thread_num >= store_paths.size()
106+
&& store_paths.size() > 0) {
107+
_scan_thread_pool = new PriorityWorkStealingThreadPool(config::doris_scanner_thread_pool_thread_num,
108+
store_paths.size(),
99109
config::doris_scanner_thread_pool_queue_size);
110+
LOG(INFO) << "scan thread pool use PriorityWorkStealingThreadPool";
111+
} else {
112+
_scan_thread_pool = new PriorityThreadPool(config::doris_scanner_thread_pool_thread_num,
113+
config::doris_scanner_thread_pool_queue_size);
114+
LOG(INFO) << "scan thread pool use PriorityThreadPool";
115+
}
100116

101117
ThreadPoolBuilder("LimitedScanThreadPool")
102118
.set_min_threads(1)

be/src/util/blocking_priority_queue.hpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,11 @@ class BlockingPriorityQueue {
4141
_total_get_wait_time(0),
4242
_total_put_wait_time(0) {}
4343

44-
// Get an element from the queue, waiting indefinitely for one to become available.
44+
// Get an element from the queue, waiting indefinitely (or until timeout) for one to become available.
4545
// Returns false if we were shut down prior to getting the element, and there
4646
// are no more elements available.
47-
bool blocking_get(T* out) {
47+
// -- timeout_ms: 0 means wait indefinitely
48+
bool blocking_get(T* out, uint32_t timeout_ms = 0) {
4849
MonotonicStopWatch timer;
4950
std::unique_lock<std::mutex> unique_lock(_lock);
5051

@@ -76,7 +77,14 @@ class BlockingPriorityQueue {
7677
}
7778

7879
timer.start();
79-
_get_cv.wait(unique_lock);
80+
if (timeout_ms != 0) {
81+
auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms);
82+
if (_get_cv.wait_until(unique_lock, deadline) == std::cv_status::timeout) {
83+
return false;
84+
}
85+
} else {
86+
_get_cv.wait(unique_lock);
87+
}
8088
timer.stop();
8189
}
8290
}

be/src/util/priority_thread_pool.hpp

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class PriorityThreadPool {
3939
public:
4040
int priority;
4141
WorkFunction work_function;
42+
int queue_id;
4243
bool operator<(const Task& o) const { return priority < o.priority; }
4344

4445
Task& operator++() {
@@ -63,7 +64,7 @@ class PriorityThreadPool {
6364

6465
// Destructor ensures that all threads are terminated before this object is freed
6566
// (otherwise they may continue to run and reference member variables)
66-
~PriorityThreadPool() {
67+
virtual ~PriorityThreadPool() {
6768
shutdown();
6869
join();
6970
}
@@ -79,32 +80,32 @@ class PriorityThreadPool {
7980
//
8081
// Returns true if the work item was successfully added to the queue, false otherwise
8182
// (which typically means that the thread pool has already been shut down).
82-
bool offer(Task task) { return _work_queue.blocking_put(task); }
83+
virtual bool offer(Task task) { return _work_queue.blocking_put(task); }
8384

84-
bool offer(WorkFunction func) {
85-
PriorityThreadPool::Task task = {0, func};
85+
virtual bool offer(WorkFunction func) {
86+
PriorityThreadPool::Task task = {0, func, 0};
8687
return _work_queue.blocking_put(task);
8788
}
8889

8990
// Shuts the thread pool down, causing the work queue to cease accepting offered work
9091
// and the worker threads to terminate once they have processed their current work item.
9192
// Returns once the shutdown flag has been set, does not wait for the threads to
9293
// terminate.
93-
void shutdown() {
94+
virtual void shutdown() {
9495
_shutdown = true;
9596
_work_queue.shutdown();
9697
}
9798

9899
// Blocks until all threads are finished. shutdown does not need to have been called,
99100
// since it may be called on a separate thread.
100-
void join() { _threads.join_all(); }
101+
virtual void join() { _threads.join_all(); }
101102

102-
uint32_t get_queue_size() const { return _work_queue.get_size(); }
103+
virtual uint32_t get_queue_size() const { return _work_queue.get_size(); }
103104

104105
// Blocks until the work queue is empty, and then calls shutdown to stop the worker
105106
// threads and Join to wait until they are finished.
106107
// Any work Offer()'ed during DrainAndshutdown may or may not be processed.
107-
void drain_and_shutdown() {
108+
virtual void drain_and_shutdown() {
108109
{
109110
std::unique_lock<std::mutex> l(_lock);
110111
while (_work_queue.get_size() != 0) {
@@ -114,7 +115,8 @@ class PriorityThreadPool {
114115
shutdown();
115116
join();
116117
}
117-
118+
protected:
119+
virtual bool is_shutdown() { return _shutdown; }
118120
private:
119121
// Driver method for each thread in the pool. Continues to read work from the queue
120122
// until the pool is shutdown.
@@ -130,8 +132,6 @@ class PriorityThreadPool {
130132
}
131133
}
132134

133-
bool is_shutdown() { return _shutdown; }
134-
135135
// Queue on which work items are held until a thread is available to process them in
136136
// FIFO order.
137137
BlockingPriorityQueue<Task> _work_queue;
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <mutex>
21+
#include <thread>
22+
23+
#include "util/blocking_priority_queue.hpp"
24+
#include "util/thread_group.h"
25+
26+
namespace doris {
27+
28+
// Work-Stealing threadpool which processes items (of type T) in parallel which were placed on multi
29+
// blocking queues by Offer(). Each item is processed by a single user-supplied method.
30+
class PriorityWorkStealingThreadPool : public PriorityThreadPool {
31+
public:
32+
33+
// Creates a new thread pool and start num_threads threads.
34+
// -- num_threads: how many threads are part of this pool
35+
// -- num_queues: how many queues are part of this pool
36+
// -- queue_size: the maximum size of the queue on which work items are offered. If the
37+
// queue exceeds this size, subsequent calls to Offer will block until there is
38+
// capacity available.
39+
// -- work_function: the function to run every time an item is consumed from the queue
40+
PriorityWorkStealingThreadPool(uint32_t num_threads, uint32_t num_queues, uint32_t queue_size)
41+
: PriorityThreadPool(0, 0) {
42+
DCHECK_GT(num_queues, 0);
43+
DCHECK_GE(num_threads, num_queues);
44+
// init _work_queues first because the work thread needs it
45+
for (int i = 0; i < num_queues; ++i) {
46+
_work_queues.emplace_back(std::make_shared<BlockingPriorityQueue<Task>>(queue_size));
47+
}
48+
for (int i = 0; i < num_threads; ++i) {
49+
_threads.create_thread(
50+
std::bind<void>(std::mem_fn(&PriorityWorkStealingThreadPool::work_thread), this, i));
51+
}
52+
}
53+
54+
// Blocking operation that puts a work item on the queue. If the queue is full, blocks
55+
// until there is capacity available.
56+
//
57+
// 'work' is copied into the work queue, but may be referenced at any time in the
58+
// future. Therefore the caller needs to ensure that any data referenced by work (if T
59+
// is, e.g., a pointer type) remains valid until work has been processed, and it's up to
60+
// the caller to provide their own signalling mechanism to detect this (or to wait until
61+
// after DrainAndshutdown returns).
62+
//
63+
// Returns true if the work item was successfully added to the queue, false otherwise
64+
// (which typically means that the thread pool has already been shut down).
65+
bool offer(Task task) override {
66+
return _work_queues[task.queue_id]->blocking_put(task);
67+
}
68+
69+
bool offer(WorkFunction func) override {
70+
PriorityThreadPool::Task task = {0, func, 0};
71+
return _work_queues[task.queue_id]->blocking_put(task);
72+
}
73+
74+
// Shuts the thread pool down, causing the work queue to cease accepting offered work
75+
// and the worker threads to terminate once they have processed their current work item.
76+
// Returns once the shutdown flag has been set, does not wait for the threads to
77+
// terminate.
78+
void shutdown() override {
79+
PriorityThreadPool::shutdown();
80+
for (auto work_queue : _work_queues) {
81+
work_queue->shutdown();
82+
}
83+
}
84+
85+
// Blocks until all threads are finished. shutdown does not need to have been called,
86+
// since it may be called on a separate thread.
87+
void join() override { _threads.join_all(); }
88+
89+
uint32_t get_queue_size() const override {
90+
uint32_t size = 0;
91+
for (auto work_queue : _work_queues) {
92+
size += work_queue->get_size();
93+
}
94+
return size;
95+
}
96+
97+
// Blocks until the work queue is empty, and then calls shutdown to stop the worker
98+
// threads and Join to wait until they are finished.
99+
// Any work Offer()'ed during DrainAndshutdown may or may not be processed.
100+
void drain_and_shutdown() override {
101+
{
102+
std::unique_lock<std::mutex> l(_lock);
103+
while (get_queue_size() != 0) {
104+
_empty_cv.wait(l);
105+
}
106+
}
107+
shutdown();
108+
join();
109+
}
110+
111+
private:
112+
// Driver method for each thread in the pool. Continues to read work from the queue
113+
// until the pool is shutdown.
114+
void work_thread(int thread_id) {
115+
auto queue_id = thread_id % _work_queues.size();
116+
auto steal_queue_id = (queue_id + 1) % _work_queues.size();
117+
while (!is_shutdown()) {
118+
Task task;
119+
// avoid blocking get
120+
bool is_other_queues_empty = true;
121+
// steal work in round-robin if nothing to do
122+
while (_work_queues[queue_id]->get_size() == 0 && queue_id != steal_queue_id && !is_shutdown()) {
123+
if (_work_queues[steal_queue_id]->non_blocking_get(&task)) {
124+
is_other_queues_empty = false;
125+
task.work_function();
126+
}
127+
steal_queue_id = (steal_queue_id + 1) % _work_queues.size();
128+
}
129+
if (queue_id == steal_queue_id) {
130+
steal_queue_id = (steal_queue_id + 1) % _work_queues.size();
131+
}
132+
if (is_other_queues_empty && _work_queues[queue_id]->blocking_get(&task, config::doris_blocking_priority_queue_wait_timeout_ms)) {
133+
task.work_function();
134+
}
135+
if (_work_queues[queue_id]->get_size() == 0) {
136+
_empty_cv.notify_all();
137+
}
138+
}
139+
}
140+
141+
// Queue on which work items are held until a thread is available to process them in
142+
// FIFO order.
143+
std::vector<std::shared_ptr<BlockingPriorityQueue<Task>>> _work_queues;
144+
145+
// Collection of worker threads that process work from the queues.
146+
ThreadGroup _threads;
147+
148+
// Guards _empty_cv
149+
std::mutex _lock;
150+
151+
// Signalled when the queue becomes empty
152+
std::condition_variable _empty_cv;
153+
};
154+
155+
} // namespace doris

be/src/vec/exec/volap_scan_node.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,7 @@ int VOlapScanNode::_start_scanner_thread_task(RuntimeState* state, int block_per
553553
PriorityThreadPool::Task task;
554554
task.work_function = std::bind(&VOlapScanNode::scanner_thread, this, *iter);
555555
task.priority = _nice;
556+
task.queue_id = state->exec_env()->store_path_to_index((*iter)->scan_disk());
556557
(*iter)->start_wait_worker_timer();
557558
if (thread_pool->offer(task)) {
558559
olap_scanners.erase(iter++);

0 commit comments

Comments
 (0)