-
-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Expand file tree
/
Copy pathExecuteTaskflow.html
More file actions
542 lines (540 loc) · 64.5 KB
/
Copy pathExecuteTaskflow.html
File metadata and controls
542 lines (540 loc) · 64.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
<!-- HTML header for doxygen 1.13.1-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en-US">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=11"/>
<meta name="generator" content="Doxygen 1.13.1"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Taskflow: A General-purpose Task-parallel Programming System: Executor</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<script type="text/javascript" src="clipboard.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="navtreedata.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="cookie.js"></script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/searchdata.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
<link href="custom.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr id="projectrow">
<td id="projectlogo"><img alt="Logo" src="taskflow_logo.png"/></td>
<td id="projectalign">
<div id="projectname"><a href="https://github.com/taskflow/taskflow" style="color:inherit; text-decoration:none;">Taskflow: A General-purpose Task-parallel Programming System</a>
</div>
</td>
</tr>
</tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.13.1 -->
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt MIT */
var searchBox = new SearchBox("searchBox", "search/",'.html');
/* @license-end */
</script>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt MIT */
$(function() { codefold.init(0); });
/* @license-end */
</script>
<script type="text/javascript" src="menudata.js"></script>
<script type="text/javascript" src="menu.js"></script>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt MIT */
$(function() {
initMenu('',true,false,'search.php','Search',true);
$(function() { init_search(); });
});
/* @license-end */
</script>
<div id="main-nav"></div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
<div id="nav-sync" class="sync"></div>
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt MIT */
$(function(){initNavTree('ExecuteTaskflow.html',''); initResizable(true); });
/* @license-end */
</script>
<div id="doc-content">
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
onkeydown="return searchBox.OnSearchSelectKey(event)">
</div>
<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<div id="MSearchResults">
<div class="SRPage">
<div id="SRIndex">
<div id="SRResults"></div>
<div class="SRStatus" id="Loading">Loading...</div>
<div class="SRStatus" id="Searching">Searching...</div>
<div class="SRStatus" id="NoMatches">No Matches</div>
</div>
</div>
</div>
</div>
<div><div class="header">
<div class="headertitle"><div class="title">Executor</div></div>
</div><!--header-->
<div class="contents">
<div class="toc"><h3>Table of Contents</h3>
<ul>
<li class="level1">
<a href="#CreateAnExecutor">Create an Executor</a>
</li>
<li class="level1">
<a href="#UnderstandWorkStealingInExecutor">Understand Work Stealing in Executor</a>
</li>
<li class="level1">
<a href="#ExecuteATaskflow">Execute a Taskflow</a>
</li>
<li class="level1">
<a href="#UnderstandTheExecutionOrder">Understand the Execution Order</a>
</li>
<li class="level1">
<a href="#UnderstandTheOwnership">Understand the Ownership</a>
</li>
<li class="level1">
<a href="#ExecuteATaskflowWithTransferredOwnership">Execute a Taskflow with Transferred Ownership</a>
</li>
<li class="level1">
<a href="#ExecuteATaskflowFromAnInternalWorker">Execute a Taskflow from an Internal Worker Cooperatively</a>
</li>
<li class="level1">
<a href="#ThreadSafetyOfExecution">Thread Safety of Executor</a>
</li>
<li class="level1">
<a href="#QueryTheWorkerID">Query the Worker ID</a>
</li>
<li class="level1">
<a href="#ObserveThreadActivities">Observe Thread Activities</a>
</li>
<li class="level1">
<a href="#ModifyWorkerProperty">Modify Worker Property</a>
</li>
</ul>
</div>
<div class="textblock"><p>After you create a task dependency graph, you need to submit it to threads for execution. In this chapter, we will show you how to execute a task dependency graph.</p>
<h1><a class="anchor" id="CreateAnExecutor"></a>
Create an Executor</h1>
<p>To execute a taskflow, you need to create an <em>executor</em> of type <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a>. An executor is a <em>thread-safe</em> object that manages a set of worker threads and executes tasks through an efficient <em>work-stealing</em> algorithm. Issuing a call to run a taskflow creates a <em>topology</em>, a data structure to keep track of the execution status of a running graph. <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> takes an unsigned integer to construct with <code>N</code> worker threads. The default value is std::thread::hardware_concurrency.</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor1; <span class="comment">// create an executor with the number of workers</span></div>
<div class="line"> <span class="comment">// equal to std::thread::hardware_concurrency</span></div>
<div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor2(4); <span class="comment">// create an executor of 4 worker threads</span></div>
<div class="ttc" id="aclasstf_1_1Executor_html"><div class="ttname"><a href="classtf_1_1Executor.html">tf::Executor</a></div><div class="ttdoc">class to create an executor</div><div class="ttdef"><b>Definition</b> executor.hpp:62</div></div>
</div><!-- fragment --><dl class="section note"><dt>Note</dt><dd>Creating a <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> has non-negligible overhead. Unless your application requires multiple executors, we recommend creating a single <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> and reusing it to run multiple taskflows.</dd></dl>
<h1><a class="anchor" id="UnderstandWorkStealingInExecutor"></a>
Understand Work Stealing in Executor</h1>
<p>Taskflow designs a highly efficient <em>work-stealing</em> algorithm to schedule and run tasks in an executor. Work-stealing is a dynamic scheduling algorithm widely used in parallel computing to distribute and balance workload among multiple threads or cores. Specifically, within an executor, each worker maintains its own local queue of tasks. When a worker finishes its own tasks, instead of becoming idle or going sleep, it (thief) tries to <em>steal</em> a task from the queue another worker (victim). The figure below illustrates the idea of work-stealing:</p>
<div class="image">
<img src="work-stealing.png" alt=""/>
</div>
<p>The key advantage of work-stealing lies in its <em>decentralized</em> nature and efficiency. Most of the time, worker threads work on their local queues without contention. Stealing only occurs when a worker becomes idle, minimizing overhead associated with synchronization and task distribution. This decentralized strategy effectively balances the workload, ensuring that idle workers are put to work and that the overall computation progresses efficiently. That being said, the internal scheduling mechanisms in <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> are not trivial, and it's not easy to explain every detail in just a few sentences. If you're interested in learning more about the technical details, please refer to our paper published in 2022 <em>IEEE Transactions on Parallel and Distributed Systems (TPDS)</em>:</p>
<ul>
<li>Tsung-Wei Huang, Dian-Lun Lin, Chun-Xun Lin, and Yibo Lin, "<a href="https://tsung-wei-huang.github.io/papers/tpds21-taskflow.pdf">Taskflow: A Lightweight Parallel and Heterogeneous Task Graph Computing System</a>," <em>IEEE Transactions on Parallel and Distributed Systems (TPDS)</em>, vol. 33, no. 6, pp. 1303-1320, June 2022</li>
</ul>
<h1><a class="anchor" id="ExecuteATaskflow"></a>
Execute a Taskflow</h1>
<p><a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> provides a set of <code>run_*</code> methods, <a class="el" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069" title="runs a taskflow once">tf::Executor::run</a>, <a class="el" href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00" title="runs a taskflow for N times">tf::Executor::run_n</a>, and <a class="el" href="classtf_1_1Executor.html#a0f52e9dd64b65aba32ca0e13c1ed300a" title="runs a taskflow multiple times until the predicate becomes true">tf::Executor::run_until</a> to run a taskflow for one time, multiple times, or until a given predicate evaluates to true. All methods accept an optional callback to invoke after the execution completes, and return a <a class="el" href="classtf_1_1Future.html" title="class to access the result of an execution">tf::Future</a> for users to access the execution status. The code below shows several ways to run a taskflow.</p>
<div class="fragment"><div class="line"> 1: <span class="comment">// Declare an executor and a taskflow</span></div>
<div class="line"> 2: <a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor;</div>
<div class="line"> 3: <a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"> 4:</div>
<div class="line"> 5: <span class="comment">// Add three tasks into the taskflow</span></div>
<div class="line"> 6: <a class="code hl_class" href="classtf_1_1Task.html">tf::Task</a> A = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"This is TaskA\n"</span>; });</div>
<div class="line"> 7: <a class="code hl_class" href="classtf_1_1Task.html">tf::Task</a> B = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"This is TaskB\n"</span>; });</div>
<div class="line"> 8: <a class="code hl_class" href="classtf_1_1Task.html">tf::Task</a> C = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"This is TaskC\n"</span>; });</div>
<div class="line"> 9: </div>
<div class="line">10: <span class="comment">// Build precedence between tasks</span></div>
<div class="line">11: A.<a class="code hl_function" href="classtf_1_1Task.html#a8c78c453295a553c1c016e4062da8588">precede</a>(B, C); </div>
<div class="line">12: </div>
<div class="line">13: <a class="code hl_class" href="classtf_1_1Future.html">tf::Future<void></a> fu = executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow);</div>
<div class="line">14: fu.wait(); <span class="comment">// block until the execution completes</span></div>
<div class="line">15:</div>
<div class="line">16: executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow, [](){ std::cout << <span class="stringliteral">"end of 1 run"</span>; }).wait();</div>
<div class="line">17: executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00">run_n</a>(taskflow, 4);</div>
<div class="line">18: executor.<a class="code hl_function" href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85">wait_for_all</a>(); <span class="comment">// block until all associated executions finish</span></div>
<div class="line">19: executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00">run_n</a>(taskflow, 4, [](){ std::cout << <span class="stringliteral">"end of 4 runs"</span>; }).wait();</div>
<div class="line">20: executor.<a class="code hl_function" href="classtf_1_1Executor.html#a0f52e9dd64b65aba32ca0e13c1ed300a">run_until</a>(taskflow, [cnt=0] () <span class="keyword">mutable</span> { <span class="keywordflow">return</span> ++cnt == 10; });</div>
<div class="ttc" id="aclasstf_1_1Executor_html_a0f52e9dd64b65aba32ca0e13c1ed300a"><div class="ttname"><a href="classtf_1_1Executor.html#a0f52e9dd64b65aba32ca0e13c1ed300a">tf::Executor::run_until</a></div><div class="ttdeci">tf::Future< void > run_until(Taskflow &taskflow, P &&pred)</div><div class="ttdoc">runs a taskflow multiple times until the predicate becomes true</div></div>
<div class="ttc" id="aclasstf_1_1Executor_html_a519777f5783981d534e9e53b99712069"><div class="ttname"><a href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">tf::Executor::run</a></div><div class="ttdeci">tf::Future< void > run(Taskflow &taskflow)</div><div class="ttdoc">runs a taskflow once</div></div>
<div class="ttc" id="aclasstf_1_1Executor_html_a6d0617eebc9421f1ba1f82ce6dd02c00"><div class="ttname"><a href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00">tf::Executor::run_n</a></div><div class="ttdeci">tf::Future< void > run_n(Taskflow &taskflow, size_t N)</div><div class="ttdoc">runs a taskflow for N times</div></div>
<div class="ttc" id="aclasstf_1_1Executor_html_ab9aa252f70e9a40020a1e5a89d485b85"><div class="ttname"><a href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85">tf::Executor::wait_for_all</a></div><div class="ttdeci">void wait_for_all()</div><div class="ttdoc">waits for all tasks to complete</div></div>
<div class="ttc" id="aclasstf_1_1FlowBuilder_html_a4d52a7fe2814b264846a2085e931652c"><div class="ttname"><a href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">tf::FlowBuilder::emplace</a></div><div class="ttdeci">Task emplace(C &&callable)</div><div class="ttdoc">creates a static task</div><div class="ttdef"><b>Definition</b> flow_builder.hpp:1571</div></div>
<div class="ttc" id="aclasstf_1_1Future_html"><div class="ttname"><a href="classtf_1_1Future.html">tf::Future</a></div><div class="ttdoc">class to access the result of an execution</div><div class="ttdef"><b>Definition</b> taskflow.hpp:630</div></div>
<div class="ttc" id="aclasstf_1_1Task_html"><div class="ttname"><a href="classtf_1_1Task.html">tf::Task</a></div><div class="ttdoc">class to create a task handle over a taskflow node</div><div class="ttdef"><b>Definition</b> task.hpp:569</div></div>
<div class="ttc" id="aclasstf_1_1Task_html_a8c78c453295a553c1c016e4062da8588"><div class="ttname"><a href="classtf_1_1Task.html#a8c78c453295a553c1c016e4062da8588">tf::Task::precede</a></div><div class="ttdeci">Task & precede(Ts &&... tasks)</div><div class="ttdoc">adds precedence links from this to other tasks</div><div class="ttdef"><b>Definition</b> task.hpp:1258</div></div>
<div class="ttc" id="aclasstf_1_1Taskflow_html"><div class="ttname"><a href="classtf_1_1Taskflow.html">tf::Taskflow</a></div><div class="ttdoc">class to create a taskflow object</div><div class="ttdef"><b>Definition</b> taskflow.hpp:64</div></div>
</div><!-- fragment --><p>Lines 6–8 create a taskflow consisting of three tasks, A, B, and C. Lines 13–14 execute the taskflow once and block until its completion. Line 16 runs the taskflow once and registers a callback that is invoked when the execution finishes. Lines 17–18 execute the taskflow four times and use <a class="el" href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85" title="waits for all tasks to complete">tf::Executor::wait_for_all</a> to wait for all executions to complete. Line 19 again runs the taskflow four times but invokes a callback at the end of the fourth execution. Finally, line 20 repeatedly runs the taskflow until the specified predicate returns true.</p>
<h1><a class="anchor" id="UnderstandTheExecutionOrder"></a>
Understand the Execution Order</h1>
<p>While it is possible to submit the same taskflow to an executor multiple times, these multiple runs will be synchronized to a sequential chain of executions in the order of their submissions. For example, the three submissions below will finish in order, where execution #1 completes before #2, and execution #2 completes before execution #3:</p>
<div class="fragment"><div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow); <span class="comment">// execution #1</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00">run_n</a>(taskflow, 10); <span class="comment">// execution #2</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow); <span class="comment">// execution #3</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85">wait_for_all</a>(); <span class="comment">// execution #1 -> execution #2 -> execution #3</span></div>
</div><!-- fragment --><p>However, there is no deterministic order for different taskflows submitted simultaneously. For example, the three submissions below may finish in an arbitrary order, since the three taskflows are distinct:</p>
<div class="fragment"><div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow1); <span class="comment">// execution 1</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6d0617eebc9421f1ba1f82ce6dd02c00">run_n</a>(taskflow2, 10); <span class="comment">// execution 2</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow3); <span class="comment">// execution 3</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85">wait_for_all</a>(); <span class="comment">// no order guarantee among the three taskflows</span></div>
</div><!-- fragment --><h1><a class="anchor" id="UnderstandTheOwnership"></a>
Understand the Ownership</h1>
<p>A running taskflow must remain alive for the duration of its execution because the executor does not take ownership of the taskflow. It is your responsibility to ensure that a taskflow is not destroyed while it is still running. For example, the code below can result in undefined behavior.</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor; <span class="comment">// create an executor</span></div>
<div class="line"> </div>
<div class="line"><span class="comment">// create a taskflow whose lifetime is restricted by the block statement</span></div>
<div class="line">{</div>
<div class="line"> <a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// adds tasks to the taskflow</span></div>
<div class="line"> <span class="comment">// ... </span></div>
<div class="line"> </div>
<div class="line"> <span class="comment">// runs the taskflow once</span></div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow);</div>
<div class="line"> </div>
<div class="line">} <span class="comment">// leaving the block statement will destroy taskflow while it is running, </span></div>
<div class="line"> <span class="comment">// resulting in undefined behavior</span></div>
</div><!-- fragment --><p>Similarly, you should avoid modifying a taskflow while it is running:</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"> </div>
<div class="line"><span class="comment">// Add tasks into the taskflow</span></div>
<div class="line"><span class="comment">// ...</span></div>
<div class="line"> </div>
<div class="line"><span class="comment">// Declare an executor</span></div>
<div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor;</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Future.html">tf::Future<void></a> future = executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow); </div>
<div class="line"> </div>
<div class="line"><span class="comment">// alter the taskflow while running leads to undefined behavior </span></div>
<div class="line">taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([](){ std::cout << <span class="stringliteral">"Add a new task\n"</span>; });</div>
</div><!-- fragment --><p>You must always keep a taskflow alive and must not modify it while it is running on an executor.</p>
<h1><a class="anchor" id="ExecuteATaskflowWithTransferredOwnership"></a>
Execute a Taskflow with Transferred Ownership</h1>
<p>You can transfer the ownership of a taskflow to an executor and run it without wrangling with the lifetime issue of that taskflow. Each <code>run_*</code> method discussed in the previous section comes with an overload that takes a <em>moved</em> taskflow object.</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor;</div>
<div class="line"> </div>
<div class="line">taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([](){});</div>
<div class="line"> </div>
<div class="line"><span class="comment">// let the executor manage the lifetime of the submitted taskflow</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(std::move(taskflow));</div>
<div class="line"> </div>
<div class="line"><span class="comment">// now taskflow has no tasks</span></div>
<div class="line">assert(taskflow.<a class="code hl_function" href="classtf_1_1Taskflow.html#af4f03bca084deb5c2228ac8936d33649">num_tasks</a>() == 0);</div>
<div class="ttc" id="aclasstf_1_1Taskflow_html_af4f03bca084deb5c2228ac8936d33649"><div class="ttname"><a href="classtf_1_1Taskflow.html#af4f03bca084deb5c2228ac8936d33649">tf::Taskflow::num_tasks</a></div><div class="ttdeci">size_t num_tasks() const</div><div class="ttdoc">queries the number of tasks in this taskflow</div><div class="ttdef"><b>Definition</b> taskflow.hpp:376</div></div>
</div><!-- fragment --><p>However, you should avoid moving a <em>running</em> taskflow which can result in undefined behavior.</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor;</div>
<div class="line"> </div>
<div class="line">taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([](){});</div>
<div class="line"> </div>
<div class="line"><span class="comment">// executor does not manage the lifetime of taskflow</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow);</div>
<div class="line"> </div>
<div class="line"><span class="comment">// error! you cannot move a taskflow while it is running</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(std::move(taskflow)); </div>
</div><!-- fragment --><p>The correct way to submit a taskflow with moved ownership to an executor is to ensure all previous runs have completed. The executor will automatically release the resources of a moved taskflow right <em>after</em> its execution completes.</p>
<div class="fragment"><div class="line"><span class="comment">// submit the taskflow and wait until it completes</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow).wait();</div>
<div class="line"> </div>
<div class="line"><span class="comment">// now it's safe to move the taskflow to the executor and run it</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(std::move(taskflow)); </div>
</div><!-- fragment --><p>Likewise, you cannot move a taskflow that is running on an executor. You must wait until all the previous fires of runs on that taskflow complete before calling move.</p>
<div class="fragment"><div class="line"><span class="comment">// submit the taskflow and wait until it completes</span></div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow).wait();</div>
<div class="line"> </div>
<div class="line"><span class="comment">// now it's safe to move the taskflow to another</span></div>
<div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> moved_taskflow(std::move(taskflow)); </div>
</div><!-- fragment --><h1><a class="anchor" id="ExecuteATaskflowFromAnInternalWorker"></a>
Execute a Taskflow from an Internal Worker Cooperatively</h1>
<p>Each run variant of <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> returns a <a class="el" href="classtf_1_1Future.html" title="class to access the result of an execution">tf::Future</a> object that allows you to wait for the associated execution to complete. When <code>tf::Future::wait</code> is called, the caller blocks without making any progress until the underlying state becomes ready. However, this blocking design can lead to potential deadlocks, particularly when multiple taskflows are launched from within the internal workers of the same executor. For example, the following code creates a taskflow of 1,000 tasks, where each task runs another taskflow of 500 tasks in a blocking manner, leading to a potential deadlock problem:</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor(2);</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line">std::array<tf::Taskflow, 1000> others;</div>
<div class="line"> </div>
<div class="line"><span class="keywordflow">for</span>(<span class="keywordtype">size_t</span> n=0; n<1000; n++) {</div>
<div class="line"> <span class="keywordflow">for</span>(<span class="keywordtype">size_t</span> i=0; i<500; i++) {</div>
<div class="line"> others[n].emplace([&](){});</div>
<div class="line"> }</div>
<div class="line"> taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([&executor, &<a class="code hl_namespace" href="namespacetf.html">tf</a>=others[n]](){</div>
<div class="line"> <span class="comment">// Blocking a worker can cause deadlock if all workers are waiting</span></div>
<div class="line"> <span class="comment">// for their taskflows to complete without making any progress</span></div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(<a class="code hl_namespace" href="namespacetf.html">tf</a>).wait();</div>
<div class="line"> });</div>
<div class="line">}</div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow).wait();</div>
<div class="ttc" id="anamespacetf_html"><div class="ttname"><a href="namespacetf.html">tf</a></div><div class="ttdoc">taskflow namespace</div><div class="ttdef"><b>Definition</b> small_vector.hpp:20</div></div>
</div><!-- fragment --><p>To avoid this deadlock issue, <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> provides a cooperative execution method, <a class="el" href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8" title="runs a target graph and waits until it completes using an internal worker of this executor">tf::Executor::corun</a>, which allows a worker to execute a taskflow <em>cooperatively</em> with other workers within the same executor. Specifically, although the worker is blocking until corun finishes, it is <em>cooperatively</em> blocked and continues to execute the taskflow alongside other tasks in the executor's work-stealing loop. For instance, the following code with <a class="el" href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8" title="runs a target graph and waits until it completes using an internal worker of this executor">tf::Executor::corun</a> is deadlock-free:</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor(2);</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line">std::array<tf::Taskflow, 1000> others;</div>
<div class="line"> </div>
<div class="line">std::atomic<size_t> counter{0};</div>
<div class="line"> </div>
<div class="line"><span class="keywordflow">for</span>(<span class="keywordtype">size_t</span> n=0; n<1000; n++) {</div>
<div class="line"> <span class="keywordflow">for</span>(<span class="keywordtype">size_t</span> i=0; i<500; i++) {</div>
<div class="line"> others[n].emplace([&](){ counter++; });</div>
<div class="line"> }</div>
<div class="line"> taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([&executor, &<a class="code hl_namespace" href="namespacetf.html">tf</a>=others[n]](){</div>
<div class="line"> <span class="comment">// calling worker coruns the taskflow cooperatively with other workers</span></div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8">corun</a>(<a class="code hl_namespace" href="namespacetf.html">tf</a>);</div>
<div class="line"> });</div>
<div class="line">}</div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow).wait();</div>
<div class="ttc" id="aclasstf_1_1Executor_html_a8fcd9e0557922bb8194999f0cd433ea8"><div class="ttname"><a href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8">tf::Executor::corun</a></div><div class="ttdeci">void corun(T &target)</div><div class="ttdoc">runs a target graph and waits until it completes using an internal worker of this executor</div></div>
</div><!-- fragment --><p>Similar to <a class="el" href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8" title="runs a target graph and waits until it completes using an internal worker of this executor">tf::Executor::corun</a>, <a class="el" href="classtf_1_1Executor.html#a0fc6eb19f168dc4a9cd0a7c6187c1d2d" title="keeps running the work-stealing loop until the predicate returns true">tf::Executor::corun_until</a> provides a more general mechanism for cooperative execution. It allows the calling worker to run tasks cooperatively with other workers until a specified condition becomes true. This function is especially useful for implementing custom synchronization or polling conditions, where the worker continues making progress by executing available tasks in the executor while periodically checking for the termination condition.</p>
<div class="fragment"><div class="line">taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([&](){</div>
<div class="line"> <span class="keyword">auto</span> fu = std::async([](){ std::sleep(100s); });</div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a0fc6eb19f168dc4a9cd0a7c6187c1d2d">corun_until</a>([](){</div>
<div class="line"> <span class="keywordflow">return</span> fu.wait_for(std::chrono::seconds(0)) == future_status::ready;</div>
<div class="line"> });</div>
<div class="line">});</div>
<div class="ttc" id="aclasstf_1_1Executor_html_a0fc6eb19f168dc4a9cd0a7c6187c1d2d"><div class="ttname"><a href="classtf_1_1Executor.html#a0fc6eb19f168dc4a9cd0a7c6187c1d2d">tf::Executor::corun_until</a></div><div class="ttdeci">void corun_until(P &&predicate)</div><div class="ttdoc">keeps running the work-stealing loop until the predicate returns true</div></div>
</div><!-- fragment --><dl class="section note"><dt>Note</dt><dd>You must call <a class="el" href="classtf_1_1Executor.html#a0fc6eb19f168dc4a9cd0a7c6187c1d2d" title="keeps running the work-stealing loop until the predicate returns true">tf::Executor::corun_until</a> and <a class="el" href="classtf_1_1Executor.html#a8fcd9e0557922bb8194999f0cd433ea8" title="runs a target graph and waits until it completes using an internal worker of this executor">tf::Executor::corun</a> from a worker of the calling executor or an exception will be thrown.</dd></dl>
<h1><a class="anchor" id="ThreadSafetyOfExecution"></a>
Thread Safety of Executor</h1>
<p>All <code>run_*</code> methods of <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> are <em>thread-safe</em>. You can safely call these methods from multiple threads to execute different taskflows concurrently. For instance, the code below launches ten threads, each executing a different taskflow concurrently on the same executor, and waits for all executions to complete:</p>
<div class="fragment"><div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor;</div>
<div class="line">std::array<tf::Taskflow, 10> taskflows;</div>
<div class="line"><span class="keywordflow">for</span>(<span class="keywordtype">int</span> i=0; i<10; ++i) {</div>
<div class="line"> std::thread([i, &](){</div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflows[i]); <span class="comment">// thread i runs taskflow i</span></div>
<div class="line"> }).detach();</div>
<div class="line">}</div>
<div class="line">executor.<a class="code hl_function" href="classtf_1_1Executor.html#ab9aa252f70e9a40020a1e5a89d485b85">wait_for_all</a>();</div>
</div><!-- fragment --><h1><a class="anchor" id="QueryTheWorkerID"></a>
Query the Worker ID</h1>
<p>Each worker thread in a <a class="el" href="classtf_1_1Executor.html" title="class to create an executor">tf::Executor</a> is assigned a <em>unique</em> integer identifier in the range <code>[0, N)</code>, where <code>N</code> is the number of worker threads in the executor. You can query the identifier of the calling thread using <a class="el" href="classtf_1_1Executor.html#a6487d589cb1f6b078b69fd3bb1082345" title="queries the id of the caller thread within this executor">tf::Executor::this_worker_id</a>. If the calling thread is not a worker of the executor, the method returns -1. This functionality is particularly useful for establishing a one-to-one mapping between worker threads and application-specific data structures.</p>
<div class="fragment"><div class="line">std::vector<int> data[8]; <span class="comment">// worker-specific data vector</span></div>
<div class="line"><a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"><a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor(8); <span class="comment">// an executor of eight workers</span></div>
<div class="line">assert(executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6487d589cb1f6b078b69fd3bb1082345">this_worker_id</a>() == -1); <span class="comment">// main thread is not a worker</span></div>
<div class="line">taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([&](){</div>
<div class="line"> <span class="keywordtype">int</span> <span class="keywordtype">id</span> = executor.<a class="code hl_function" href="classtf_1_1Executor.html#a6487d589cb1f6b078b69fd3bb1082345">this_worker_id</a>(); <span class="comment">// in the range [0, 8)</span></div>
<div class="line"> <span class="keyword">auto</span>& vec = data[id]; <span class="comment">// worker id process data[id]</span></div>
<div class="line">});</div>
<div class="ttc" id="aclasstf_1_1Executor_html_a6487d589cb1f6b078b69fd3bb1082345"><div class="ttname"><a href="classtf_1_1Executor.html#a6487d589cb1f6b078b69fd3bb1082345">tf::Executor::this_worker_id</a></div><div class="ttdeci">int this_worker_id() const</div><div class="ttdoc">queries the id of the caller thread within this executor</div></div>
</div><!-- fragment --><h1><a class="anchor" id="ObserveThreadActivities"></a>
Observe Thread Activities</h1>
<p>You can observe thread activities in an executor when a worker thread participates in executing a task and leaves the execution using <a class="el" href="classtf_1_1ObserverInterface.html" title="class to derive an executor observer">tf::ObserverInterface</a> – an <em>interface</em> class that provides a set of methods for you to define what to do when a thread enters and leaves the execution context of a task.</p>
<div class="fragment"><div class="line"><span class="keyword">class </span>ObserverInterface {</div>
<div class="line"> <span class="keyword">virtual</span> ~ObserverInterface() = <span class="keywordflow">default</span>;</div>
<div class="line"> <span class="keyword">virtual</span> <span class="keywordtype">void</span> set_up(<span class="keywordtype">size_t</span> num_workers) = 0;</div>
<div class="line"> <span class="keyword">virtual</span> <span class="keywordtype">void</span> on_entry(<a class="code hl_class" href="classtf_1_1WorkerView.html">tf::WorkerView</a> worker_view, <a class="code hl_class" href="classtf_1_1TaskView.html">tf::TaskView</a> task_view) = 0;</div>
<div class="line"> <span class="keyword">virtual</span> <span class="keywordtype">void</span> on_exit(<a class="code hl_class" href="classtf_1_1WorkerView.html">tf::WorkerView</a> worker_view, <a class="code hl_class" href="classtf_1_1TaskView.html">tf::TaskView</a> task_view) = 0;</div>
<div class="line">};</div>
<div class="ttc" id="aclasstf_1_1TaskView_html"><div class="ttname"><a href="classtf_1_1TaskView.html">tf::TaskView</a></div><div class="ttdoc">class to access task information from the observer interface</div><div class="ttdef"><b>Definition</b> task.hpp:1546</div></div>
<div class="ttc" id="aclasstf_1_1WorkerView_html"><div class="ttname"><a href="classtf_1_1WorkerView.html">tf::WorkerView</a></div><div class="ttdoc">class to create an immutable view of a worker</div><div class="ttdef"><b>Definition</b> worker.hpp:116</div></div>
</div><!-- fragment --><p>There are three methods you must define in your derived class, <a class="el" href="classtf_1_1ObserverInterface.html#a41e6e62f12bf9d9dc4fa74632f6825d9" title="constructor-like method to call when the executor observer is fully created">tf::ObserverInterface::set_up</a>, <a class="el" href="classtf_1_1ObserverInterface.html#a8225fcacb03089677a1efc4b16b734cc" title="method to call before a worker thread executes a closure">tf::ObserverInterface::on_entry</a>, and <a class="el" href="classtf_1_1ObserverInterface.html#aa22f5378154653f08d9a58326bda4754" title="method to call after a worker thread executed a closure">tf::ObserverInterface::on_exit</a>. The method, <a class="el" href="classtf_1_1ObserverInterface.html#a41e6e62f12bf9d9dc4fa74632f6825d9" title="constructor-like method to call when the executor observer is fully created">tf::ObserverInterface::set_up</a>, is a constructor-like method that will be called by the executor when the observer is constructed. It passes an argument of the number of workers to observer in the executor. You may use it to preallocate or initialize data storage, e.g., an independent vector for each worker. The methods, <a class="el" href="classtf_1_1ObserverInterface.html#a8225fcacb03089677a1efc4b16b734cc" title="method to call before a worker thread executes a closure">tf::ObserverInterface::on_entry</a> and <a class="el" href="classtf_1_1ObserverInterface.html#aa22f5378154653f08d9a58326bda4754" title="method to call after a worker thread executed a closure">tf::ObserverInterface::on_exit</a>, are called by a worker thread before and after the execution context of a task, respectively. Both methods provide immutable access to the underlying worker and the running task using <a class="el" href="classtf_1_1WorkerView.html" title="class to create an immutable view of a worker">tf::WorkerView</a> and <a class="el" href="classtf_1_1TaskView.html" title="class to access task information from the observer interface">tf::TaskView</a>. You may use them to record timepoints and calculate the elapsed time of a task.</p>
<p>You can associate an executor with one or multiple observers using <a class="el" href="classtf_1_1Executor.html#aff77def96ae740d648dd84e571237c83" title="constructs an observer to inspect the activities of worker threads">tf::Executor::make_observer</a>. We use std::shared_ptr to manage the ownership of an observer. The executor loops through each observer and invoke the corresponding methods accordingly.</p>
<div class="fragment"><div class="line"><span class="preprocessor">#include <taskflow/taskflow.hpp></span></div>
<div class="line"> </div>
<div class="line"><span class="keyword">struct </span>MyObserver : <span class="keyword">public</span> <a class="code hl_class" href="classtf_1_1ObserverInterface.html">tf::ObserverInterface</a> {</div>
<div class="line"> </div>
<div class="line"> MyObserver(<span class="keyword">const</span> std::string& name) {</div>
<div class="line"> std::cout << <span class="stringliteral">"constructing observer "</span> << name << <span class="charliteral">'\n'</span>;</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line"> <span class="keywordtype">void</span> <a class="code hl_function" href="classtf_1_1ObserverInterface.html#a41e6e62f12bf9d9dc4fa74632f6825d9">set_up</a>(<span class="keywordtype">size_t</span> num_workers) <span class="keyword">override</span> <span class="keyword">final</span> {</div>
<div class="line"> std::cout << <span class="stringliteral">"setting up observer with "</span> << num_workers << <span class="stringliteral">" workers\n"</span>;</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line"> <span class="keywordtype">void</span> <a class="code hl_function" href="classtf_1_1ObserverInterface.html#a8225fcacb03089677a1efc4b16b734cc">on_entry</a>(<a class="code hl_class" href="classtf_1_1WorkerView.html">tf::WorkerView</a> w, <a class="code hl_class" href="classtf_1_1TaskView.html">tf::TaskView</a> tv) <span class="keyword">override</span> <span class="keyword">final</span> {</div>
<div class="line"> std::ostringstream oss;</div>
<div class="line"> oss << <span class="stringliteral">"worker "</span> << w.id() << <span class="stringliteral">" ready to run "</span> << tv.name() << <span class="charliteral">'\n'</span>;</div>
<div class="line"> std::cout << oss.str();</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line"> <span class="keywordtype">void</span> <a class="code hl_function" href="classtf_1_1ObserverInterface.html#aa22f5378154653f08d9a58326bda4754">on_exit</a>(tf::WorkerView w, tf::TaskView tv) <span class="keyword">override</span> <span class="keyword">final</span> {</div>
<div class="line"> std::ostringstream oss;</div>
<div class="line"> oss << <span class="stringliteral">"worker "</span> << w.id() << <span class="stringliteral">" finished running "</span> << tv.name() << <span class="charliteral">'\n'</span>;</div>
<div class="line"> std::cout << oss.str();</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line">};</div>
<div class="line"> </div>
<div class="line"><span class="keywordtype">int</span> main(){</div>
<div class="line"> </div>
<div class="line"> <a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor(4);</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// Create a taskflow of eight tasks</span></div>
<div class="line"> <a class="code hl_class" href="classtf_1_1Taskflow.html">tf::Taskflow</a> taskflow;</div>
<div class="line"> </div>
<div class="line"> <span class="keyword">auto</span> A = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"1\n"</span>; }).name(<span class="stringliteral">"A"</span>);</div>
<div class="line"> <span class="keyword">auto</span> B = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"2\n"</span>; }).name(<span class="stringliteral">"B"</span>);</div>
<div class="line"> <span class="keyword">auto</span> C = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"3\n"</span>; }).name(<span class="stringliteral">"C"</span>);</div>
<div class="line"> <span class="keyword">auto</span> D = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"4\n"</span>; }).name(<span class="stringliteral">"D"</span>);</div>
<div class="line"> <span class="keyword">auto</span> E = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"5\n"</span>; }).name(<span class="stringliteral">"E"</span>);</div>
<div class="line"> <span class="keyword">auto</span> F = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"6\n"</span>; }).name(<span class="stringliteral">"F"</span>);</div>
<div class="line"> <span class="keyword">auto</span> G = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"7\n"</span>; }).name(<span class="stringliteral">"G"</span>);</div>
<div class="line"> <span class="keyword">auto</span> H = taskflow.<a class="code hl_function" href="classtf_1_1FlowBuilder.html#a4d52a7fe2814b264846a2085e931652c">emplace</a>([] () { std::cout << <span class="stringliteral">"8\n"</span>; }).name(<span class="stringliteral">"H"</span>);</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// create an observer</span></div>
<div class="line"> std::shared_ptr<MyObserver> observer = executor.<a class="code hl_function" href="classtf_1_1Executor.html#aff77def96ae740d648dd84e571237c83">make_observer</a><MyObserver>(</div>
<div class="line"> <span class="stringliteral">"MyObserver"</span></div>
<div class="line"> );</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// run the taskflow</span></div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a519777f5783981d534e9e53b99712069">run</a>(taskflow).get();</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// remove the observer (optional)</span></div>
<div class="line"> executor.<a class="code hl_function" href="classtf_1_1Executor.html#a31081f492c376f7b798de0e430534531">remove_observer</a>(std::move(observer));</div>
<div class="line"> </div>
<div class="line"> <span class="keywordflow">return</span> 0;</div>
<div class="line">}</div>
<div class="ttc" id="aclasstf_1_1Executor_html_a31081f492c376f7b798de0e430534531"><div class="ttname"><a href="classtf_1_1Executor.html#a31081f492c376f7b798de0e430534531">tf::Executor::remove_observer</a></div><div class="ttdeci">void remove_observer(std::shared_ptr< Observer > observer)</div><div class="ttdoc">removes an observer from the executor</div></div>
<div class="ttc" id="aclasstf_1_1Executor_html_aff77def96ae740d648dd84e571237c83"><div class="ttname"><a href="classtf_1_1Executor.html#aff77def96ae740d648dd84e571237c83">tf::Executor::make_observer</a></div><div class="ttdeci">std::shared_ptr< Observer > make_observer(ArgsT &&... args)</div><div class="ttdoc">constructs an observer to inspect the activities of worker threads</div></div>
<div class="ttc" id="aclasstf_1_1ObserverInterface_html"><div class="ttname"><a href="classtf_1_1ObserverInterface.html">tf::ObserverInterface</a></div><div class="ttdoc">class to derive an executor observer</div><div class="ttdef"><b>Definition</b> interface.hpp:81</div></div>
<div class="ttc" id="aclasstf_1_1ObserverInterface_html_a41e6e62f12bf9d9dc4fa74632f6825d9"><div class="ttname"><a href="classtf_1_1ObserverInterface.html#a41e6e62f12bf9d9dc4fa74632f6825d9">tf::ObserverInterface::set_up</a></div><div class="ttdeci">virtual void set_up(size_t num_workers)=0</div><div class="ttdoc">constructor-like method to call when the executor observer is fully created</div></div>
<div class="ttc" id="aclasstf_1_1ObserverInterface_html_a8225fcacb03089677a1efc4b16b734cc"><div class="ttname"><a href="classtf_1_1ObserverInterface.html#a8225fcacb03089677a1efc4b16b734cc">tf::ObserverInterface::on_entry</a></div><div class="ttdeci">virtual void on_entry(WorkerView wv, TaskView task_view)=0</div><div class="ttdoc">method to call before a worker thread executes a closure</div></div>
<div class="ttc" id="aclasstf_1_1ObserverInterface_html_aa22f5378154653f08d9a58326bda4754"><div class="ttname"><a href="classtf_1_1ObserverInterface.html#aa22f5378154653f08d9a58326bda4754">tf::ObserverInterface::on_exit</a></div><div class="ttdeci">virtual void on_exit(WorkerView wv, TaskView task_view)=0</div><div class="ttdoc">method to call after a worker thread executed a closure</div></div>
</div><!-- fragment --><p>The above code produces the following output:</p>
<div class="fragment"><div class="line">constructing observer MyObserver</div>
<div class="line">setting up observer with 4 workers</div>
<div class="line">worker 2 ready to run A</div>
<div class="line">1</div>
<div class="line">worker 2 finished running A</div>
<div class="line">worker 2 ready to run B</div>
<div class="line">2</div>
<div class="line">worker 1 ready to run C</div>
<div class="line">worker 2 finished running B</div>
<div class="line">3</div>
<div class="line">worker 2 ready to run D</div>
<div class="line">worker 3 ready to run E</div>
<div class="line">worker 1 finished running C</div>
<div class="line">4</div>
<div class="line">5</div>
<div class="line">worker 1 ready to run F</div>
<div class="line">worker 2 finished running D</div>
<div class="line">worker 3 finished running E</div>
<div class="line">6</div>
<div class="line">worker 2 ready to run G</div>
<div class="line">worker 3 ready to run H</div>
<div class="line">worker 1 finished running F</div>
<div class="line">7</div>
<div class="line">8</div>
<div class="line">worker 2 finished running G</div>
<div class="line">worker 3 finished running H</div>
</div><!-- fragment --><p>It is expected each line of std::cout interleaves with each other as there are four workers participating in task scheduling. However, the <em>ready</em> message always appears before the corresponding task message (e.g., numbers) and then the <em>finished</em> message.</p>
<h1><a class="anchor" id="ModifyWorkerProperty"></a>
Modify Worker Property</h1>
<p>You can change the property of each worker thread from its executor, such as assigning thread-processor affinity before the worker enters the scheduler loop and post-processing additional information after the worker leaves the scheduler loop, by passing an instance derived from <a class="el" href="classtf_1_1WorkerInterface.html" title="class to configure worker behavior in an executor">tf::WorkerInterface</a> to the executor. The example demonstrates the usage of <a class="el" href="classtf_1_1WorkerInterface.html" title="class to configure worker behavior in an executor">tf::WorkerInterface</a> to affine a worker to a specific CPU core equal to its id on a Linux platform:</p>
<div class="fragment"><div class="line"><span class="comment">// affine the given thread to the given core index (linux-specific)</span></div>
<div class="line"><span class="keywordtype">bool</span> affine(std::thread& thread, <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> core_id) {</div>
<div class="line"> cpu_set_t cpuset;</div>
<div class="line"> CPU_ZERO(&cpuset);</div>
<div class="line"> CPU_SET(core_id, &cpuset);</div>
<div class="line"> pthread_t native_handle = thread.native_handle();</div>
<div class="line"> <span class="keywordflow">return</span> pthread_setaffinity_np(native_handle, <span class="keyword">sizeof</span>(cpu_set_t), &cpuset) == 0;</div>
<div class="line">}</div>
<div class="line"> </div>
<div class="line"><span class="keyword">class </span>CustomWorkerBehavior : <span class="keyword">public</span> <a class="code hl_class" href="classtf_1_1WorkerInterface.html">tf::WorkerInterface</a> {</div>
<div class="line"> </div>
<div class="line"> <span class="keyword">public</span>:</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// to call before the worker enters the scheduling loop</span></div>
<div class="line"> <span class="keywordtype">void</span> <a class="code hl_function" href="classtf_1_1WorkerInterface.html#a41c3b931a36bde8eff4aa8d375e8888a">scheduler_prologue</a>(<a class="code hl_class" href="classtf_1_1Worker.html">tf::Worker</a>& w)<span class="keyword"> override </span>{</div>
<div class="line"> printf(<span class="stringliteral">"worker %lu prepares to enter the work-stealing loop\n"</span>, w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>());</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// now affine the worker to a particular CPU core equal to its id</span></div>
<div class="line"> <span class="keywordflow">if</span>(affine(w.<a class="code hl_function" href="classtf_1_1Worker.html#a6158f91db3b980e3072cc0329cbe3c14">thread</a>(), w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>())) {</div>
<div class="line"> printf(<span class="stringliteral">"successfully affines worker %lu to CPU core %lu\n"</span>, w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>(), w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>());</div>
<div class="line"> }</div>
<div class="line"> <span class="keywordflow">else</span> {</div>
<div class="line"> printf(<span class="stringliteral">"failed to affine worker %lu to CPU core %lu\n"</span>, w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>(), w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>());</div>
<div class="line"> }</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// to call after the worker leaves the scheduling loop</span></div>
<div class="line"> <span class="keywordtype">void</span> <a class="code hl_function" href="classtf_1_1WorkerInterface.html#a3e6d68fd4041f433d1b7ca9e5786b57c">scheduler_epilogue</a>(tf::Worker& w, std::exception_ptr)<span class="keyword"> override </span>{</div>
<div class="line"> printf(<span class="stringliteral">"worker %lu left the work-stealing loop\n"</span>, w.<a class="code hl_function" href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">id</a>());</div>
<div class="line"> }</div>
<div class="line">};</div>
<div class="line"> </div>
<div class="line"><span class="keywordtype">int</span> main() {</div>
<div class="line"> <a class="code hl_class" href="classtf_1_1Executor.html">tf::Executor</a> executor(4, <a class="code hl_function" href="namespacetf.html#a277bf0914351f67a59f54d6b1fa0b08f">tf::make_worker_interface<CustomWorkerBehavior></a>());</div>
<div class="line"> <span class="keywordflow">return</span> 0;</div>
<div class="line">}</div>
<div class="ttc" id="aclasstf_1_1WorkerInterface_html"><div class="ttname"><a href="classtf_1_1WorkerInterface.html">tf::WorkerInterface</a></div><div class="ttdoc">class to configure worker behavior in an executor</div><div class="ttdef"><b>Definition</b> worker.hpp:275</div></div>
<div class="ttc" id="aclasstf_1_1WorkerInterface_html_a3e6d68fd4041f433d1b7ca9e5786b57c"><div class="ttname"><a href="classtf_1_1WorkerInterface.html#a3e6d68fd4041f433d1b7ca9e5786b57c">tf::WorkerInterface::scheduler_epilogue</a></div><div class="ttdeci">virtual void scheduler_epilogue(Worker &worker, std::exception_ptr ptr)=0</div><div class="ttdoc">method to call after a worker leaves the scheduling loop</div></div>
<div class="ttc" id="aclasstf_1_1WorkerInterface_html_a41c3b931a36bde8eff4aa8d375e8888a"><div class="ttname"><a href="classtf_1_1WorkerInterface.html#a41c3b931a36bde8eff4aa8d375e8888a">tf::WorkerInterface::scheduler_prologue</a></div><div class="ttdeci">virtual void scheduler_prologue(Worker &worker)=0</div><div class="ttdoc">method to call before a worker enters the scheduling loop</div></div>
<div class="ttc" id="aclasstf_1_1Worker_html"><div class="ttname"><a href="classtf_1_1Worker.html">tf::Worker</a></div><div class="ttdoc">class to create a worker in an executor</div><div class="ttdef"><b>Definition</b> worker.hpp:55</div></div>
<div class="ttc" id="aclasstf_1_1Worker_html_a0180ea51cc46551157eaae451b50c7d8"><div class="ttname"><a href="classtf_1_1Worker.html#a0180ea51cc46551157eaae451b50c7d8">tf::Worker::id</a></div><div class="ttdeci">size_t id() const</div><div class="ttdoc">queries the worker id associated with its parent executor</div><div class="ttdef"><b>Definition</b> worker.hpp:72</div></div>
<div class="ttc" id="aclasstf_1_1Worker_html_a6158f91db3b980e3072cc0329cbe3c14"><div class="ttname"><a href="classtf_1_1Worker.html#a6158f91db3b980e3072cc0329cbe3c14">tf::Worker::thread</a></div><div class="ttdeci">std::thread & thread()</div><div class="ttdoc">acquires the associated thread</div><div class="ttdef"><b>Definition</b> worker.hpp:88</div></div>
<div class="ttc" id="anamespacetf_html_a277bf0914351f67a59f54d6b1fa0b08f"><div class="ttname"><a href="namespacetf.html#a277bf0914351f67a59f54d6b1fa0b08f">tf::make_worker_interface</a></div><div class="ttdeci">std::shared_ptr< T > make_worker_interface(ArgsT &&... args)</div><div class="ttdoc">helper function to create an instance derived from tf::WorkerInterface</div><div class="ttdef"><b>Definition</b> worker.hpp:314</div></div>
</div><!-- fragment --><p>When running the program, we see the following one possible output:</p>
<div class="fragment"><div class="line">worker 3 prepares to enter the work-stealing loop</div>
<div class="line">successfully affines worker 3 to CPU core 3</div>
<div class="line">worker 3 left the work-stealing loop</div>
<div class="line">worker 0 prepares to enter the work-stealing loop</div>
<div class="line">successfully affines worker 0 to CPU core 0</div>
<div class="line">worker 0 left the work-stealing loop</div>
<div class="line">worker 1 prepares to enter the work-stealing loop</div>
<div class="line">worker 2 prepares to enter the work-stealing loop</div>
<div class="line">successfully affines worker 1 to CPU core 1</div>
<div class="line">worker 1 left the work-stealing loop</div>
<div class="line">successfully affines worker 2 to CPU core 2</div>
<div class="line">worker 2 left the work-stealing loop</div>
</div><!-- fragment --><p>When you create an executor, it spawns a set of worker threads to run tasks using a work-stealing scheduling algorithm. The execution logic of the scheduler and its interaction with each spawned worker via <a class="el" href="classtf_1_1WorkerInterface.html" title="class to configure worker behavior in an executor">tf::WorkerInterface</a> is given below:</p>
<div class="fragment"><div class="line"><span class="keywordflow">for</span>(<span class="keywordtype">size_t</span> n=0; n<num_workers; n++) {</div>
<div class="line"> create_thread([](Worker& worker){</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// enter the scheduling loop</span></div>
<div class="line"> <span class="comment">// Here, WorkerInterface::scheduler_prologue is invoked, if any</span></div>
<div class="line"> worker_interface->scheduler_prologue(worker);</div>
<div class="line"> </div>
<div class="line"> <span class="keywordflow">try</span> {</div>
<div class="line"> <span class="keywordflow">while</span>(1) {</div>
<div class="line"> perform_work_stealing_algorithm();</div>
<div class="line"> <span class="keywordflow">if</span>(stop) {</div>
<div class="line"> <span class="keywordflow">break</span>;</div>
<div class="line"> }</div>
<div class="line"> }</div>
<div class="line"> } <span class="keywordflow">catch</span>(...) {</div>
<div class="line"> exception_ptr = std::current_exception();</div>
<div class="line"> }</div>
<div class="line"> </div>
<div class="line"> <span class="comment">// leaves the scheduling loop and joins this worker thread</span></div>
<div class="line"> <span class="comment">// Here, WorkerInterface::scheduler_epilogue is invoked, if any</span></div>
<div class="line"> worker_interface->scheduler_epilogue(worker, exception_ptr);</div>
<div class="line"> });</div>
<div class="line">}</div>
</div><!-- fragment --><dl class="section note"><dt>Note</dt><dd><a class="el" href="classtf_1_1WorkerInterface.html#a41c3b931a36bde8eff4aa8d375e8888a" title="method to call before a worker enters the scheduling loop">tf::WorkerInterface::scheduler_prologue</a> and <a class="el" href="classtf_1_1WorkerInterface.html#a3e6d68fd4041f433d1b7ca9e5786b57c" title="method to call after a worker leaves the scheduling loop">tf::WorkerInterface::scheduler_epilogue</a> are invoked by each worker simultaneously. It is your responsibility to ensure that no data race can occur during their invocation. </dd></dl>
</div></div><!-- contents -->
</div><!-- PageDoc -->
</div><!-- doc-content -->
<!-- HTML footer for doxygen 1.13.1-->
<!-- start footer part -->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
<ul>
<li class="navelem"><a class="el" href="Cookbook.html">Cookbook</a></li>
<li class="footer">
Maintained by <a href="https://tsung-wei-huang.github.io/">Dr. Tsung-Wei Huang</a>
—
Generated by <a href="https://www.doxygen.org/index.html"><img class="footer" src="doxygen.svg" width="104" height="31" alt="doxygen"/></a> 1.13.1
</li>
</ul>
</div>