Repository navigation
Expand file tree
/
Copy pathmessage_passing.cpp
More file actions
120 lines (102 loc) · 3.5 KB
/
Copy pathmessage_passing.cpp
File metadata and controls
120 lines (102 loc) · 3.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#include "openmc/message_passing.h"
#include <algorithm>
#include <climits>
#include <limits>
namespace openmc {
namespace mpi {
int rank {0};
int n_procs {1};
bool master {true};
#ifdef OPENMC_MPI
MPI_Comm intracomm {MPI_COMM_NULL};
MPI_Datatype source_site {MPI_DATATYPE_NULL};
MPI_Datatype collision_track_site {MPI_DATATYPE_NULL};
#endif
extern "C" bool openmc_master()
{
return mpi::master;
}
vector<int64_t> calculate_parallel_index_vector(int64_t size)
{
vector<int64_t> result;
result.resize(n_procs + 1);
result[0] = 0;
#ifdef OPENMC_MPI
// Populate the result with cumulative sum of the number of
// surface source banks per process
int64_t scan_total;
MPI_Scan(&size, &scan_total, 1, MPI_INT64_T, MPI_SUM, intracomm);
MPI_Allgather(
&scan_total, 1, MPI_INT64_T, result.data() + 1, 1, MPI_INT64_T, intracomm);
#else
result[1] = size;
#endif
return result;
}
#ifdef OPENMC_MPI
void broadcast_buffer(void* buffer, std::size_t count, MPI_Datatype datatype,
std::size_t type_size, int root, MPI_Comm comm)
{
#ifdef OPENMC_HAVE_MPI_LARGE_COUNT
const std::size_t chunk_limit {
static_cast<std::size_t>(std::numeric_limits<MPI_Count>::max())};
#else
constexpr std::size_t chunk_limit {INT_MAX};
#endif
// Bound the count before converting it to the type accepted by MPI.
for (std::size_t offset = 0; offset < count;) {
const std::size_t chunk_size = std::min(count - offset, chunk_limit);
void* chunk = static_cast<char*>(buffer) + offset * type_size;
#ifdef OPENMC_HAVE_MPI_LARGE_COUNT
MPI_Bcast_c(
chunk, static_cast<MPI_Count>(chunk_size), datatype, root, comm);
#else
MPI_Bcast(chunk, static_cast<int>(chunk_size), datatype, root, comm);
#endif
offset += chunk_size;
}
}
void reduce_buffer(const void* sendbuf, void* recvbuf, std::size_t count,
MPI_Datatype datatype, std::size_t type_size, MPI_Op op, int root,
MPI_Comm comm)
{
// Determine the maximum number of elements accepted by the selected API.
#ifdef OPENMC_HAVE_MPI_LARGE_COUNT
const std::size_t chunk_limit {
static_cast<std::size_t>(std::numeric_limits<MPI_Count>::max())};
#else
constexpr std::size_t chunk_limit {INT_MAX};
#endif
// A legacy MPI reduction accepts an int count, so split the logical buffer
// into bounded chunks before converting the count.
for (std::size_t offset = 0; offset < count; offset += chunk_limit) {
const std::size_t chunk_size = std::min(count - offset, chunk_limit);
// MPI_IN_PLACE must be passed unchanged on the root. Other buffers are
// advanced by the chunk offset in bytes because their element type is
// represented by an MPI datatype rather than a C++ pointer type here.
const void* send_chunk =
sendbuf == MPI_IN_PLACE
? MPI_IN_PLACE
: static_cast<const char*>(sendbuf) + offset * type_size;
void* recv_chunk = recvbuf == nullptr
? nullptr
: static_cast<char*>(recvbuf) + offset * type_size;
#ifdef OPENMC_HAVE_MPI_LARGE_COUNT
MPI_Reduce_c(send_chunk, recv_chunk, static_cast<MPI_Count>(chunk_size),
datatype, op, root, comm);
#else
MPI_Reduce(send_chunk, recv_chunk, static_cast<int>(chunk_size), datatype,
op, root, comm);
#endif
}
}
// Specializations of the MPITypeMap template struct
template<>
const MPI_Datatype MPITypeMap<int>::mpi_type = MPI_INT;
template<>
const MPI_Datatype MPITypeMap<double>::mpi_type = MPI_DOUBLE;
template<>
const MPI_Datatype MPITypeMap<int64_t>::mpi_type = MPI_INT64_T;
#endif
} // namespace mpi
} // namespace openmc