| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #include "sinev_a_allreduce/mpi/include/ops_mpi.hpp" | ||
| 2 | |||
| 3 | #include <mpi.h> | ||
| 4 | |||
| 5 | #include <cstddef> | ||
| 6 | #include <cstring> | ||
| 7 | #include <variant> | ||
| 8 | #include <vector> | ||
| 9 | |||
| 10 | #include "sinev_a_allreduce/common/include/common.hpp" | ||
| 11 | // #include "util/include/util.hpp" | ||
| 12 | |||
| 13 | namespace sinev_a_allreduce { | ||
| 14 | |||
| 15 |
1/2✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
|
18 | SinevAAllreduce::SinevAAllreduce(const InType &in) { |
| 16 | SetTypeOfTask(GetStaticTypeOfTask()); | ||
| 17 | GetInput() = in; | ||
| 18 | GetOutput() = in; | ||
| 19 | 18 | } | |
| 20 | |||
| 21 | 18 | bool SinevAAllreduce::ValidationImpl() { | |
| 22 | 18 | int initialized = 0; | |
| 23 | 18 | MPI_Initialized(&initialized); | |
| 24 | 18 | return initialized == 1; | |
| 25 | } | ||
| 26 | |||
| 27 | 18 | bool SinevAAllreduce::PreProcessingImpl() { | |
| 28 | 18 | return true; | |
| 29 | } | ||
| 30 | |||
| 31 | ✗ | int SinevAAllreduce::GetTypeSize(MPI_Datatype datatype) { | |
| 32 |
2/6✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 10 times.
✓ Branch 3 taken 8 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
18 | if (datatype == MPI_INT) { |
| 33 | return sizeof(int); | ||
| 34 | } | ||
| 35 |
2/6✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
10 | if (datatype == MPI_FLOAT) { |
| 36 | return sizeof(float); | ||
| 37 | } | ||
| 38 |
1/6✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
4 | if (datatype == MPI_DOUBLE) { |
| 39 | 4 | return sizeof(double); | |
| 40 | } | ||
| 41 | return 1; | ||
| 42 | } | ||
| 43 | |||
| 44 | namespace { | ||
| 45 | template <typename T> | ||
| 46 | void PerformSumTemplate(T *out, const T *in, int count) { | ||
| 47 |
6/6✓ Branch 0 taken 1111 times.
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 111 times.
✓ Branch 3 taken 3 times.
✓ Branch 4 taken 11 times.
✓ Branch 5 taken 2 times.
|
1242 | for (int i = 0; i < count; i++) { |
| 48 | 1233 | out[i] += in[i]; | |
| 49 | } | ||
| 50 | } | ||
| 51 | } // namespace | ||
| 52 | |||
| 53 | 9 | void SinevAAllreduce::PerformOperation(void *inout, const void *in, int count, MPI_Datatype datatype, MPI_Op op) { | |
| 54 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (op != MPI_SUM) { |
| 55 | return; | ||
| 56 | } | ||
| 57 | |||
| 58 |
2/2✓ Branch 0 taken 4 times.
✓ Branch 1 taken 5 times.
|
9 | if (datatype == MPI_INT) { |
| 59 | PerformSumTemplate(static_cast<int *>(inout), static_cast<const int *>(in), count); | ||
| 60 |
2/2✓ Branch 0 taken 3 times.
✓ Branch 1 taken 2 times.
|
5 | } else if (datatype == MPI_FLOAT) { |
| 61 | PerformSumTemplate(static_cast<float *>(inout), static_cast<const float *>(in), count); | ||
| 62 |
1/2✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
|
2 | } else if (datatype == MPI_DOUBLE) { |
| 63 | PerformSumTemplate(static_cast<double *>(inout), static_cast<const double *>(in), count); | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | namespace { | ||
| 68 | |||
| 69 | 18 | void PerformReducePhase(int rank, int size, int total_bytes, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, | |
| 70 | std::vector<char> &local_buffer) { | ||
| 71 | int mask = 1; | ||
| 72 |
2/2✓ Branch 0 taken 18 times.
✓ Branch 1 taken 9 times.
|
27 | while (mask < size) { |
| 73 | 18 | int partner = rank ^ mask; | |
| 74 | |||
| 75 |
1/2✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
|
18 | if (partner < size) { |
| 76 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
|
18 | if ((rank & mask) == 0) { |
| 77 |
1/2✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
|
9 | std::vector<char> recv_buffer(total_bytes); |
| 78 |
1/2✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
|
9 | MPI_Recv(recv_buffer.data(), total_bytes, MPI_BYTE, partner, 0, comm, MPI_STATUS_IGNORE); |
| 79 | 9 | SinevAAllreduce::PerformOperation(local_buffer.data(), recv_buffer.data(), count, datatype, op); | |
| 80 | } else { | ||
| 81 | 9 | MPI_Send(local_buffer.data(), total_bytes, MPI_BYTE, partner, 0, comm); | |
| 82 | 9 | break; | |
| 83 | } | ||
| 84 | } | ||
| 85 | 9 | mask <<= 1; | |
| 86 | } | ||
| 87 | 18 | } | |
| 88 | |||
| 89 | 18 | void BroadcastViaBinaryTree(int rank, int size, int count, MPI_Datatype datatype, MPI_Comm comm, void *recvbuf) { | |
| 90 | int tree_size = 1; | ||
| 91 |
2/2✓ Branch 0 taken 18 times.
✓ Branch 1 taken 18 times.
|
36 | while (tree_size < size) { |
| 92 | 18 | tree_size <<= 1; | |
| 93 | } | ||
| 94 | |||
| 95 |
2/2✓ Branch 0 taken 18 times.
✓ Branch 1 taken 18 times.
|
36 | for (int level = tree_size / 2; level > 0; level >>= 1) { |
| 96 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
|
18 | if (rank < level) { |
| 97 | 9 | int dest = rank + level; | |
| 98 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (dest < size) { |
| 99 | 9 | MPI_Send(recvbuf, count, datatype, dest, 1, comm); | |
| 100 | } | ||
| 101 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | } else if (rank < 2 * level && rank >= level) { |
| 102 | 9 | int source = rank - level; | |
| 103 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (source < size) { |
| 104 | 9 | MPI_Recv(recvbuf, count, datatype, source, 1, comm, MPI_STATUS_IGNORE); | |
| 105 | } | ||
| 106 | } | ||
| 107 | } | ||
| 108 | 18 | } | |
| 109 | |||
| 110 | 18 | void BroadcastRemainingProcesses(int rank, int size, int count, MPI_Datatype datatype, MPI_Comm comm, void *recvbuf) { | |
| 111 |
1/2✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
|
18 | if (size <= 1) { |
| 112 | return; | ||
| 113 | } | ||
| 114 | |||
| 115 |
2/2✓ Branch 0 taken 18 times.
✓ Branch 1 taken 18 times.
|
36 | for (int step = 1; step < size; step *= 2) { |
| 116 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
|
18 | if (rank < step) { |
| 117 | 9 | int dest = rank + step; | |
| 118 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (dest < size && dest >= step) { |
| 119 | 9 | MPI_Send(recvbuf, count, datatype, dest, 2, comm); | |
| 120 | } | ||
| 121 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | } else if (rank < 2 * step && rank >= step) { |
| 122 | 9 | int source = rank - step; | |
| 123 | if (source >= 0) { | ||
| 124 | 9 | MPI_Recv(recvbuf, count, datatype, source, 2, comm, MPI_STATUS_IGNORE); | |
| 125 | } | ||
| 126 | } | ||
| 127 | } | ||
| 128 | } | ||
| 129 | |||
| 130 | } // namespace | ||
| 131 | |||
| 132 | 18 | int SinevAAllreduce::MpiAllreduceCustom(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, | |
| 133 | MPI_Comm comm) { | ||
| 134 | 18 | int rank = 0; | |
| 135 | 18 | int size = 0; | |
| 136 | 18 | MPI_Comm_rank(comm, &rank); | |
| 137 | 18 | MPI_Comm_size(comm, &size); | |
| 138 | |||
| 139 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
|
18 | if (size == 1) { |
| 140 | int type_size = GetTypeSize(datatype); | ||
| 141 | ✗ | size_t total_size = static_cast<size_t>(count) * static_cast<size_t>(type_size); | |
| 142 | std::memcpy(recvbuf, sendbuf, total_size); | ||
| 143 | ✗ | return 0; | |
| 144 | } | ||
| 145 | |||
| 146 | int type_size = GetTypeSize(datatype); | ||
| 147 | 18 | int total_bytes = count * type_size; | |
| 148 | |||
| 149 | 18 | std::vector<char> local_buffer(total_bytes); | |
| 150 | |||
| 151 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 18 times.
|
18 | if (sendbuf == MPI_IN_PLACE) { |
| 152 | std::memcpy(local_buffer.data(), recvbuf, total_bytes); | ||
| 153 | } else { | ||
| 154 | std::memcpy(local_buffer.data(), sendbuf, total_bytes); | ||
| 155 | } | ||
| 156 | |||
| 157 |
1/2✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
|
18 | PerformReducePhase(rank, size, total_bytes, count, datatype, op, comm, local_buffer); |
| 158 | |||
| 159 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
|
18 | if (rank == 0) { |
| 160 | std::memcpy(recvbuf, local_buffer.data(), total_bytes); | ||
| 161 | } | ||
| 162 | |||
| 163 |
3/4✓ Branch 0 taken 9 times.
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
|
18 | if (rank != 0 && sendbuf != MPI_IN_PLACE) { |
| 164 | std::memcpy(recvbuf, sendbuf, total_bytes); | ||
| 165 | } | ||
| 166 | |||
| 167 |
1/2✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
|
18 | BroadcastViaBinaryTree(rank, size, count, datatype, comm, recvbuf); |
| 168 | |||
| 169 |
1/2✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
|
18 | BroadcastRemainingProcesses(rank, size, count, datatype, comm, recvbuf); |
| 170 | |||
| 171 | return 0; | ||
| 172 | } | ||
| 173 | |||
| 174 |
2/2✓ Branch 0 taken 8 times.
✓ Branch 1 taken 10 times.
|
18 | bool SinevAAllreduce::RunImpl() { |
| 175 | auto &input_variant = GetInput(); | ||
| 176 | auto &output_variant = GetOutput(); | ||
| 177 | |||
| 178 | try { | ||
| 179 |
2/2✓ Branch 0 taken 8 times.
✓ Branch 1 taken 10 times.
|
18 | if (std::holds_alternative<std::vector<int>>(input_variant)) { |
| 180 | auto &input = std::get<std::vector<int>>(input_variant); | ||
| 181 | auto &output = std::get<std::vector<int>>(output_variant); | ||
| 182 | |||
| 183 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
|
8 | if (output.size() != input.size()) { |
| 184 | ✗ | output.resize(input.size()); | |
| 185 | } | ||
| 186 | |||
| 187 |
1/2✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
|
8 | MpiAllreduceCustom(input.data(), output.data(), static_cast<int>(input.size()), MPI_INT, MPI_SUM, MPI_COMM_WORLD); |
| 188 | |||
| 189 |
2/2✓ Branch 0 taken 6 times.
✓ Branch 1 taken 4 times.
|
10 | } else if (std::holds_alternative<std::vector<float>>(input_variant)) { |
| 190 | auto &input = std::get<std::vector<float>>(input_variant); | ||
| 191 | auto &output = std::get<std::vector<float>>(output_variant); | ||
| 192 | |||
| 193 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
|
6 | if (output.size() != input.size()) { |
| 194 | ✗ | output.resize(input.size()); | |
| 195 | } | ||
| 196 | |||
| 197 |
1/2✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
|
6 | MpiAllreduceCustom(input.data(), output.data(), static_cast<int>(input.size()), MPI_FLOAT, MPI_SUM, |
| 198 | MPI_COMM_WORLD); | ||
| 199 | |||
| 200 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | } else if (std::holds_alternative<std::vector<double>>(input_variant)) { |
| 201 | auto &input = std::get<std::vector<double>>(input_variant); | ||
| 202 | auto &output = std::get<std::vector<double>>(output_variant); | ||
| 203 | |||
| 204 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (output.size() != input.size()) { |
| 205 | ✗ | output.resize(input.size()); | |
| 206 | } | ||
| 207 | |||
| 208 |
1/2✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
|
4 | MpiAllreduceCustom(input.data(), output.data(), static_cast<int>(input.size()), MPI_DOUBLE, MPI_SUM, |
| 209 | MPI_COMM_WORLD); | ||
| 210 | } | ||
| 211 | |||
| 212 | return true; | ||
| 213 | ✗ | } catch (...) { | |
| 214 | return false; | ||
| 215 | ✗ | } | |
| 216 | } | ||
| 217 | |||
| 218 | 18 | bool SinevAAllreduce::PostProcessingImpl() { | |
| 219 | 18 | return true; | |
| 220 | } | ||
| 221 | |||
| 222 | } // namespace sinev_a_allreduce | ||
| 223 |