| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #include "zorin_d_strassen_alg_matrix_seq/all/include/ops_all.hpp" | ||
| 2 | |||
| 3 | #include <mpi.h> | ||
| 4 | #include <omp.h> | ||
| 5 | |||
| 6 | #include <algorithm> | ||
| 7 | #include <cstddef> | ||
| 8 | #include <cstdint> | ||
| 9 | #include <vector> | ||
| 10 | |||
| 11 | #include "util/include/util.hpp" | ||
| 12 | #include "zorin_d_strassen_alg_matrix_seq/common/include/common.hpp" | ||
| 13 | |||
| 14 | namespace zorin_d_strassen_alg_matrix_seq { | ||
| 15 | |||
| 16 | namespace { | ||
| 17 | |||
| 18 | constexpr std::size_t kCutoff = 128; | ||
| 19 | constexpr std::size_t kBlockSize = 64; | ||
| 20 | |||
| 21 | std::size_t NextPow2(std::size_t x) { | ||
| 22 | 14 | if (x <= 1) { | |
| 23 | return 1; | ||
| 24 | } | ||
| 25 | std::size_t p = 1; | ||
| 26 |
2/2✓ Branch 0 taken 34 times.
✓ Branch 1 taken 12 times.
|
46 | while (p < x) { |
| 27 | 34 | p <<= 1; | |
| 28 | } | ||
| 29 | return p; | ||
| 30 | } | ||
| 31 | |||
| 32 | void ZeroMatrix(double *dst, std::size_t stride, std::size_t n) { | ||
| 33 |
2/2✓ Branch 0 taken 55 times.
✓ Branch 1 taken 7 times.
|
62 | for (std::size_t i = 0; i < n; ++i) { |
| 34 | 55 | std::fill_n(dst + (i * stride), n, 0.0); | |
| 35 | } | ||
| 36 | } | ||
| 37 | |||
| 38 | ✗ | void AddToBuffer(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *dst, | |
| 39 | std::size_t n, double b_coeff) { | ||
| 40 | ✗ | for (std::size_t i = 0; i < n; ++i) { | |
| 41 | ✗ | const double *a_row = a + (i * a_stride); | |
| 42 | ✗ | const double *b_row = b + (i * b_stride); | |
| 43 | ✗ | double *dst_row = dst + (i * n); | |
| 44 | ✗ | for (std::size_t j = 0; j < n; ++j) { | |
| 45 | ✗ | dst_row[j] = a_row[j] + (b_coeff * b_row[j]); | |
| 46 | } | ||
| 47 | } | ||
| 48 | ✗ | } | |
| 49 | |||
| 50 | void MulMicroBlock(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *c, | ||
| 51 | std::size_t c_stride, std::size_t i_begin, std::size_t i_end, std::size_t k_begin, std::size_t k_end, | ||
| 52 | std::size_t j_begin, std::size_t j_end) { | ||
| 53 | for (std::size_t i = i_begin; i < i_end; ++i) { | ||
| 54 | double *c_row = c + (i * c_stride); | ||
| 55 | const double *a_row = a + (i * a_stride); | ||
| 56 | for (std::size_t k = k_begin; k < k_end; ++k) { | ||
| 57 | const double aik = a_row[k]; | ||
| 58 | const double *b_row = b + (k * b_stride); | ||
| 59 | for (std::size_t j = j_begin; j < j_end; ++j) { | ||
| 60 | c_row[j] += aik * b_row[j]; | ||
| 61 | } | ||
| 62 | } | ||
| 63 | } | ||
| 64 | } | ||
| 65 | |||
| 66 | 7 | void NaiveMulBlocked(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *c, | |
| 67 | std::size_t c_stride, std::size_t n) { | ||
| 68 | 7 | ZeroMatrix(c, c_stride, n); | |
| 69 | |||
| 70 | 7 | const auto n_signed = static_cast<std::ptrdiff_t>(n); | |
| 71 | const auto block_signed = static_cast<std::ptrdiff_t>(kBlockSize); | ||
| 72 | |||
| 73 | 7 | #pragma omp parallel for schedule(static) default(none) \ | |
| 74 | shared(a, a_stride, b, b_stride, c, c_stride, n, n_signed, block_signed) | ||
| 75 | for (std::ptrdiff_t ii = 0; ii < n_signed; ii += block_signed) { | ||
| 76 | const auto ii_usize = static_cast<std::size_t>(ii); | ||
| 77 | const std::size_t i_end = std::min(ii_usize + kBlockSize, n); | ||
| 78 | for (std::size_t kk = 0; kk < n; kk += kBlockSize) { | ||
| 79 | const std::size_t k_end = std::min(kk + kBlockSize, n); | ||
| 80 | for (std::size_t jj = 0; jj < n; jj += kBlockSize) { | ||
| 81 | const std::size_t j_end = std::min(jj + kBlockSize, n); | ||
| 82 | MulMicroBlock(a, a_stride, b, b_stride, c, c_stride, ii_usize, i_end, kk, k_end, jj, j_end); | ||
| 83 | } | ||
| 84 | } | ||
| 85 | } | ||
| 86 | 7 | } | |
| 87 | |||
| 88 | ✗ | void CombineQuadrants(const std::vector<double> &m1, const std::vector<double> &m2, const std::vector<double> &m3, | |
| 89 | const std::vector<double> &m4, const std::vector<double> &m5, const std::vector<double> &m6, | ||
| 90 | const std::vector<double> &m7, double *c, std::size_t c_stride, std::size_t half) { | ||
| 91 | ✗ | for (std::size_t i = 0; i < half; ++i) { | |
| 92 | ✗ | double *c11 = c + (i * c_stride); | |
| 93 | double *c12 = c11 + half; | ||
| 94 | ✗ | double *c21 = c + ((i + half) * c_stride); | |
| 95 | double *c22 = c21 + half; | ||
| 96 | |||
| 97 | ✗ | const double *m1_row = m1.data() + (i * half); | |
| 98 | const double *m2_row = m2.data() + (i * half); | ||
| 99 | const double *m3_row = m3.data() + (i * half); | ||
| 100 | const double *m4_row = m4.data() + (i * half); | ||
| 101 | const double *m5_row = m5.data() + (i * half); | ||
| 102 | const double *m6_row = m6.data() + (i * half); | ||
| 103 | const double *m7_row = m7.data() + (i * half); | ||
| 104 | |||
| 105 | ✗ | for (std::size_t j = 0; j < half; ++j) { | |
| 106 | ✗ | c11[j] = m1_row[j] + m4_row[j] - m5_row[j] + m7_row[j]; | |
| 107 | ✗ | c12[j] = m3_row[j] + m5_row[j]; | |
| 108 | ✗ | c21[j] = m2_row[j] + m4_row[j]; | |
| 109 | ✗ | c22[j] = m1_row[j] - m2_row[j] + m3_row[j] + m6_row[j]; | |
| 110 | } | ||
| 111 | } | ||
| 112 | ✗ | } | |
| 113 | |||
| 114 | using StrassenFn = void (*)(const double *, std::size_t, const double *, std::size_t, double *, std::size_t, | ||
| 115 | std::size_t); | ||
| 116 | |||
| 117 | void StrassenSeqImpl(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *c, | ||
| 118 | std::size_t c_stride, std::size_t n); | ||
| 119 | |||
| 120 | constexpr StrassenFn kStrassenSeqFn = &StrassenSeqImpl; | ||
| 121 | |||
| 122 | ✗ | void ComputeProduct(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, double a2_coeff, | |
| 123 | const double *b1, std::size_t b1_stride, const double *b2, std::size_t b2_stride, double b2_coeff, | ||
| 124 | std::vector<double> &out, std::size_t n) { | ||
| 125 | ✗ | std::vector<double> lhs(n * n); | |
| 126 | ✗ | std::vector<double> rhs(n * n); | |
| 127 | ✗ | AddToBuffer(a1, a1_stride, a2, a2_stride, lhs.data(), n, a2_coeff); | |
| 128 | ✗ | AddToBuffer(b1, b1_stride, b2, b2_stride, rhs.data(), n, b2_coeff); | |
| 129 | ✗ | out.assign(n * n, 0.0); | |
| 130 | ✗ | kStrassenSeqFn(lhs.data(), n, rhs.data(), n, out.data(), n, n); | |
| 131 | ✗ | } | |
| 132 | |||
| 133 | ✗ | void ComputeProductSingle(const double *a, std::size_t a_stride, const double *b1, std::size_t b1_stride, | |
| 134 | const double *b2, std::size_t b2_stride, double b2_coeff, std::vector<double> &out, | ||
| 135 | std::size_t n) { | ||
| 136 | ✗ | std::vector<double> rhs(n * n); | |
| 137 | ✗ | AddToBuffer(b1, b1_stride, b2, b2_stride, rhs.data(), n, b2_coeff); | |
| 138 | ✗ | out.assign(n * n, 0.0); | |
| 139 | ✗ | kStrassenSeqFn(a, a_stride, rhs.data(), n, out.data(), n, n); | |
| 140 | ✗ | } | |
| 141 | |||
| 142 | ✗ | void ComputeProductSingleLeft(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, | |
| 143 | double a2_coeff, const double *b, std::size_t b_stride, std::vector<double> &out, | ||
| 144 | std::size_t n) { | ||
| 145 | ✗ | std::vector<double> lhs(n * n); | |
| 146 | ✗ | AddToBuffer(a1, a1_stride, a2, a2_stride, lhs.data(), n, a2_coeff); | |
| 147 | ✗ | out.assign(n * n, 0.0); | |
| 148 | ✗ | kStrassenSeqFn(lhs.data(), n, b, b_stride, out.data(), n, n); | |
| 149 | ✗ | } | |
| 150 | |||
| 151 | void ComputeProductOmp(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, | ||
| 152 | double a2_coeff, const double *b1, std::size_t b1_stride, const double *b2, | ||
| 153 | std::size_t b2_stride, double b2_coeff, std::vector<double> &out, std::size_t n); | ||
| 154 | |||
| 155 | void ComputeProductSingleOmp(const double *a, std::size_t a_stride, const double *b1, std::size_t b1_stride, | ||
| 156 | const double *b2, std::size_t b2_stride, double b2_coeff, std::vector<double> &out, | ||
| 157 | std::size_t n); | ||
| 158 | |||
| 159 | void ComputeProductSingleLeftOmp(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, | ||
| 160 | double a2_coeff, const double *b, std::size_t b_stride, std::vector<double> &out, | ||
| 161 | std::size_t n); | ||
| 162 | |||
| 163 | 7 | void StrassenOmpLocal(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *c, | |
| 164 | std::size_t c_stride, std::size_t n) { | ||
| 165 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
7 | if (n <= kCutoff || ppc::util::GetNumThreads() <= 1) { |
| 166 | 7 | NaiveMulBlocked(a, a_stride, b, b_stride, c, c_stride, n); | |
| 167 | 7 | return; | |
| 168 | } | ||
| 169 | |||
| 170 | ✗ | const std::size_t half = n / 2; | |
| 171 | |||
| 172 | const double *a11 = a; | ||
| 173 | ✗ | const double *a12 = a + half; | |
| 174 | ✗ | const double *a21 = a + (half * a_stride); | |
| 175 | ✗ | const double *a22 = a21 + half; | |
| 176 | |||
| 177 | const double *b11 = b; | ||
| 178 | ✗ | const double *b12 = b + half; | |
| 179 | ✗ | const double *b21 = b + (half * b_stride); | |
| 180 | ✗ | const double *b22 = b21 + half; | |
| 181 | |||
| 182 | ✗ | std::vector<double> m1; | |
| 183 | ✗ | std::vector<double> m2; | |
| 184 | ✗ | std::vector<double> m3; | |
| 185 | ✗ | std::vector<double> m4; | |
| 186 | ✗ | std::vector<double> m5; | |
| 187 | ✗ | std::vector<double> m6; | |
| 188 | ✗ | std::vector<double> m7; | |
| 189 | |||
| 190 | ✗ | #pragma omp parallel default(none) \ | |
| 191 | shared(m1, m2, m3, m4, m5, m6, m7, a11, a12, a21, a22, b11, b12, b21, b22, a_stride, b_stride, half) | ||
| 192 | { | ||
| 193 | #pragma omp single nowait | ||
| 194 | { | ||
| 195 | #pragma omp task default(none) shared(m1, a11, a22, b11, b22, a_stride, b_stride, half) | ||
| 196 | ComputeProduct(a11, a_stride, a22, a_stride, 1.0, b11, b_stride, b22, b_stride, 1.0, m1, half); | ||
| 197 | #pragma omp task default(none) shared(m2, a21, a22, b11, a_stride, b_stride, half) | ||
| 198 | ComputeProductSingleLeft(a21, a_stride, a22, a_stride, 1.0, b11, b_stride, m2, half); | ||
| 199 | #pragma omp task default(none) shared(m3, a11, b12, b22, a_stride, b_stride, half) | ||
| 200 | ComputeProductSingle(a11, a_stride, b12, b_stride, b22, b_stride, -1.0, m3, half); | ||
| 201 | #pragma omp task default(none) shared(m4, a22, b21, b11, a_stride, b_stride, half) | ||
| 202 | ComputeProductSingle(a22, a_stride, b21, b_stride, b11, b_stride, -1.0, m4, half); | ||
| 203 | #pragma omp task default(none) shared(m5, a11, a12, b22, a_stride, b_stride, half) | ||
| 204 | ComputeProductSingleLeft(a11, a_stride, a12, a_stride, 1.0, b22, b_stride, m5, half); | ||
| 205 | #pragma omp task default(none) shared(m6, a21, a11, b11, b12, a_stride, b_stride, half) | ||
| 206 | ComputeProduct(a21, a_stride, a11, a_stride, -1.0, b11, b_stride, b12, b_stride, 1.0, m6, half); | ||
| 207 | #pragma omp task default(none) shared(m7, a12, a22, b21, b22, a_stride, b_stride, half) | ||
| 208 | ComputeProduct(a12, a_stride, a22, a_stride, -1.0, b21, b_stride, b22, b_stride, 1.0, m7, half); | ||
| 209 | #pragma omp taskwait | ||
| 210 | } | ||
| 211 | } | ||
| 212 | |||
| 213 | ✗ | CombineQuadrants(m1, m2, m3, m4, m5, m6, m7, c, c_stride, half); | |
| 214 | } | ||
| 215 | |||
| 216 | ✗ | void ComputeProductOmp(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, | |
| 217 | double a2_coeff, const double *b1, std::size_t b1_stride, const double *b2, | ||
| 218 | std::size_t b2_stride, double b2_coeff, std::vector<double> &out, std::size_t n) { | ||
| 219 | ✗ | std::vector<double> lhs(n * n); | |
| 220 | ✗ | std::vector<double> rhs(n * n); | |
| 221 | ✗ | AddToBuffer(a1, a1_stride, a2, a2_stride, lhs.data(), n, a2_coeff); | |
| 222 | ✗ | AddToBuffer(b1, b1_stride, b2, b2_stride, rhs.data(), n, b2_coeff); | |
| 223 | ✗ | out.assign(n * n, 0.0); | |
| 224 | ✗ | StrassenOmpLocal(lhs.data(), n, rhs.data(), n, out.data(), n, n); | |
| 225 | ✗ | } | |
| 226 | |||
| 227 | ✗ | void ComputeProductSingleOmp(const double *a, std::size_t a_stride, const double *b1, std::size_t b1_stride, | |
| 228 | const double *b2, std::size_t b2_stride, double b2_coeff, std::vector<double> &out, | ||
| 229 | std::size_t n) { | ||
| 230 | ✗ | std::vector<double> rhs(n * n); | |
| 231 | ✗ | AddToBuffer(b1, b1_stride, b2, b2_stride, rhs.data(), n, b2_coeff); | |
| 232 | ✗ | out.assign(n * n, 0.0); | |
| 233 | ✗ | StrassenOmpLocal(a, a_stride, rhs.data(), n, out.data(), n, n); | |
| 234 | ✗ | } | |
| 235 | |||
| 236 | ✗ | void ComputeProductSingleLeftOmp(const double *a1, std::size_t a1_stride, const double *a2, std::size_t a2_stride, | |
| 237 | double a2_coeff, const double *b, std::size_t b_stride, std::vector<double> &out, | ||
| 238 | std::size_t n) { | ||
| 239 | ✗ | std::vector<double> lhs(n * n); | |
| 240 | ✗ | AddToBuffer(a1, a1_stride, a2, a2_stride, lhs.data(), n, a2_coeff); | |
| 241 | ✗ | out.assign(n * n, 0.0); | |
| 242 | ✗ | StrassenOmpLocal(lhs.data(), n, b, b_stride, out.data(), n, n); | |
| 243 | ✗ | } | |
| 244 | |||
| 245 | ✗ | void StrassenSeqImpl(const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, double *c, | |
| 246 | std::size_t c_stride, std::size_t n) { | ||
| 247 | ✗ | if (n <= kCutoff) { | |
| 248 | ✗ | NaiveMulBlocked(a, a_stride, b, b_stride, c, c_stride, n); | |
| 249 | ✗ | return; | |
| 250 | } | ||
| 251 | |||
| 252 | ✗ | const std::size_t half = n / 2; | |
| 253 | |||
| 254 | const double *a11 = a; | ||
| 255 | ✗ | const double *a12 = a + half; | |
| 256 | ✗ | const double *a21 = a + (half * a_stride); | |
| 257 | ✗ | const double *a22 = a21 + half; | |
| 258 | |||
| 259 | const double *b11 = b; | ||
| 260 | ✗ | const double *b12 = b + half; | |
| 261 | ✗ | const double *b21 = b + (half * b_stride); | |
| 262 | ✗ | const double *b22 = b21 + half; | |
| 263 | |||
| 264 | ✗ | std::vector<double> m1(half * half); | |
| 265 | ✗ | std::vector<double> m2(half * half); | |
| 266 | ✗ | std::vector<double> m3(half * half); | |
| 267 | ✗ | std::vector<double> m4(half * half); | |
| 268 | ✗ | std::vector<double> m5(half * half); | |
| 269 | ✗ | std::vector<double> m6(half * half); | |
| 270 | ✗ | std::vector<double> m7(half * half); | |
| 271 | ✗ | std::vector<double> lhs(half * half); | |
| 272 | ✗ | std::vector<double> rhs(half * half); | |
| 273 | |||
| 274 | ✗ | AddToBuffer(a11, a_stride, a22, a_stride, lhs.data(), half, 1.0); | |
| 275 | ✗ | AddToBuffer(b11, b_stride, b22, b_stride, rhs.data(), half, 1.0); | |
| 276 | ✗ | kStrassenSeqFn(lhs.data(), half, rhs.data(), half, m1.data(), half, half); | |
| 277 | |||
| 278 | ✗ | AddToBuffer(a21, a_stride, a22, a_stride, lhs.data(), half, 1.0); | |
| 279 | ✗ | kStrassenSeqFn(lhs.data(), half, b11, b_stride, m2.data(), half, half); | |
| 280 | |||
| 281 | ✗ | AddToBuffer(b12, b_stride, b22, b_stride, rhs.data(), half, -1.0); | |
| 282 | ✗ | kStrassenSeqFn(a11, a_stride, rhs.data(), half, m3.data(), half, half); | |
| 283 | |||
| 284 | ✗ | AddToBuffer(b21, b_stride, b11, b_stride, rhs.data(), half, -1.0); | |
| 285 | ✗ | kStrassenSeqFn(a22, a_stride, rhs.data(), half, m4.data(), half, half); | |
| 286 | |||
| 287 | ✗ | AddToBuffer(a11, a_stride, a12, a_stride, lhs.data(), half, 1.0); | |
| 288 | ✗ | kStrassenSeqFn(lhs.data(), half, b22, b_stride, m5.data(), half, half); | |
| 289 | |||
| 290 | ✗ | AddToBuffer(a21, a_stride, a11, a_stride, lhs.data(), half, -1.0); | |
| 291 | ✗ | AddToBuffer(b11, b_stride, b12, b_stride, rhs.data(), half, 1.0); | |
| 292 | ✗ | kStrassenSeqFn(lhs.data(), half, rhs.data(), half, m6.data(), half, half); | |
| 293 | |||
| 294 | ✗ | AddToBuffer(a12, a_stride, a22, a_stride, lhs.data(), half, -1.0); | |
| 295 | ✗ | AddToBuffer(b21, b_stride, b22, b_stride, rhs.data(), half, 1.0); | |
| 296 | ✗ | kStrassenSeqFn(lhs.data(), half, rhs.data(), half, m7.data(), half, half); | |
| 297 | |||
| 298 | ✗ | CombineQuadrants(m1, m2, m3, m4, m5, m6, m7, c, c_stride, half); | |
| 299 | } | ||
| 300 | |||
| 301 | void AddContribution(double *accum, std::size_t stride, const std::vector<double> &block, std::size_t row_offset, | ||
| 302 | std::size_t col_offset, std::size_t half, double coeff) { | ||
| 303 | ✗ | for (std::size_t i = 0; i < half; ++i) { | |
| 304 | ✗ | double *dst_row = accum + ((row_offset + i) * stride) + col_offset; | |
| 305 | ✗ | const double *src_row = block.data() + (i * half); | |
| 306 | ✗ | for (std::size_t j = 0; j < half; ++j) { | |
| 307 | ✗ | dst_row[j] += coeff * src_row[j]; | |
| 308 | } | ||
| 309 | } | ||
| 310 | } | ||
| 311 | |||
| 312 | ✗ | void ComputeAssignedProduct(int task_id, const double *a, std::size_t a_stride, const double *b, std::size_t b_stride, | |
| 313 | std::size_t half, std::vector<double> &local_accum) { | ||
| 314 | const double *a11 = a; | ||
| 315 | ✗ | const double *a12 = a + half; | |
| 316 | ✗ | const double *a21 = a + (half * a_stride); | |
| 317 | ✗ | const double *a22 = a21 + half; | |
| 318 | |||
| 319 | const double *b11 = b; | ||
| 320 | ✗ | const double *b12 = b + half; | |
| 321 | ✗ | const double *b21 = b + (half * b_stride); | |
| 322 | ✗ | const double *b22 = b21 + half; | |
| 323 | |||
| 324 | ✗ | std::vector<double> m(half * half, 0.0); | |
| 325 | |||
| 326 | ✗ | switch (task_id) { | |
| 327 | ✗ | case 0: | |
| 328 | ✗ | ComputeProductOmp(a11, a_stride, a22, a_stride, 1.0, b11, b_stride, b22, b_stride, 1.0, m, half); | |
| 329 | AddContribution(local_accum.data(), half * 2, m, 0, 0, half, 1.0); | ||
| 330 | AddContribution(local_accum.data(), half * 2, m, half, half, half, 1.0); | ||
| 331 | break; | ||
| 332 | ✗ | case 1: | |
| 333 | ✗ | ComputeProductSingleLeftOmp(a21, a_stride, a22, a_stride, 1.0, b11, b_stride, m, half); | |
| 334 | AddContribution(local_accum.data(), half * 2, m, half, 0, half, 1.0); | ||
| 335 | AddContribution(local_accum.data(), half * 2, m, half, half, half, -1.0); | ||
| 336 | break; | ||
| 337 | ✗ | case 2: | |
| 338 | ✗ | ComputeProductSingleOmp(a11, a_stride, b12, b_stride, b22, b_stride, -1.0, m, half); | |
| 339 | AddContribution(local_accum.data(), half * 2, m, 0, half, half, 1.0); | ||
| 340 | AddContribution(local_accum.data(), half * 2, m, half, half, half, 1.0); | ||
| 341 | break; | ||
| 342 | ✗ | case 3: | |
| 343 | ✗ | ComputeProductSingleOmp(a22, a_stride, b21, b_stride, b11, b_stride, -1.0, m, half); | |
| 344 | AddContribution(local_accum.data(), half * 2, m, 0, 0, half, 1.0); | ||
| 345 | AddContribution(local_accum.data(), half * 2, m, half, 0, half, 1.0); | ||
| 346 | break; | ||
| 347 | ✗ | case 4: | |
| 348 | ✗ | ComputeProductSingleLeftOmp(a11, a_stride, a12, a_stride, 1.0, b22, b_stride, m, half); | |
| 349 | AddContribution(local_accum.data(), half * 2, m, 0, 0, half, -1.0); | ||
| 350 | AddContribution(local_accum.data(), half * 2, m, 0, half, half, 1.0); | ||
| 351 | break; | ||
| 352 | ✗ | case 5: | |
| 353 | ✗ | ComputeProductOmp(a21, a_stride, a11, a_stride, -1.0, b11, b_stride, b12, b_stride, 1.0, m, half); | |
| 354 | AddContribution(local_accum.data(), half * 2, m, half, half, half, 1.0); | ||
| 355 | break; | ||
| 356 | ✗ | case 6: | |
| 357 | ✗ | ComputeProductOmp(a12, a_stride, a22, a_stride, -1.0, b21, b_stride, b22, b_stride, 1.0, m, half); | |
| 358 | AddContribution(local_accum.data(), half * 2, m, 0, 0, half, 1.0); | ||
| 359 | break; | ||
| 360 | default: | ||
| 361 | break; | ||
| 362 | } | ||
| 363 | ✗ | } | |
| 364 | |||
| 365 | } // namespace | ||
| 366 | |||
| 367 |
1/2✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
|
14 | ZorinDStrassenAlgMatrixALL::ZorinDStrassenAlgMatrixALL(const InType &in) { |
| 368 | SetTypeOfTask(GetStaticTypeOfTask()); | ||
| 369 | GetInput() = in; | ||
| 370 | GetOutput().clear(); | ||
| 371 | 14 | } | |
| 372 | |||
| 373 | 14 | bool ZorinDStrassenAlgMatrixALL::ValidationImpl() { | |
| 374 | 14 | int rank = 0; | |
| 375 | 14 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); | |
| 376 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank != 0) { |
| 377 | return true; | ||
| 378 | } | ||
| 379 | |||
| 380 | const auto &in = GetInput(); | ||
| 381 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (in.n == 0) { |
| 382 | return false; | ||
| 383 | } | ||
| 384 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (in.a.size() != in.n * in.n) { |
| 385 | return false; | ||
| 386 | } | ||
| 387 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (in.b.size() != in.n * in.n) { |
| 388 | return false; | ||
| 389 | } | ||
| 390 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | if (!GetOutput().empty()) { |
| 391 | ✗ | return false; | |
| 392 | } | ||
| 393 | return true; | ||
| 394 | } | ||
| 395 | |||
| 396 | 14 | bool ZorinDStrassenAlgMatrixALL::PreProcessingImpl() { | |
| 397 | 14 | int rank = 0; | |
| 398 | 14 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); | |
| 399 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank == 0) { |
| 400 | 7 | const auto n = GetInput().n; | |
| 401 | 7 | GetOutput().assign(n * n, 0.0); | |
| 402 | } else { | ||
| 403 | GetOutput().clear(); | ||
| 404 | } | ||
| 405 | 14 | return true; | |
| 406 | } | ||
| 407 | |||
| 408 | 14 | bool ZorinDStrassenAlgMatrixALL::RunImpl() { | |
| 409 | 14 | int rank = 0; | |
| 410 | 14 | int world_size = 1; | |
| 411 | 14 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); | |
| 412 | 14 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); | |
| 413 | |||
| 414 | 14 | std::uint64_t n_u64 = 0; | |
| 415 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank == 0) { |
| 416 | 7 | n_u64 = static_cast<std::uint64_t>(GetInput().n); | |
| 417 | } | ||
| 418 | 14 | MPI_Bcast(&n_u64, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD); | |
| 419 | |||
| 420 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 2 times.
|
14 | const auto n = static_cast<std::size_t>(n_u64); |
| 421 | const std::size_t padded = NextPow2(n); | ||
| 422 | |||
| 423 | 14 | std::vector<double> a_pad(padded * padded, 0.0); | |
| 424 |
1/4✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
14 | std::vector<double> b_pad(padded * padded, 0.0); |
| 425 | |||
| 426 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank == 0) { |
| 427 | const auto &in = GetInput(); | ||
| 428 |
2/2✓ Branch 0 taken 44 times.
✓ Branch 1 taken 7 times.
|
51 | for (std::size_t i = 0; i < n; ++i) { |
| 429 | 44 | std::copy_n(in.a.data() + (i * n), n, a_pad.data() + (i * padded)); | |
| 430 | 44 | std::copy_n(in.b.data() + (i * n), n, b_pad.data() + (i * padded)); | |
| 431 | } | ||
| 432 | } | ||
| 433 | |||
| 434 |
1/2✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
|
14 | MPI_Bcast(a_pad.data(), static_cast<int>(a_pad.size()), MPI_DOUBLE, 0, MPI_COMM_WORLD); |
| 435 |
1/2✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
|
14 | MPI_Bcast(b_pad.data(), static_cast<int>(b_pad.size()), MPI_DOUBLE, 0, MPI_COMM_WORLD); |
| 436 | |||
| 437 |
1/4✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
14 | std::vector<double> global_c(padded * padded, 0.0); |
| 438 | |||
| 439 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
14 | if (padded <= kCutoff || world_size == 1) { |
| 440 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank == 0) { |
| 441 |
1/2✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
|
7 | StrassenOmpLocal(a_pad.data(), padded, b_pad.data(), padded, global_c.data(), padded, padded); |
| 442 | } | ||
| 443 | } else { | ||
| 444 | ✗ | const std::size_t half = padded / 2; | |
| 445 | ✗ | std::vector<double> local_c(padded * padded, 0.0); | |
| 446 | |||
| 447 | ✗ | for (int task_id = rank; task_id < 7; task_id += world_size) { | |
| 448 | ✗ | ComputeAssignedProduct(task_id, a_pad.data(), padded, b_pad.data(), padded, half, local_c); | |
| 449 | } | ||
| 450 | |||
| 451 | ✗ | MPI_Reduce(local_c.data(), global_c.data(), static_cast<int>(global_c.size()), MPI_DOUBLE, MPI_SUM, 0, | |
| 452 | MPI_COMM_WORLD); | ||
| 453 | } | ||
| 454 | |||
| 455 | auto &out = GetOutput(); | ||
| 456 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank != 0) { |
| 457 |
1/4✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
|
7 | out.assign(n * n, 0.0); |
| 458 | } | ||
| 459 |
2/2✓ Branch 0 taken 7 times.
✓ Branch 1 taken 7 times.
|
14 | if (rank == 0) { |
| 460 |
2/2✓ Branch 0 taken 44 times.
✓ Branch 1 taken 7 times.
|
51 | for (std::size_t i = 0; i < n; ++i) { |
| 461 | 44 | std::copy_n(global_c.data() + (i * padded), n, out.data() + (i * n)); | |
| 462 | } | ||
| 463 | } | ||
| 464 | |||
| 465 |
1/2✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
|
14 | MPI_Bcast(out.data(), static_cast<int>(out.size()), MPI_DOUBLE, 0, MPI_COMM_WORLD); |
| 466 | 14 | return true; | |
| 467 | } | ||
| 468 | |||
| 469 | 14 | bool ZorinDStrassenAlgMatrixALL::PostProcessingImpl() { | |
| 470 | 14 | return true; | |
| 471 | } | ||
| 472 | |||
| 473 | } // namespace zorin_d_strassen_alg_matrix_seq | ||
| 474 |