removing the *Hip files from the unsupported/Eigen/CXX11/src/Tensor and unsupported/test directories
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionHip.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionHip.h
deleted file mode 100644
index 7561846..0000000
--- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionHip.h
+++ /dev/null
@@ -1,1521 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014-2015 Benoit Steiner <benoit.steiner.goog@gmail.com>
-// Copyright (C) 2015 Navdeep Jaitly <ndjaitly@google.com>
-// Copyright (C) 2014 Eric Martin <eric@ericmart.in>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_HIP_H
-#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_HIP_H
-
-#if defined(EIGEN_USE_GPU) && defined(EIGEN_HIPCC)
-
-namespace Eigen {
-
-template<typename Scalar, typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper, bool needs_edge_check>
-__device__ EIGEN_STRONG_INLINE void
-EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output, Scalar* lhs_shmem, Scalar* rhs_shmem,
- const Index m_size, const Index n_size, const Index k_size) {
-
- const Index m_block_idx = hipBlockIdx_x;
- const Index n_block_idx = hipBlockIdx_y;
-
- const Index base_m = 64 * m_block_idx;
- const Index base_n = 64 * n_block_idx;
-
- // declare and initialize 64 registers for output 8x8 block
-
- // prefetch registers
- Scalar lhs_pf0;
- Scalar lhs_pf1;
- Scalar lhs_pf2;
- Scalar lhs_pf3;
- Scalar lhs_pf4;
- Scalar lhs_pf5;
- Scalar lhs_pf6;
- Scalar lhs_pf7;
-
- Scalar rhs_pf0;
- Scalar rhs_pf1;
- Scalar rhs_pf2;
- Scalar rhs_pf3;
- Scalar rhs_pf4;
- Scalar rhs_pf5;
- Scalar rhs_pf6;
- Scalar rhs_pf7;
-
- // shared memory is formatted
- // (contract idx in block, nocontract idx in block, block idx)
- // where block idx is column major. This transposition limits the number of
- // bank conflicts when reading the LHS. The core idea is that since the contracting
- // index is shared by both sides, then the contracting index should be in hipThreadIdx_x.
-
- // On the LHS, we pad each row inside of each block with an extra element. This makes
- // each block 8 rows of 9 elements, which is 72 elements. This gives no bank conflicts
- // on writes and very few 2-way conflicts on reads. There is an 8x8 grid of these blocks.
-
- // On the RHS we just add 8 padding elements to the end of each block. This gives no bank
- // conflicts on writes and also none on reads.
-
- // storage indices
- const Index lhs_store_idx_base = hipThreadIdx_y * 72 + hipThreadIdx_x * 9 + hipThreadIdx_z;
- const Index rhs_store_idx_base = hipThreadIdx_y * 72 + hipThreadIdx_z * 8 + hipThreadIdx_x;
-
- const Index lhs_store_idx_0 = lhs_store_idx_base + 576 * 0;
- const Index lhs_store_idx_1 = lhs_store_idx_base + 576 * 1;
- const Index lhs_store_idx_2 = lhs_store_idx_base + 576 * 2;
- const Index lhs_store_idx_3 = lhs_store_idx_base + 576 * 3;
- const Index lhs_store_idx_4 = lhs_store_idx_base + 576 * 4;
- const Index lhs_store_idx_5 = lhs_store_idx_base + 576 * 5;
- const Index lhs_store_idx_6 = lhs_store_idx_base + 576 * 6;
- const Index lhs_store_idx_7 = lhs_store_idx_base + 576 * 7;
-
- const Index rhs_store_idx_0 = rhs_store_idx_base + 576 * 0;
- const Index rhs_store_idx_1 = rhs_store_idx_base + 576 * 1;
- const Index rhs_store_idx_2 = rhs_store_idx_base + 576 * 2;
- const Index rhs_store_idx_3 = rhs_store_idx_base + 576 * 3;
- const Index rhs_store_idx_4 = rhs_store_idx_base + 576 * 4;
- const Index rhs_store_idx_5 = rhs_store_idx_base + 576 * 5;
- const Index rhs_store_idx_6 = rhs_store_idx_base + 576 * 6;
- const Index rhs_store_idx_7 = rhs_store_idx_base + 576 * 7;
-
- // in the loading code, the following variables are important:
- // hipThreadIdx_x: the vertical position in an 8x8 block
- // hipThreadIdx_y: the vertical index of the 8x8 block in the grid
- // hipThreadIdx_z: the horizontal position in an 8x8 block
- // k: the horizontal index of the 8x8 block in the grid
- //
- // The k parameter is implicit (it was the loop counter for a loop that went
- // from 0 to <8, but now that loop is unrolled in the below code.
-
- const Index load_idx_vert = hipThreadIdx_x + 8 * hipThreadIdx_y;
- const Index lhs_vert = base_m + load_idx_vert;
-
-#define prefetchIntoRegisters(base_k) \
- { \
- lhs_pf0 = conv(0); \
- lhs_pf1 = conv(0); \
- lhs_pf2 = conv(0); \
- lhs_pf3 = conv(0); \
- lhs_pf4 = conv(0); \
- lhs_pf5 = conv(0); \
- lhs_pf6 = conv(0); \
- lhs_pf7 = conv(0); \
- \
- rhs_pf0 = conv(0); \
- rhs_pf1 = conv(0); \
- rhs_pf2 = conv(0); \
- rhs_pf3 = conv(0); \
- rhs_pf4 = conv(0); \
- rhs_pf5 = conv(0); \
- rhs_pf6 = conv(0); \
- rhs_pf7 = conv(0); \
- \
- if (!needs_edge_check || lhs_vert < m_size) { \
- const Index lhs_horiz_0 = base_k + hipThreadIdx_z + 0 * 8; \
- const Index lhs_horiz_1 = base_k + hipThreadIdx_z + 1 * 8; \
- const Index lhs_horiz_2 = base_k + hipThreadIdx_z + 2 * 8; \
- const Index lhs_horiz_3 = base_k + hipThreadIdx_z + 3 * 8; \
- const Index lhs_horiz_4 = base_k + hipThreadIdx_z + 4 * 8; \
- const Index lhs_horiz_5 = base_k + hipThreadIdx_z + 5 * 8; \
- const Index lhs_horiz_6 = base_k + hipThreadIdx_z + 6 * 8; \
- const Index lhs_horiz_7 = base_k + hipThreadIdx_z + 7 * 8; \
- \
- if (!needs_edge_check || lhs_horiz_7 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
- lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
- lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
- lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \
- lhs_pf7 = lhs(lhs_vert, lhs_horiz_7); \
- } else if (lhs_horiz_6 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
- lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
- lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
- lhs_pf6 = lhs(lhs_vert, lhs_horiz_6); \
- } else if (lhs_horiz_5 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
- lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
- lhs_pf5 = lhs(lhs_vert, lhs_horiz_5); \
- } else if (lhs_horiz_4 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
- lhs_pf4 = lhs(lhs_vert, lhs_horiz_4); \
- } else if (lhs_horiz_3 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- lhs_pf3 = lhs(lhs_vert, lhs_horiz_3); \
- } else if (lhs_horiz_2 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- lhs_pf2 = lhs(lhs_vert, lhs_horiz_2); \
- } else if (lhs_horiz_1 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- lhs_pf1 = lhs(lhs_vert, lhs_horiz_1); \
- } else if (lhs_horiz_0 < k_size) { \
- lhs_pf0 = lhs(lhs_vert, lhs_horiz_0); \
- } \
- } \
- \
- const Index rhs_vert = base_k + load_idx_vert; \
- if (!needs_edge_check || rhs_vert < k_size) { \
- const Index rhs_horiz_0 = base_n + hipThreadIdx_z + 0 * 8; \
- const Index rhs_horiz_1 = base_n + hipThreadIdx_z + 1 * 8; \
- const Index rhs_horiz_2 = base_n + hipThreadIdx_z + 2 * 8; \
- const Index rhs_horiz_3 = base_n + hipThreadIdx_z + 3 * 8; \
- const Index rhs_horiz_4 = base_n + hipThreadIdx_z + 4 * 8; \
- const Index rhs_horiz_5 = base_n + hipThreadIdx_z + 5 * 8; \
- const Index rhs_horiz_6 = base_n + hipThreadIdx_z + 6 * 8; \
- const Index rhs_horiz_7 = base_n + hipThreadIdx_z + 7 * 8; \
- \
- if (rhs_horiz_7 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
- rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
- rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
- rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \
- rhs_pf7 = rhs(rhs_vert, rhs_horiz_7); \
- } else if (rhs_horiz_6 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
- rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
- rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
- rhs_pf6 = rhs(rhs_vert, rhs_horiz_6); \
- } else if (rhs_horiz_5 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
- rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
- rhs_pf5 = rhs(rhs_vert, rhs_horiz_5); \
- } else if (rhs_horiz_4 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
- rhs_pf4 = rhs(rhs_vert, rhs_horiz_4); \
- } else if (rhs_horiz_3 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- rhs_pf3 = rhs(rhs_vert, rhs_horiz_3); \
- } else if (rhs_horiz_2 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- rhs_pf2 = rhs(rhs_vert, rhs_horiz_2); \
- } else if (rhs_horiz_1 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- rhs_pf1 = rhs(rhs_vert, rhs_horiz_1); \
- } else if (rhs_horiz_0 < n_size) { \
- rhs_pf0 = rhs(rhs_vert, rhs_horiz_0); \
- } \
- } \
- } \
-
-#define writeRegToShmem(_) \
- lhs_shmem[lhs_store_idx_0] = lhs_pf0; \
- rhs_shmem[rhs_store_idx_0] = rhs_pf0; \
- \
- lhs_shmem[lhs_store_idx_1] = lhs_pf1; \
- rhs_shmem[rhs_store_idx_1] = rhs_pf1; \
- \
- lhs_shmem[lhs_store_idx_2] = lhs_pf2; \
- rhs_shmem[rhs_store_idx_2] = rhs_pf2; \
- \
- lhs_shmem[lhs_store_idx_3] = lhs_pf3; \
- rhs_shmem[rhs_store_idx_3] = rhs_pf3; \
- \
- lhs_shmem[lhs_store_idx_4] = lhs_pf4; \
- rhs_shmem[rhs_store_idx_4] = rhs_pf4; \
- \
- lhs_shmem[lhs_store_idx_5] = lhs_pf5; \
- rhs_shmem[rhs_store_idx_5] = rhs_pf5; \
- \
- lhs_shmem[lhs_store_idx_6] = lhs_pf6; \
- rhs_shmem[rhs_store_idx_6] = rhs_pf6; \
- \
- lhs_shmem[lhs_store_idx_7] = lhs_pf7; \
- rhs_shmem[rhs_store_idx_7] = rhs_pf7; \
-
- // declare and initialize result array
-#define res(i, j) _res_##i##j
-#define initResultRow(i) \
- Scalar res(i, 0) = conv(0); \
- Scalar res(i, 1) = conv(0); \
- Scalar res(i, 2) = conv(0); \
- Scalar res(i, 3) = conv(0); \
- Scalar res(i, 4) = conv(0); \
- Scalar res(i, 5) = conv(0); \
- Scalar res(i, 6) = conv(0); \
- Scalar res(i, 7) = conv(0); \
-
- internal::scalar_cast_op<int, Scalar> conv;
- initResultRow(0);
- initResultRow(1);
- initResultRow(2);
- initResultRow(3);
- initResultRow(4);
- initResultRow(5);
- initResultRow(6);
- initResultRow(7);
-#undef initResultRow
-
- for (Index base_k = 0; base_k < k_size; base_k += 64) {
- // wait for previous iteration to finish with shmem. Despite common sense,
- // the code is a bit faster with this here then at bottom of loop
- __syncthreads();
-
- prefetchIntoRegisters(base_k);
- writeRegToShmem();
-
- #undef prefetchIntoRegisters
- #undef writeRegToShmem
-
- // wait for shared mem packing to be done before starting computation
- __syncthreads();
-
- // compute 8x8 matrix product by outer product. This involves packing one column
- // of LHS and one row of RHS into registers (takes 16 registers).
-
-#define lcol(i) _lcol##i
- Scalar lcol(0);
- Scalar lcol(1);
- Scalar lcol(2);
- Scalar lcol(3);
- Scalar lcol(4);
- Scalar lcol(5);
- Scalar lcol(6);
- Scalar lcol(7);
-
-#define rrow(j) _rrow##j
- Scalar rrow(0);
- Scalar rrow(1);
- Scalar rrow(2);
- Scalar rrow(3);
- Scalar rrow(4);
- Scalar rrow(5);
- Scalar rrow(6);
- Scalar rrow(7);
-
- // Now x corresponds to k, y to m, and z to n
- const Scalar* lhs_block = &lhs_shmem[hipThreadIdx_x + 9 * hipThreadIdx_y];
- const Scalar* rhs_block = &rhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_z];
-
-#define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))]
-#define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))]
-
-#define loadData(i, j) \
- lcol(0) = lhs_element(0, j); \
- rrow(0) = rhs_element(i, 0); \
- lcol(1) = lhs_element(1, j); \
- rrow(1) = rhs_element(i, 1); \
- lcol(2) = lhs_element(2, j); \
- rrow(2) = rhs_element(i, 2); \
- lcol(3) = lhs_element(3, j); \
- rrow(3) = rhs_element(i, 3); \
- lcol(4) = lhs_element(4, j); \
- rrow(4) = rhs_element(i, 4); \
- lcol(5) = lhs_element(5, j); \
- rrow(5) = rhs_element(i, 5); \
- lcol(6) = lhs_element(6, j); \
- rrow(6) = rhs_element(i, 6); \
- lcol(7) = lhs_element(7, j); \
- rrow(7) = rhs_element(i, 7); \
-
-#define computeCol(j) \
- res(0, j) += lcol(0) * rrow(j); \
- res(1, j) += lcol(1) * rrow(j); \
- res(2, j) += lcol(2) * rrow(j); \
- res(3, j) += lcol(3) * rrow(j); \
- res(4, j) += lcol(4) * rrow(j); \
- res(5, j) += lcol(5) * rrow(j); \
- res(6, j) += lcol(6) * rrow(j); \
- res(7, j) += lcol(7) * rrow(j); \
-
-#define computePass(i) \
- loadData(i, i); \
- \
- computeCol(0); \
- computeCol(1); \
- computeCol(2); \
- computeCol(3); \
- computeCol(4); \
- computeCol(5); \
- computeCol(6); \
- computeCol(7); \
-
- computePass(0);
- computePass(1);
- computePass(2);
- computePass(3);
- computePass(4);
- computePass(5);
- computePass(6);
- computePass(7);
-
-#undef lcol
-#undef rrow
-#undef lhs_element
-#undef rhs_element
-#undef loadData
-#undef computeCol
-#undef computePass
- } // end loop over k
-
- // we've now iterated over all of the large (ie width 64) k blocks and
- // accumulated results in registers. At this point thread (x, y, z) contains
- // the sum across all big k blocks of the product of little k block of index (x, y)
- // with block of index (y, z). To compute the final output, we need to reduce
- // the 8 threads over y by summation.
-#define shuffleInc(i, j, mask) res(i, j) += __shfl_xor(res(i, j), mask)
-
-#define reduceRow(i, mask) \
- shuffleInc(i, 0, mask); \
- shuffleInc(i, 1, mask); \
- shuffleInc(i, 2, mask); \
- shuffleInc(i, 3, mask); \
- shuffleInc(i, 4, mask); \
- shuffleInc(i, 5, mask); \
- shuffleInc(i, 6, mask); \
- shuffleInc(i, 7, mask); \
-
-#define reduceMatrix(mask) \
- reduceRow(0, mask); \
- reduceRow(1, mask); \
- reduceRow(2, mask); \
- reduceRow(3, mask); \
- reduceRow(4, mask); \
- reduceRow(5, mask); \
- reduceRow(6, mask); \
- reduceRow(7, mask); \
-
- // actually perform the reduction, now each thread of index (_, y, z)
- // contains the correct values in its registers that belong in the output
- // block
- reduceMatrix(1);
- reduceMatrix(2);
- reduceMatrix(4);
-
-#undef shuffleInc
-#undef reduceRow
-#undef reduceMatrix
-
- // now we need to copy the 64 values into main memory. We can't split work
- // among threads because all variables are in registers. There's 2 ways
- // to do this:
- // (1) have 1 thread do 64 writes from registers into global memory
- // (2) have 1 thread do 64 writes into shared memory, and then 8 threads
- // each do 8 writes into global memory. We can just overwrite the shared
- // memory from the problem we just solved.
- // (2) is slightly faster than (1) due to less branching and more ILP
-
- // TODO: won't yield much gain, but could just use currently unused shared mem
- // and then we won't have to sync
- // wait for shared mem to be out of use
- __syncthreads();
-
-#define writeResultShmem(i, j) \
- lhs_shmem[i + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * j] = res(i, j); \
-
-#define writeRow(i) \
- writeResultShmem(i, 0); \
- writeResultShmem(i, 1); \
- writeResultShmem(i, 2); \
- writeResultShmem(i, 3); \
- writeResultShmem(i, 4); \
- writeResultShmem(i, 5); \
- writeResultShmem(i, 6); \
- writeResultShmem(i, 7); \
-
- if (hipThreadIdx_x == 0) {
- writeRow(0);
- writeRow(1);
- writeRow(2);
- writeRow(3);
- writeRow(4);
- writeRow(5);
- writeRow(6);
- writeRow(7);
- }
-#undef writeResultShmem
-#undef writeRow
-
- const int max_i_write = numext::mini((int)((m_size - base_m - hipThreadIdx_y + 7) / 8), 8);
- const int max_j_write = numext::mini((int)((n_size - base_n - hipThreadIdx_z + 7) / 8), 8);
-
- if (hipThreadIdx_x < max_i_write) {
- if (max_j_write == 8) {
- // TODO: can i trade bank conflicts for coalesced writes?
- Scalar val0 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 0];
- Scalar val1 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 1];
- Scalar val2 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 2];
- Scalar val3 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 3];
- Scalar val4 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 4];
- Scalar val5 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 5];
- Scalar val6 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 6];
- Scalar val7 = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * 7];
-
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 0) = val0;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 1) = val1;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 2) = val2;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 3) = val3;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 4) = val4;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 5) = val5;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 6) = val6;
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * 7) = val7;
- } else {
-#pragma unroll 7
- for (int j = 0; j < max_j_write; j++) {
- Scalar val = lhs_shmem[hipThreadIdx_x + 8 * hipThreadIdx_y + 64 * hipThreadIdx_z + 512 * j];
- output(base_m + hipThreadIdx_y + 8 * hipThreadIdx_x, base_n + hipThreadIdx_z + 8 * j) = val;
- }
- }
- }
-#undef res
-}
-
-
-template<typename Scalar, typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper>
-__global__ void
-__launch_bounds__(512, 1)
-EigenContractionKernel(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output,
- const Index m_size, const Index n_size, const Index k_size) {
- __shared__ Scalar lhs_shmem[72 * 64];
- __shared__ Scalar rhs_shmem[72 * 64];
-
- const Index m_block_idx = hipBlockIdx_x;
- const Index n_block_idx = hipBlockIdx_y;
-
- const Index base_m = 64 * m_block_idx;
- const Index base_n = 64 * n_block_idx;
-
- if (base_m + 63 < m_size && base_n + 63 < n_size) {
- EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size);
- } else {
- EigenContractionKernelInternal<Scalar, Index, LhsMapper, RhsMapper, OutputMapper, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size);
- }
-}
-
-
-template<typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY,
- bool CHECK_RHS_BOUNDARY>
-__device__ EIGEN_STRONG_INLINE void
-EigenFloatContractionKernelInternal16x16(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output, float2 lhs_shmem2[][16],
- float2 rhs_shmem2[][8], const Index m_size,
- const Index n_size, const Index k_size,
- const Index base_m, const Index base_n) {
-
- // prefetch registers
- float4 lhs_pf0, rhs_pf0;
-
- float4 results[4];
- for (int i=0; i < 4; i++) {
- results[i].x = results[i].y = results[i].z = results[i].w = 0;
- }
-
-
-#define prefetch_lhs(reg, row, col) \
- if (!CHECK_LHS_BOUNDARY) { \
- if (col < k_size) { \
- /*reg = lhs.template loadPacket<Unaligned>(row, col);*/ \
- reg.x =lhs(row + 0, col); \
- reg.y =lhs(row + 1, col); \
- reg.z =lhs(row + 2, col); \
- reg.w =lhs(row + 3, col); \
- } \
- } else { \
- if (col < k_size) { \
- if (row + 3 < m_size) { \
- /*reg =lhs.template loadPacket<Unaligned>(row, col);*/ \
- reg.x =lhs(row + 0, col); \
- reg.y =lhs(row + 1, col); \
- reg.z =lhs(row + 2, col); \
- reg.w =lhs(row + 3, col); \
- } else if (row + 2 < m_size) { \
- reg.x =lhs(row + 0, col); \
- reg.y =lhs(row + 1, col); \
- reg.z =lhs(row + 2, col); \
- } else if (row + 1 < m_size) { \
- reg.x =lhs(row + 0, col); \
- reg.y =lhs(row + 1, col); \
- } else if (row < m_size) { \
- reg.x =lhs(row + 0, col); \
- } \
- } \
- } \
-
-
- Index lhs_vert = base_m+hipThreadIdx_x*4;
-
- for (Index k = 0; k < k_size; k += 16) {
- //lhs_pf0 = internal::pset1<float4>(0);
- //rhs_pf0 = internal::pset1<float4>(0);
- lhs_pf0 = make_float4(0, 0, 0, 0);
- rhs_pf0 = make_float4(0, 0, 0, 0);
-
- Index lhs_horiz = hipThreadIdx_y+k;
- prefetch_lhs(lhs_pf0, lhs_vert, lhs_horiz)
-
- Index rhs_vert = k+(hipThreadIdx_x%4)*4;
- Index rhs_horiz0 = (hipThreadIdx_x>>2)+hipThreadIdx_y*4+base_n;
-
- if (!CHECK_RHS_BOUNDARY) {
- if ((rhs_vert + 3) < k_size) {
- // just CHECK_RHS_BOUNDARY
- //rhs_pf0 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz0);
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf0.w = rhs(rhs_vert + 3, rhs_horiz0);
- } else if (rhs_vert + 2 < k_size) {
- // just CHECK_RHS_BOUNDARY
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- } else if (rhs_vert + 1 < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- } else if (rhs_vert < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- }
- } else {
- if (rhs_horiz0 < n_size) {
- if ((rhs_vert + 3) < k_size) {
- //rhs_pf0 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz0);
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf0.w = rhs(rhs_vert + 3, rhs_horiz0);
- } else if ((rhs_vert + 2) < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- } else if ((rhs_vert + 1) < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- } else if (rhs_vert < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- }
- }
- }
- float x1, x2 ;
- // the following can be a bitwise operation..... some day.
- if((hipThreadIdx_x%8) < 4) {
- x1 = rhs_pf0.y;
- x2 = rhs_pf0.w;
- } else {
- x1 = rhs_pf0.x;
- x2 = rhs_pf0.z;
- }
- x1 = __shfl_xor(x1, 4);
- x2 = __shfl_xor(x2, 4);
- if((hipThreadIdx_x%8) < 4) {
- rhs_pf0.y = x1;
- rhs_pf0.w = x2;
- } else {
- rhs_pf0.x = x1;
- rhs_pf0.z = x2;
- }
-
- // We have 64 features.
- // Row 0 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 0, 1.
- // Row 1 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 2, 3.
- // ...
- // Row 31 -> times (0, 4, 8, 12, 1, 5, 9, 13) for features 62, 63
- // Row 32 -> times (2, 6, 10, 14, 3, 7, 11, 15) for features 0, 1
- // ...
- rhs_shmem2[(hipThreadIdx_x>>3)+ hipThreadIdx_y*2][hipThreadIdx_x%8] = make_float2(rhs_pf0.x, rhs_pf0.y);
- rhs_shmem2[(hipThreadIdx_x>>3)+ hipThreadIdx_y*2+32][hipThreadIdx_x%8] = make_float2(rhs_pf0.z, rhs_pf0.w);
-
- // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61)
- // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61)
- // ...
- // Row 15 (time 15) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61)
- // Row 16 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63)
- // ...
-
- lhs_shmem2[hipThreadIdx_y][hipThreadIdx_x] = make_float2(lhs_pf0.x, lhs_pf0.y);
- lhs_shmem2[hipThreadIdx_y+16][hipThreadIdx_x] = make_float2(lhs_pf0.z, lhs_pf0.w);
-
-
-#define add_vals(fl1, fl2, fr1, fr2)\
- results[0].x += fl1.x * fr1.x;\
- results[0].y += fl1.y * fr1.x;\
- results[0].z += fl2.x * fr1.x;\
- results[0].w += fl2.y * fr1.x;\
-\
- results[1].x += fl1.x * fr1.y;\
- results[1].y += fl1.y * fr1.y;\
- results[1].z += fl2.x * fr1.y;\
- results[1].w += fl2.y * fr1.y;\
-\
- results[2].x += fl1.x * fr2.x;\
- results[2].y += fl1.y * fr2.x;\
- results[2].z += fl2.x * fr2.x;\
- results[2].w += fl2.y * fr2.x;\
-\
- results[3].x += fl1.x * fr2.y;\
- results[3].y += fl1.y * fr2.y;\
- results[3].z += fl2.x * fr2.y;\
- results[3].w += fl2.y * fr2.y;\
-
- __syncthreads();
-
- // Do the multiplies.
- #pragma unroll
- for (int koff = 0; koff < 16; koff ++) {
- // 32 x threads.
- float2 fl1 = lhs_shmem2[koff][hipThreadIdx_x];
- float2 fl2 = lhs_shmem2[koff + 16][hipThreadIdx_x];
-
- int start_feature = hipThreadIdx_y * 4;
- float2 fr1 = rhs_shmem2[(start_feature>>1) + 32*((koff%4)/2)][koff/4 + (koff%2)*4];
- float2 fr2 = rhs_shmem2[(start_feature>>1) + 1 + 32*((koff%4)/2)][koff/4 + (koff%2)*4];
-
- add_vals(fl1, fl2, fr1, fr2)
- }
- __syncthreads();
- }
-
-#undef prefetch_lhs
-#undef add_vals
-
- Index horiz_base = hipThreadIdx_y*4+base_n;
- if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) {
- for (int i = 0; i < 4; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- } else if (!CHECK_RHS_BOUNDARY) {
- // CHECK LHS
- if (lhs_vert + 3 < m_size) {
- for (int i = 0; i < 4; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- } else if (lhs_vert + 2 < m_size) {
- for (int i = 0; i < 4; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- }
- } else if (lhs_vert + 1 < m_size) {
- for (int i = 0; i < 4; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- }
- } else if (lhs_vert < m_size) {
- for (int i = 0; i < 4; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- }
- }
- } else if (!CHECK_LHS_BOUNDARY) {
- // CHECK RHS
- /*
- int ncols_rem = fminf(n_size- horiz_base, 4);
- for (int i = 0; i < ncols_rem; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }*/
- for (int i = 0; i < 4; i++) {
- if (horiz_base+i < n_size) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- }
- } else {
- // CHECK both boundaries.
- for (int i = 0; i < 4; i++) {
- if (horiz_base+i < n_size) {
- if (lhs_vert < m_size)
- output(lhs_vert, horiz_base + i) = results[i].x;
- if (lhs_vert + 1 < m_size)
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- if (lhs_vert + 2 < m_size)
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- if (lhs_vert + 3 < m_size)
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- }
- }
-}
-
-
-template<typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper, bool CHECK_LHS_BOUNDARY,
- bool CHECK_RHS_BOUNDARY>
-__device__ EIGEN_STRONG_INLINE void
-EigenFloatContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output, float2 lhs_shmem2[][32],
- float2 rhs_shmem2[][8], const Index m_size,
- const Index n_size, const Index k_size,
- const Index base_m, const Index base_n) {
-
- // prefetch registers
- float4 lhs_pf0, lhs_pf1, lhs_pf2, lhs_pf3;
- float4 rhs_pf0, rhs_pf1;
-
- float4 results[8];
- for (int i=0; i < 8; i++) {
- results[i].x = results[i].y = results[i].z = results[i].w = 0;
- }
-
-
- Index lhs_vert = base_m+hipThreadIdx_x*4+(hipThreadIdx_y%4)*32;
- for (Index k = 0; k < k_size; k += 32) {
- /*lhs_pf0 = internal::pset1<float4>(0);
- lhs_pf1 = internal::pset1<float4>(0);
- lhs_pf2 = internal::pset1<float4>(0);
- lhs_pf3 = internal::pset1<float4>(0);
-
- rhs_pf0 = internal::pset1<float4>(0);
- rhs_pf1 = internal::pset1<float4>(0);*/
-
-
- lhs_pf0 = make_float4(0, 0, 0, 0);
- lhs_pf1 = make_float4(0, 0, 0, 0);
- lhs_pf2 = make_float4(0, 0, 0, 0);
- lhs_pf3 = make_float4(0, 0, 0, 0);
-
- rhs_pf0 = make_float4(0, 0, 0, 0);
- rhs_pf1 = make_float4(0, 0, 0, 0);
-
- if (!CHECK_LHS_BOUNDARY) {
- if ((hipThreadIdx_y/4+k+24) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- //lhs_pf2 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+16));
- //lhs_pf3 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+24));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf2.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf3.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+24));
- lhs_pf3.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+24));
- lhs_pf3.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+24));
- lhs_pf3.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+24));
- } else if ((hipThreadIdx_y/4+k+16) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- //lhs_pf2 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+16));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf2.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- } else if ((hipThreadIdx_y/4+k+8) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- } else if ((hipThreadIdx_y/4+k) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- }
- } else {
- // just CHECK_LHS_BOUNDARY
- if (lhs_vert + 3 < m_size) {
- if ((hipThreadIdx_y/4+k+24) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- //lhs_pf2 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+16));
- //lhs_pf3 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+24));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf2.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf3.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+24));
- lhs_pf3.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+24));
- lhs_pf3.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+24));
- lhs_pf3.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+24));
- } else if ((hipThreadIdx_y/4+k+16) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- //lhs_pf2 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+16));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf2.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- } else if ((hipThreadIdx_y/4+k+8) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- //lhs_pf1 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k+8));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf1.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- } else if ((hipThreadIdx_y/4+k) < k_size) {
- //lhs_pf0 =lhs.template loadPacket<Unaligned>(lhs_vert, (hipThreadIdx_y/4+k));
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf0.w =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- }
- } else if (lhs_vert + 2 < m_size) {
- if ((hipThreadIdx_y/4+k+24) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- lhs_pf3.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+24));
- lhs_pf3.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+24));
- lhs_pf3.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+24));
- } else if ((hipThreadIdx_y/4+k+16) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf2.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+16));
- } else if ((hipThreadIdx_y/4+k+8) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf1.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k+8));
- } else if ((hipThreadIdx_y/4+k) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf0.z =lhs(lhs_vert + 2, (hipThreadIdx_y/4+k));
- }
- } else if (lhs_vert + 1 < m_size) {
- if ((hipThreadIdx_y/4+k+24) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- lhs_pf3.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+24));
- lhs_pf3.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+24));
- } else if ((hipThreadIdx_y/4+k+16) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf2.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+16));
- } else if ((hipThreadIdx_y/4+k+8) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf1.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k+8));
- } else if ((hipThreadIdx_y/4+k) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf0.y =lhs(lhs_vert + 1, (hipThreadIdx_y/4+k));
- }
- } else if (lhs_vert < m_size) {
- if ((hipThreadIdx_y/4+k+24) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- lhs_pf3.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+24));
- } else if ((hipThreadIdx_y/4+k+16) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- lhs_pf2.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+16));
- } else if ((hipThreadIdx_y/4+k+8) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- lhs_pf1.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k+8));
- } else if ((hipThreadIdx_y/4+k) < k_size) {
- lhs_pf0.x =lhs(lhs_vert + 0, (hipThreadIdx_y/4+k));
- }
- }
- }
- __syncthreads();
- Index rhs_vert = k+hipThreadIdx_x*4;
- Index rhs_horiz0 = hipThreadIdx_y*2+base_n;
- Index rhs_horiz1 = hipThreadIdx_y*2+1+base_n;
- if (!CHECK_RHS_BOUNDARY) {
- if ((rhs_vert + 3) < k_size) {
- // just CHECK_RHS_BOUNDARY
- //rhs_pf0 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz0);
- //rhs_pf1 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz1);
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf0.w = rhs(rhs_vert + 3, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1);
- rhs_pf1.w = rhs(rhs_vert + 3, rhs_horiz1);
- } else if (rhs_vert + 2 < k_size) {
- // just CHECK_RHS_BOUNDARY
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1);
- } else if (rhs_vert + 1 < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- } else if (rhs_vert < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- }
- } else {
- if (rhs_horiz1 < n_size) {
- if ((rhs_vert + 3) < k_size) {
- // just CHECK_RHS_BOUNDARY
- //rhs_pf0 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz0);
- //rhs_pf1 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz1);
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf0.w = rhs(rhs_vert + 3, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1);
- rhs_pf1.w = rhs(rhs_vert + 3, rhs_horiz1);
- } else if (rhs_vert + 2 < k_size) {
- // just CHECK_RHS_BOUNDARY
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- rhs_pf1.z = rhs(rhs_vert + 2, rhs_horiz1);
- } else if (k+hipThreadIdx_x*4 + 1 < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- rhs_pf1.y = rhs(rhs_vert + 1, rhs_horiz1);
- } else if (k+hipThreadIdx_x*4 < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf1.x = rhs(rhs_vert, rhs_horiz1);
- }
- } else if (rhs_horiz0 < n_size) {
- if ((rhs_vert + 3) < k_size) {
- // just CHECK_RHS_BOUNDARY
- //rhs_pf0 = rhs.template loadPacket<Unaligned>(rhs_vert, rhs_horiz0);
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- rhs_pf0.w = rhs(rhs_vert + 3, rhs_horiz0);
- } else if ((rhs_vert + 2) < k_size) {
- // just CHECK_RHS_BOUNDARY
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- rhs_pf0.z = rhs(rhs_vert + 2, rhs_horiz0);
- } else if ((rhs_vert + 1) < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- rhs_pf0.y = rhs(rhs_vert + 1, rhs_horiz0);
- } else if (rhs_vert < k_size) {
- rhs_pf0.x = rhs(rhs_vert, rhs_horiz0);
- }
- }
- }
- __syncthreads();
- // Loaded. Do computation
- // Row 0 -> times (0, 4, 8, .. 28) for features 0, 1.
- // Row 1 -> times (0, 4, 8, .. 28) for features 2, 3.
- // ..
- // Row 31 -> times (0, 4, 8, .. 28) for features 62, 63
- rhs_shmem2[hipThreadIdx_y][hipThreadIdx_x] = make_float2(rhs_pf0.x, rhs_pf1.x);
- // Row 32 -> times (1, 5, 9, .. 29) for features 0, 1.
- // Row 33 -> times (1, 5, 9, .. 29) for features 2, 3.
- // ..
- rhs_shmem2[hipThreadIdx_y+32][hipThreadIdx_x] = make_float2(rhs_pf0.y, rhs_pf1.y);
- // Row 64 -> times (2, 6, 10, .. 30) for features 0, 1.
- // Row 65 -> times (2, 6, 10, .. 30) for features 2, 3.
- rhs_shmem2[hipThreadIdx_y+64][hipThreadIdx_x] = make_float2(rhs_pf0.z, rhs_pf1.z);
- // Row 96 -> times (3, 7, 11, .. 31) for features 0, 1.
- // Row 97 -> times (3, 7, 11, .. 31) for features 2, 3.
- rhs_shmem2[hipThreadIdx_y+96][hipThreadIdx_x] = make_float2(rhs_pf0.w, rhs_pf1.w);
-
- // LHS.
- // Row 0 (time 0) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125)
- // Row 1 (time 1) -> features (0, 1), (4, 5), .. (28, 29), (32, 33), .. (60, 61) .. (124, 125)
- // ...
- // Row 8 (time 0) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127)
- // Row 15 (time 7) -> features (2, 3), (6, 7), .. (30, 31), (34, 35), .. (62, 63) .. (126, 127)
-
-
-#define add_vals(a_feat1, a_feat2, f1, f2, f3, f4)\
- results[0].x += a_feat1.x * f1.x;\
- results[1].x += a_feat1.x * f1.y;\
- results[2].x += a_feat1.x * f2.x;\
- results[3].x += a_feat1.x * f2.y;\
- results[4].x += a_feat1.x * f3.x;\
- results[5].x += a_feat1.x * f3.y;\
- results[6].x += a_feat1.x * f4.x;\
- results[7].x += a_feat1.x * f4.y;\
-\
- results[0].y += a_feat1.y * f1.x;\
- results[1].y += a_feat1.y * f1.y;\
- results[2].y += a_feat1.y * f2.x;\
- results[3].y += a_feat1.y * f2.y;\
- results[4].y += a_feat1.y * f3.x;\
- results[5].y += a_feat1.y * f3.y;\
- results[6].y += a_feat1.y * f4.x;\
- results[7].y += a_feat1.y * f4.y;\
-\
- results[0].z += a_feat2.x * f1.x;\
- results[1].z += a_feat2.x * f1.y;\
- results[2].z += a_feat2.x * f2.x;\
- results[3].z += a_feat2.x * f2.y;\
- results[4].z += a_feat2.x * f3.x;\
- results[5].z += a_feat2.x * f3.y;\
- results[6].z += a_feat2.x * f4.x;\
- results[7].z += a_feat2.x * f4.y;\
-\
- results[0].w += a_feat2.y * f1.x;\
- results[1].w += a_feat2.y * f1.y;\
- results[2].w += a_feat2.y * f2.x;\
- results[3].w += a_feat2.y * f2.y;\
- results[4].w += a_feat2.y * f3.x;\
- results[5].w += a_feat2.y * f3.y;\
- results[6].w += a_feat2.y * f4.x;\
- results[7].w += a_feat2.y * f4.y;\
-
- lhs_shmem2[hipThreadIdx_y/4][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf0.x, lhs_pf0.y);
- lhs_shmem2[hipThreadIdx_y/4+8][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf1.x, lhs_pf1.y);
- lhs_shmem2[hipThreadIdx_y/4+16][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf2.x, lhs_pf2.y);
- lhs_shmem2[hipThreadIdx_y/4+24][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf3.x, lhs_pf3.y);
-
- lhs_shmem2[hipThreadIdx_y/4 + 32][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf0.z, lhs_pf0.w);
- lhs_shmem2[hipThreadIdx_y/4 + 40][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf1.z, lhs_pf1.w);
- lhs_shmem2[hipThreadIdx_y/4 + 48][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf2.z, lhs_pf2.w);
- lhs_shmem2[hipThreadIdx_y/4 + 56][hipThreadIdx_x+(hipThreadIdx_y%4)*8] = make_float2(lhs_pf3.z, lhs_pf3.w);
-
- __syncthreads();
-
- // Do the multiplies.
- #pragma unroll
- for (int koff = 0; koff < 32; koff ++) {
- float2 a3 = lhs_shmem2[koff][hipThreadIdx_x + (hipThreadIdx_y % 4) * 8];
- float2 a4 = lhs_shmem2[koff + 32][hipThreadIdx_x + (hipThreadIdx_y % 4) * 8];
-
- // first feature is at (hipThreadIdx_y/4) * 8 last is at start + 8.
- int start_feature = (hipThreadIdx_y / 4) * 8;
-
- float2 br1 = rhs_shmem2[start_feature/2 + (koff % 4) * 32][koff/4];
- float2 br2 = rhs_shmem2[start_feature/2 + 1 + (koff % 4) * 32][koff/4];
- float2 br3 = rhs_shmem2[start_feature/2 + 2 + (koff % 4) * 32][koff/4];
- float2 br4 = rhs_shmem2[start_feature/2 + 3 + (koff % 4) * 32][koff/4];
-
- add_vals(a3, a4, br1, br2, br3, br4)
- }
- __syncthreads();
- } // end loop over k
-
-
- __syncthreads();
- Index horiz_base = (hipThreadIdx_y/4)*8+base_n;
- if (!CHECK_LHS_BOUNDARY && !CHECK_RHS_BOUNDARY) {
- for (int i = 0; i < 8; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- } else if (!CHECK_RHS_BOUNDARY) {
- if (lhs_vert + 3 < m_size) {
- for (int i = 0; i < 8; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- } else if (lhs_vert + 2 < m_size) {
- for (int i = 0; i < 8; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- }
- } else if (lhs_vert + 1 < m_size) {
- for (int i = 0; i < 8; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- }
- } else if (lhs_vert < m_size) {
- for (int i = 0; i < 8; i++) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- }
- }
- } else if (!CHECK_LHS_BOUNDARY) {
- // CHECK BOUNDARY_B
- for (int i = 0; i < 8; i++) {
- if (horiz_base + i < n_size) {
- output(lhs_vert, horiz_base + i) = results[i].x;
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- }
- } else {
- // CHECK both boundaries.
- for (int i = 0; i < 8; i++) {
- if (horiz_base + i < n_size) {
- if (lhs_vert < m_size)
- output(lhs_vert, horiz_base + i) = results[i].x;
- if (lhs_vert + 1 < m_size)
- output(lhs_vert + 1, horiz_base + i) = results[i].y;
- if (lhs_vert + 2 < m_size)
- output(lhs_vert + 2, horiz_base + i) = results[i].z;
- if (lhs_vert + 3 < m_size)
- output(lhs_vert + 3, horiz_base + i) = results[i].w;
- }
- }
- }
-}
-
-
-template<typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper>
-__global__ void
-__launch_bounds__(256, 1)
-EigenFloatContractionKernel(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output,
- const Index m_size, const Index n_size, const Index k_size) {
- __shared__ float2 lhs_shmem[64*32];
- __shared__ float2 rhs_shmem[128*8];
-
- typedef float2 LHS_MEM[64][32];
- typedef float2 RHS_MEM[128][8];
-
- typedef float2 LHS_MEM16x16[32][16];
- typedef float2 RHS_MEM16x16[64][8];
-
- const Index m_block_idx = hipBlockIdx_x;
- const Index n_block_idx = hipBlockIdx_y;
-
- const Index base_m = 128 * m_block_idx;
- const Index base_n = 64 * n_block_idx;
-
- bool check_rhs = (base_n + 63) >= n_size;
- bool check_lhs128 = (base_m + 127) >= m_size;
-
- if (!check_rhs) {
- if (!check_lhs128) {
- // >= 128 rows left
- EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, false>(
- lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n);
- } else {
- EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, false>(
- lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n);
- }
- } else {
- if (!check_lhs128) {
- // >= 128 rows left
- EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, false, true>(
- lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n);
- } else {
- EigenFloatContractionKernelInternal<Index, LhsMapper, RhsMapper, OutputMapper, true, true>(
- lhs, rhs, output, *((LHS_MEM *) lhs_shmem), *((RHS_MEM *) rhs_shmem), m_size, n_size, k_size, base_m, base_n);
- }
- }
-}
-
-template<typename Index, typename LhsMapper,
- typename RhsMapper, typename OutputMapper>
-__global__ void
-__launch_bounds__(256, 1)
-EigenFloatContractionKernel16x16(const LhsMapper lhs, const RhsMapper rhs,
- const OutputMapper output,
- const Index m_size, const Index n_size, const Index k_size) {
- __shared__ float2 lhs_shmem[32][16];
- __shared__ float2 rhs_shmem[64][8];
-
- const Index m_block_idx = hipBlockIdx_x;
- const Index n_block_idx = hipBlockIdx_y;
-
- const Index base_m = 64 * m_block_idx;
- const Index base_n = 64 * n_block_idx;
-
- if (base_m + 63 < m_size) {
- if (base_n + 63 < n_size) {
- EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n);
- } else {
- EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, false, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n);
- }
- } else {
- if (base_n + 63 < n_size) {
- EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, false>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n);
- } else {
- EigenFloatContractionKernelInternal16x16<Index, LhsMapper, RhsMapper, OutputMapper, true, true>(lhs, rhs, output, lhs_shmem, rhs_shmem, m_size, n_size, k_size, base_m, base_n);
- }
- }
-}
-
-
-template<typename Indices, typename LeftArgType, typename RightArgType>
-struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, GpuDevice> :
- public TensorContractionEvaluatorBase<TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, GpuDevice> > {
-
- typedef GpuDevice Device;
-
- typedef TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgType>, Device> Self;
- typedef TensorContractionEvaluatorBase<Self> Base;
-
- typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType;
- typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar;
- typedef typename XprType::Index Index;
- typedef typename XprType::CoeffReturnType CoeffReturnType;
- typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType;
-
- enum {
- Layout = TensorEvaluator<LeftArgType, Device>::Layout,
- };
-
- // Most of the code is assuming that both input tensors are ColMajor. If the
- // inputs are RowMajor, we will "cheat" by swapping the LHS and RHS:
- // If we want to compute A * B = C, where A is LHS and B is RHS, the code
- // will pretend B is LHS and A is RHS.
- typedef typename internal::conditional<
- static_cast<int>(Layout) == static_cast<int>(ColMajor), LeftArgType, RightArgType>::type EvalLeftArgType;
- typedef typename internal::conditional<
- static_cast<int>(Layout) == static_cast<int>(ColMajor), RightArgType, LeftArgType>::type EvalRightArgType;
-
- static const int LDims =
- internal::array_size<typename TensorEvaluator<EvalLeftArgType, Device>::Dimensions>::value;
- static const int RDims =
- internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value;
- static const int ContractDims = internal::array_size<Indices>::value;
-
- typedef array<Index, LDims> left_dim_mapper_t;
- typedef array<Index, RDims> right_dim_mapper_t;
-
- typedef array<Index, ContractDims> contract_t;
- typedef array<Index, LDims - ContractDims> left_nocontract_t;
- typedef array<Index, RDims - ContractDims> right_nocontract_t;
-
- static const int NumDims = LDims + RDims - 2 * ContractDims;
-
- typedef DSizes<Index, NumDims> Dimensions;
-
- // typedefs needed in evalTo
- typedef typename internal::remove_const<typename EvalLeftArgType::Scalar>::type LhsScalar;
- typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar;
-
- typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator;
- typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator;
-
- typedef typename LeftEvaluator::Dimensions LeftDimensions;
- typedef typename RightEvaluator::Dimensions RightDimensions;
-
- EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) :
- Base(op, device) {}
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~TensorEvaluator() {}
-
- // We need to redefine this method to make hipcc happy
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) {
- this->m_leftImpl.evalSubExprsIfNeeded(NULL);
- this->m_rightImpl.evalSubExprsIfNeeded(NULL);
- if (data) {
- evalTo(data);
- return false;
- } else {
- this->m_result = static_cast<Scalar *>(this->m_device.allocate(this->dimensions().TotalSize() * sizeof(Scalar)));
- evalTo(this->m_result);
- return true;
- }
- }
-
- void evalTo(Scalar* buffer) const {
- if (this->m_lhs_inner_dim_contiguous) {
- if (this->m_rhs_inner_dim_contiguous) {
- if (this->m_rhs_inner_dim_reordered) {
- evalTyped<true, true, true, Unaligned>(buffer);
- }
- else {
- evalTyped<true, true, false, Unaligned>(buffer);
- }
- }
- else {
- if (this->m_rhs_inner_dim_reordered) {
- evalTyped<true, false, true, Unaligned>(buffer);
- }
- else {
- evalTyped<true, false, false, Unaligned>(buffer);
- }
- }
- }
- else {
- if (this->m_rhs_inner_dim_contiguous) {
- if (this->m_rhs_inner_dim_reordered) {
- evalTyped<false, true, true, Unaligned>(buffer);
- }
- else {
- evalTyped<false, true, false, Unaligned>(buffer);
- }
- }
- else {
- if (this->m_rhs_inner_dim_reordered) {
- evalTyped<false, false, true, Unaligned>(buffer);
- }
- else {
- evalTyped<false, false, false, Unaligned>(buffer);
- }
- }
- }
- }
-
- template <typename LhsScalar, typename RhsScalar, typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels {
- static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) {
- const Index m_blocks = (m + 63) / 64;
- const Index n_blocks = (n + 63) / 64;
- const dim3 num_blocks(m_blocks, n_blocks, 1);
- const dim3 block_size(8, 8, 8);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenContractionKernel<Scalar, Index, LhsMapper, RhsMapper, OutputMapper>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), lhs, rhs, output, m, n, k);
- }
- };
-
- template <typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels<float, float, Index, LhsMapper, RhsMapper, OutputMapper> {
- static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) {
- if (m < 768 || n < 768) {
- const Index m_blocks = (m + 63) / 64;
- const Index n_blocks = (n + 63) / 64;
- const dim3 num_blocks(m_blocks, n_blocks, 1);
- const dim3 block_size(16, 16, 1);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenFloatContractionKernel16x16<Index, LhsMapper, RhsMapper, OutputMapper>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), lhs, rhs, output, m, n, k);
- } else {
- const Index m_blocks = (m + 127) / 128;
- const Index n_blocks = (n + 63) / 64;
- const dim3 num_blocks(m_blocks, n_blocks, 1);
- const dim3 block_size(8, 32, 1);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenFloatContractionKernel<Index, LhsMapper, RhsMapper, OutputMapper>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), lhs, rhs, output, m, n, k);
- }
- }
- };
-
- template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
- void evalTyped(Scalar* buffer) const {
- // columns in left side, rows in right side
- const Index k = this->m_k_size;
- EIGEN_UNUSED_VARIABLE(k)
-
- // rows in left side
- const Index m = this->m_i_size;
-
- // columns in right side
- const Index n = this->m_j_size;
-
- // zero out the result buffer (which must be of size at least m * n * sizeof(Scalar)
- this->m_device.memset(buffer, 0, m * n * sizeof(Scalar));
-
- typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs,
- LeftEvaluator, left_nocontract_t,
- contract_t, 4,
- lhs_inner_dim_contiguous,
- false, Unaligned> LhsMapper;
-
- typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs,
- RightEvaluator, right_nocontract_t,
- contract_t, 4,
- rhs_inner_dim_contiguous,
- rhs_inner_dim_reordered, Unaligned> RhsMapper;
-
- typedef internal::blas_data_mapper<Scalar, Index, ColMajor> OutputMapper;
-
-
- // initialize data mappers
- LhsMapper lhs(this->m_leftImpl, this->m_left_nocontract_strides, this->m_i_strides,
- this->m_left_contracting_strides, this->m_k_strides);
-
- RhsMapper rhs(this->m_rightImpl, this->m_right_nocontract_strides, this->m_j_strides,
- this->m_right_contracting_strides, this->m_k_strides);
-
- OutputMapper output(buffer, m);
-
- setHipSharedMemConfig(hipSharedMemBankSizeEightByte);
- LaunchKernels<LhsScalar, RhsScalar, Index, LhsMapper, RhsMapper, OutputMapper>::Run(lhs, rhs, output, m, n, k, this->m_device);
- }
-};
-
-} // end namespace Eigen
-
-#endif // EIGEN_USE_GPU and EIGEN_HIPCC
-#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_HIP_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionHip.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionHip.h
deleted file mode 100644
index ba99710..0000000
--- a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolutionHip.h
+++ /dev/null
@@ -1,1119 +0,0 @@
-//#include "hip/hip_runtime.h"
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_H
-#define EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_H
-
-namespace Eigen {
-
-/** \class TensorConvolution
- * \ingroup CXX11_Tensor_Module
- *
- * \brief Tensor convolution class.
- *
- *
- */
-namespace internal {
-
-template <typename Index, typename InputDims, int NumKernelDims, int Layout>
-class IndexMapper {
- public:
- IndexMapper(const InputDims& input_dims, const array<Index, NumKernelDims>& kernel_dims,
- const array<Index, NumKernelDims>& indices) {
-
- array<Index, NumDims> dimensions = input_dims;
- for (int i = 0; i < NumKernelDims; ++i) {
- const Index index = indices[i];
- const Index input_dim = input_dims[index];
- const Index kernel_dim = kernel_dims[i];
- const Index result_dim = input_dim - kernel_dim + 1;
- dimensions[index] = result_dim;
- }
-
- array<Index, NumDims> inputStrides;
- array<Index, NumDims> outputStrides;
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- inputStrides[0] = 1;
- outputStrides[0] = 1;
- for (int i = 1; i < NumDims; ++i) {
- inputStrides[i] = inputStrides[i-1] * input_dims[i-1];
- outputStrides[i] = outputStrides[i-1] * dimensions[i-1];
- }
- } else {
- inputStrides[NumDims - 1] = 1;
- outputStrides[NumDims - 1] = 1;
- for (int i = static_cast<int>(NumDims) - 2; i >= 0; --i) {
- inputStrides[i] = inputStrides[i + 1] * input_dims[i + 1];
- outputStrides[i] = outputStrides[i + 1] * dimensions[i + 1];
- }
- }
-
- array<Index, NumDims> hipInputDimensions;
- array<Index, NumDims> hipOutputDimensions;
- array<Index, NumDims> tmp = dimensions;
- array<Index, NumDims> ordering;
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- for (int i = 0; i < NumKernelDims; ++i) {
- const Index index = i + offset;
- ordering[index] = indices[i];
- tmp[indices[i]] = -1;
- hipInputDimensions[index] = input_dims[indices[i]];
- hipOutputDimensions[index] = dimensions[indices[i]];
- }
-
- int written = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? NumKernelDims
- : 0;
- for (int i = 0; i < NumDims; ++i) {
- if (tmp[i] >= 0) {
- ordering[written] = i;
- hipInputDimensions[written] = input_dims[i];
- hipOutputDimensions[written] = dimensions[i];
- ++written;
- }
- }
-
- for (int i = 0; i < NumDims; ++i) {
- m_inputStrides[i] = inputStrides[ordering[i]];
- m_outputStrides[i] = outputStrides[ordering[i]];
- }
-
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int i = 0; i < NumDims; ++i) {
- if (i > NumKernelDims) {
- m_hipInputStrides[i] =
- m_hipInputStrides[i - 1] * hipInputDimensions[i - 1];
- m_hipOutputStrides[i] =
- m_hipOutputStrides[i - 1] * hipOutputDimensions[i - 1];
- } else {
- m_hipInputStrides[i] = 1;
- m_hipOutputStrides[i] = 1;
- }
- }
- } else {
- for (int i = NumDims - 1; i >= 0; --i) {
- if (i + 1 < offset) {
- m_hipInputStrides[i] =
- m_hipInputStrides[i + 1] * hipInputDimensions[i + 1];
- m_hipOutputStrides[i] =
- m_hipOutputStrides[i + 1] * hipOutputDimensions[i + 1];
- } else {
- m_hipInputStrides[i] = 1;
- m_hipOutputStrides[i] = 1;
- }
- }
- }
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipInputPlaneToTensorInputOffset(Index p) const {
- Index inputIndex = 0;
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int d = NumDims - 1; d > NumKernelDims; --d) {
- const Index idx = p / m_hipInputStrides[d];
- inputIndex += idx * m_inputStrides[d];
- p -= idx * m_hipInputStrides[d];
- }
- inputIndex += p * m_inputStrides[NumKernelDims];
- } else {
- std::ptrdiff_t limit = 0;
- if (NumKernelDims < NumDims) {
- limit = NumDims - NumKernelDims - 1;
- }
- for (int d = 0; d < limit; ++d) {
- const Index idx = p / m_hipInputStrides[d];
- inputIndex += idx * m_inputStrides[d];
- p -= idx * m_hipInputStrides[d];
- }
- inputIndex += p * m_inputStrides[limit];
- }
- return inputIndex;
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipOutputPlaneToTensorOutputOffset(Index p) const {
- Index outputIndex = 0;
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int d = NumDims - 1; d > NumKernelDims; --d) {
- const Index idx = p / m_hipOutputStrides[d];
- outputIndex += idx * m_outputStrides[d];
- p -= idx * m_hipOutputStrides[d];
- }
- outputIndex += p * m_outputStrides[NumKernelDims];
- } else {
- std::ptrdiff_t limit = 0;
- if (NumKernelDims < NumDims) {
- limit = NumDims - NumKernelDims - 1;
- }
- for (int d = 0; d < limit; ++d) {
- const Index idx = p / m_hipOutputStrides[d];
- outputIndex += idx * m_outputStrides[d];
- p -= idx * m_hipOutputStrides[d];
- }
- outputIndex += p * m_outputStrides[limit];
- }
- return outputIndex;
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipInputKernelToTensorInputOffset(Index i) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_inputStrides[offset];
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipOutputKernelToTensorOutputOffset(Index i) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_outputStrides[offset];
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipInputKernelToTensorInputOffset(Index i, Index j) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1];
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipOutputKernelToTensorOutputOffset(Index i, Index j) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1];
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipInputKernelToTensorInputOffset(Index i, Index j, Index k) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1] +
- k * m_inputStrides[offset + 2];
- }
-
- EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapHipOutputKernelToTensorOutputOffset(Index i, Index j, Index k) const {
- const size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : NumDims - NumKernelDims;
- return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1] +
- k * m_outputStrides[offset + 2];
- }
-
- private:
- static const int NumDims = internal::array_size<InputDims>::value;
- array<Index, NumDims> m_inputStrides;
- array<Index, NumDims> m_outputStrides;
- array<Index, NumDims> m_hipInputStrides;
- array<Index, NumDims> m_hipOutputStrides;
-};
-
-
-
-template<typename Dimensions, typename InputXprType, typename KernelXprType>
-struct traits<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> >
-{
- // Type promotion to handle the case where the types of the lhs and the rhs are different.
- typedef typename promote_storage_type<typename InputXprType::Scalar,
- typename KernelXprType::Scalar>::ret Scalar;
- typedef typename promote_storage_type<typename traits<InputXprType>::StorageKind,
- typename traits<KernelXprType>::StorageKind>::ret StorageKind;
- typedef typename promote_index_type<typename traits<InputXprType>::Index,
- typename traits<KernelXprType>::Index>::type Index;
- typedef typename InputXprType::Nested LhsNested;
- typedef typename KernelXprType::Nested RhsNested;
- typedef typename remove_reference<LhsNested>::type _LhsNested;
- typedef typename remove_reference<RhsNested>::type _RhsNested;
- static const int NumDimensions = traits<InputXprType>::NumDimensions;
- static const int Layout = traits<InputXprType>::Layout;
-
- enum {
- Flags = 0
- };
-};
-
-template<typename Dimensions, typename InputXprType, typename KernelXprType>
-struct eval<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>, Eigen::Dense>
-{
- typedef const TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>& type;
-};
-
-template<typename Dimensions, typename InputXprType, typename KernelXprType>
-struct nested<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>, 1, typename eval<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> >::type>
-{
- typedef TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> type;
-};
-
-} // end namespace internal
-
-
-
-template<typename Indices, typename InputXprType, typename KernelXprType>
-class TensorConvolutionOp : public TensorBase<TensorConvolutionOp<Indices, InputXprType, KernelXprType>, ReadOnlyAccessors>
-{
- public:
- typedef typename Eigen::internal::traits<TensorConvolutionOp>::Scalar Scalar;
- typedef typename Eigen::NumTraits<Scalar>::Real RealScalar;
- typedef typename internal::promote_storage_type<typename InputXprType::CoeffReturnType,
- typename KernelXprType::CoeffReturnType>::ret CoeffReturnType;
- typedef typename Eigen::internal::nested<TensorConvolutionOp>::type Nested;
- typedef typename Eigen::internal::traits<TensorConvolutionOp>::StorageKind StorageKind;
- typedef typename Eigen::internal::traits<TensorConvolutionOp>::Index Index;
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConvolutionOp(const InputXprType& input, const KernelXprType& kernel, const Indices& dims)
- : m_input_xpr(input), m_kernel_xpr(kernel), m_indices(dims) {}
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- const Indices& indices() const { return m_indices; }
-
- /** \returns the nested expressions */
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- const typename internal::remove_all<typename InputXprType::Nested>::type&
- inputExpression() const { return m_input_xpr; }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- const typename internal::remove_all<typename KernelXprType::Nested>::type&
- kernelExpression() const { return m_kernel_xpr; }
-
- protected:
- typename InputXprType::Nested m_input_xpr;
- typename KernelXprType::Nested m_kernel_xpr;
- const Indices m_indices;
-};
-
-
-template<typename Indices, typename InputArgType, typename KernelArgType, typename Device>
-struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelArgType>, Device>
-{
- typedef TensorConvolutionOp<Indices, InputArgType, KernelArgType> XprType;
-
- static const int NumDims = internal::array_size<typename TensorEvaluator<InputArgType, Device>::Dimensions>::value;
- static const int NumKernelDims = internal::array_size<Indices>::value;
- typedef typename XprType::Index Index;
- typedef DSizes<Index, NumDims> Dimensions;
-
- typedef typename XprType::Scalar Scalar;
- typedef typename XprType::CoeffReturnType CoeffReturnType;
- typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
- static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size;
-
- enum {
- IsAligned = TensorEvaluator<InputArgType, Device>::IsAligned & TensorEvaluator<KernelArgType, Device>::IsAligned,
- PacketAccess = TensorEvaluator<InputArgType, Device>::PacketAccess & TensorEvaluator<KernelArgType, Device>::PacketAccess,
- Layout = TensorEvaluator<InputArgType, Device>::Layout,
- CoordAccess = false, // to be implemented
- RawAccess = false
- };
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
- : m_inputImpl(op.inputExpression(), device), m_kernelImpl(op.kernelExpression(), device), m_kernelArg(op.kernelExpression()), m_kernel(NULL), m_local_kernel(false), m_device(device)
- {
- EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, Device>::Layout) == static_cast<int>(TensorEvaluator<KernelArgType, Device>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE);
-
- const typename TensorEvaluator<InputArgType, Device>::Dimensions& input_dims = m_inputImpl.dimensions();
- const typename TensorEvaluator<KernelArgType, Device>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
-
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- m_inputStride[0] = 1;
- for (int i = 1; i < NumDims; ++i) {
- m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1];
- }
- } else {
- m_inputStride[NumDims - 1] = 1;
- for (int i = NumDims - 2; i >= 0; --i) {
- m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1];
- }
- }
-
- m_dimensions = m_inputImpl.dimensions();
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int i = 0; i < NumKernelDims; ++i) {
- const Index index = op.indices()[i];
- const Index input_dim = input_dims[index];
- const Index kernel_dim = kernel_dims[i];
- const Index result_dim = input_dim - kernel_dim + 1;
- m_dimensions[index] = result_dim;
- if (i > 0) {
- m_kernelStride[i] = m_kernelStride[i - 1] * kernel_dims[i - 1];
- } else {
- m_kernelStride[0] = 1;
- }
- m_indexStride[i] = m_inputStride[index];
- }
-
- m_outputStride[0] = 1;
- for (int i = 1; i < NumDims; ++i) {
- m_outputStride[i] = m_outputStride[i - 1] * m_dimensions[i - 1];
- }
- } else {
- for (int i = NumKernelDims - 1; i >= 0; --i) {
- const Index index = op.indices()[i];
- const Index input_dim = input_dims[index];
- const Index kernel_dim = kernel_dims[i];
- const Index result_dim = input_dim - kernel_dim + 1;
- m_dimensions[index] = result_dim;
- if (i < NumKernelDims - 1) {
- m_kernelStride[i] = m_kernelStride[i + 1] * kernel_dims[i + 1];
- } else {
- m_kernelStride[NumKernelDims - 1] = 1;
- }
- m_indexStride[i] = m_inputStride[index];
- }
-
- m_outputStride[NumDims - 1] = 1;
- for (int i = NumDims - 2; i >= 0; --i) {
- m_outputStride[i] = m_outputStride[i + 1] * m_dimensions[i + 1];
- }
- }
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~TensorEvaluator() {}
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar*) {
- m_inputImpl.evalSubExprsIfNeeded(NULL);
- preloadKernel();
- return true;
- }
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() {
- m_inputImpl.cleanup();
- if (m_local_kernel) {
- m_device.deallocate((void*)m_kernel);
- m_local_kernel = false;
- }
- m_kernel = NULL;
- }
-
- void evalTo(typename XprType::Scalar* buffer) {
- evalSubExprsIfNeeded(NULL);
- for (int i = 0; i < dimensions().TotalSize(); ++i) {
- buffer[i] += coeff(i);
- }
- cleanup();
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const
- {
- CoeffReturnType result = CoeffReturnType(0);
- convolve(firstInput(index), 0, NumKernelDims-1, result);
- return result;
- }
-
- template<int LoadMode>
- EIGEN_DEVICE_FUNC PacketReturnType packet(const Index index) const
- {
- Index indices[2] = {index, index+PacketSize-1};
- Index startInputs[2] = {0, 0};
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int i = NumDims - 1; i > 0; --i) {
- const Index idx0 = indices[0] / m_outputStride[i];
- const Index idx1 = indices[1] / m_outputStride[i];
- startInputs[0] += idx0 * m_inputStride[i];
- startInputs[1] += idx1 * m_inputStride[i];
- indices[0] -= idx0 * m_outputStride[i];
- indices[1] -= idx1 * m_outputStride[i];
- }
- } else {
- for (int i = 0; i < NumDims - 1; ++i) {
- const Index idx0 = indices[0] / m_outputStride[i];
- const Index idx1 = indices[1] / m_outputStride[i];
- startInputs[0] += idx0 * m_inputStride[i];
- startInputs[1] += idx1 * m_inputStride[i];
- indices[0] -= idx0 * m_outputStride[i];
- indices[1] -= idx1 * m_outputStride[i];
- }
- }
- startInputs[0] += indices[0];
- startInputs[1] += indices[1];
-
- if (startInputs[1]-startInputs[0] == PacketSize-1) {
- PacketReturnType result = internal::pset1<PacketReturnType>(0);
- convolvePacket(startInputs[0], 0, NumKernelDims-1, result);
- return result;
- } else {
- EIGEN_ALIGN_MAX Scalar data[PacketSize];
- data[0] = Scalar(0);
- convolve(startInputs[0], 0, NumKernelDims-1, data[0]);
- for (int i = 1; i < PacketSize-1; ++i) {
- data[i] = Scalar(0);
- convolve(firstInput(index+i), 0, NumKernelDims-1, data[i]);
- }
- data[PacketSize-1] = Scalar(0);
- convolve(startInputs[1], 0, NumKernelDims-1, data[PacketSize-1]);
- return internal::pload<PacketReturnType>(data);
- }
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost
- costPerCoeff(bool vectorized) const {
- const double kernel_size = m_kernelImpl.dimensions().TotalSize();
- // We ignore the use of fused multiply-add.
- const double convolve_compute_cost =
- TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
- const double firstIndex_compute_cost =
- NumDims *
- (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
- TensorOpCost::DivCost<Index>());
- return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
- kernel_size * (m_inputImpl.costPerCoeff(vectorized) +
- m_kernelImpl.costPerCoeff(vectorized) +
- TensorOpCost(0, 0, convolve_compute_cost, vectorized,
- PacketSize));
- }
-
- EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; }
-
- private:
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
- Index startInput = 0;
- if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
- for (int i = NumDims - 1; i > 0; --i) {
- const Index idx = index / m_outputStride[i];
- startInput += idx * m_inputStride[i];
- index -= idx * m_outputStride[i];
- }
- } else {
- for (int i = 0; i < NumDims - 1; ++i) {
- const Index idx = index / m_outputStride[i];
- startInput += idx * m_inputStride[i];
- index -= idx * m_outputStride[i];
- }
- }
- startInput += index;
- return startInput;
- }
-
- EIGEN_DEVICE_FUNC void convolve(Index firstIndex, Index firstKernel, int DimIndex, CoeffReturnType& accum) const {
- for (int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
- const Index input = firstIndex + j * m_indexStride[DimIndex];
- const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
- if (DimIndex > 0) {
- convolve(input, kernel, DimIndex-1, accum);
- } else {
- accum += m_inputImpl.coeff(input) * m_kernel[kernel];
- }
- }
- }
-
- template <typename Packet>
- EIGEN_DEVICE_FUNC void convolvePacket(Index firstIndex, Index firstKernel, int DimIndex, Packet& accum) const {
- for (int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
- const Index input = firstIndex + j * m_indexStride[DimIndex];
- const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
- if (DimIndex > 0) {
- convolvePacket(input, kernel, DimIndex-1, accum);
- } else {
- accum = internal::pmadd<Packet>(m_inputImpl.template packet<Unaligned>(input), internal::pset1<Packet>(m_kernel[kernel]), accum);
- }
- }
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void preloadKernel() {
- // Don't make a local copy of the kernel unless we have to (i.e. it's an
- // expression that needs to be evaluated)
- const Scalar* in_place = m_kernelImpl.data();
- if (in_place) {
- m_kernel = in_place;
- m_local_kernel = false;
- } else {
- size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() * sizeof(Scalar);
- Scalar* local = (Scalar*)m_device.allocate(kernel_sz);
- typedef TensorEvalToOp<const KernelArgType> EvalTo;
- EvalTo evalToTmp(local, m_kernelArg);
- const bool PacketAccess = internal::IsVectorizable<Device, KernelArgType>::value;
- internal::TensorExecutor<const EvalTo, Device, PacketAccess>::run(evalToTmp, m_device);
-
- m_kernel = local;
- m_local_kernel = true;
- }
- }
-
- array<Index, NumDims> m_inputStride;
- array<Index, NumDims> m_outputStride;
-
- array<Index, NumKernelDims> m_indexStride;
- array<Index, NumKernelDims> m_kernelStride;
- TensorEvaluator<InputArgType, Device> m_inputImpl;
- TensorEvaluator<KernelArgType, Device> m_kernelImpl;
- Dimensions m_dimensions;
-
- KernelArgType m_kernelArg;
- const Scalar* m_kernel;
- bool m_local_kernel;
- const Device& m_device;
-};
-
-
-
-
-// Use an optimized implementation of the evaluation code for GPUs whenever possible.
-#if defined(EIGEN_USE_GPU) && defined(EIGEN_HIPCC)
-
-template <int StaticKernelSize>
-struct GetKernelSize {
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int operator() (const int /*kernelSize*/) const {
- return StaticKernelSize;
- }
-};
-template <>
-struct GetKernelSize<Dynamic> {
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int operator() (const int kernelSize) const {
- return kernelSize;
- }
-};
-
-template <typename InputEvaluator, typename Index, typename InputDims,
- int StaticKernelSize>
-__global__ void EigenConvolutionKernel1D(
- InputEvaluator eval,
- const internal::IndexMapper<Index, InputDims, 1, InputEvaluator::Layout>
- indexMapper,
- const float* __restrict kernel, const int numPlanes, const int numX,
- const int maxX, const int kernelSize, float* buffer) {
- HIP_DYNAMIC_SHARED( float, s)
-
- const int first_x = hipBlockIdx_x * maxX;
- const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
- const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSize>()(kernelSize);
- const int num_x_output = last_x - first_x + 1;
-
- const int first_plane = hipBlockIdx_y * hipBlockDim_y;
- const int plane_stride = hipBlockDim_y * hipGridDim_y;
-
- for (int p = first_plane + hipThreadIdx_y; p < numPlanes; p += plane_stride) {
- // Load inputs to shared memory
- const int plane_input_offset = indexMapper.mapHipInputPlaneToTensorInputOffset(p);
- const int plane_kernel_offset = hipThreadIdx_y * num_x_input;
- #pragma unroll
- for (int i = hipThreadIdx_x; i < num_x_input; i += hipBlockDim_x) {
- const int tensor_index = plane_input_offset + indexMapper.mapHipInputKernelToTensorInputOffset(i+first_x);
- s[i + plane_kernel_offset] = eval.coeff(tensor_index);
- }
-
- __syncthreads();
-
- // Compute the convolution
- const int plane_output_offset = indexMapper.mapHipOutputPlaneToTensorOutputOffset(p);
-
- #pragma unroll
- for (int i = hipThreadIdx_x; i < num_x_output; i += hipBlockDim_x) {
- const int kernel_offset = plane_kernel_offset + i;
- float result = 0.0f;
- #pragma unroll
- for (int k = 0; k < GetKernelSize<StaticKernelSize>()(kernelSize); ++k) {
- result += s[k + kernel_offset] * kernel[k];
- }
- const int tensor_index = plane_output_offset + indexMapper.mapHipOutputKernelToTensorOutputOffset(i+first_x);
- buffer[tensor_index] = result;
- }
- __syncthreads();
- }
-};
-
-template <typename InputEvaluator, typename Index, typename InputDims,
- int StaticKernelSizeX, int StaticKernelSizeY>
-__global__ void EigenConvolutionKernel2D(
- InputEvaluator eval,
- const internal::IndexMapper<Index, InputDims, 2, InputEvaluator::Layout>
- indexMapper,
- const float* __restrict kernel, const int numPlanes, const int numX,
- const int maxX, const int numY, const int maxY, const int kernelSizeX,
- const int kernelSizeY, float* buffer) {
- HIP_DYNAMIC_SHARED( float, s)
-
- const int first_x = hipBlockIdx_x * maxX;
- const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
- const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSizeX>()(kernelSizeX);
- const int num_x_output = last_x - first_x + 1;
-
- const int first_y = hipBlockIdx_y * maxY;
- const int last_y = (first_y + maxY < numY ? first_y + maxY : numY) - 1;
- const int num_y_input = last_y - first_y + GetKernelSize<StaticKernelSizeY>()(kernelSizeY);
- const int num_y_output = last_y - first_y + 1;
-
- const int first_plane = hipBlockIdx_z * hipBlockDim_z;
- const int plane_stride = hipBlockDim_z * hipGridDim_z;
-
- for (int p = first_plane + hipThreadIdx_z; p < numPlanes; p += plane_stride) {
-
- const int plane_input_offset = indexMapper.mapHipInputPlaneToTensorInputOffset(p);
- const int plane_kernel_offset = hipThreadIdx_z * num_y_input;
-
- // Load inputs to shared memory
- #pragma unroll
- for (int j = hipThreadIdx_y; j < num_y_input; j += hipBlockDim_y) {
- const int input_offset = num_x_input * (j + plane_kernel_offset);
- #pragma unroll
- for (int i = hipThreadIdx_x; i < num_x_input; i += hipBlockDim_x) {
- const int tensor_index = plane_input_offset + indexMapper.mapHipInputKernelToTensorInputOffset(i+first_x, j+first_y);
- s[i + input_offset] = eval.coeff(tensor_index);
- }
- }
-
- __syncthreads();
-
- // Convolution
- const int plane_output_offset = indexMapper.mapHipOutputPlaneToTensorOutputOffset(p);
-
- #pragma unroll
- for (int j = hipThreadIdx_y; j < num_y_output; j += hipBlockDim_y) {
- #pragma unroll
- for (int i = hipThreadIdx_x; i < num_x_output; i += hipBlockDim_x) {
- float result = 0.0f;
- #pragma unroll
- for (int l = 0; l < GetKernelSize<StaticKernelSizeY>()(kernelSizeY); ++l) {
- const int kernel_offset = kernelSizeX * l;
- const int input_offset = i + num_x_input * (j + l + plane_kernel_offset);
- #pragma unroll
- for (int k = 0; k < GetKernelSize<StaticKernelSizeX>()(kernelSizeX); ++k) {
- result += s[k + input_offset] * kernel[k + kernel_offset];
- }
- }
- const int tensor_index = plane_output_offset + indexMapper.mapHipOutputKernelToTensorOutputOffset(i+first_x, j+first_y);
- buffer[tensor_index] = result;
- }
- }
-
- __syncthreads();
- }
-};
-
-template <typename InputEvaluator, typename Index, typename InputDims>
-__global__ void EigenConvolutionKernel3D(
- InputEvaluator eval,
- const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout>
- indexMapper,
- const float* __restrict kernel, const size_t numPlanes, const size_t numX,
- const size_t maxX, const size_t numY, const size_t maxY, const size_t numZ,
- const size_t maxZ, const size_t kernelSizeX, const size_t kernelSizeY,
- const size_t kernelSizeZ, float* buffer) {
- HIP_DYNAMIC_SHARED( float, s)
-
- // Load inputs to shared memory
- const int first_x = hipBlockIdx_x * maxX;
- const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
- const int num_x_input = last_x - first_x + kernelSizeX;
-
- const int first_y = hipBlockIdx_y * maxY;
- const int last_y = (first_y + maxY < numY ? first_y + maxY : numY) - 1;
- const int num_y_input = last_y - first_y + kernelSizeY;
-
- const int first_z = hipBlockIdx_z * maxZ;
- const int last_z = (first_z + maxZ < numZ ? first_z + maxZ : numZ) - 1;
- const int num_z_input = last_z - first_z + kernelSizeZ;
-
- for (int p = 0; p < numPlanes; ++p) {
-
- const int plane_input_offset = indexMapper.mapHipInputPlaneToTensorInputOffset(p);
- const int plane_kernel_offset = 0;
-
- for (int k = hipThreadIdx_z; k < num_z_input; k += hipBlockDim_z) {
- for (int j = hipThreadIdx_y; j < num_y_input; j += hipBlockDim_y) {
- for (int i = hipThreadIdx_x; i < num_x_input; i += hipBlockDim_x) {
- const int tensor_index = plane_input_offset + indexMapper.mapHipInputKernelToTensorInputOffset(i+first_x, j+first_y, k+first_z);
- s[i + num_x_input * (j + num_y_input * (k + plane_kernel_offset))] = eval.coeff(tensor_index);
- }
- }
- }
-
- __syncthreads();
-
- // Convolution
- const int num_z_output = last_z - first_z + 1;
- const int num_y_output = last_y - first_y + 1;
- const int num_x_output = last_x - first_x + 1;
- const int plane_output_offset = indexMapper.mapHipOutputPlaneToTensorOutputOffset(p);
-
- for (int k = hipThreadIdx_z; k < num_z_output; k += hipBlockDim_z) {
- for (int j = hipThreadIdx_y; j < num_y_output; j += hipBlockDim_y) {
- for (int i = hipThreadIdx_x; i < num_x_output; i += hipBlockDim_x) {
- float result = 0.0f;
- for (int n = 0; n < kernelSizeZ; ++n) {
- for (int m = 0; m < kernelSizeY; ++m) {
- for (int l = 0; l < kernelSizeX; ++l) {
- result += s[i + l + num_x_input * (j + m + num_y_input * (k + n + plane_kernel_offset))] * kernel[l + kernelSizeX * (m + kernelSizeY * n)];
- }
- }
- }
- const int tensor_index = plane_output_offset + indexMapper.mapHipOutputKernelToTensorOutputOffset(i+first_x, j+first_y, k+first_z);
- buffer[tensor_index] = result;
- }
- }
- }
- __syncthreads();
- }
-};
-
-
-
-template<typename Indices, typename InputArgType, typename KernelArgType>
-struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelArgType>, GpuDevice>
-{
- typedef TensorConvolutionOp<Indices, InputArgType, KernelArgType> XprType;
-
- static const int NumDims = internal::array_size<typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions>::value;
- static const int NumKernelDims = internal::array_size<Indices>::value;
- typedef typename XprType::Index Index;
- typedef DSizes<Index, NumDims> Dimensions;
- typedef typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions KernelDimensions;
-
- enum {
- IsAligned = TensorEvaluator<InputArgType, GpuDevice>::IsAligned & TensorEvaluator<KernelArgType, GpuDevice>::IsAligned,
- PacketAccess = false,
- Layout = TensorEvaluator<InputArgType, GpuDevice>::Layout,
- CoordAccess = false, // to be implemented
- RawAccess = false
- };
-
- EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const GpuDevice& device)
- : m_inputImpl(op.inputExpression(), device), m_kernelArg(op.kernelExpression()), m_kernelImpl(op.kernelExpression(), device), m_indices(op.indices()), m_buf(NULL), m_kernel(NULL), m_local_kernel(false), m_device(device)
- {
- EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, GpuDevice>::Layout) == static_cast<int>(TensorEvaluator<KernelArgType, GpuDevice>::Layout)), YOU_MADE_A_PROGRAMMING_MISTAKE);
-
- const typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions& input_dims = m_inputImpl.dimensions();
- const typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
-
- m_dimensions = m_inputImpl.dimensions();
- for (int i = 0; i < NumKernelDims; ++i) {
- const Index index = op.indices()[i];
- const Index input_dim = input_dims[index];
- const Index kernel_dim = kernel_dims[i];
- const Index result_dim = input_dim - kernel_dim + 1;
- m_dimensions[index] = result_dim;
- }
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~TensorEvaluator() {}
-
- typedef typename XprType::CoeffReturnType CoeffReturnType;
- typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType;
- typedef typename InputArgType::Scalar Scalar;
- static const int PacketSize = internal::unpacket_traits<PacketReturnType>::size;
-
- EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_dimensions; }
-
- EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) {
- preloadKernel();
- m_inputImpl.evalSubExprsIfNeeded(NULL);
- if (data) {
- executeEval(data);
- return false;
- } else {
- m_buf = (Scalar*)m_device.allocate(dimensions().TotalSize() * sizeof(Scalar));
- executeEval(m_buf);
- return true;
- }
- }
-
- EIGEN_STRONG_INLINE void cleanup() {
- m_inputImpl.cleanup();
- if (m_buf) {
- m_device.deallocate(m_buf);
- m_buf = NULL;
- }
- if (m_local_kernel) {
- m_device.deallocate((void*)m_kernel);
- m_local_kernel = false;
- }
- m_kernel = NULL;
- }
-
- EIGEN_STRONG_INLINE void preloadKernel() {
- // Don't make a local copy of the kernel unless we have to (i.e. it's an
- // expression that needs to be evaluated)
- const Scalar* in_place = m_kernelImpl.data();
- if (in_place) {
- m_kernel = in_place;
- m_local_kernel = false;
- } else {
- size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() * sizeof(Scalar);
- Scalar* local = (Scalar*)m_device.allocate(kernel_sz);
- typedef TensorEvalToOp<const KernelArgType> EvalTo;
- EvalTo evalToTmp(local, m_kernelArg);
- const bool PacketAccess = internal::IsVectorizable<GpuDevice, KernelArgType>::value;
- internal::TensorExecutor<const EvalTo, GpuDevice, PacketAccess>::run(evalToTmp, m_device);
-
- m_kernel = local;
- m_local_kernel = true;
- }
- }
-
- static unsigned int ceil(unsigned int num, unsigned int denom) {
- const unsigned int rounded_toward_zero = num / denom;
- if (num > rounded_toward_zero * denom) {
- return rounded_toward_zero + 1;
- }
- return rounded_toward_zero;
- }
-
- void executeEval(Scalar* data) const {
- typedef typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions InputDims;
-
- const int maxSharedMem = m_device.sharedMemPerBlock();
- const int maxThreadsPerBlock = m_device.maxHipThreadsPerBlock();
- const int maxBlocksPerProcessor = m_device.maxHipThreadsPerMultiProcessor() / maxThreadsPerBlock;
- const int numMultiProcessors = m_device.getNumHipMultiProcessors();
- const int hipWarpSize = 32;
-
- switch (NumKernelDims) {
- case 1: {
- const int kernel_size = m_kernelImpl.dimensions().TotalSize();
-
- const int numX = dimensions()[m_indices[0]];
- const int numP = dimensions().TotalSize() / numX;
- int maxX;
- dim3 block_size;
-
- const int single_stride_dim =
- static_cast<int>(Layout) == static_cast<int>(ColMajor)
- ? 0
- : m_inputImpl.dimensions().rank() - 1;
- if (m_indices[0] == single_stride_dim) {
- // Maximum the reuse
- const int inner_dim = ((maxSharedMem / (sizeof(Scalar)) - kernel_size + 1 + 31) / 32) * 32;
- maxX = numext::mini<int>(inner_dim, numX);
- const int maxP = numext::mini<int>(maxSharedMem / ((kernel_size - 1 + maxX) * sizeof(Scalar)), numP);
- block_size.x = numext::mini(maxThreadsPerBlock, maxX);
- block_size.y = numext::mini<int>(maxThreadsPerBlock / block_size.x, maxP);
- }
- else {
- // Read as much as possible alongside the inner most dimension, that is the plane
- const int inner_dim = maxSharedMem / ((hipWarpSize + kernel_size) * sizeof(Scalar));
- const int maxP = numext::mini<int>(inner_dim, numP);
- maxX = numext::mini<int>(maxSharedMem / (inner_dim * sizeof(Scalar)) - kernel_size + 1, numX);
-
- block_size.x = numext::mini(hipWarpSize, maxX);
- block_size.y = numext::mini<int>(maxThreadsPerBlock/block_size.x, maxP);
- }
-
- const int shared_mem = block_size.y * (maxX + kernel_size - 1) * sizeof(Scalar);
- assert(shared_mem <= maxSharedMem);
-
- const int num_x_blocks = ceil(numX, maxX);
- const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
- const int num_y_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks);
-
- dim3 num_blocks(num_x_blocks, numext::mini<int>(num_y_blocks, ceil(numP, block_size.y)));
-
-
- //cout << "launching 1D kernel with block_size.x: " << block_size.x << " block_size.y: " << block_size.y << " num_blocks.x: " << num_blocks.x << " num_blocks.y: " << num_blocks.y << " maxX: " << maxX << " shared_mem: " << shared_mem << " in stream " << m_device.stream() << endl;
-
- const array<Index, 1> indices(m_indices[0]);
- const array<Index, 1> kernel_dims(m_kernelImpl.dimensions()[0]);
- internal::IndexMapper<Index, InputDims, 1, Layout> indexMapper(
- m_inputImpl.dimensions(), kernel_dims, indices);
- switch(kernel_size) {
- case 4: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 4, data);
- break;
- }
- case 7: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, 7, data);
- break;
- }
- default: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, kernel_size, data);
- }
- }
- break;
- }
-
- case 2: {
- const int idxX =
- static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 1;
- const int idxY =
- static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 0;
- const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
- const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
-
- const int numX = dimensions()[m_indices[idxX]];
- const int numY = dimensions()[m_indices[idxY]];
- const int numP = dimensions().TotalSize() / (numX*numY);
-
- const float scaling_factor = sqrtf(static_cast<float>(maxSharedMem) / (sizeof(Scalar) * kernel_size_y * kernel_size_x));
-
- // Snap maxX to warp size
- int inner_dim = ((static_cast<int>(scaling_factor * kernel_size_x) - kernel_size_x + 1 + 32) / 32) * 32;
- const int maxX = numext::mini<int>(inner_dim, numX);
- const int maxY = numext::mini<int>(maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1)) - kernel_size_y + 1, numY);
- const int maxP = numext::mini<int>(maxSharedMem / ((kernel_size_x - 1 + maxX) * (kernel_size_y - 1 + maxY) * sizeof(Scalar)), numP);
-
- dim3 block_size;
- block_size.x = numext::mini(1024, maxX);
- block_size.y = numext::mini<int>(1024/block_size.x, maxY);
- block_size.z = numext::mini<int>(1024/(block_size.x*block_size.y), maxP);
-
- const int shared_mem = block_size.z * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * sizeof(Scalar);
- assert(shared_mem <= maxSharedMem);
-
- const int num_x_blocks = ceil(numX, maxX);
- const int num_y_blocks = ceil(numY, maxY);
- const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
- const int num_z_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks * num_y_blocks);
-
- dim3 num_blocks(num_x_blocks, num_y_blocks, numext::mini<int>(num_z_blocks, ceil(numP, block_size.z)));
-
-
- //cout << "launching 2D kernel with block_size.x: " << block_size.x << " block_size.y: " << block_size.y << " block_size.z: " << block_size.z << " num_blocks.x: " << num_blocks.x << " num_blocks.y: " << num_blocks.y << " num_blocks.z: " << num_blocks.z << " maxX: " << maxX << " maxY: " << maxY << " maxP: " << maxP << " shared_mem: " << shared_mem << " in stream " << m_device.stream() << endl;
-
- const array<Index, 2> indices(m_indices[idxX], m_indices[idxY]);
- const array<Index, 2> kernel_dims(m_kernelImpl.dimensions()[idxX],
- m_kernelImpl.dimensions()[idxY]);
- internal::IndexMapper<Index, InputDims, 2, Layout> indexMapper(
- m_inputImpl.dimensions(), kernel_dims, indices);
- switch (kernel_size_x) {
- case 4: {
- switch (kernel_size_y) {
- case 7: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, 7>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, 7, data);
- break;
- }
- default: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, Dynamic>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 4, kernel_size_y, data);
- break;
- }
- }
- break;
- }
- case 7: {
- switch (kernel_size_y) {
- case 4: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, 4>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, 4, data);
- break;
- }
- default: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, Dynamic>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, 7, kernel_size_y, data);
- break;
- }
- }
- break;
- }
- default: {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic, Dynamic>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel, numP, numX, maxX, numY, maxY, kernel_size_x, kernel_size_y, data);
- break;
- }
- }
- break;
- }
-
- case 3: {
- const int idxX =
- static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 2;
- const int idxY =
- static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 1;
- const int idxZ =
- static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 0;
-
- const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
- const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
- const int kernel_size_z = m_kernelImpl.dimensions()[idxZ];
-
- const int numX = dimensions()[m_indices[idxX]];
- const int numY = dimensions()[m_indices[idxY]];
- const int numZ = dimensions()[m_indices[idxZ]];
- const int numP = dimensions().TotalSize() / (numX*numY*numZ);
-
- const int maxX = numext::mini<int>(128, numext::mini<int>(maxSharedMem / (sizeof(Scalar) * kernel_size_y * kernel_size_z) - kernel_size_x + 1, numX));
- const int maxY = numext::mini<int>(128, numext::mini<int>(maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1) * kernel_size_z) - kernel_size_y + 1, numY));
- const int maxZ = numext::mini<int>(128, numext::mini<int>(maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1)) - kernel_size_z + 1, numZ));
-
- dim3 block_size;
- block_size.x = numext::mini(32, maxX);
- block_size.y = numext::mini(32, maxY);
- block_size.z = numext::mini<int>(1024/(block_size.x*block_size.y), maxZ);
- dim3 num_blocks(ceil(numX, maxX), ceil(numY, maxY), ceil(numZ, maxZ));
-
- const int shared_mem = (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * (maxZ + kernel_size_z - 1) * sizeof(Scalar);
- assert(shared_mem <= maxSharedMem);
-
- //cout << "launching 3D kernel with block_size.x: " << block_size.x << " block_size.y: " << block_size.y << " block_size.z: " << block_size.z << " num_blocks.x: " << num_blocks.x << " num_blocks.y: " << num_blocks.y << " num_blocks.z: " << num_blocks.z << " shared_mem: " << shared_mem << " in stream " << m_device.stream() << endl;
- const array<Index, 3> indices(m_indices[idxX], m_indices[idxY],
- m_indices[idxZ]);
- const array<Index, 3> kernel_dims(m_kernelImpl.dimensions()[idxX],
- m_kernelImpl.dimensions()[idxY],
- m_kernelImpl.dimensions()[idxZ]);
- internal::IndexMapper<Index, InputDims, 3, Layout> indexMapper(
- m_inputImpl.dimensions(), kernel_dims, indices);
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(EigenConvolutionKernel3D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>),
- dim3(num_blocks), dim3(block_size), shared_mem, m_device.stream(), m_inputImpl, indexMapper, m_kernel,
- numP, numX, maxX, numY, maxY, numZ, maxZ, kernel_size_x, kernel_size_y, kernel_size_z, data);
- break;
- }
-
- default: {
- EIGEN_STATIC_ASSERT((NumKernelDims >= 1 && NumKernelDims <= 3), THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE);
- }
- }
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const
- {
- eigen_assert(m_buf);
- eigen_assert(index < m_dimensions.TotalSize());
- return m_buf[index];
- }
-
- template<int LoadMode>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(const Index index) const
- {
- eigen_assert(m_buf);
- eigen_assert(index < m_dimensions.TotalSize());
- return internal::ploadt<PacketReturnType, LoadMode>(m_buf+index);
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost
- costPerCoeff(bool vectorized) const {
- // TODO(rmlarsen): FIXME: For now, this is just a copy of the CPU cost
- // model.
- const double kernel_size = m_kernelImpl.dimensions().TotalSize();
- // We ignore the use of fused multiply-add.
- const double convolve_compute_cost =
- TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
- const double firstIndex_compute_cost =
- NumDims *
- (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
- TensorOpCost::DivCost<Index>());
- return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
- kernel_size * (m_inputImpl.costPerCoeff(vectorized) +
- m_kernelImpl.costPerCoeff(vectorized) +
- TensorOpCost(0, 0, convolve_compute_cost, vectorized,
- PacketSize));
- }
-
- private:
- // No assignment (copies are needed by the kernels)
- TensorEvaluator& operator = (const TensorEvaluator&);
-
- TensorEvaluator<InputArgType, GpuDevice> m_inputImpl;
- TensorEvaluator<KernelArgType, GpuDevice> m_kernelImpl;
- KernelArgType m_kernelArg;
- Indices m_indices;
- Dimensions m_dimensions;
- Scalar* m_buf;
- const Scalar* m_kernel;
- bool m_local_kernel;
-
- const GpuDevice& m_device;
-};
-#endif
-
-
-} // end namespace Eigen
-
-#endif // EIGEN_CXX11_TENSOR_TENSOR_CONVOLUTION_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceHip.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceHip.h
deleted file mode 100644
index c0e1109..0000000
--- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceHip.h
+++ /dev/null
@@ -1,352 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_HIP_H)
-#define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_HIP_H
-
-#if defined(EIGEN_HIPCC)
-#include "hip/hip_runtime.h"
-#include "hip/hip_runtime_api.h"
-#endif
-#include <unistd.h> //for sleep function
-
-namespace Eigen {
-
-static const int kHipScratchSize = 1024;
-
-// This defines an interface that GPUDevice can take to use
-// HIP streams underneath.
-class StreamInterface {
- public:
- virtual ~StreamInterface() {}
-
- virtual const hipStream_t& stream() const = 0;
- virtual const hipDeviceProp_t& deviceProperties() const = 0;
-
- // Allocate memory on the actual device where the computation will run
- virtual void* allocate(size_t num_bytes) const = 0;
- virtual void deallocate(void* buffer) const = 0;
-
- // Return a scratchpad buffer of size 1k
- virtual void* scratchpad() const = 0;
-
- // Return a semaphore. The semaphore is initially initialized to 0, and
- // each kernel using it is responsible for resetting to 0 upon completion
- // to maintain the invariant that the semaphore is always equal to 0 upon
- // each kernel start.
- virtual unsigned int* semaphore() const = 0;
-};
-
-static hipDeviceProp_t* m_deviceProperties;
-static bool m_devicePropInitialized = false;
-
-static void initializeDeviceProp() {
- if (!m_devicePropInitialized) {
- // Attempts to ensure proper behavior in the case of multiple threads
- // calling this function simultaneously. This would be trivial to
- // implement if we could use std::mutex, but unfortunately mutex don't
- // compile with nvcc, so we resort to atomics and thread fences instead.
- // Note that if the caller uses a compiler that doesn't support c++11 we
- // can't ensure that the initialization is thread safe.
-#if 0 && __cplusplus >= 201103L
- static std::atomic<bool> first(true);
- if (first.exchange(false)) {
-#else
- static bool first = true;
- if (first) {
- first = false;
-#endif
- // We're the first thread to reach this point.
- int num_devices;
- hipError_t status = hipGetDeviceCount(&num_devices);
- if (status != hipSuccess) {
- std::cerr << "Failed to get the number of HIP devices: "
- << hipGetErrorString(status)
- << std::endl;
- assert(status == hipSuccess);
- }
- m_deviceProperties = new hipDeviceProp_t[num_devices];
- for (int i = 0; i < num_devices; ++i) {
- status = hipGetDeviceProperties(&m_deviceProperties[i], i);
- if (status != hipSuccess) {
- std::cerr << "Failed to initialize HIP device #"
- << i
- << ": "
- << hipGetErrorString(status)
- << std::endl;
- assert(status == hipSuccess);
- }
- }
-
-#if 0 && __cplusplus >= 201103L
- std::atomic_thread_fence(std::memory_order_release);
-#endif
- m_devicePropInitialized = true;
- } else {
- // Wait for the other thread to inititialize the properties.
- while (!m_devicePropInitialized) {
-#if 0 && __cplusplus >= 201103L
- std::atomic_thread_fence(std::memory_order_acquire);
-#endif
- sleep(1);
- }
- }
- }
-}
-
-static const hipStream_t default_stream = 0x00;//TODO: Use hipStreamDefault instead of 0x00;
-
-class HipStreamDevice : public StreamInterface {
- public:
- // Use the default stream on the current device
- HipStreamDevice() : stream_(&default_stream), scratch_(NULL), semaphore_(NULL) {
- hipGetDevice(&device_);
- initializeDeviceProp();
- }
- // Use the default stream on the specified device
- HipStreamDevice(int device) : stream_(&default_stream), device_(device), scratch_(NULL), semaphore_(NULL) {
- initializeDeviceProp();
- }
- // Use the specified stream. Note that it's the
- // caller responsibility to ensure that the stream can run on
- // the specified device. If no device is specified the code
- // assumes that the stream is associated to the current gpu device.
- HipStreamDevice(const hipStream_t* stream, int device = -1)
- : stream_(stream), device_(device), scratch_(NULL), semaphore_(NULL) {
- if (device < 0) {
- hipGetDevice(&device_);
- } else {
- int num_devices;
- hipError_t err = hipGetDeviceCount(&num_devices);
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- assert(device < num_devices);
- device_ = device;
- }
- initializeDeviceProp();
- }
-
- virtual ~HipStreamDevice() {
- if (scratch_) {
- deallocate(scratch_);
- }
- }
-
- const hipStream_t& stream() const { return *stream_; }
- const hipDeviceProp_t& deviceProperties() const {
- return m_deviceProperties[device_];
- }
- virtual void* allocate(size_t num_bytes) const {
- hipError_t err = hipSetDevice(device_);
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- void* result;
- err = hipMalloc(&result, num_bytes);
- assert(err == hipSuccess);
- assert(result != NULL);
- return result;
- }
- virtual void deallocate(void* buffer) const {
- hipError_t err = hipSetDevice(device_);
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- assert(buffer != NULL);
- err = hipFree(buffer);
- assert(err == hipSuccess);
- }
-
- virtual void* scratchpad() const {
- if (scratch_ == NULL) {
- scratch_ = allocate(kHipScratchSize + sizeof(unsigned int));
- }
- return scratch_;
- }
-
- virtual unsigned int* semaphore() const {
- if (semaphore_ == NULL) {
- char* scratch = static_cast<char*>(scratchpad()) + kHipScratchSize;
- semaphore_ = reinterpret_cast<unsigned int*>(scratch);
- //hipError_t err = hipMemsetAsync(semaphore_, 0, sizeof(unsigned int), *stream_);
- hipError_t err = hipMemset(semaphore_, 0, sizeof(unsigned int));
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- }
- return semaphore_;
- }
-
- private:
- const hipStream_t* stream_;
- int device_;
- mutable void* scratch_;
- mutable unsigned int* semaphore_;
-};
-
-struct GpuDevice {
- // The StreamInterface is not owned: the caller is
- // responsible for its initialization and eventual destruction.
- explicit GpuDevice(const StreamInterface* stream) : stream_(stream), max_blocks_(INT_MAX) {
- eigen_assert(stream);
- }
- explicit GpuDevice(const StreamInterface* stream, int num_blocks) : stream_(stream), max_blocks_(num_blocks) {
- eigen_assert(stream);
- }
- // TODO(bsteiner): This is an internal API, we should not expose it.
- EIGEN_STRONG_INLINE const hipStream_t& stream() const {
- return stream_->stream();
- }
-
- EIGEN_STRONG_INLINE void* allocate(size_t num_bytes) const {
- return stream_->allocate(num_bytes);
- }
-
- EIGEN_STRONG_INLINE void deallocate(void* buffer) const {
- stream_->deallocate(buffer);
- }
-
- EIGEN_STRONG_INLINE void* scratchpad() const {
- return stream_->scratchpad();
- }
-
- EIGEN_STRONG_INLINE unsigned int* semaphore() const {
- return stream_->semaphore();
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memcpy(void* dst, const void* src, size_t n) const {
-#if !defined(EIGEN_HIP_DEVICE_COMPILE)
- hipError_t err = hipMemcpyAsync(dst, src, n, hipMemcpyDeviceToDevice,
- stream_->stream());
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
-#else
- eigen_assert(false && "The default device should be used instead to generate kernel code");
-#endif
- }
-
- EIGEN_STRONG_INLINE void memcpyHostToDevice(void* dst, const void* src, size_t n) const {
- hipError_t err =
- hipMemcpyAsync(dst, src, n, hipMemcpyHostToDevice, stream_->stream());
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- }
-
- EIGEN_STRONG_INLINE void memcpyDeviceToHost(void* dst, const void* src, size_t n) const {
- hipError_t err =
- hipMemcpyAsync(dst, src, n, hipMemcpyDeviceToHost, stream_->stream());
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
- }
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void memset(void* buffer, int c, size_t n) const {
-#if !defined(EIGEN_HIP_DEVICE_COMPILE)
- //TODO:hipError_t err = hipMemsetAsync(buffer, c, n, stream_->stream());
- hipError_t err = hipMemset(buffer, c, n);
- EIGEN_UNUSED_VARIABLE(err)
- assert(err == hipSuccess);
-#else
- eigen_assert(false && "The default device should be used instead to generate kernel code");
-#endif
- }
-
- EIGEN_STRONG_INLINE size_t numThreads() const {
- // FIXME
- return 32;
- }
-
- EIGEN_STRONG_INLINE size_t firstLevelCacheSize() const {
- // FIXME
- return 48*1024;
- }
-
- EIGEN_STRONG_INLINE size_t lastLevelCacheSize() const {
- // We won't try to take advantage of the l2 cache for the time being, and
- // there is no l3 cache on hip devices.
- return firstLevelCacheSize();
- }
-
-// FIXME - this will move into HIP
-#if defined(EIGEN_HIP_DEVICE_COMPILE)
-#undef assert
-#define assert(COND)
-#endif
-
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void synchronize() const {
-#if defined(EIGEN_HIPCC) && \
- !defined(EIGEN_HIP_DEVICE_COMPILE)
- hipError_t err = hipStreamSynchronize(stream_->stream());
- if (err != hipSuccess) {
- std::cerr << "Error detected in HIP stream: "
- << hipGetErrorString(err)
- << std::endl;
- assert(err == hipSuccess);
- }
-#else
- assert(false && "The default device should be used instead to generate kernel code");
-#endif
- }
-
- EIGEN_STRONG_INLINE int getNumHipMultiProcessors() const {
- return stream_->deviceProperties().multiProcessorCount;
- }
- EIGEN_STRONG_INLINE int maxHipThreadsPerBlock() const {
- return stream_->deviceProperties().maxThreadsPerBlock;
- }
- EIGEN_STRONG_INLINE int maxHipThreadsPerMultiProcessor() const {
- return stream_->deviceProperties().maxThreadsPerMultiProcessor;
- }
- EIGEN_STRONG_INLINE int sharedMemPerBlock() const {
- return stream_->deviceProperties().sharedMemPerBlock;
- }
- EIGEN_STRONG_INLINE int majorDeviceVersion() const {
- return stream_->deviceProperties().major;
- }
- EIGEN_STRONG_INLINE int minorDeviceVersion() const {
- return stream_->deviceProperties().minor;
- }
-
- EIGEN_STRONG_INLINE int maxBlocks() const {
- return max_blocks_;
- }
-
- // This function checks if the HIP runtime recorded an error for the
- // underlying stream device.
- inline bool ok() const {
-#if defined(EIGEN_HIPCC)
- hipError_t error = hipStreamQuery(stream_->stream());
- return (error == hipSuccess) || (error == hipErrorNotReady);
-#else
- return false;
-#endif
- }
-
- private:
- const StreamInterface* stream_;
- int max_blocks_;
-};
-
-#define LAUNCH_HIP_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \
- hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), dim3(gridsize), dim3(blocksize), (sharedmem), (device).stream(), (__VA_ARGS__)); \
- assert(hipGetLastError() == hipSuccess);
-
-
-// FIXME: Should be device and kernel specific.
-#if defined(EIGEN_HIPCC)
-static EIGEN_DEVICE_FUNC inline void setHipSharedMemConfig(hipSharedMemConfig config) {
-#if !defined(EIGEN_HIP_DEVICE_COMPILE)
- hipError_t status = hipDeviceSetSharedMemConfig(config);
- EIGEN_UNUSED_VARIABLE(status)
- assert(status == hipSuccess);
-#else
- EIGEN_UNUSED_VARIABLE(config)
-#endif
-}
-#endif
-
-} // end namespace Eigen
-
-#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_HIP_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionHip.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionHip.h
deleted file mode 100644
index 5304a22..0000000
--- a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionHip.h
+++ /dev/null
@@ -1,815 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_HIP_H
-#define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_HIP_H
-
-#if defined(EIGEN_HIP_DEVICE_COMPILE)
-#include "Eigen/src/Core/arch/HIP/hcc/math_constants.h"
-#endif
-
-#if defined(EIGEN_HIPCC)
-#define HIP_WARP_SIZE 64
-#endif
-
-namespace Eigen {
-namespace internal {
-
-
-#if defined(EIGEN_USE_GPU) && defined(EIGEN_HIPCC)
-// Full reducers for GPU, don't vectorize for now
-
-// Reducer function that enables multiple hip thread to safely accumulate at the same
-// output address. It basically reads the current value of the output variable, and
-// attempts to update it with the new value. If in the meantime another hip thread
-// updated the content of the output address it will try again.
-template <typename T, typename R>
-__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) {
-#if defined(EIGEN_HIP_DEVICE_COMPILE) && defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)
- if (sizeof(T) == 4)
- {
- unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
- unsigned int newval = oldval;
- reducer.reduce(accum, reinterpret_cast<T*>(&newval));
- if (newval == oldval) {
- return;
- }
- unsigned int readback;
- while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) {
- oldval = readback;
- newval = oldval;
- reducer.reduce(accum, reinterpret_cast<T*>(&newval));
- if (newval == oldval) {
- return;
- }
- }
- }
- else if (sizeof(T) == 8) {
- unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output);
- unsigned long long newval = oldval;
- reducer.reduce(accum, reinterpret_cast<T*>(&newval));
- if (newval == oldval) {
- return;
- }
- unsigned long long readback;
- while ((readback = atomicCAS((unsigned long long*)output, oldval, newval)) != oldval) {
- oldval = readback;
- newval = oldval;
- reducer.reduce(accum, reinterpret_cast<T*>(&newval));
- if (newval == oldval) {
- return;
- }
- }
- }
- else {
- assert(0 && "Wordsize not supported");
- }
-#else
- assert(0 && "Shouldn't be called on unsupported device");
-#endif
-}
-
-// We extend atomicExch to support extra data types
-template <typename Type>
-__device__ inline Type atomicExchCustom(Type* address, Type val) {
- return atomicExch(address, val);
-}
-
-template <>
-__device__ inline double atomicExchCustom(double* address, double val) {
- unsigned long long int* address_as_ull = reinterpret_cast<unsigned long long int*>(address);
- return __longlong_as_double(atomicExch(address_as_ull, __double_as_longlong(val)));
-}
-
-#if defined(EIGEN_HAS_HIP_FP16)
-template <template <typename T> class R>
-__device__ inline void atomicReduce(half2* output, half2 accum, R<half>& reducer) {
- unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
- unsigned int newval = oldval;
- reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval));
- if (newval == oldval) {
- return;
- }
- unsigned int readback;
- while ((readback = atomicCAS((unsigned int*)output, oldval, newval)) != oldval) {
- oldval = readback;
- newval = oldval;
- reducer.reducePacket(accum, reinterpret_cast<half2*>(&newval));
- if (newval == oldval) {
- return;
- }
- }
-}
-#endif
-
-template <>
-__device__ inline void atomicReduce(float* output, float accum, SumReducer<float>&) {
-#if defined(EIGEN_HIP_DEVICE_COMPILE) && (__HIP_DEVICE_COMPILE__ == 1) &&\
- defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)
- atomicAdd(output, accum);
-#else
- assert(0 && "Shouldn't be called on unsupported device");
-#endif
-}
-
-
-template <typename CoeffType, typename Index>
-__global__ void ReductionInitKernel(const CoeffType val, Index num_preserved_coeffs, CoeffType* output) {
- const Index thread_id = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x;
- const Index num_threads = hipBlockDim_x * hipGridDim_x;
- for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
- output[i] = val;
- }
-}
-
-
-template <int BlockSize, int NumPerThread, typename Self,
- typename Reducer, typename Index>
-__global__ void FullReductionKernel(const Self input, Index num_coeffs,
- typename Self::CoeffReturnType* output, unsigned int* semaphore, Reducer reducer) {
-#if defined(EIGEN_HIP_DEVICE_COMPILE) && (__HIP_DEVICE_COMPILE__ == 1) &&\
- defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)
- // Initialize the output value
- const Index first_index = hipBlockIdx_x * BlockSize * NumPerThread + hipThreadIdx_x;
- if (hipGridDim_x == 1) {
- if (first_index == 0) {
- *output = reducer.initialize();
- }
- }
- else {
- if (hipThreadIdx_x == 0) {
- unsigned int block = atomicCAS(semaphore, 0u, 1u);
- if (block == 0) {
- // We're the first block to run, initialize the output value
- atomicExchCustom(output, reducer.initialize());
- __threadfence();
- atomicExch(semaphore, 2u);
- }
- else {
- // Wait for the first block to initialize the output value.
- // Use atomicCAS here to ensure that the reads aren't cached
- unsigned int val;
- do {
- val = atomicCAS(semaphore, 2u, 2u);
- }
- while (val < 2u);
- }
- }
- }
-
- __syncthreads();
-
- eigen_assert(hipGridDim_x == 1 || *semaphore >= 2u);
-
- typename Self::CoeffReturnType accum = reducer.initialize();
- Index max_iter = numext::mini<Index>(num_coeffs - first_index, NumPerThread*BlockSize);
- for (Index i = 0; i < max_iter; i+=BlockSize) {
- const Index index = first_index + i;
- eigen_assert(index < num_coeffs);
- typename Self::CoeffReturnType val = input.m_impl.coeff(index);
- reducer.reduce(val, &accum);
- }
-
-#pragma unroll
- for (int offset = HIP_WARP_SIZE/2; offset > 0; offset /= 2) {
- // XXX use std::is_floating_point to determine the type of accum
- if (std::is_floating_point<typename Self::CoeffReturnType>::value) {
- reducer.reduce(__shfl_down(static_cast<float>(accum), offset, HIP_WARP_SIZE), &accum);
- } else {
- reducer.reduce(__shfl_down(static_cast<int>(accum), offset, HIP_WARP_SIZE), &accum);
- }
- }
-
- if ((hipThreadIdx_x & (HIP_WARP_SIZE - 1)) == 0) {
- atomicReduce(output, accum, reducer);
- }
-
- if (hipGridDim_x > 1 && hipThreadIdx_x == 0) {
- // Let the last block reset the semaphore
- atomicInc(semaphore, hipGridDim_x + 1);
- __threadfence_system();
- }
-
-#else
- assert(0 && "Shouldn't be called on unsupported device");
-#endif
-}
-
-
-#if defined(EIGEN_HAS_HIP_FP16)
-template <typename Self,
- typename Reducer, typename Index>
-__global__ void ReductionInitFullReduxKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs, half2* scratch) {
- eigen_assert(hipBlockDim_x == 1);
- eigen_assert(hipGridDim_x == 1);
- if (num_coeffs % 2 != 0) {
- half last = input.m_impl.coeff(num_coeffs-1);
- *scratch = __halves2half2(last, reducer.initialize());
- } else {
- *scratch = reducer.template initializePacket<half2>();
- }
-}
-
-template <typename Self,
- typename Reducer, typename Index>
-__global__ void ReductionInitKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs, half* output) {
- const Index thread_id = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x;
- const Index num_threads = hipBlockDim_x * hipGridDim_x;
- const Index num_packets = num_coeffs / 2;
- for (Index i = thread_id; i < num_packets; i += num_threads) {
- ((half2*)output)[i] = reducer.template initializePacket<half2>();
- }
-
- if (thread_id == 0 && num_coeffs % 2 != 0) {
- output[num_coeffs-1] = reducer.initialize();
- }
-}
-
-template <int BlockSize, int NumPerThread, typename Self,
- typename Reducer, typename Index>
-__global__ void FullReductionKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs,
- half* output, half2* scratch) {
- eigen_assert(NumPerThread % 2 == 0);
-
- const Index first_index = hipBlockIdx_x * BlockSize * NumPerThread + 2*hipThreadIdx_x;
-
- // Initialize the output value if it wasn't initialized by the ReductionInitKernel
- if (hipGridDim_x == 1 && first_index == 0) {
- if (num_coeffs % 2 != 0) {
- half last = input.m_impl.coeff(num_coeffs-1);
- *scratch = __halves2half2(last, reducer.initialize());
- } else {
- *scratch = reducer.template initializePacket<half2>();
- }
- __syncthreads();
- }
-
- half2 accum = reducer.template initializePacket<half2>();
- const Index max_iter = numext::mini<Index>((num_coeffs - first_index) / 2, NumPerThread*BlockSize / 2);
- for (Index i = 0; i < max_iter; i += BlockSize) {
- const Index index = first_index + 2*i;
- eigen_assert(index + 1 < num_coeffs);
- half2 val = input.m_impl.template packet<Unaligned>(index);
- reducer.reducePacket(val, &accum);
- }
-
-#pragma unroll
- for (int offset = HIP_WARP_SIZE/2; offset > 0; offset /= 2) {
- // FIXME : remove this workaround once we have native half/half2 support for __shfl_down
- union { int i; half2 h; } wka_in, wka_out;
- wka_in.h = accum;
- wka_out.i = __shfl_down(wka_in.i, offset, HIP_WARP_SIZE);
- reducer.reducePacket(wka_out.h, &accum);
- }
-
- if ((hipThreadIdx_x & (HIP_WARP_SIZE - 1)) == 0) {
- atomicReduce(scratch, accum, reducer);
- }
-
- __syncthreads();
-
- if (hipGridDim_x == 1 && first_index == 0) {
- half tmp = __low2half(*scratch);
- reducer.reduce(__high2half(*scratch), &tmp);
- *output = tmp;
- }
-}
-
-template <typename Op>
-__global__ void ReductionCleanupKernelHalfFloat(Op& reducer, half* output, half2* scratch) {
- eigen_assert(hipThreadIdx_x == 1);
- half tmp = __low2half(*scratch);
- reducer.reduce(__high2half(*scratch), &tmp);
- *output = tmp;
-}
-
-#endif
-
-template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void>
-struct FullReductionLauncher {
- static void run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index) {
- assert(false && "Should only be called on doubles, floats and half floats");
- }
-};
-
-namespace {
- std::mutex __eigen_reduction_hip_mutex;
-}
-
-// Specialization for float and double
-template <typename Self, typename Op, typename OutputType, bool PacketAccess>
-struct FullReductionLauncher<
- Self, Op, OutputType, PacketAccess,
- typename internal::enable_if<
- internal::is_same<float, OutputType>::value ||
- internal::is_same<double, OutputType>::value,
- void>::type> {
- static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs) {
- // guard FullReductionLauncher with a mutex so only 1 FullReductionKernel
- // is dispatched at a time
- std::lock_guard<std::mutex> lock(__eigen_reduction_hip_mutex);
-
- typedef typename Self::Index Index;
- typedef typename Self::CoeffReturnType Scalar;
- const int block_size = 256;
- const int num_per_thread = 128;
- const int num_blocks = divup<int>(num_coeffs, block_size * num_per_thread);
-
- unsigned int* semaphore = NULL;
- if (num_blocks > 1) {
- semaphore = device.semaphore();
-
- unsigned int semaphore_host = 0xFF;
- hipMemcpy(&semaphore_host, semaphore, sizeof(unsigned int), hipMemcpyDeviceToHost);
- if (semaphore_host != 0) {
- std::cerr << "[WARN][EIGEN][FullReductionLauncher] incorrect semaphore value: "
- << semaphore_host << "\n";
- // wait for all commands on the device to complete so semaphore value
- // is reset to 0
- hipDeviceSynchronize();
-
- // read again
- hipMemcpy(&semaphore_host, semaphore, sizeof(unsigned int), hipMemcpyDeviceToHost);
- if (semaphore_host != 0) {
- std::cerr << "[ERROR][EIGEN][FullReductionLauncher] CRITICAL incorrect semaphore value: "
- << semaphore_host << ", apply manual override to 0\n";
-
- // force set semaphore value to be 0
- semaphore_host = 0;
- hipMemcpy(semaphore, &semaphore_host, sizeof(unsigned int), hipMemcpyHostToDevice);
- }
- }
- }
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(FullReductionKernel<block_size, num_per_thread, Self, Op, Index>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), self, num_coeffs, output, semaphore, reducer);
- }
-};
-
-#if defined(EIGEN_HAS_HIP_FP16)
-template <typename Self, typename Op>
-struct FullReductionLauncher<Self, Op, Eigen::half, false> {
- static void run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index) {
- assert(false && "Should not be called since there is no packet accessor");
- }
-};
-
-template <typename Self, typename Op>
-struct FullReductionLauncher<Self, Op, Eigen::half, true> {
- static void run(const Self& self, Op& reducer, const GpuDevice& device, half* output, typename Self::Index num_coeffs) {
- typedef typename Self::Index Index;
-
- const int block_size = 256;
- const int num_per_thread = 128;
- const int num_blocks = divup<int>(num_coeffs, block_size * num_per_thread);
- half2* scratch = static_cast<half2*>(device.scratchpad());
-
- if (num_blocks > 1) {
- // We initialize the output and the scrathpad outside the reduction kernel when we can't be sure that there
- // won't be a race conditions between multiple thread blocks.
- hipLaunchKernelGGL(HIP_KERNEL_NAME(ReductionInitFullReduxKernelHalfFloat<Self, Op, Index>),
- dim3(1), dim3(1), 0, device.stream(), reducer, self, num_coeffs, scratch);
- }
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(FullReductionKernelHalfFloat<block_size, num_per_thread, Self, Op, Index>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), reducer, self, num_coeffs, output, scratch);
-
- if (num_blocks > 1) {
- hipLaunchKernelGGL(HIP_KERNEL_NAME(ReductionCleanupKernelHalfFloat<Op>),
- dim3(1), dim3(1), 0, device.stream(), reducer, output, scratch);
- }
- }
-};
-#endif
-
-
-template <typename Self, typename Op, bool Vectorizable>
-struct FullReducer<Self, Op, GpuDevice, Vectorizable> {
- // Unfortunately nvidia doesn't support well exotic types such as complex,
- // so reduce the scope of the optimized version of the code to the simple cases
- // of doubles, floats and half floats
-#if defined(EIGEN_HAS_HIP_FP16)
- static const bool HasOptimizedImplementation = !Op::IsStateful &&
- (internal::is_same<typename Self::CoeffReturnType, float>::value ||
- internal::is_same<typename Self::CoeffReturnType, double>::value ||
- (internal::is_same<typename Self::CoeffReturnType, Eigen::half>::value && reducer_traits<Op, GpuDevice>::PacketAccess));
-#else
- static const bool HasOptimizedImplementation = !Op::IsStateful &&
- (internal::is_same<typename Self::CoeffReturnType, float>::value ||
- internal::is_same<typename Self::CoeffReturnType, double>::value);
-#endif
-
- template <typename OutputType>
- static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output) {
- assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats");
- const Index num_coeffs = array_prod(self.m_impl.dimensions());
- // Don't crash when we're called with an input tensor of size 0.
- if (num_coeffs == 0) {
- return;
- }
-
- FullReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(self, reducer, device, output, num_coeffs);
- }
-};
-
-
-template <int NumPerThread, typename Self,
- typename Reducer, typename Index>
-__global__ void InnerReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs,
- typename Self::CoeffReturnType* output) {
-#if defined(EIGEN_HIP_DEVICE_COMPILE) && (__HIP_DEVICE_COMPILE__ == 1) &&\
- defined(__HIP_ARCH_HAS_WARP_SHUFFLE__)
- typedef typename Self::CoeffReturnType Type;
- eigen_assert(hipBlockDim_y == 1);
- eigen_assert(hipBlockDim_z == 1);
- eigen_assert(hipGridDim_y == 1);
- eigen_assert(hipGridDim_z == 1);
-
- const int unroll_times = 16;
- eigen_assert(NumPerThread % unroll_times == 0);
-
- const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, hipBlockDim_x * NumPerThread);
- const Index num_input_blocks = input_col_blocks * num_preserved_coeffs;
-
- const Index num_threads = hipBlockDim_x * hipGridDim_x;
- const Index thread_id = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x;
-
- // Initialize the output values if they weren't initialized by the ReductionInitKernel
- if (hipGridDim_x == 1) {
- for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
- output[i] = reducer.initialize();
- }
- __syncthreads();
- }
-
- for (Index i = hipBlockIdx_x; i < num_input_blocks; i += hipGridDim_x) {
- const Index row = i / input_col_blocks;
-
- if (row < num_preserved_coeffs) {
- const Index col_block = i % input_col_blocks;
- const Index col_begin = col_block * hipBlockDim_x * NumPerThread + hipThreadIdx_x;
-
- Type reduced_val = reducer.initialize();
-
- for (Index j = 0; j < NumPerThread; j += unroll_times) {
- const Index last_col = col_begin + hipBlockDim_x * (j + unroll_times - 1);
- if (last_col >= num_coeffs_to_reduce) {
- for (Index col = col_begin + hipBlockDim_x * j; col < num_coeffs_to_reduce; col += hipBlockDim_x) {
- const Type val = input.m_impl.coeff(row * num_coeffs_to_reduce + col);
- reducer.reduce(val, &reduced_val);
- }
- break;
- } else {
- // Faster version of the loop with no branches after unrolling.
-#pragma unroll
- for (int k = 0; k < unroll_times; ++k) {
- const Index col = col_begin + hipBlockDim_x * (j + k);
- reducer.reduce(input.m_impl.coeff(row * num_coeffs_to_reduce + col), &reduced_val);
- }
- }
- }
-
-#pragma unroll
- for (int offset = HIP_WARP_SIZE/2; offset > 0; offset /= 2) {
- // XXX use std::is_floating_point to determine the type of reduced_val
- if (std::is_floating_point<Type>::value) {
- reducer.reduce(__shfl_down(static_cast<float>(reduced_val), offset), &reduced_val);
- } else {
- reducer.reduce(__shfl_down(static_cast<int>(reduced_val), offset), &reduced_val);
- }
- }
-
- if ((hipThreadIdx_x & (HIP_WARP_SIZE - 1)) == 0) {
- atomicReduce(&(output[row]), reduced_val, reducer);
- }
- }
- }
-#else
- assert(0 && "Shouldn't be called on unsupported device");
-#endif
-}
-
-#if defined(EIGEN_HAS_HIP_FP16)
-
-template <int NumPerThread, typename Self,
- typename Reducer, typename Index>
-__global__ void InnerReductionKernelHalfFloat(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs,
- half* output) {
- eigen_assert(hipBlockDim_y == 1);
- eigen_assert(hipBlockDim_z == 1);
- eigen_assert(hipGridDim_y == 1);
- eigen_assert(hipGridDim_z == 1);
-
- const int unroll_times = 16;
- eigen_assert(NumPerThread % unroll_times == 0);
- eigen_assert(unroll_times % 2 == 0);
-
- const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, hipBlockDim_x * NumPerThread * 2);
- const Index num_input_blocks = divup<Index>(input_col_blocks * num_preserved_coeffs, 2);
-
- const Index num_threads = hipBlockDim_x * hipGridDim_x;
- const Index thread_id = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x;
-
- // Initialize the output values if they weren't initialized by the ReductionInitKernel
- if (hipGridDim_x == 1) {
- Index i = 2*thread_id;
- for (; i + 1 < num_preserved_coeffs; i += 2*num_threads) {
- half* loc = output + i;
- *((half2*)loc) = reducer.template initializePacket<half2>();
- }
- if (i < num_preserved_coeffs) {
- output[i] = reducer.initialize();
- }
- __syncthreads();
- }
-
- for (Index i = hipBlockIdx_x; i < num_input_blocks; i += hipGridDim_x) {
- const Index row = 2 * (i / input_col_blocks);
-
- if (row + 1 < num_preserved_coeffs) {
- const Index col_block = i % input_col_blocks;
- const Index col_begin = 2 * (col_block * hipBlockDim_x * NumPerThread + hipThreadIdx_x);
-
- half2 reduced_val1 = reducer.template initializePacket<half2>();
- half2 reduced_val2 = reducer.template initializePacket<half2>();
-
- for (Index j = 0; j < NumPerThread; j += unroll_times) {
- const Index last_col = col_begin + hipBlockDim_x * (j + unroll_times - 1) * 2;
- if (last_col >= num_coeffs_to_reduce) {
- Index col = col_begin + hipBlockDim_x * j;
- for (; col + 1 < num_coeffs_to_reduce; col += hipBlockDim_x) {
- const half2 val1 = input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col);
- reducer.reducePacket(val1, &reduced_val1);
- const half2 val2 = input.m_impl.template packet<Unaligned>((row+1) * num_coeffs_to_reduce + col);
- reducer.reducePacket(val2, &reduced_val2);
- }
- if (col < num_coeffs_to_reduce) {
- // Peel;
- const half last1 = input.m_impl.coeff(row * num_coeffs_to_reduce + col);
- const half2 val1 = __halves2half2(last1, reducer.initialize());
- reducer.reducePacket(val1, &reduced_val1);
- const half last2 = input.m_impl.coeff((row+1) * num_coeffs_to_reduce + col);
- const half2 val2 = __halves2half2(last2, reducer.initialize());
- reducer.reducePacket(val2, &reduced_val2);
- }
- break;
- } else {
- // Faster version of the loop with no branches after unrolling.
-#pragma unroll
- for (int k = 0; k < unroll_times; ++k) {
- const Index col = col_begin + hipBlockDim_x * (j + k) * 2;
- reducer.reducePacket(input.m_impl.template packet<Unaligned>(row * num_coeffs_to_reduce + col), &reduced_val1);
- reducer.reducePacket(input.m_impl.template packet<Unaligned>((row + 1)* num_coeffs_to_reduce + col), &reduced_val2);
- }
- }
- }
-
-#pragma unroll
- for (int offset = HIP_WARP_SIZE/2; offset > 0; offset /= 2) {
- // FIXME : remove this workaround once we have native half/half2 support for __shfl_down
- union { int i; half2 h; } wka_in, wka_out;
-
- wka_in.h = reduced_val1;
- wka_out.i = __shfl_down(wka_in.i, offset, HIP_WARP_SIZE);
- reducer.reducePacket(wka_out.h, &reduced_val1);
-
- wka_in.h = reduced_val2;
- wka_out.i = __shfl_down(wka_in.i, offset, HIP_WARP_SIZE);
- reducer.reducePacket(wka_out.h, &reduced_val2);
- }
-
- half val1 = __low2half(reduced_val1);
- reducer.reduce(__high2half(reduced_val1), &val1);
- half val2 = __low2half(reduced_val2);
- reducer.reduce(__high2half(reduced_val2), &val2);
- half2 val = __halves2half2(val1, val2);
-
- if ((hipThreadIdx_x & (HIP_WARP_SIZE - 1)) == 0) {
- half* loc = output + row;
- atomicReduce((half2*)loc, val, reducer);
- }
- }
- }
-}
-
-#endif
-
-template <typename Self, typename Op, typename OutputType, bool PacketAccess, typename Enabled = void>
-struct InnerReductionLauncher {
- static bool run(const Self&, Op&, const GpuDevice&, OutputType*, typename Self::Index, typename Self::Index) {
- assert(false && "Should only be called to reduce doubles, floats and half floats on a gpu device");
- return true;
- }
-};
-
-// Specialization for float and double
-template <typename Self, typename Op, typename OutputType, bool PacketAccess>
-struct InnerReductionLauncher<
- Self, Op, OutputType, PacketAccess,
- typename internal::enable_if<
- internal::is_same<float, OutputType>::value ||
- internal::is_same<double, OutputType>::value,
- void>::type> {
- static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
- typedef typename Self::Index Index;
-
- const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
- const int block_size = 256;
- const int num_per_thread = 128;
- const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / block_size;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
-
- if (num_blocks > 1) {
- // We initialize the outputs outside the reduction kernel when we can't be sure that there
- // won't be a race conditions between multiple thread blocks.
- const int dyn_blocks = divup<int>(num_preserved_vals, 1024);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / 1024;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(ReductionInitKernel<OutputType, Index>),
- dim3(num_blocks), dim3(1024), 0, device.stream(),
- reducer.initialize(), num_preserved_vals, output);
- }
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(InnerReductionKernel<num_per_thread, Self, Op, Index>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), reducer, self,
- num_coeffs_to_reduce, num_preserved_vals, output);
-
- return false;
- }
-};
-
-#if defined(EIGEN_HAS_HIP_FP16)
-template <typename Self, typename Op>
-struct InnerReductionLauncher<Self, Op, Eigen::half, false> {
- static bool run(const Self&, Op&, const GpuDevice&, half*, typename Self::Index, typename Self::Index) {
- assert(false && "Should not be called since there is no packet accessor");
- return true;
- }
-};
-
-template <typename Self, typename Op>
-struct InnerReductionLauncher<Self, Op, Eigen::half, true> {
- static bool run(const Self& self, Op& reducer, const GpuDevice& device, half* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
- typedef typename Self::Index Index;
-
- if (num_preserved_vals % 2 != 0) {
- // Not supported yet, revert to the slower code path
- return true;
- }
-
- const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
- const int block_size = /*256*/128;
- const int num_per_thread = /*128*/64;
- const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / block_size;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
-
- if (num_blocks > 1) {
- // We initialize the outputs outside the reduction kernel when we can't be sure that there
- // won't be a race conditions between multiple thread blocks.
- const int dyn_blocks = divup<int>(num_preserved_vals, 1024);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / 1024;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(ReductionInitKernelHalfFloat<Self, Op, Index>),
- dim3(1), dim3(1), 0, device.stream(), reducer, self, num_preserved_vals, output);
- }
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(InnerReductionKernelHalfFloat<num_per_thread, Self, Op, Index>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), reducer, self, num_coeffs_to_reduce, num_preserved_vals, output);
-
- return false;
- }
-};
-#endif
-
-
-template <typename Self, typename Op>
-struct InnerReducer<Self, Op, GpuDevice> {
- // Unfortunately nvidia doesn't support well exotic types such as complex,
- // so reduce the scope of the optimized version of the code to the simple case
- // of floats and half floats.
-#if defined(EIGEN_HAS_HIP_FP16)
- static const bool HasOptimizedImplementation = !Op::IsStateful &&
- (internal::is_same<typename Self::CoeffReturnType, float>::value ||
- internal::is_same<typename Self::CoeffReturnType, double>::value ||
- (internal::is_same<typename Self::CoeffReturnType, Eigen::half>::value && reducer_traits<Op, GpuDevice>::PacketAccess));
-#else
- static const bool HasOptimizedImplementation = !Op::IsStateful &&
- (internal::is_same<typename Self::CoeffReturnType, float>::value ||
- internal::is_same<typename Self::CoeffReturnType, double>::value);
-#endif
-
- template <typename OutputType>
- static bool run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
- assert(HasOptimizedImplementation && "Should only be called on doubles, floats or half floats");
- const Index num_coeffs = array_prod(self.m_impl.dimensions());
- // Don't crash when we're called with an input tensor of size 0.
- if (num_coeffs == 0) {
- return true;
- }
- // It's faster to use the usual code.
- if (num_coeffs_to_reduce <= 128) {
- return true;
- }
-
- return InnerReductionLauncher<Self, Op, OutputType, reducer_traits<Op, GpuDevice>::PacketAccess>::run(self, reducer, device, output, num_coeffs_to_reduce, num_preserved_vals);
- }
-};
-
-template <int NumPerThread, typename Self,
- typename Reducer, typename Index>
-__global__ void OuterReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs,
- typename Self::CoeffReturnType* output) {
- const Index num_threads = hipBlockDim_x * hipGridDim_x;
- const Index thread_id = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x;
- // Initialize the output values if they weren't initialized by the ReductionInitKernel
- if (hipGridDim_x == 1) {
- for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) {
- output[i] = reducer.initialize();
- }
- __syncthreads();
- }
-
- // Do the reduction.
- const Index max_iter = num_preserved_coeffs * divup<Index>(num_coeffs_to_reduce, NumPerThread);
- for (Index i = thread_id; i < max_iter; i += num_threads) {
- const Index input_col = i % num_preserved_coeffs;
- const Index input_row = (i / num_preserved_coeffs) * NumPerThread;
- typename Self::CoeffReturnType reduced_val = reducer.initialize();
- const Index max_row = numext::mini(input_row + NumPerThread, num_coeffs_to_reduce);
- for (Index j = input_row; j < max_row; j++) {
- typename Self::CoeffReturnType val = input.m_impl.coeff(j * num_preserved_coeffs + input_col);
- reducer.reduce(val, &reduced_val);
- }
- atomicReduce(&(output[input_col]), reduced_val, reducer);
- }
-}
-
-
-template <typename Self, typename Op>
-struct OuterReducer<Self, Op, GpuDevice> {
- // Unfortunately nvidia doesn't support well exotic types such as complex,
- // so reduce the scope of the optimized version of the code to the simple case
- // of floats.
- static const bool HasOptimizedImplementation = !Op::IsStateful &&
- (internal::is_same<typename Self::CoeffReturnType, float>::value ||
- internal::is_same<typename Self::CoeffReturnType, double>::value);
- template <typename Device, typename OutputType>
- static bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) {
- assert(false && "Should only be called to reduce doubles or floats on a gpu device");
- return true;
- }
-
- static bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) {
- typedef typename Self::Index Index;
-
- // It's faster to use the usual code.
- if (num_coeffs_to_reduce <= 32) {
- return true;
- }
-
- const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals;
- const int block_size = 256;
- const int num_per_thread = 16;
- const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / block_size;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
-
- if (num_blocks > 1) {
- // We initialize the outputs in the reduction kernel itself when we don't have to worry
- // about race conditions between multiple thread blocks.
- const int dyn_blocks = divup<int>(num_preserved_vals, 1024);
- const int max_blocks = device.getNumHipMultiProcessors() *
- device.maxHipThreadsPerMultiProcessor() / 1024;
- const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks);
- hipLaunchKernelGGL(HIP_KERNEL_NAME(ReductionInitKernel<float, Index>),
- dim3(num_blocks), dim3(1024), 0, device.stream(),
- reducer.initialize(), num_preserved_vals, output);
- }
-
- hipLaunchKernelGGL(HIP_KERNEL_NAME(OuterReductionKernel<num_per_thread, Self, Op, Index>),
- dim3(num_blocks), dim3(block_size), 0, device.stream(), reducer, self, num_coeffs_to_reduce, num_preserved_vals, output);
-
- return false;
- }
-};
-
-#endif
-
-
-} // end namespace internal
-} // end namespace Eigen
-
-#endif // EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_HIP_H
diff --git a/unsupported/test/cxx11_tensor_argmax_hip.cu b/unsupported/test/cxx11_tensor_argmax_hip.cu
deleted file mode 100644
index 57d6ca0..0000000
--- a/unsupported/test/cxx11_tensor_argmax_hip.cu
+++ /dev/null
@@ -1,251 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_FUNC cxx11_tensor_hip
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-using Eigen::Tensor;
-
-template <int Layout>
-void test_hip_simple_argmax()
-{
- Tensor<double, 3, Layout> in(Eigen::array<DenseIndex, 3>(72,53,97));
- Tensor<DenseIndex, 1, Layout> out_max(Eigen::array<DenseIndex, 1>(1));
- Tensor<DenseIndex, 1, Layout> out_min(Eigen::array<DenseIndex, 1>(1));
- in.setRandom();
- in *= in.constant(100.0);
- in(0, 0, 0) = -1000.0;
- in(71, 52, 96) = 1000.0;
-
- std::size_t in_bytes = in.size() * sizeof(double);
- std::size_t out_bytes = out_max.size() * sizeof(DenseIndex);
-
- double* d_in;
- DenseIndex* d_out_max;
- DenseIndex* d_out_min;
- hipMalloc((void**)(&d_in), in_bytes);
- hipMalloc((void**)(&d_out_max), out_bytes);
- hipMalloc((void**)(&d_out_min), out_bytes);
-
- hipMemcpy(d_in, in.data(), in_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<double, 3, Layout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 3>(72,53,97));
- Eigen::TensorMap<Eigen::Tensor<DenseIndex, 1, Layout>, Aligned > gpu_out_max(d_out_max, Eigen::array<DenseIndex, 1>(1));
- Eigen::TensorMap<Eigen::Tensor<DenseIndex, 1, Layout>, Aligned > gpu_out_min(d_out_min, Eigen::array<DenseIndex, 1>(1));
-
- gpu_out_max.device(gpu_device) = gpu_in.argmax();
- gpu_out_min.device(gpu_device) = gpu_in.argmin();
-
- assert(hipMemcpyAsync(out_max.data(), d_out_max, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipMemcpyAsync(out_min.data(), d_out_min, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- VERIFY_IS_EQUAL(out_max(Eigen::array<DenseIndex, 1>(0)), 72*53*97 - 1);
- VERIFY_IS_EQUAL(out_min(Eigen::array<DenseIndex, 1>(0)), 0);
-
- hipFree(d_in);
- hipFree(d_out_max);
- hipFree(d_out_min);
-}
-
-template <int DataLayout>
-void test_hip_argmax_dim()
-{
- Tensor<float, 4, DataLayout> tensor(2,3,5,7);
- std::vector<int> dims;
- dims.push_back(2); dims.push_back(3); dims.push_back(5); dims.push_back(7);
-
- for (int dim = 0; dim < 4; ++dim) {
- tensor.setRandom();
- tensor = (tensor + tensor.constant(0.5)).log();
-
- array<DenseIndex, 3> out_shape;
- for (int d = 0; d < 3; ++d) out_shape[d] = (d < dim) ? dims[d] : dims[d+1];
-
- Tensor<DenseIndex, 3, DataLayout> tensor_arg(out_shape);
-
- array<DenseIndex, 4> ix;
- for (int i = 0; i < 2; ++i) {
- for (int j = 0; j < 3; ++j) {
- for (int k = 0; k < 5; ++k) {
- for (int l = 0; l < 7; ++l) {
- ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l;
- if (ix[dim] != 0) continue;
- // suppose dim == 1, then for all i, k, l, set tensor(i, 0, k, l) = 10.0
- tensor(ix) = 10.0;
- }
- }
- }
- }
-
- std::size_t in_bytes = tensor.size() * sizeof(float);
- std::size_t out_bytes = tensor_arg.size() * sizeof(DenseIndex);
-
- float* d_in;
- DenseIndex* d_out;
- hipMalloc((void**)(&d_in), in_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in, tensor.data(), in_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 4>(2, 3, 5, 7));
- Eigen::TensorMap<Eigen::Tensor<DenseIndex, 3, DataLayout>, Aligned > gpu_out(d_out, out_shape);
-
- gpu_out.device(gpu_device) = gpu_in.argmax(dim);
-
- assert(hipMemcpyAsync(tensor_arg.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- VERIFY_IS_EQUAL(tensor_arg.size(),
- size_t(2*3*5*7 / tensor.dimension(dim)));
-
- for (DenseIndex n = 0; n < tensor_arg.size(); ++n) {
- // Expect max to be in the first index of the reduced dimension
- VERIFY_IS_EQUAL(tensor_arg.data()[n], 0);
- }
-
- for (int i = 0; i < 2; ++i) {
- for (int j = 0; j < 3; ++j) {
- for (int k = 0; k < 5; ++k) {
- for (int l = 0; l < 7; ++l) {
- ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l;
- if (ix[dim] != tensor.dimension(dim) - 1) continue;
- // suppose dim == 1, then for all i, k, l, set tensor(i, 2, k, l) = 20.0
- tensor(ix) = 20.0;
- }
- }
- }
- }
-
- hipMemcpy(d_in, tensor.data(), in_bytes, hipMemcpyHostToDevice);
-
- gpu_out.device(gpu_device) = gpu_in.argmax(dim);
-
- assert(hipMemcpyAsync(tensor_arg.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (DenseIndex n = 0; n < tensor_arg.size(); ++n) {
- // Expect max to be in the last index of the reduced dimension
- VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1);
- }
-
- hipFree(d_in);
- hipFree(d_out);
- }
-}
-
-template <int DataLayout>
-void test_hip_argmin_dim()
-{
- Tensor<float, 4, DataLayout> tensor(2,3,5,7);
- std::vector<int> dims;
- dims.push_back(2); dims.push_back(3); dims.push_back(5); dims.push_back(7);
-
- for (int dim = 0; dim < 4; ++dim) {
- tensor.setRandom();
- tensor = (tensor + tensor.constant(0.5)).log();
-
- array<DenseIndex, 3> out_shape;
- for (int d = 0; d < 3; ++d) out_shape[d] = (d < dim) ? dims[d] : dims[d+1];
-
- Tensor<DenseIndex, 3, DataLayout> tensor_arg(out_shape);
-
- array<DenseIndex, 4> ix;
- for (int i = 0; i < 2; ++i) {
- for (int j = 0; j < 3; ++j) {
- for (int k = 0; k < 5; ++k) {
- for (int l = 0; l < 7; ++l) {
- ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l;
- if (ix[dim] != 0) continue;
- // suppose dim == 1, then for all i, k, l, set tensor(i, 0, k, l) = 10.0
- tensor(ix) = -10.0;
- }
- }
- }
- }
-
- std::size_t in_bytes = tensor.size() * sizeof(float);
- std::size_t out_bytes = tensor_arg.size() * sizeof(DenseIndex);
-
- float* d_in;
- DenseIndex* d_out;
- hipMalloc((void**)(&d_in), in_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in, tensor.data(), in_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout>, Aligned > gpu_in(d_in, Eigen::array<DenseIndex, 4>(2, 3, 5, 7));
- Eigen::TensorMap<Eigen::Tensor<DenseIndex, 3, DataLayout>, Aligned > gpu_out(d_out, out_shape);
-
- gpu_out.device(gpu_device) = gpu_in.argmin(dim);
-
- assert(hipMemcpyAsync(tensor_arg.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- VERIFY_IS_EQUAL(tensor_arg.size(),
- 2*3*5*7 / tensor.dimension(dim));
-
- for (DenseIndex n = 0; n < tensor_arg.size(); ++n) {
- // Expect min to be in the first index of the reduced dimension
- VERIFY_IS_EQUAL(tensor_arg.data()[n], 0);
- }
-
- for (int i = 0; i < 2; ++i) {
- for (int j = 0; j < 3; ++j) {
- for (int k = 0; k < 5; ++k) {
- for (int l = 0; l < 7; ++l) {
- ix[0] = i; ix[1] = j; ix[2] = k; ix[3] = l;
- if (ix[dim] != tensor.dimension(dim) - 1) continue;
- // suppose dim == 1, then for all i, k, l, set tensor(i, 2, k, l) = 20.0
- tensor(ix) = -20.0;
- }
- }
- }
- }
-
- hipMemcpy(d_in, tensor.data(), in_bytes, hipMemcpyHostToDevice);
-
- gpu_out.device(gpu_device) = gpu_in.argmin(dim);
-
- assert(hipMemcpyAsync(tensor_arg.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (DenseIndex n = 0; n < tensor_arg.size(); ++n) {
- // Expect max to be in the last index of the reduced dimension
- VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1);
- }
-
- hipFree(d_in);
- hipFree(d_out);
- }
-}
-
-void test_cxx11_tensor_hip()
-{
- CALL_SUBTEST(test_hip_simple_argmax<RowMajor>());
- CALL_SUBTEST(test_hip_simple_argmax<ColMajor>());
- CALL_SUBTEST(test_hip_argmax_dim<RowMajor>());
- CALL_SUBTEST(test_hip_argmax_dim<ColMajor>());
- CALL_SUBTEST(test_hip_argmin_dim<RowMajor>());
- CALL_SUBTEST(test_hip_argmin_dim<ColMajor>());
-}
diff --git a/unsupported/test/cxx11_tensor_cast_float16_hip.cu b/unsupported/test/cxx11_tensor_cast_float16_hip.cu
deleted file mode 100644
index bf6a49d..0000000
--- a/unsupported/test/cxx11_tensor_cast_float16_hip.cu
+++ /dev/null
@@ -1,79 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_cast_float16_hip
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-using Eigen::Tensor;
-
-void test_hip_conversion() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- Tensor<float, 1> floats(num_elem);
- floats.setRandom();
-
- float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- float* d_conv = (float*)gpu_device.allocate(num_elem * sizeof(float));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float(
- d_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_half(
- d_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_conv(
- d_conv, num_elem);
-
- gpu_device.memcpyHostToDevice(d_float, floats.data(), num_elem*sizeof(float));
-
- gpu_half.device(gpu_device) = gpu_float.cast<Eigen::half>();
- gpu_conv.device(gpu_device) = gpu_half.cast<float>();
-
- Tensor<float, 1> initial(num_elem);
- Tensor<float, 1> final(num_elem);
- gpu_device.memcpyDeviceToHost(initial.data(), d_float, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(final.data(), d_conv, num_elem*sizeof(float));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- VERIFY_IS_APPROX(initial(i), final(i));
- }
-
- gpu_device.deallocate(d_float);
- gpu_device.deallocate(d_half);
- gpu_device.deallocate(d_conv);
-}
-
-
-void test_fallback_conversion() {
- int num_elem = 101;
- Tensor<float, 1> floats(num_elem);
- floats.setRandom();
-
- Eigen::Tensor<Eigen::half, 1> halfs = floats.cast<Eigen::half>();
- Eigen::Tensor<float, 1> conv = halfs.cast<float>();
-
- for (int i = 0; i < num_elem; ++i) {
- VERIFY_IS_APPROX(floats(i), conv(i));
- }
-}
-
-
-void test_cxx11_tensor_cast_float16_hip()
-{
- CALL_SUBTEST(test_hip_conversion());
- CALL_SUBTEST(test_fallback_conversion());
-}
diff --git a/unsupported/test/cxx11_tensor_contract_hip.cu b/unsupported/test/cxx11_tensor_contract_hip.cu
deleted file mode 100644
index 652af0a..0000000
--- a/unsupported/test/cxx11_tensor_contract_hip.cu
+++ /dev/null
@@ -1,215 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-// Copyright (C) 2014 Navdeep Jaitly <ndjaitly@google.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_hip
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-
-using Eigen::Tensor;
-typedef Tensor<float, 1>::DimensionPair DimPair;
-
-template<int DataLayout>
-void test_hip_contraction(int m_size, int k_size, int n_size)
-{
- std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl;
- // with these dimensions, the output has 300 * 140 elements, which is
- // more than 30 * 1024, which is the number of threads in blocks on
- // a 15 SM GK110 GPU
- Tensor<float, 2, DataLayout> t_left(m_size, k_size);
- Tensor<float, 2, DataLayout> t_right(k_size, n_size);
- Tensor<float, 2, DataLayout> t_result(m_size, n_size);
- Tensor<float, 2, DataLayout> t_result_gpu(m_size, n_size);
- Eigen::array<DimPair, 1> dims(DimPair(1, 0));
-
- t_left.setRandom();
- t_right.setRandom();
-
- std::size_t t_left_bytes = t_left.size() * sizeof(float);
- std::size_t t_right_bytes = t_right.size() * sizeof(float);
- std::size_t t_result_bytes = t_result.size() * sizeof(float);
-
- float* d_t_left;
- float* d_t_right;
- float* d_t_result;
-
- hipMalloc((void**)(&d_t_left), t_left_bytes);
- hipMalloc((void**)(&d_t_right), t_right_bytes);
- hipMalloc((void**)(&d_t_result), t_result_bytes);
-
- hipMemcpy(d_t_left, t_left.data(), t_left_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_t_right, t_right.data(), t_right_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> >
- gpu_t_left(d_t_left, Eigen::array<int, 2>(m_size, k_size));
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> >
- gpu_t_right(d_t_right, Eigen::array<int, 2>(k_size, n_size));
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> >
- gpu_t_result(d_t_result, Eigen::array<int, 2>(m_size, n_size));
-
-
- gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims);
- t_result = t_left.contract(t_right, dims);
-
- hipMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, hipMemcpyDeviceToHost);
- for (DenseIndex i = 0; i < t_result.size(); i++) {
- if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) {
- continue;
- }
- if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) {
- continue;
- }
- std::cout << "mismatch detected at index " << i << ": " << t_result(i)
- << " vs " << t_result_gpu(i) << std::endl;
- assert(false);
- }
-
- hipFree((void*)d_t_left);
- hipFree((void*)d_t_right);
- hipFree((void*)d_t_result);
-}
-
-
-template<int DataLayout>
-void test_scalar(int m_size, int k_size, int n_size)
-{
- std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl;
- // with these dimensions, the output has 300 * 140 elements, which is
- // more than 30 * 1024, which is the number of threads in blocks on
- // a 15 SM GK110 GPU
- Tensor<float, 2, DataLayout> t_left(m_size, k_size);
- Tensor<float, 2, DataLayout> t_right(k_size, n_size);
- Tensor<float, 0, DataLayout> t_result;
- Tensor<float, 0, DataLayout> t_result_gpu;
- Eigen::array<DimPair, 2> dims(DimPair(0, 0), DimPair(1, 1));
-
- t_left.setRandom();
- t_right.setRandom();
-
- std::size_t t_left_bytes = t_left.size() * sizeof(float);
- std::size_t t_right_bytes = t_right.size() * sizeof(float);
- std::size_t t_result_bytes = sizeof(float);
-
- float* d_t_left;
- float* d_t_right;
- float* d_t_result;
-
- hipMalloc((void**)(&d_t_left), t_left_bytes);
- hipMalloc((void**)(&d_t_right), t_right_bytes);
- hipMalloc((void**)(&d_t_result), t_result_bytes);
-
- hipMemcpy(d_t_left, t_left.data(), t_left_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_t_right, t_right.data(), t_right_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> >
- gpu_t_left(d_t_left, m_size, k_size);
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> >
- gpu_t_right(d_t_right, k_size, n_size);
- Eigen::TensorMap<Eigen::Tensor<float, 0, DataLayout> >
- gpu_t_result(d_t_result);
-
- gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims);
- t_result = t_left.contract(t_right, dims);
-
- hipMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, hipMemcpyDeviceToHost);
- if (fabs(t_result() - t_result_gpu()) > 1e-4f &&
- !Eigen::internal::isApprox(t_result(), t_result_gpu(), 1e-4f)) {
- std::cout << "mismatch detected: " << t_result()
- << " vs " << t_result_gpu() << std::endl;
- assert(false);
- }
-
- hipFree((void*)d_t_left);
- hipFree((void*)d_t_right);
- hipFree((void*)d_t_result);
-}
-
-
-template<int DataLayout>
-void test_hip_contraction_m() {
- for (int k = 32; k < 256; k++) {
- test_hip_contraction<ColMajor>(k, 128, 128);
- test_hip_contraction<RowMajor>(k, 128, 128);
- }
-}
-
-template<int DataLayout>
-void test_hip_contraction_k() {
- for (int k = 32; k < 256; k++) {
- test_hip_contraction<ColMajor>(128, k, 128);
- test_hip_contraction<RowMajor>(128, k, 128);
- }
-}
-
-template<int DataLayout>
-void test_hip_contraction_n() {
- for (int k = 32; k < 256; k++) {
- test_hip_contraction<ColMajor>(128, 128, k);
- test_hip_contraction<RowMajor>(128, 128, k);
- }
-}
-
-
-template<int DataLayout>
-void test_hip_contraction_sizes() {
- int m_sizes[] = { 31, 39, 63, 64, 65,
- 127, 129, 255, 257 , 511,
- 512, 513, 1023, 1024, 1025};
-
- int n_sizes[] = { 31, 39, 63, 64, 65,
- 127, 129, 255, 257, 511,
- 512, 513, 1023, 1024, 1025};
-
- int k_sizes[] = { 31, 39, 63, 64, 65,
- 95, 96, 127, 129, 255,
- 257, 511, 512, 513, 1023,
- 1024, 1025};
-
- for (int i = 0; i < 15; i++) {
- for (int j = 0; j < 15; j++) {
- for (int k = 0; k < 17; k++) {
- test_hip_contraction<DataLayout>(m_sizes[i], n_sizes[j], k_sizes[k]);
- }
- }
- }
-}
-
-void test_cxx11_tensor_hip()
-{
- CALL_SUBTEST(test_hip_contraction<ColMajor>(128, 128, 128));
- CALL_SUBTEST(test_hip_contraction<RowMajor>(128, 128, 128));
-
- CALL_SUBTEST(test_scalar<ColMajor>(128, 128, 128));
- CALL_SUBTEST(test_scalar<RowMajor>(128, 128, 128));
-
- CALL_SUBTEST(test_hip_contraction_m<ColMajor>());
- CALL_SUBTEST(test_hip_contraction_m<RowMajor>());
-
- CALL_SUBTEST(test_hip_contraction_k<ColMajor>());
- CALL_SUBTEST(test_hip_contraction_k<RowMajor>());
-
- CALL_SUBTEST(test_hip_contraction_n<ColMajor>());
- CALL_SUBTEST(test_hip_contraction_n<RowMajor>());
-
- // Commenting out these tests due to long runtimes
- // CALL_SUBTEST(test_hip_contraction_sizes<ColMajor>());
- // CALL_SUBTEST(test_hip_contraction_sizes<RowMajor>());
-}
diff --git a/unsupported/test/cxx11_tensor_device_hip.cu b/unsupported/test/cxx11_tensor_device_hip.cu
deleted file mode 100644
index b98c481..0000000
--- a/unsupported/test/cxx11_tensor_device_hip.cu
+++ /dev/null
@@ -1,389 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_device
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-using Eigen::Tensor;
-using Eigen::RowMajor;
-
-// Context for evaluation on cpu
-struct CPUContext {
- CPUContext(const Eigen::Tensor<float, 3>& in1, Eigen::Tensor<float, 3>& in2, Eigen::Tensor<float, 3>& out) : in1_(in1), in2_(in2), out_(out), kernel_1d_(2), kernel_2d_(2,2), kernel_3d_(2,2,2) {
- kernel_1d_(0) = 3.14f;
- kernel_1d_(1) = 2.7f;
-
- kernel_2d_(0,0) = 3.14f;
- kernel_2d_(1,0) = 2.7f;
- kernel_2d_(0,1) = 0.2f;
- kernel_2d_(1,1) = 7.0f;
-
- kernel_3d_(0,0,0) = 3.14f;
- kernel_3d_(0,1,0) = 2.7f;
- kernel_3d_(0,0,1) = 0.2f;
- kernel_3d_(0,1,1) = 7.0f;
- kernel_3d_(1,0,0) = -1.0f;
- kernel_3d_(1,1,0) = -0.3f;
- kernel_3d_(1,0,1) = -0.7f;
- kernel_3d_(1,1,1) = -0.5f;
- }
-
- const Eigen::DefaultDevice& device() const { return cpu_device_; }
-
- const Eigen::Tensor<float, 3>& in1() const { return in1_; }
- const Eigen::Tensor<float, 3>& in2() const { return in2_; }
- Eigen::Tensor<float, 3>& out() { return out_; }
- const Eigen::Tensor<float, 1>& kernel1d() const { return kernel_1d_; }
- const Eigen::Tensor<float, 2>& kernel2d() const { return kernel_2d_; }
- const Eigen::Tensor<float, 3>& kernel3d() const { return kernel_3d_; }
-
- private:
- const Eigen::Tensor<float, 3>& in1_;
- const Eigen::Tensor<float, 3>& in2_;
- Eigen::Tensor<float, 3>& out_;
-
- Eigen::Tensor<float, 1> kernel_1d_;
- Eigen::Tensor<float, 2> kernel_2d_;
- Eigen::Tensor<float, 3> kernel_3d_;
-
- Eigen::DefaultDevice cpu_device_;
-};
-
-
-// Context for evaluation on GPU
-struct GPUContext {
- GPUContext(const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in1, Eigen::TensorMap<Eigen::Tensor<float, 3> >& in2, Eigen::TensorMap<Eigen::Tensor<float, 3> >& out) : in1_(in1), in2_(in2), out_(out), gpu_device_(&stream_) {
- assert(hipMalloc((void**)(&kernel_1d_), 2*sizeof(float)) == hipSuccess);
- float kernel_1d_val[] = {3.14f, 2.7f};
- assert(hipMemcpy(kernel_1d_, kernel_1d_val, 2*sizeof(float), hipMemcpyHostToDevice) == hipSuccess);
-
- assert(hipMalloc((void**)(&kernel_2d_), 4*sizeof(float)) == hipSuccess);
- float kernel_2d_val[] = {3.14f, 2.7f, 0.2f, 7.0f};
- assert(hipMemcpy(kernel_2d_, kernel_2d_val, 4*sizeof(float), hipMemcpyHostToDevice) == hipSuccess);
-
- assert(hipMalloc((void**)(&kernel_3d_), 8*sizeof(float)) == hipSuccess);
- float kernel_3d_val[] = {3.14f, -1.0f, 2.7f, -0.3f, 0.2f, -0.7f, 7.0f, -0.5f};
- assert(hipMemcpy(kernel_3d_, kernel_3d_val, 8*sizeof(float), hipMemcpyHostToDevice) == hipSuccess);
- }
- ~GPUContext() {
- assert(hipFree(kernel_1d_) == hipSuccess);
- assert(hipFree(kernel_2d_) == hipSuccess);
- assert(hipFree(kernel_3d_) == hipSuccess);
- }
-
- const Eigen::GpuDevice& device() const { return gpu_device_; }
-
- const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in1() const { return in1_; }
- const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in2() const { return in2_; }
- Eigen::TensorMap<Eigen::Tensor<float, 3> >& out() { return out_; }
- Eigen::TensorMap<Eigen::Tensor<float, 1> > kernel1d() const { return Eigen::TensorMap<Eigen::Tensor<float, 1> >(kernel_1d_, 2); }
- Eigen::TensorMap<Eigen::Tensor<float, 2> > kernel2d() const { return Eigen::TensorMap<Eigen::Tensor<float, 2> >(kernel_2d_, 2, 2); }
- Eigen::TensorMap<Eigen::Tensor<float, 3> > kernel3d() const { return Eigen::TensorMap<Eigen::Tensor<float, 3> >(kernel_3d_, 2, 2, 2); }
-
- private:
- const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in1_;
- const Eigen::TensorMap<Eigen::Tensor<float, 3> >& in2_;
- Eigen::TensorMap<Eigen::Tensor<float, 3> >& out_;
-
- float* kernel_1d_;
- float* kernel_2d_;
- float* kernel_3d_;
-
- Eigen::HipStreamDevice stream_;
- Eigen::GpuDevice gpu_device_;
-};
-
-
-// The actual expression to evaluate
-template <typename Context>
-void test_contextual_eval(Context* context)
-{
- context->out().device(context->device()) = context->in1() + context->in2() * 3.14f + context->in1().constant(2.718f);
-}
-
-template <typename Context>
-void test_forced_contextual_eval(Context* context)
-{
- context->out().device(context->device()) = (context->in1() + context->in2()).eval() * 3.14f + context->in1().constant(2.718f);
-}
-
-template <typename Context>
-void test_compound_assignment(Context* context)
-{
- context->out().device(context->device()) = context->in1().constant(2.718f);
- context->out().device(context->device()) += context->in1() + context->in2() * 3.14f;
-}
-
-
-template <typename Context>
-void test_contraction(Context* context)
-{
- Eigen::array<std::pair<int, int>, 2> dims;
- dims[0] = std::make_pair(1, 1);
- dims[1] = std::make_pair(2, 2);
-
- Eigen::array<int, 2> shape(40, 50*70);
-
- Eigen::DSizes<int, 2> indices(0,0);
- Eigen::DSizes<int, 2> sizes(40,40);
-
- context->out().reshape(shape).slice(indices, sizes).device(context->device()) = context->in1().contract(context->in2(), dims);
-}
-
-
-template <typename Context>
-void test_1d_convolution(Context* context)
-{
- Eigen::DSizes<int, 3> indices(0,0,0);
- Eigen::DSizes<int, 3> sizes(40,49,70);
-
- Eigen::array<int, 1> dims(1);
- context->out().slice(indices, sizes).device(context->device()) = context->in1().convolve(context->kernel1d(), dims);
-}
-
-template <typename Context>
-void test_2d_convolution(Context* context)
-{
- Eigen::DSizes<int, 3> indices(0,0,0);
- Eigen::DSizes<int, 3> sizes(40,49,69);
-
- Eigen::array<int, 2> dims(1,2);
- context->out().slice(indices, sizes).device(context->device()) = context->in1().convolve(context->kernel2d(), dims);
-}
-
-template <typename Context>
-void test_3d_convolution(Context* context)
-{
- Eigen::DSizes<int, 3> indices(0,0,0);
- Eigen::DSizes<int, 3> sizes(39,49,69);
-
- Eigen::array<int, 3> dims(0,1,2);
- context->out().slice(indices, sizes).device(context->device()) = context->in1().convolve(context->kernel3d(), dims);
-}
-
-
-void test_cpu() {
- Eigen::Tensor<float, 3> in1(40,50,70);
- Eigen::Tensor<float, 3> in2(40,50,70);
- Eigen::Tensor<float, 3> out(40,50,70);
-
- in1 = in1.random() + in1.constant(10.0f);
- in2 = in2.random() + in2.constant(10.0f);
-
- CPUContext context(in1, in2, out);
- test_contextual_eval(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), in1(i,j,k) + in2(i,j,k) * 3.14f + 2.718f);
- }
- }
- }
-
- test_forced_contextual_eval(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), (in1(i,j,k) + in2(i,j,k)) * 3.14f + 2.718f);
- }
- }
- }
-
- test_compound_assignment(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), in1(i,j,k) + in2(i,j,k) * 3.14f + 2.718f);
- }
- }
- }
-
- test_contraction(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 40; ++j) {
- const float result = out(i,j,0);
- float expected = 0;
- for (int k = 0; k < 50; ++k) {
- for (int l = 0; l < 70; ++l) {
- expected += in1(i, k, l) * in2(j, k, l);
- }
- }
- VERIFY_IS_APPROX(expected, result);
- }
- }
-
- test_1d_convolution(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f));
- }
- }
- }
-
- test_2d_convolution(&context);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 69; ++k) {
- const float result = out(i,j,k);
- const float expected = (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f) +
- (in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f);
- if (fabs(expected) < 1e-4f && fabs(result) < 1e-4f) {
- continue;
- }
- VERIFY_IS_APPROX(expected, result);
- }
- }
- }
-
- test_3d_convolution(&context);
- for (int i = 0; i < 39; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 69; ++k) {
- const float result = out(i,j,k);
- const float expected = (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f +
- in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f) +
- (in1(i+1,j,k) * -1.0f + in1(i+1,j+1,k) * -0.3f +
- in1(i+1,j,k+1) * -0.7f + in1(i+1,j+1,k+1) * -0.5f);
- if (fabs(expected) < 1e-4f && fabs(result) < 1e-4f) {
- continue;
- }
- VERIFY_IS_APPROX(expected, result);
- }
- }
- }
-}
-
-void test_gpu() {
- Eigen::Tensor<float, 3> in1(40,50,70);
- Eigen::Tensor<float, 3> in2(40,50,70);
- Eigen::Tensor<float, 3> out(40,50,70);
- in1 = in1.random() + in1.constant(10.0f);
- in2 = in2.random() + in2.constant(10.0f);
-
- std::size_t in1_bytes = in1.size() * sizeof(float);
- std::size_t in2_bytes = in2.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_in1;
- float* d_in2;
- float* d_out;
- hipMalloc((void**)(&d_in1), in1_bytes);
- hipMalloc((void**)(&d_in2), in2_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in1, in1.data(), in1_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in2, in2.data(), in2_bytes, hipMemcpyHostToDevice);
-
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in1(d_in1, 40,50,70);
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in2(d_in2, 40,50,70);
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_out(d_out, 40,50,70);
-
- GPUContext context(gpu_in1, gpu_in2, gpu_out);
- test_contextual_eval(&context);
- assert(hipMemcpy(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), in1(i,j,k) + in2(i,j,k) * 3.14f + 2.718f);
- }
- }
- }
-
- test_forced_contextual_eval(&context);
- assert(hipMemcpy(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), (in1(i,j,k) + in2(i,j,k)) * 3.14f + 2.718f);
- }
- }
- }
-
- test_compound_assignment(&context);
- assert(hipMemcpy(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 50; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), in1(i,j,k) + in2(i,j,k) * 3.14f + 2.718f);
- }
- }
- }
-
- test_contraction(&context);
- assert(hipMemcpy(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 40; ++j) {
- const float result = out(i,j,0);
- float expected = 0;
- for (int k = 0; k < 50; ++k) {
- for (int l = 0; l < 70; ++l) {
- expected += in1(i, k, l) * in2(j, k, l);
- }
- }
- VERIFY_IS_APPROX(expected, result);
- }
- }
-
- test_1d_convolution(&context);
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, context.device().stream()) == hipSuccess);
- assert(hipStreamSynchronize(context.device().stream()) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 70; ++k) {
- VERIFY_IS_APPROX(out(i,j,k), (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f));
- }
- }
- }
-
- test_2d_convolution(&context);
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, context.device().stream()) == hipSuccess);
- assert(hipStreamSynchronize(context.device().stream()) == hipSuccess);
- for (int i = 0; i < 40; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 69; ++k) {
- const float result = out(i,j,k);
- const float expected = (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f +
- in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f);
- VERIFY_IS_APPROX(expected, result);
- }
- }
- }
-
- /*
- test_3d_convolution(&context);
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, context.device().stream()) == hipSuccess);
- assert(hipStreamSynchronize(context.device().stream()) == hipSuccess);
- for (int i = 0; i < 39; ++i) {
- for (int j = 0; j < 49; ++j) {
- for (int k = 0; k < 69; ++k) {
- const float result = out(i,j,k);
- const float expected = (in1(i,j,k) * 3.14f + in1(i,j+1,k) * 2.7f +
- in1(i,j,k+1) * 0.2f + in1(i,j+1,k+1) * 7.0f +
- in1(i+1,j,k) * -1.0f + in1(i+1,j+1,k) * -0.3f +
- in1(i+1,j,k+1) * -0.7f + in1(i+1,j+1,k+1) * -0.5f);
- VERIFY_IS_APPROX(expected, result);
- }
- }
- }
- */
-}
-
-
-void test_cxx11_tensor_device()
-{
- CALL_SUBTEST(test_cpu());
- CALL_SUBTEST(test_gpu());
-}
diff --git a/unsupported/test/cxx11_tensor_hip.cu b/unsupported/test/cxx11_tensor_hip.cu
deleted file mode 100644
index b288402..0000000
--- a/unsupported/test/cxx11_tensor_hip.cu
+++ /dev/null
@@ -1,1295 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_hip
-#define EIGEN_USE_GPU
-
-#ifdef __NVCC__
-#if defined __CUDACC_VER__ && __CUDACC_VER__ >= 70500
-#include <cuda_fp16.h>
-#endif
-#endif
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-using Eigen::Tensor;
-
-void test_hip_nullary() {
- Tensor<float, 1, 0, int> in1(2);
- Tensor<float, 1, 0, int> in2(2);
- in1.setRandom();
- in2.setRandom();
-
- std::size_t tensor_bytes = in1.size() * sizeof(float);
-
- float* d_in1;
- float* d_in2;
- hipMalloc((void**)(&d_in1), tensor_bytes);
- hipMalloc((void**)(&d_in2), tensor_bytes);
- hipMemcpy(d_in1, in1.data(), tensor_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in2, in2.data(), tensor_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 1, 0, int>, Eigen::Aligned> gpu_in1(
- d_in1, 2);
- Eigen::TensorMap<Eigen::Tensor<float, 1, 0, int>, Eigen::Aligned> gpu_in2(
- d_in2, 2);
-
- gpu_in1.device(gpu_device) = gpu_in1.constant(3.14f);
- gpu_in2.device(gpu_device) = gpu_in2.random();
-
- Tensor<float, 1, 0, int> new1(2);
- Tensor<float, 1, 0, int> new2(2);
-
- assert(hipMemcpyAsync(new1.data(), d_in1, tensor_bytes, hipMemcpyDeviceToHost,
- gpu_device.stream()) == hipSuccess);
- assert(hipMemcpyAsync(new2.data(), d_in2, tensor_bytes, hipMemcpyDeviceToHost,
- gpu_device.stream()) == hipSuccess);
-
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 2; ++i) {
- VERIFY_IS_APPROX(new1(i), 3.14f);
- VERIFY_IS_NOT_EQUAL(new2(i), in2(i));
- }
-
- hipFree(d_in1);
- hipFree(d_in2);
-}
-
-void test_hip_elementwise_small() {
- Tensor<float, 1> in1(Eigen::array<Eigen::DenseIndex, 1>{2});
- Tensor<float, 1> in2(Eigen::array<Eigen::DenseIndex, 1>{2});
- Tensor<float, 1> out(Eigen::array<Eigen::DenseIndex, 1>{2});
- in1.setRandom();
- in2.setRandom();
-
- std::size_t in1_bytes = in1.size() * sizeof(float);
- std::size_t in2_bytes = in2.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_in1;
- float* d_in2;
- float* d_out;
- hipMalloc((void**)(&d_in1), in1_bytes);
- hipMalloc((void**)(&d_in2), in2_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in1, in1.data(), in1_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in2, in2.data(), in2_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1(
- d_in1, Eigen::array<Eigen::DenseIndex, 1>{2});
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in2(
- d_in2, Eigen::array<Eigen::DenseIndex, 1>{2});
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_out(
- d_out, Eigen::array<Eigen::DenseIndex, 1>{2});
-
- gpu_out.device(gpu_device) = gpu_in1 + gpu_in2;
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost,
- gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 2; ++i) {
- VERIFY_IS_APPROX(
- out(Eigen::array<Eigen::DenseIndex, 1>{i}),
- in1(Eigen::array<Eigen::DenseIndex, 1>{i}) + in2(Eigen::array<Eigen::DenseIndex, 1>{i}));
- }
-
- hipFree(d_in1);
- hipFree(d_in2);
- hipFree(d_out);
-}
-
-void test_hip_elementwise()
-{
- Tensor<float, 3> in1(Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Tensor<float, 3> in2(Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Tensor<float, 3> in3(Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Tensor<float, 3> out(Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- in1.setRandom();
- in2.setRandom();
- in3.setRandom();
-
- std::size_t in1_bytes = in1.size() * sizeof(float);
- std::size_t in2_bytes = in2.size() * sizeof(float);
- std::size_t in3_bytes = in3.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_in1;
- float* d_in2;
- float* d_in3;
- float* d_out;
- hipMalloc((void**)(&d_in1), in1_bytes);
- hipMalloc((void**)(&d_in2), in2_bytes);
- hipMalloc((void**)(&d_in3), in3_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in1, in1.data(), in1_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in2, in2.data(), in2_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in3, in3.data(), in3_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in1(d_in1, Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in2(d_in2, Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_in3(d_in3, Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
- Eigen::TensorMap<Eigen::Tensor<float, 3> > gpu_out(d_out, Eigen::array<Eigen::DenseIndex, 3>{72,53,97});
-
- gpu_out.device(gpu_device) = gpu_in1 + gpu_in2 * gpu_in3;
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 72; ++i) {
- for (int j = 0; j < 53; ++j) {
- for (int k = 0; k < 97; ++k) {
- VERIFY_IS_APPROX(out(Eigen::array<Eigen::DenseIndex, 3>{i,j,k}), in1(Eigen::array<Eigen::DenseIndex, 3>{i,j,k}) + in2(Eigen::array<Eigen::DenseIndex, 3>{i,j,k}) * in3(Eigen::array<Eigen::DenseIndex, 3>{i,j,k}));
- }
- }
- }
-
- hipFree(d_in1);
- hipFree(d_in2);
- hipFree(d_in3);
- hipFree(d_out);
-}
-
-void test_hip_props() {
- Tensor<float, 1> in1(200);
- Tensor<bool, 1> out(200);
- in1.setRandom();
-
- std::size_t in1_bytes = in1.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(bool);
-
- float* d_in1;
- bool* d_out;
- hipMalloc((void**)(&d_in1), in1_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in1, in1.data(), in1_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_in1(
- d_in1, 200);
- Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_out(
- d_out, 200);
-
- gpu_out.device(gpu_device) = (gpu_in1.isnan)();
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost,
- gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 200; ++i) {
- VERIFY_IS_EQUAL(out(i), (std::isnan)(in1(i)));
- }
-
- hipFree(d_in1);
- hipFree(d_out);
-}
-
-void test_hip_reduction()
-{
- Tensor<float, 4> in1(72,53,97,113);
- Tensor<float, 2> out(72,97);
- in1.setRandom();
-
- std::size_t in1_bytes = in1.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_in1;
- float* d_out;
- hipMalloc((void**)(&d_in1), in1_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_in1, in1.data(), in1_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4> > gpu_in1(d_in1, 72,53,97,113);
- Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97);
-
- array<Eigen::DenseIndex, 2> reduction_axis;
- reduction_axis[0] = 1;
- reduction_axis[1] = 3;
-
- gpu_out.device(gpu_device) = gpu_in1.maximum(reduction_axis);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 72; ++i) {
- for (int j = 0; j < 97; ++j) {
- float expected = 0;
- for (int k = 0; k < 53; ++k) {
- for (int l = 0; l < 113; ++l) {
- expected =
- std::max<float>(expected, in1(i, k, j, l));
- }
- }
- VERIFY_IS_APPROX(out(i,j), expected);
- }
- }
-
- hipFree(d_in1);
- hipFree(d_out);
-}
-
-template<int DataLayout>
-void test_hip_contraction()
-{
- // with these dimensions, the output has 300 * 140 elements, which is
- // more than 30 * 1024, which is the number of threads in blocks on
- // a 15 SM GK110 GPU
- Tensor<float, 4, DataLayout> t_left(6, 50, 3, 31);
- Tensor<float, 5, DataLayout> t_right(Eigen::array<Eigen::DenseIndex, 5>{3, 31, 7, 20, 1});
- Tensor<float, 5, DataLayout> t_result(Eigen::array<Eigen::DenseIndex, 5>{6, 50, 7, 20, 1});
-
- t_left.setRandom();
- t_right.setRandom();
-
- std::size_t t_left_bytes = t_left.size() * sizeof(float);
- std::size_t t_right_bytes = t_right.size() * sizeof(float);
- std::size_t t_result_bytes = t_result.size() * sizeof(float);
-
- float* d_t_left;
- float* d_t_right;
- float* d_t_result;
-
- hipMalloc((void**)(&d_t_left), t_left_bytes);
- hipMalloc((void**)(&d_t_right), t_right_bytes);
- hipMalloc((void**)(&d_t_result), t_result_bytes);
-
- hipMemcpy(d_t_left, t_left.data(), t_left_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_t_right, t_right.data(), t_right_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_t_left(d_t_left, 6, 50, 3, 31);
- Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_right(d_t_right, 3, 31, 7, 20, 1);
- Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_t_result(d_t_result, 6, 50, 7, 20, 1);
-
- typedef Eigen::Map<Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> > MapXf;
- MapXf m_left(t_left.data(), 300, 93);
- MapXf m_right(t_right.data(), 93, 140);
- Eigen::Matrix<float, Dynamic, Dynamic, DataLayout> m_result(300, 140);
-
- typedef Tensor<float, 1>::DimensionPair DimPair;
- Eigen::array<DimPair, 2> dims;
- dims[0] = DimPair(2, 0);
- dims[1] = DimPair(3, 1);
-
- m_result = m_left * m_right;
- gpu_t_result.device(gpu_device) = gpu_t_left.contract(gpu_t_right, dims);
-
- hipMemcpy(t_result.data(), d_t_result, t_result_bytes, hipMemcpyDeviceToHost);
-
- for (DenseIndex i = 0; i < t_result.size(); i++) {
- if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4f) {
- std::cout << "mismatch detected at index " << i << ": " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl;
- assert(false);
- }
- }
-
- hipFree(d_t_left);
- hipFree(d_t_right);
- hipFree(d_t_result);
-}
-
-
-template<int DataLayout>
-void test_hip_convolution_1d()
-{
- Tensor<float, 4, DataLayout> input(74,37,11,137);
- Tensor<float, 1, DataLayout> kernel(4);
- Tensor<float, 4, DataLayout> out(74,34,11,137);
- input = input.constant(10.0f) + input.random();
- kernel = kernel.constant(7.0f) + kernel.random();
-
- std::size_t input_bytes = input.size() * sizeof(float);
- std::size_t kernel_bytes = kernel.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_input;
- float* d_kernel;
- float* d_out;
- hipMalloc((void**)(&d_input), input_bytes);
- hipMalloc((void**)(&d_kernel), kernel_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_input, input.data(), input_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_kernel, kernel.data(), kernel_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input, 74,37,11,137);
- Eigen::TensorMap<Eigen::Tensor<float, 1, DataLayout> > gpu_kernel(d_kernel, 4);
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out, 74,34,11,137);
-
- Eigen::array<Eigen::DenseIndex, 1> dims{1};
- gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 74; ++i) {
- for (int j = 0; j < 34; ++j) {
- for (int k = 0; k < 11; ++k) {
- for (int l = 0; l < 137; ++l) {
- const float result = out(i,j,k,l);
- const float expected = input(i,j+0,k,l) * kernel(0) + input(i,j+1,k,l) * kernel(1) +
- input(i,j+2,k,l) * kernel(2) + input(i,j+3,k,l) * kernel(3);
- VERIFY_IS_APPROX(result, expected);
- }
- }
- }
- }
-
- hipFree(d_input);
- hipFree(d_kernel);
- hipFree(d_out);
-}
-
-void test_hip_convolution_inner_dim_col_major_1d()
-{
- Tensor<float, 4, ColMajor> input(74,9,11,7);
- Tensor<float, 1, ColMajor> kernel(4);
- Tensor<float, 4, ColMajor> out(71,9,11,7);
- input = input.constant(10.0f) + input.random();
- kernel = kernel.constant(7.0f) + kernel.random();
-
- std::size_t input_bytes = input.size() * sizeof(float);
- std::size_t kernel_bytes = kernel.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_input;
- float* d_kernel;
- float* d_out;
- hipMalloc((void**)(&d_input), input_bytes);
- hipMalloc((void**)(&d_kernel), kernel_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_input, input.data(), input_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_kernel, kernel.data(), kernel_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_input(d_input,74,9,11,7);
- Eigen::TensorMap<Eigen::Tensor<float, 1, ColMajor> > gpu_kernel(d_kernel,4);
- Eigen::TensorMap<Eigen::Tensor<float, 4, ColMajor> > gpu_out(d_out,71,9,11,7);
-
- Eigen::array<Eigen::DenseIndex, 1> dims{0};
- gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 71; ++i) {
- for (int j = 0; j < 9; ++j) {
- for (int k = 0; k < 11; ++k) {
- for (int l = 0; l < 7; ++l) {
- const float result = out(i,j,k,l);
- const float expected = input(i+0,j,k,l) * kernel(0) + input(i+1,j,k,l) * kernel(1) +
- input(i+2,j,k,l) * kernel(2) + input(i+3,j,k,l) * kernel(3);
- VERIFY_IS_APPROX(result, expected);
- }
- }
- }
- }
-
- hipFree(d_input);
- hipFree(d_kernel);
- hipFree(d_out);
-}
-
-void test_hip_convolution_inner_dim_row_major_1d()
-{
- Tensor<float, 4, RowMajor> input(7,9,11,74);
- Tensor<float, 1, RowMajor> kernel(4);
- Tensor<float, 4, RowMajor> out(7,9,11,71);
- input = input.constant(10.0f) + input.random();
- kernel = kernel.constant(7.0f) + kernel.random();
-
- std::size_t input_bytes = input.size() * sizeof(float);
- std::size_t kernel_bytes = kernel.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_input;
- float* d_kernel;
- float* d_out;
- hipMalloc((void**)(&d_input), input_bytes);
- hipMalloc((void**)(&d_kernel), kernel_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_input, input.data(), input_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_kernel, kernel.data(), kernel_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_input(d_input, 7,9,11,74);
- Eigen::TensorMap<Eigen::Tensor<float, 1, RowMajor> > gpu_kernel(d_kernel, 4);
- Eigen::TensorMap<Eigen::Tensor<float, 4, RowMajor> > gpu_out(d_out, 7,9,11,71);
-
- Eigen::array<Eigen::DenseIndex, 1> dims{3};
- gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 7; ++i) {
- for (int j = 0; j < 9; ++j) {
- for (int k = 0; k < 11; ++k) {
- for (int l = 0; l < 71; ++l) {
- const float result = out(i,j,k,l);
- const float expected = input(i,j,k,l+0) * kernel(0) + input(i,j,k,l+1) * kernel(1) +
- input(i,j,k,l+2) * kernel(2) + input(i,j,k,l+3) * kernel(3);
- VERIFY_IS_APPROX(result, expected);
- }
- }
- }
- }
-
- hipFree(d_input);
- hipFree(d_kernel);
- hipFree(d_out);
-}
-
-template<int DataLayout>
-void test_hip_convolution_2d()
-{
- Tensor<float, 4, DataLayout> input(74,37,11,137);
- Tensor<float, 2, DataLayout> kernel(3,4);
- Tensor<float, 4, DataLayout> out(74,35,8,137);
- input = input.constant(10.0f) + input.random();
- kernel = kernel.constant(7.0f) + kernel.random();
-
- std::size_t input_bytes = input.size() * sizeof(float);
- std::size_t kernel_bytes = kernel.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_input;
- float* d_kernel;
- float* d_out;
- hipMalloc((void**)(&d_input), input_bytes);
- hipMalloc((void**)(&d_kernel), kernel_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_input, input.data(), input_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_kernel, kernel.data(), kernel_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_input(d_input,74,37,11,137);
- Eigen::TensorMap<Eigen::Tensor<float, 2, DataLayout> > gpu_kernel(d_kernel,3,4);
- Eigen::TensorMap<Eigen::Tensor<float, 4, DataLayout> > gpu_out(d_out,74,35,8,137);
-
- Eigen::array<Eigen::DenseIndex, 2> dims{1,2};
- gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 74; ++i) {
- for (int j = 0; j < 35; ++j) {
- for (int k = 0; k < 8; ++k) {
- for (int l = 0; l < 137; ++l) {
- const float result = out(i,j,k,l);
- const float expected = input(i,j+0,k+0,l) * kernel(0,0) +
- input(i,j+1,k+0,l) * kernel(1,0) +
- input(i,j+2,k+0,l) * kernel(2,0) +
- input(i,j+0,k+1,l) * kernel(0,1) +
- input(i,j+1,k+1,l) * kernel(1,1) +
- input(i,j+2,k+1,l) * kernel(2,1) +
- input(i,j+0,k+2,l) * kernel(0,2) +
- input(i,j+1,k+2,l) * kernel(1,2) +
- input(i,j+2,k+2,l) * kernel(2,2) +
- input(i,j+0,k+3,l) * kernel(0,3) +
- input(i,j+1,k+3,l) * kernel(1,3) +
- input(i,j+2,k+3,l) * kernel(2,3);
- VERIFY_IS_APPROX(result, expected);
- }
- }
- }
- }
-
- hipFree(d_input);
- hipFree(d_kernel);
- hipFree(d_out);
-}
-
-template<int DataLayout>
-void test_hip_convolution_3d()
-{
- Tensor<float, 5, DataLayout> input(Eigen::array<Eigen::DenseIndex, 5>{74,37,11,137,17});
- Tensor<float, 3, DataLayout> kernel(3,4,2);
- Tensor<float, 5, DataLayout> out(Eigen::array<Eigen::DenseIndex, 5>{74,35,8,136,17});
- input = input.constant(10.0f) + input.random();
- kernel = kernel.constant(7.0f) + kernel.random();
-
- std::size_t input_bytes = input.size() * sizeof(float);
- std::size_t kernel_bytes = kernel.size() * sizeof(float);
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_input;
- float* d_kernel;
- float* d_out;
- hipMalloc((void**)(&d_input), input_bytes);
- hipMalloc((void**)(&d_kernel), kernel_bytes);
- hipMalloc((void**)(&d_out), out_bytes);
-
- hipMemcpy(d_input, input.data(), input_bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_kernel, kernel.data(), kernel_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_input(d_input,74,37,11,137,17);
- Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> > gpu_kernel(d_kernel,3,4,2);
- Eigen::TensorMap<Eigen::Tensor<float, 5, DataLayout> > gpu_out(d_out,74,35,8,136,17);
-
- Eigen::array<Eigen::DenseIndex, 3> dims{1,2,3};
- gpu_out.device(gpu_device) = gpu_input.convolve(gpu_kernel, dims);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 74; ++i) {
- for (int j = 0; j < 35; ++j) {
- for (int k = 0; k < 8; ++k) {
- for (int l = 0; l < 136; ++l) {
- for (int m = 0; m < 17; ++m) {
- const float result = out(i,j,k,l,m);
- const float expected = input(i,j+0,k+0,l+0,m) * kernel(0,0,0) +
- input(i,j+1,k+0,l+0,m) * kernel(1,0,0) +
- input(i,j+2,k+0,l+0,m) * kernel(2,0,0) +
- input(i,j+0,k+1,l+0,m) * kernel(0,1,0) +
- input(i,j+1,k+1,l+0,m) * kernel(1,1,0) +
- input(i,j+2,k+1,l+0,m) * kernel(2,1,0) +
- input(i,j+0,k+2,l+0,m) * kernel(0,2,0) +
- input(i,j+1,k+2,l+0,m) * kernel(1,2,0) +
- input(i,j+2,k+2,l+0,m) * kernel(2,2,0) +
- input(i,j+0,k+3,l+0,m) * kernel(0,3,0) +
- input(i,j+1,k+3,l+0,m) * kernel(1,3,0) +
- input(i,j+2,k+3,l+0,m) * kernel(2,3,0) +
- input(i,j+0,k+0,l+1,m) * kernel(0,0,1) +
- input(i,j+1,k+0,l+1,m) * kernel(1,0,1) +
- input(i,j+2,k+0,l+1,m) * kernel(2,0,1) +
- input(i,j+0,k+1,l+1,m) * kernel(0,1,1) +
- input(i,j+1,k+1,l+1,m) * kernel(1,1,1) +
- input(i,j+2,k+1,l+1,m) * kernel(2,1,1) +
- input(i,j+0,k+2,l+1,m) * kernel(0,2,1) +
- input(i,j+1,k+2,l+1,m) * kernel(1,2,1) +
- input(i,j+2,k+2,l+1,m) * kernel(2,2,1) +
- input(i,j+0,k+3,l+1,m) * kernel(0,3,1) +
- input(i,j+1,k+3,l+1,m) * kernel(1,3,1) +
- input(i,j+2,k+3,l+1,m) * kernel(2,3,1);
- VERIFY_IS_APPROX(result, expected);
- }
- }
- }
- }
- }
-
- hipFree(d_input);
- hipFree(d_kernel);
- hipFree(d_out);
-}
-
-
-template <typename Scalar>
-void test_hip_lgamma(const Scalar stddev)
-{
- Tensor<Scalar, 2> in(72,97);
- in.setRandom();
- in *= in.constant(stddev);
- Tensor<Scalar, 2> out(72,97);
- out.setZero();
-
- std::size_t bytes = in.size() * sizeof(Scalar);
-
- Scalar* d_in;
- Scalar* d_out;
- hipMalloc((void**)(&d_in), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in, in.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97);
-
- gpu_out.device(gpu_device) = gpu_in.lgamma();
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 72; ++i) {
- for (int j = 0; j < 97; ++j) {
- VERIFY_IS_APPROX(out(i,j), (std::lgamma)(in(i,j)));
- }
- }
-
- hipFree(d_in);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_digamma()
-{
- Tensor<Scalar, 1> in(7);
- Tensor<Scalar, 1> out(7);
- Tensor<Scalar, 1> expected_out(7);
- out.setZero();
-
- in(0) = Scalar(1);
- in(1) = Scalar(1.5);
- in(2) = Scalar(4);
- in(3) = Scalar(-10.5);
- in(4) = Scalar(10000.5);
- in(5) = Scalar(0);
- in(6) = Scalar(-1);
-
- expected_out(0) = Scalar(-0.5772156649015329);
- expected_out(1) = Scalar(0.03648997397857645);
- expected_out(2) = Scalar(1.2561176684318);
- expected_out(3) = Scalar(2.398239129535781);
- expected_out(4) = Scalar(9.210340372392849);
- expected_out(5) = std::numeric_limits<Scalar>::infinity();
- expected_out(6) = std::numeric_limits<Scalar>::infinity();
-
- std::size_t bytes = in.size() * sizeof(Scalar);
-
- Scalar* d_in;
- Scalar* d_out;
- hipMalloc((void**)(&d_in), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in, in.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in(d_in, 7);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7);
-
- gpu_out.device(gpu_device) = gpu_in.digamma();
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 5; ++i) {
- VERIFY_IS_APPROX(out(i), expected_out(i));
- }
- for (int i = 5; i < 7; ++i) {
- VERIFY_IS_EQUAL(out(i), expected_out(i));
- }
-
- hipFree(d_in);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_zeta()
-{
- Tensor<Scalar, 1> in_x(6);
- Tensor<Scalar, 1> in_q(6);
- Tensor<Scalar, 1> out(6);
- Tensor<Scalar, 1> expected_out(6);
- out.setZero();
-
- in_x(0) = Scalar(1);
- in_x(1) = Scalar(1.5);
- in_x(2) = Scalar(4);
- in_x(3) = Scalar(-10.5);
- in_x(4) = Scalar(10000.5);
- in_x(5) = Scalar(3);
-
- in_q(0) = Scalar(1.2345);
- in_q(1) = Scalar(2);
- in_q(2) = Scalar(1.5);
- in_q(3) = Scalar(3);
- in_q(4) = Scalar(1.0001);
- in_q(5) = Scalar(-2.5);
-
- expected_out(0) = std::numeric_limits<Scalar>::infinity();
- expected_out(1) = Scalar(1.61237534869);
- expected_out(2) = Scalar(0.234848505667);
- expected_out(3) = Scalar(1.03086757337e-5);
- expected_out(4) = Scalar(0.367879440865);
- expected_out(5) = Scalar(0.054102025820864097);
-
- std::size_t bytes = in_x.size() * sizeof(Scalar);
-
- Scalar* d_in_x;
- Scalar* d_in_q;
- Scalar* d_out;
- hipMalloc((void**)(&d_in_x), bytes);
- hipMalloc((void**)(&d_in_q), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in_x, in_x.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in_q, in_q.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_q(d_in_q, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 6);
-
- gpu_out.device(gpu_device) = gpu_in_x.zeta(gpu_in_q);
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- VERIFY_IS_EQUAL(out(0), expected_out(0));
- VERIFY((std::isnan)(out(3)));
-
- for (int i = 1; i < 6; ++i) {
- if (i != 3) {
- VERIFY_IS_APPROX(out(i), expected_out(i));
- }
- }
-
- hipFree(d_in_x);
- hipFree(d_in_q);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_polygamma()
-{
- Tensor<Scalar, 1> in_x(7);
- Tensor<Scalar, 1> in_n(7);
- Tensor<Scalar, 1> out(7);
- Tensor<Scalar, 1> expected_out(7);
- out.setZero();
-
- in_n(0) = Scalar(1);
- in_n(1) = Scalar(1);
- in_n(2) = Scalar(1);
- in_n(3) = Scalar(17);
- in_n(4) = Scalar(31);
- in_n(5) = Scalar(28);
- in_n(6) = Scalar(8);
-
- in_x(0) = Scalar(2);
- in_x(1) = Scalar(3);
- in_x(2) = Scalar(25.5);
- in_x(3) = Scalar(4.7);
- in_x(4) = Scalar(11.8);
- in_x(5) = Scalar(17.7);
- in_x(6) = Scalar(30.2);
-
- expected_out(0) = Scalar(0.644934066848);
- expected_out(1) = Scalar(0.394934066848);
- expected_out(2) = Scalar(0.0399946696496);
- expected_out(3) = Scalar(293.334565435);
- expected_out(4) = Scalar(0.445487887616);
- expected_out(5) = Scalar(-2.47810300902e-07);
- expected_out(6) = Scalar(-8.29668781082e-09);
-
- std::size_t bytes = in_x.size() * sizeof(Scalar);
-
- Scalar* d_in_x;
- Scalar* d_in_n;
- Scalar* d_out;
- hipMalloc((void**)(&d_in_x), bytes);
- hipMalloc((void**)(&d_in_n), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in_x, in_x.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in_n, in_n.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 7);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_n(d_in_n, 7);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 7);
-
- gpu_out.device(gpu_device) = gpu_in_n.polygamma(gpu_in_x);
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 7; ++i) {
- VERIFY_IS_APPROX(out(i), expected_out(i));
- }
-
- hipFree(d_in_x);
- hipFree(d_in_n);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_igamma()
-{
- Tensor<Scalar, 2> a(6, 6);
- Tensor<Scalar, 2> x(6, 6);
- Tensor<Scalar, 2> out(6, 6);
- out.setZero();
-
- Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)};
- Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)};
-
- for (int i = 0; i < 6; ++i) {
- for (int j = 0; j < 6; ++j) {
- a(i, j) = a_s[i];
- x(i, j) = x_s[j];
- }
- }
-
- Scalar nan = std::numeric_limits<Scalar>::quiet_NaN();
- Scalar igamma_s[][6] = {{0.0, nan, nan, nan, nan, nan},
- {0.0, 0.6321205588285578, 0.7768698398515702,
- 0.9816843611112658, 9.999500016666262e-05, 1.0},
- {0.0, 0.4275932955291202, 0.608374823728911,
- 0.9539882943107686, 7.522076445089201e-07, 1.0},
- {0.0, 0.01898815687615381, 0.06564245437845008,
- 0.5665298796332909, 4.166333347221828e-18, 1.0},
- {0.0, 0.9999780593618628, 0.9999899967080838,
- 0.9999996219837988, 0.9991370418689945, 1.0},
- {0.0, 0.0, 0.0, 0.0, 0.0, 0.5042041932513908}};
-
-
-
- std::size_t bytes = a.size() * sizeof(Scalar);
-
- Scalar* d_a;
- Scalar* d_x;
- Scalar* d_out;
- assert(hipMalloc((void**)(&d_a), bytes) == hipSuccess);
- assert(hipMalloc((void**)(&d_x), bytes) == hipSuccess);
- assert(hipMalloc((void**)(&d_out), bytes) == hipSuccess);
-
- hipMemcpy(d_a, a.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_x, x.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6);
-
- gpu_out.device(gpu_device) = gpu_a.igamma(gpu_x);
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 6; ++i) {
- for (int j = 0; j < 6; ++j) {
- if ((std::isnan)(igamma_s[i][j])) {
- VERIFY((std::isnan)(out(i, j)));
- } else {
- VERIFY_IS_APPROX(out(i, j), igamma_s[i][j]);
- }
- }
- }
-
- hipFree(d_a);
- hipFree(d_x);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_igammac()
-{
- Tensor<Scalar, 2> a(6, 6);
- Tensor<Scalar, 2> x(6, 6);
- Tensor<Scalar, 2> out(6, 6);
- out.setZero();
-
- Scalar a_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)};
- Scalar x_s[] = {Scalar(0), Scalar(1), Scalar(1.5), Scalar(4), Scalar(0.0001), Scalar(1000.5)};
-
- for (int i = 0; i < 6; ++i) {
- for (int j = 0; j < 6; ++j) {
- a(i, j) = a_s[i];
- x(i, j) = x_s[j];
- }
- }
-
- Scalar nan = std::numeric_limits<Scalar>::quiet_NaN();
- Scalar igammac_s[][6] = {{nan, nan, nan, nan, nan, nan},
- {1.0, 0.36787944117144233, 0.22313016014842982,
- 0.018315638888734182, 0.9999000049998333, 0.0},
- {1.0, 0.5724067044708798, 0.3916251762710878,
- 0.04601170568923136, 0.9999992477923555, 0.0},
- {1.0, 0.9810118431238462, 0.9343575456215499,
- 0.4334701203667089, 1.0, 0.0},
- {1.0, 2.1940638138146658e-05, 1.0003291916285e-05,
- 3.7801620118431334e-07, 0.0008629581310054535,
- 0.0},
- {1.0, 1.0, 1.0, 1.0, 1.0, 0.49579580674813944}};
-
- std::size_t bytes = a.size() * sizeof(Scalar);
-
- Scalar* d_a;
- Scalar* d_x;
- Scalar* d_out;
- hipMalloc((void**)(&d_a), bytes);
- hipMalloc((void**)(&d_x), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_a, a.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_x, x.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_a(d_a, 6, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_x(d_x, 6, 6);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 6, 6);
-
- gpu_out.device(gpu_device) = gpu_a.igammac(gpu_x);
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 6; ++i) {
- for (int j = 0; j < 6; ++j) {
- if ((std::isnan)(igammac_s[i][j])) {
- VERIFY((std::isnan)(out(i, j)));
- } else {
- VERIFY_IS_APPROX(out(i, j), igammac_s[i][j]);
- }
- }
- }
-
- hipFree(d_a);
- hipFree(d_x);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_erf(const Scalar stddev)
-{
- Tensor<Scalar, 2> in(72,97);
- in.setRandom();
- in *= in.constant(stddev);
- Tensor<Scalar, 2> out(72,97);
- out.setZero();
-
- std::size_t bytes = in.size() * sizeof(Scalar);
-
- Scalar* d_in;
- Scalar* d_out;
- assert(hipMalloc((void**)(&d_in), bytes) == hipSuccess);
- assert(hipMalloc((void**)(&d_out), bytes) == hipSuccess);
-
- hipMemcpy(d_in, in.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97);
-
- gpu_out.device(gpu_device) = gpu_in.erf();
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 72; ++i) {
- for (int j = 0; j < 97; ++j) {
- VERIFY_IS_APPROX(out(i,j), (std::erf)(in(i,j)));
- }
- }
-
- hipFree(d_in);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_erfc(const Scalar stddev)
-{
- Tensor<Scalar, 2> in(72,97);
- in.setRandom();
- in *= in.constant(stddev);
- Tensor<Scalar, 2> out(72,97);
- out.setZero();
-
- std::size_t bytes = in.size() * sizeof(Scalar);
-
- Scalar* d_in;
- Scalar* d_out;
- hipMalloc((void**)(&d_in), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in, in.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_in(d_in, 72, 97);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 2> > gpu_out(d_out, 72, 97);
-
- gpu_out.device(gpu_device) = gpu_in.erfc();
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 0; i < 72; ++i) {
- for (int j = 0; j < 97; ++j) {
- VERIFY_IS_APPROX(out(i,j), (std::erfc)(in(i,j)));
- }
- }
-
- hipFree(d_in);
- hipFree(d_out);
-}
-
-template <typename Scalar>
-void test_hip_betainc()
-{
- Tensor<Scalar, 1> in_x(125);
- Tensor<Scalar, 1> in_a(125);
- Tensor<Scalar, 1> in_b(125);
- Tensor<Scalar, 1> out(125);
- Tensor<Scalar, 1> expected_out(125);
- out.setZero();
-
- Scalar nan = std::numeric_limits<Scalar>::quiet_NaN();
-
- Array<Scalar, 1, Dynamic> x(125);
- Array<Scalar, 1, Dynamic> a(125);
- Array<Scalar, 1, Dynamic> b(125);
- Array<Scalar, 1, Dynamic> v(125);
-
- a << 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
- 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999,
- 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999,
- 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999,
- 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999,
- 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999,
- 999.999, 999.999, 999.999, 999.999, 999.999, 999.999, 999.999;
-
- b << 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999,
- 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999,
- 999.999, 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 999.999, 999.999, 999.999, 999.999, 999.999, 0.0, 0.0,
- 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999,
- 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999,
- 999.999, 999.999, 999.999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.999, 0.999, 0.999, 0.999, 0.999, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 999.999, 999.999, 999.999, 999.999, 999.999, 0.0, 0.0,
- 0.0, 0.0, 0.0, 0.03062277660168379, 0.03062277660168379,
- 0.03062277660168379, 0.03062277660168379, 0.03062277660168379, 0.999,
- 0.999, 0.999, 0.999, 0.999, 31.62177660168379, 31.62177660168379,
- 31.62177660168379, 31.62177660168379, 31.62177660168379, 999.999, 999.999,
- 999.999, 999.999, 999.999;
-
- x << -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8,
- 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5,
- 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2,
- 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1,
- 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1,
- -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8,
- 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5,
- 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2,
- 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1, -0.1, 0.2, 0.5, 0.8, 1.1;
-
- v << nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
- nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
- nan, nan, 0.47972119876364683, 0.5, 0.5202788012363533, nan, nan,
- 0.9518683957740043, 0.9789663010413743, 0.9931729188073435, nan, nan,
- 0.999995949033062, 0.9999999999993698, 0.9999999999999999, nan, nan,
- 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, nan, nan, nan,
- nan, nan, nan, nan, 0.006827081192655869, 0.0210336989586256,
- 0.04813160422599567, nan, nan, 0.20014344256217678, 0.5000000000000001,
- 0.7998565574378232, nan, nan, 0.9991401428435834, 0.999999999698403,
- 0.9999999999999999, nan, nan, 0.9999999999999999, 0.9999999999999999,
- 0.9999999999999999, nan, nan, nan, nan, nan, nan, nan,
- 1.0646600232370887e-25, 6.301722877826246e-13, 4.050966937974938e-06, nan,
- nan, 7.864342668429763e-23, 3.015969667594166e-10, 0.0008598571564165444,
- nan, nan, 6.031987710123844e-08, 0.5000000000000007, 0.9999999396801229,
- nan, nan, 0.9999999999999999, 0.9999999999999999, 0.9999999999999999, nan,
- nan, nan, nan, nan, nan, nan, 0.0, 7.029920380986636e-306,
- 2.2450728208591345e-101, nan, nan, 0.0, 9.275871147869727e-302,
- 1.2232913026152827e-97, nan, nan, 0.0, 3.0891393081932924e-252,
- 2.9303043666183996e-60, nan, nan, 2.248913486879199e-196,
- 0.5000000000004947, 0.9999999999999999, nan;
-
- for (int i = 0; i < 125; ++i) {
- in_x(i) = x(i);
- in_a(i) = a(i);
- in_b(i) = b(i);
- expected_out(i) = v(i);
- }
-
- std::size_t bytes = in_x.size() * sizeof(Scalar);
-
- Scalar* d_in_x;
- Scalar* d_in_a;
- Scalar* d_in_b;
- Scalar* d_out;
- hipMalloc((void**)(&d_in_x), bytes);
- hipMalloc((void**)(&d_in_a), bytes);
- hipMalloc((void**)(&d_in_b), bytes);
- hipMalloc((void**)(&d_out), bytes);
-
- hipMemcpy(d_in_x, in_x.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in_a, in_a.data(), bytes, hipMemcpyHostToDevice);
- hipMemcpy(d_in_b, in_b.data(), bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_x(d_in_x, 125);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_a(d_in_a, 125);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_in_b(d_in_b, 125);
- Eigen::TensorMap<Eigen::Tensor<Scalar, 1> > gpu_out(d_out, 125);
-
- gpu_out.device(gpu_device) = betainc(gpu_in_a, gpu_in_b, gpu_in_x);
-
- assert(hipMemcpyAsync(out.data(), d_out, bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- for (int i = 1; i < 125; ++i) {
- if ((std::isnan)(expected_out(i))) {
- VERIFY((std::isnan)(out(i)));
- } else {
- VERIFY_IS_APPROX(out(i), expected_out(i));
- }
- }
-
- hipFree(d_in_x);
- hipFree(d_in_a);
- hipFree(d_in_b);
- hipFree(d_out);
-}
-
-
-void test_cxx11_tensor_hip()
-{
- CALL_SUBTEST(test_hip_nullary());
- CALL_SUBTEST(test_hip_elementwise_small());
- CALL_SUBTEST(test_hip_elementwise());
- CALL_SUBTEST(test_hip_props());
- CALL_SUBTEST(test_hip_reduction());
- CALL_SUBTEST(test_hip_contraction<ColMajor>());
- CALL_SUBTEST(test_hip_contraction<RowMajor>());
- CALL_SUBTEST(test_hip_convolution_1d<ColMajor>());
- CALL_SUBTEST(test_hip_convolution_1d<RowMajor>());
- CALL_SUBTEST(test_hip_convolution_inner_dim_col_major_1d());
- CALL_SUBTEST(test_hip_convolution_inner_dim_row_major_1d());
- CALL_SUBTEST(test_hip_convolution_2d<ColMajor>());
- CALL_SUBTEST(test_hip_convolution_2d<RowMajor>());
- // The following two tests commented out due to long runtime
- // CALL_SUBTEST(test_hip_convolution_3d<ColMajor>());
- // CALL_SUBTEST(test_hip_convolution_3d<RowMajor>());
-
-#if __cplusplus > 199711L
- // std::erf, std::erfc, and so on where only added in c++11. We use them
- // as a golden reference to validate the results produced by Eigen. Therefore
- // we can only run these tests if we use a c++11 compiler.
-
- CALL_SUBTEST(test_hip_lgamma<float>(1.0f));
- CALL_SUBTEST(test_hip_lgamma<float>(100.0f));
- CALL_SUBTEST(test_hip_lgamma<float>(0.01f));
- CALL_SUBTEST(test_hip_lgamma<float>(0.001f));
-
- CALL_SUBTEST(test_hip_lgamma<double>(1.0));
- CALL_SUBTEST(test_hip_lgamma<double>(100.0));
- CALL_SUBTEST(test_hip_lgamma<double>(0.01));
- CALL_SUBTEST(test_hip_lgamma<double>(0.001));
-
- CALL_SUBTEST(test_hip_erf<float>(1.0f));
- CALL_SUBTEST(test_hip_erf<float>(100.0f));
- CALL_SUBTEST(test_hip_erf<float>(0.01f));
- CALL_SUBTEST(test_hip_erf<float>(0.001f));
-
- CALL_SUBTEST(test_hip_erfc<float>(1.0f));
- // CALL_SUBTEST(test_hip_erfc<float>(100.0f)); // HIP erfc lacks precision for large inputs
- CALL_SUBTEST(test_hip_erfc<float>(5.0f));
- CALL_SUBTEST(test_hip_erfc<float>(0.01f));
- CALL_SUBTEST(test_hip_erfc<float>(0.001f));
-
- CALL_SUBTEST(test_hip_erf<double>(1.0));
- CALL_SUBTEST(test_hip_erf<double>(100.0));
- CALL_SUBTEST(test_hip_erf<double>(0.01));
- CALL_SUBTEST(test_hip_erf<double>(0.001));
-
- CALL_SUBTEST(test_hip_erfc<double>(1.0));
- // CALL_SUBTEST(test_hip_erfc<double>(100.0)); // HIP erfc lacks precision for large inputs
- CALL_SUBTEST(test_hip_erfc<double>(5.0));
- CALL_SUBTEST(test_hip_erfc<double>(0.01));
- CALL_SUBTEST(test_hip_erfc<double>(0.001));
-
- // Following tests have functional failures on some seeds
- // CALL_SUBTEST(test_hip_digamma<float>());
- // CALL_SUBTEST(test_hip_digamma<double>());
-
- // Following tests have functional failures on some seeds
- // CALL_SUBTEST(test_hip_polygamma<float>());
- // CALL_SUBTEST(test_hip_polygamma<double>());
-
- // Following tests have functional failures on some seeds
- // CALL_SUBTEST(test_hip_zeta<float>());
- // CALL_SUBTEST(test_hip_zeta<double>());
-
- CALL_SUBTEST(test_hip_igamma<float>());
- CALL_SUBTEST(test_hip_igammac<float>());
-
- CALL_SUBTEST(test_hip_igamma<double>());
- CALL_SUBTEST(test_hip_igammac<double>());
-
- CALL_SUBTEST(test_hip_betainc<float>());
- CALL_SUBTEST(test_hip_betainc<double>());
-#endif
-}
diff --git a/unsupported/test/cxx11_tensor_of_float16_hip.cu b/unsupported/test/cxx11_tensor_of_float16_hip.cu
deleted file mode 100644
index 6c1a401..0000000
--- a/unsupported/test/cxx11_tensor_of_float16_hip.cu
+++ /dev/null
@@ -1,498 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_of_float16_hip
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-
-using Eigen::Tensor;
-
-template<typename>
-void test_hip_numext() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
- bool* d_res_half = (bool*)gpu_device.allocate(num_elem * sizeof(bool));
- bool* d_res_float = (bool*)gpu_device.allocate(num_elem * sizeof(bool));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float(
- d_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_res_half(
- d_res_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<bool, 1>, Eigen::Aligned> gpu_res_float(
- d_res_float, num_elem);
-
- gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f);
- gpu_res_float.device(gpu_device) = gpu_float.unaryExpr(Eigen::internal::scalar_isnan_op<float>());
- gpu_res_half.device(gpu_device) = gpu_float.cast<Eigen::half>().unaryExpr(Eigen::internal::scalar_isnan_op<Eigen::half>());
-
- Tensor<bool, 1> half_prec(num_elem);
- Tensor<bool, 1> full_prec(num_elem);
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(bool));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(bool));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking numext " << i << std::endl;
- VERIFY_IS_EQUAL(full_prec(i), half_prec(i));
- }
-
- gpu_device.deallocate(d_float);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-
-#ifdef EIGEN_HAS_HIP_FP16
-
-template<typename>
-void test_hip_conversion() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- float* d_conv = (float*)gpu_device.allocate(num_elem * sizeof(float));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float(
- d_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_half(
- d_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_conv(
- d_conv, num_elem);
-
- gpu_float.device(gpu_device) = gpu_float.random();
- gpu_half.device(gpu_device) = gpu_float.cast<Eigen::half>();
- gpu_conv.device(gpu_device) = gpu_half.cast<float>();
-
- Tensor<float, 1> initial(num_elem);
- Tensor<float, 1> final(num_elem);
- gpu_device.memcpyDeviceToHost(initial.data(), d_float, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(final.data(), d_conv, num_elem*sizeof(float));
-
- for (int i = 0; i < num_elem; ++i) {
- VERIFY_IS_APPROX(initial(i), final(i));
- }
-
- gpu_device.deallocate(d_float);
- gpu_device.deallocate(d_half);
- gpu_device.deallocate(d_conv);
-}
-
-template<typename>
-void test_hip_unary() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float(
- d_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half(
- d_res_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float(
- d_res_float, num_elem);
-
- gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f);
- gpu_res_float.device(gpu_device) = gpu_float.abs();
- gpu_res_half.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().cast<float>();
-
- Tensor<float, 1> half_prec(num_elem);
- Tensor<float, 1> full_prec(num_elem);
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking unary " << i << std::endl;
- VERIFY_IS_APPROX(full_prec(i), half_prec(i));
- }
-
- gpu_device.deallocate(d_float);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-template<typename>
-void test_hip_elementwise() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_half = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float1(
- d_float1, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float2(
- d_float2, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half(
- d_res_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float(
- d_res_float, num_elem);
-
- gpu_float1.device(gpu_device) = gpu_float1.random();
- gpu_float2.device(gpu_device) = gpu_float2.random();
- gpu_res_float.device(gpu_device) = (gpu_float1 + gpu_float2) * gpu_float1;
- gpu_res_half.device(gpu_device) = ((gpu_float1.cast<Eigen::half>() + gpu_float2.cast<Eigen::half>()) * gpu_float1.cast<Eigen::half>()).cast<float>();
-
- Tensor<float, 1> half_prec(num_elem);
- Tensor<float, 1> full_prec(num_elem);
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking elemwise " << i << ": full prec = " << full_prec(i) << " vs half prec = " << half_prec(i) << std::endl;
- VERIFY_IS_APPROX(static_cast<Eigen::half>(full_prec(i)), static_cast<Eigen::half>(half_prec(i)));
- }
-
- gpu_device.deallocate(d_float1);
- gpu_device.deallocate(d_float2);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-template<typename>
-void test_hip_trancendental() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float3 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_res1_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res1_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res2_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res2_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res3_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res3_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float1(d_float1, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float2(d_float2, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float3(d_float3, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res1_half(d_res1_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res1_float(d_res1_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res2_half(d_res2_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res2_float(d_res2_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res3_half(d_res3_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res3_float(d_res3_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res4_half(d_res3_half, num_elem);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res4_float(d_res3_float, num_elem);
-
- gpu_float1.device(gpu_device) = gpu_float1.random() - gpu_float1.constant(0.5f);
- gpu_float2.device(gpu_device) = gpu_float2.random() + gpu_float1.constant(0.5f);
- gpu_float3.device(gpu_device) = gpu_float3.random();
- gpu_res1_float.device(gpu_device) = gpu_float1.exp().cast<Eigen::half>();
- gpu_res2_float.device(gpu_device) = gpu_float2.log().cast<Eigen::half>();
- gpu_res3_float.device(gpu_device) = gpu_float3.log1p().cast<Eigen::half>();
- gpu_res4_float.device(gpu_device) = gpu_float3.expm1().cast<Eigen::half>();
-
- gpu_res1_half.device(gpu_device) = gpu_float1.cast<Eigen::half>();
- gpu_res1_half.device(gpu_device) = gpu_res1_half.exp();
-
- gpu_res2_half.device(gpu_device) = gpu_float2.cast<Eigen::half>();
- gpu_res2_half.device(gpu_device) = gpu_res2_half.log();
-
- gpu_res3_half.device(gpu_device) = gpu_float3.cast<Eigen::half>();
- gpu_res3_half.device(gpu_device) = gpu_res3_half.log1p();
-
- gpu_res3_half.device(gpu_device) = gpu_float3.cast<Eigen::half>();
- gpu_res3_half.device(gpu_device) = gpu_res3_half.expm1();
-
- Tensor<float, 1> input1(num_elem);
- Tensor<Eigen::half, 1> half_prec1(num_elem);
- Tensor<Eigen::half, 1> full_prec1(num_elem);
- Tensor<float, 1> input2(num_elem);
- Tensor<Eigen::half, 1> half_prec2(num_elem);
- Tensor<Eigen::half, 1> full_prec2(num_elem);
- Tensor<float, 1> input3(num_elem);
- Tensor<Eigen::half, 1> half_prec3(num_elem);
- Tensor<Eigen::half, 1> full_prec3(num_elem);
- gpu_device.memcpyDeviceToHost(input1.data(), d_float1, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(input2.data(), d_float2, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(input3.data(), d_float3, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(half_prec1.data(), d_res1_half, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec1.data(), d_res1_float, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(half_prec2.data(), d_res2_half, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec2.data(), d_res2_float, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(half_prec3.data(), d_res3_half, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec3.data(), d_res3_float, num_elem*sizeof(Eigen::half));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking elemwise exp " << i << " input = " << input1(i) << " full = " << full_prec1(i) << " half = " << half_prec1(i) << std::endl;
- VERIFY_IS_APPROX(full_prec1(i), half_prec1(i));
- }
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking elemwise log " << i << " input = " << input2(i) << " full = " << full_prec2(i) << " half = " << half_prec2(i) << std::endl;
- if(std::abs(input2(i)-1.f)<0.05f) // log lacks accurary nearby 1
- VERIFY_IS_APPROX(full_prec2(i)+Eigen::half(0.1f), half_prec2(i)+Eigen::half(0.1f));
- else
- VERIFY_IS_APPROX(full_prec2(i), half_prec2(i));
- }
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking elemwise plog1 " << i << " input = " << input3(i) << " full = " << full_prec3(i) << " half = " << half_prec3(i) << std::endl;
- VERIFY_IS_APPROX(full_prec3(i), half_prec3(i));
- }
- gpu_device.deallocate(d_float1);
- gpu_device.deallocate(d_float2);
- gpu_device.deallocate(d_float3);
- gpu_device.deallocate(d_res1_half);
- gpu_device.deallocate(d_res1_float);
- gpu_device.deallocate(d_res2_half);
- gpu_device.deallocate(d_res2_float);
- gpu_device.deallocate(d_res3_float);
- gpu_device.deallocate(d_res3_half);
-}
-
-template<typename>
-void test_hip_contractions() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int rows = 23;
- int cols = 23;
- int num_elem = rows*cols;
-
- float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
- Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(num_elem * sizeof(Eigen::half));
-
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1(
- d_float1, rows, cols);
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2(
- d_float2, rows, cols);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 2>, Eigen::Aligned> gpu_res_half(
- d_res_half, rows, cols);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 2>, Eigen::Aligned> gpu_res_float(
- d_res_float, rows, cols);
-
- gpu_float1.device(gpu_device) = gpu_float1.random() - gpu_float1.constant(0.5f);
- gpu_float2.device(gpu_device) = gpu_float2.random() - gpu_float2.constant(0.5f);
-
- typedef Tensor<float, 2>::DimensionPair DimPair;
- Eigen::array<DimPair, 1> dims(DimPair(1, 0));
- gpu_res_float.device(gpu_device) = gpu_float1.contract(gpu_float2, dims).cast<Eigen::half>();
- gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().contract(gpu_float2.cast<Eigen::half>(), dims);
-
- Tensor<Eigen::half, 2> half_prec(rows, cols);
- Tensor<Eigen::half, 2> full_prec(rows, cols);
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, num_elem*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(Eigen::half));
- gpu_device.synchronize();
-
- for (int i = 0; i < rows; ++i) {
- for (int j = 0; j < cols; ++j) {
- std::cout << "Checking contract " << i << " " << j << full_prec(i, j) << " " << half_prec(i, j) << std::endl;
- if (numext::abs(full_prec(i, j) - half_prec(i, j)) > Eigen::half(1e-2f)) {
- VERIFY_IS_APPROX(full_prec(i, j), half_prec(i, j));
- }
- }
- }
-
- gpu_device.deallocate(d_float1);
- gpu_device.deallocate(d_float2);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-template<typename>
-void test_hip_reductions(int size1, int size2, int redux) {
-
- std::cout << "Reducing " << size1 << " by " << size2
- << " tensor along dim " << redux << std::endl;
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = size1*size2;
- int result_size = (redux == 1 ? size1 : size2);
-
- float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(result_size * sizeof(Eigen::half));
- Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(result_size * sizeof(Eigen::half));
-
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1(
- d_float1, size1, size2);
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2(
- d_float2, size1, size2);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res_half(
- d_res_half, result_size);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 1>, Eigen::Aligned> gpu_res_float(
- d_res_float, result_size);
-
- gpu_float1.device(gpu_device) = gpu_float1.random() * 2.0f;
- gpu_float2.device(gpu_device) = gpu_float2.random() * 2.0f;
-
- Eigen::array<int, 1> redux_dim(redux);
- gpu_res_float.device(gpu_device) = gpu_float1.sum(redux_dim).cast<Eigen::half>();
- gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().sum(redux_dim);
-
- Tensor<Eigen::half, 1> half_prec(result_size);
- Tensor<Eigen::half, 1> full_prec(result_size);
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, result_size*sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, result_size*sizeof(Eigen::half));
- gpu_device.synchronize();
-
- for (int i = 0; i < result_size; ++i) {
- std::cout << "EXPECTED " << full_prec(i) << " GOT " << half_prec(i) << std::endl;
- VERIFY_IS_APPROX(full_prec(i), half_prec(i));
- }
-
- gpu_device.deallocate(d_float1);
- gpu_device.deallocate(d_float2);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-template<typename>
-void test_hip_reductions() {
- test_hip_reductions<void>(13, 13, 0);
- test_hip_reductions<void>(13, 13, 1);
-
- test_hip_reductions<void>(35, 36, 0);
- test_hip_reductions<void>(35, 36, 1);
-
- test_hip_reductions<void>(36, 35, 0);
- test_hip_reductions<void>(36, 35, 1);
-}
-
-template<typename>
-void test_hip_full_reductions() {
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int size = 13;
- int num_elem = size*size;
-
- float* d_float1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_float2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- Eigen::half* d_res_half = (Eigen::half*)gpu_device.allocate(1 * sizeof(Eigen::half));
- Eigen::half* d_res_float = (Eigen::half*)gpu_device.allocate(1 * sizeof(Eigen::half));
-
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float1(
- d_float1, size, size);
- Eigen::TensorMap<Eigen::Tensor<float, 2>, Eigen::Aligned> gpu_float2(
- d_float2, size, size);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 0>, Eigen::Aligned> gpu_res_half(
- d_res_half);
- Eigen::TensorMap<Eigen::Tensor<Eigen::half, 0>, Eigen::Aligned> gpu_res_float(
- d_res_float);
-
- gpu_float1.device(gpu_device) = gpu_float1.random();
- gpu_float2.device(gpu_device) = gpu_float2.random();
-
- gpu_res_float.device(gpu_device) = gpu_float1.sum().cast<Eigen::half>();
- gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().sum();
-
- Tensor<Eigen::half, 0> half_prec;
- Tensor<Eigen::half, 0> full_prec;
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, sizeof(Eigen::half));
- gpu_device.synchronize();
-
- VERIFY_IS_APPROX(full_prec(), half_prec());
-
- gpu_res_float.device(gpu_device) = gpu_float1.maximum().cast<Eigen::half>();
- gpu_res_half.device(gpu_device) = gpu_float1.cast<Eigen::half>().maximum();
- gpu_device.memcpyDeviceToHost(half_prec.data(), d_res_half, sizeof(Eigen::half));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, sizeof(Eigen::half));
- gpu_device.synchronize();
-
- VERIFY_IS_APPROX(full_prec(), half_prec());
-
- gpu_device.deallocate(d_float1);
- gpu_device.deallocate(d_float2);
- gpu_device.deallocate(d_res_half);
- gpu_device.deallocate(d_res_float);
-}
-
-template<typename>
-void test_hip_forced_evals() {
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
- int num_elem = 101;
-
- float* d_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_half1 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_half2 = (float*)gpu_device.allocate(num_elem * sizeof(float));
- float* d_res_float = (float*)gpu_device.allocate(num_elem * sizeof(float));
-
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_float(
- d_float, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_half1(
- d_res_half1, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Unaligned> gpu_res_half2(
- d_res_half2, num_elem);
- Eigen::TensorMap<Eigen::Tensor<float, 1>, Eigen::Aligned> gpu_res_float(
- d_res_float, num_elem);
-
- Eigen::array<int, 1> no_bcast;
- no_bcast[0] = 1;
-
- gpu_float.device(gpu_device) = gpu_float.random() - gpu_float.constant(0.5f);
- gpu_res_float.device(gpu_device) = gpu_float.abs();
- gpu_res_half1.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().eval().cast<float>();
- gpu_res_half2.device(gpu_device) = gpu_float.cast<Eigen::half>().abs().broadcast(no_bcast).eval().cast<float>();
-
- Tensor<float, 1> half_prec1(num_elem);
- Tensor<float, 1> half_prec2(num_elem);
- Tensor<float, 1> full_prec(num_elem);
- gpu_device.memcpyDeviceToHost(half_prec1.data(), d_res_half1, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(half_prec2.data(), d_res_half1, num_elem*sizeof(float));
- gpu_device.memcpyDeviceToHost(full_prec.data(), d_res_float, num_elem*sizeof(float));
- gpu_device.synchronize();
-
- for (int i = 0; i < num_elem; ++i) {
- std::cout << "Checking forced eval " << i << full_prec(i) << " vs " << half_prec1(i) << " vs " << half_prec2(i) << std::endl;
- VERIFY_IS_APPROX(full_prec(i), half_prec1(i));
- VERIFY_IS_APPROX(full_prec(i), half_prec2(i));
- }
-
- gpu_device.deallocate(d_float);
- gpu_device.deallocate(d_res_half1);
- gpu_device.deallocate(d_res_half2);
- gpu_device.deallocate(d_res_float);
-}
-#endif
-
-
-void test_cxx11_tensor_of_float16_hip()
-{
- CALL_SUBTEST(test_hip_numext<void>());
-
-#ifdef EIGEN_HAS_HIP_FP16
- CALL_SUBTEST(test_hip_conversion<void>());
- CALL_SUBTEST(test_hip_unary<void>());
- CALL_SUBTEST(test_hip_elementwise<void>());
- CALL_SUBTEST(test_hip_trancendental<void>());
- CALL_SUBTEST(test_hip_contractions<void>());
- CALL_SUBTEST(test_hip_reductions<void>());
- CALL_SUBTEST(test_hip_full_reductions<void>());
- CALL_SUBTEST(test_hip_forced_evals<void>());
-#else
- std::cout << "Half floats are not supported by this version of hip: skipping the test" << std::endl;
-#endif
-}
diff --git a/unsupported/test/cxx11_tensor_random_hip.cu b/unsupported/test/cxx11_tensor_random_hip.cu
deleted file mode 100644
index 7d7e72c..0000000
--- a/unsupported/test/cxx11_tensor_random_hip.cu
+++ /dev/null
@@ -1,85 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_random_hip
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <Eigen/CXX11/Tensor>
-
-
-void test_hip_random_uniform()
-{
- Tensor<float, 2> out(72,97);
- out.setZero();
-
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_out;
- hipMalloc((void**)(&d_out), out_bytes);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97);
-
- gpu_out.device(gpu_device) = gpu_out.random();
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-
- // For now we just check thes code doesn't crash.
- // TODO: come up with a valid test of randomness
-}
-
-
-void test_hip_random_normal()
-{
- Tensor<float, 2> out(72,97);
- out.setZero();
-
- std::size_t out_bytes = out.size() * sizeof(float);
-
- float* d_out;
- hipMalloc((void**)(&d_out), out_bytes);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 2> > gpu_out(d_out, 72,97);
-
- Eigen::internal::NormalRandomGenerator<float> gen(true);
- gpu_out.device(gpu_device) = gpu_out.random(gen);
-
- assert(hipMemcpyAsync(out.data(), d_out, out_bytes, hipMemcpyDeviceToHost, gpu_device.stream()) == hipSuccess);
- assert(hipStreamSynchronize(gpu_device.stream()) == hipSuccess);
-}
-
-static void test_complex()
-{
- Tensor<std::complex<float>, 1> vec(6);
- vec.setRandom();
-
- // Fixme: we should check that the generated numbers follow a uniform
- // distribution instead.
- for (int i = 1; i < 6; ++i) {
- VERIFY_IS_NOT_EQUAL(vec(i), vec(i-1));
- }
-}
-
-
-void test_cxx11_tensor_random_hip()
-{
- CALL_SUBTEST(test_hip_random_uniform());
- CALL_SUBTEST(test_hip_random_normal());
- CALL_SUBTEST(test_complex());
-}
diff --git a/unsupported/test/cxx11_tensor_reduction_hip.cu b/unsupported/test/cxx11_tensor_reduction_hip.cu
deleted file mode 100644
index c5aad05..0000000
--- a/unsupported/test/cxx11_tensor_reduction_hip.cu
+++ /dev/null
@@ -1,154 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2015 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_reduction_hip
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-
-template<typename Type, int DataLayout>
-static void test_full_reductions() {
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- const int num_rows = internal::random<int>(1024, 5*1024);
- const int num_cols = internal::random<int>(1024, 5*1024);
-
- Tensor<Type, 2, DataLayout> in(num_rows, num_cols);
- in.setRandom();
-
- Tensor<Type, 0, DataLayout> full_redux;
- full_redux = in.sum();
-
- std::size_t in_bytes = in.size() * sizeof(Type);
- std::size_t out_bytes = full_redux.size() * sizeof(Type);
- Type* gpu_in_ptr = static_cast<Type*>(gpu_device.allocate(in_bytes));
- Type* gpu_out_ptr = static_cast<Type*>(gpu_device.allocate(out_bytes));
- gpu_device.memcpyHostToDevice(gpu_in_ptr, in.data(), in_bytes);
-
- TensorMap<Tensor<Type, 2, DataLayout> > in_gpu(gpu_in_ptr, num_rows, num_cols);
- TensorMap<Tensor<Type, 0, DataLayout> > out_gpu(gpu_out_ptr);
-
- out_gpu.device(gpu_device) = in_gpu.sum();
-
- Tensor<Type, 0, DataLayout> full_redux_gpu;
- gpu_device.memcpyDeviceToHost(full_redux_gpu.data(), gpu_out_ptr, out_bytes);
- gpu_device.synchronize();
-
- // Check that the CPU and GPU reductions return the same result.
- VERIFY_IS_APPROX(full_redux(), full_redux_gpu());
-
- gpu_device.deallocate(gpu_in_ptr);
- gpu_device.deallocate(gpu_out_ptr);
-}
-
-template<typename Type, int DataLayout>
-static void test_first_dim_reductions() {
- int dim_x = 33;
- int dim_y = 1;
- int dim_z = 128;
-
- Tensor<Type, 3, DataLayout> in(dim_x, dim_y, dim_z);
- in.setRandom();
-
- Eigen::array<int, 1> red_axis;
- red_axis[0] = 0;
- Tensor<Type, 2, DataLayout> redux = in.sum(red_axis);
-
- // Create device
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice dev(&stream);
-
- // Create data(T)
- Type* in_data = (Type*)dev.allocate(dim_x*dim_y*dim_z*sizeof(Type));
- Type* out_data = (Type*)dev.allocate(dim_z*dim_y*sizeof(Type));
- Eigen::TensorMap<Eigen::Tensor<Type, 3, DataLayout> > gpu_in(in_data, dim_x, dim_y, dim_z);
- Eigen::TensorMap<Eigen::Tensor<Type, 2, DataLayout> > gpu_out(out_data, dim_y, dim_z);
-
- // Perform operation
- dev.memcpyHostToDevice(in_data, in.data(), in.size()*sizeof(Type));
- gpu_out.device(dev) = gpu_in.sum(red_axis);
- gpu_out.device(dev) += gpu_in.sum(red_axis);
- Tensor<Type, 2, DataLayout> redux_gpu(dim_y, dim_z);
- dev.memcpyDeviceToHost(redux_gpu.data(), out_data, gpu_out.size()*sizeof(Type));
- dev.synchronize();
-
- // Check that the CPU and GPU reductions return the same result.
- for (int i = 0; i < gpu_out.size(); ++i) {
- VERIFY_IS_APPROX(2*redux(i), redux_gpu(i));
- }
-
- dev.deallocate(in_data);
- dev.deallocate(out_data);
-}
-
-template<typename Type, int DataLayout>
-static void test_last_dim_reductions() {
- int dim_x = 128;
- int dim_y = 1;
- int dim_z = 33;
-
- Tensor<Type, 3, DataLayout> in(dim_x, dim_y, dim_z);
- in.setRandom();
-
- Eigen::array<int, 1> red_axis;
- red_axis[0] = 2;
- Tensor<Type, 2, DataLayout> redux = in.sum(red_axis);
-
- // Create device
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice dev(&stream);
-
- // Create data
- Type* in_data = (Type*)dev.allocate(dim_x*dim_y*dim_z*sizeof(Type));
- Type* out_data = (Type*)dev.allocate(dim_x*dim_y*sizeof(Type));
- Eigen::TensorMap<Eigen::Tensor<Type, 3, DataLayout> > gpu_in(in_data, dim_x, dim_y, dim_z);
- Eigen::TensorMap<Eigen::Tensor<Type, 2, DataLayout> > gpu_out(out_data, dim_x, dim_y);
-
- // Perform operation
- dev.memcpyHostToDevice(in_data, in.data(), in.size()*sizeof(Type));
- gpu_out.device(dev) = gpu_in.sum(red_axis);
- gpu_out.device(dev) += gpu_in.sum(red_axis);
- Tensor<Type, 2, DataLayout> redux_gpu(dim_x, dim_y);
- dev.memcpyDeviceToHost(redux_gpu.data(), out_data, gpu_out.size()*sizeof(Type));
- dev.synchronize();
-
- // Check that the CPU and GPU reductions return the same result.
- for (int i = 0; i < gpu_out.size(); ++i) {
- VERIFY_IS_APPROX(2*redux(i), redux_gpu(i));
- }
-
- dev.deallocate(in_data);
- dev.deallocate(out_data);
-}
-
-
-void test_cxx11_tensor_reduction_hip() {
- CALL_SUBTEST((test_full_reductions<float, ColMajor>()));
- CALL_SUBTEST((test_full_reductions<double, ColMajor>()));
- CALL_SUBTEST((test_full_reductions<float, RowMajor>()));
- CALL_SUBTEST((test_full_reductions<double, RowMajor>()));
-
- CALL_SUBTEST((test_first_dim_reductions<float, ColMajor>()));
- CALL_SUBTEST((test_first_dim_reductions<double, ColMajor>()));
- CALL_SUBTEST((test_first_dim_reductions<float, RowMajor>()));
-// Outer reductions of doubles aren't supported just yet.
-// CALL_SUBTEST((test_first_dim_reductions<double, RowMajor>()))
-
- CALL_SUBTEST((test_last_dim_reductions<float, ColMajor>()));
-// Outer reductions of doubles aren't supported just yet.
-// CALL_SUBTEST((test_last_dim_reductions<double, ColMajor>()));
- CALL_SUBTEST((test_last_dim_reductions<float, RowMajor>()));
- CALL_SUBTEST((test_last_dim_reductions<double, RowMajor>()));
-}
diff --git a/unsupported/test/cxx11_tensor_scan_hip.cu b/unsupported/test/cxx11_tensor_scan_hip.cu
deleted file mode 100644
index f4c4f59..0000000
--- a/unsupported/test/cxx11_tensor_scan_hip.cu
+++ /dev/null
@@ -1,76 +0,0 @@
-// This file is part of Eigen, a lightweight C++ template library
-// for linear algebra.
-//
-// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com>
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#define EIGEN_TEST_NO_LONGDOUBLE
-#define EIGEN_TEST_NO_COMPLEX
-#define EIGEN_TEST_FUNC cxx11_tensor_scan_hip
-#define EIGEN_DEFAULT_DENSE_INDEX_TYPE int
-#define EIGEN_USE_GPU
-
-#include "main.h"
-#include <unsupported/Eigen/CXX11/Tensor>
-
-using Eigen::Tensor;
-typedef Tensor<float, 1>::DimensionPair DimPair;
-
-template<int DataLayout>
-void test_hip_cumsum(int m_size, int k_size, int n_size)
-{
- std::cout << "Testing for (" << m_size << "," << k_size << "," << n_size << ")" << std::endl;
- Tensor<float, 3, DataLayout> t_input(m_size, k_size, n_size);
- Tensor<float, 3, DataLayout> t_result(m_size, k_size, n_size);
- Tensor<float, 3, DataLayout> t_result_gpu(m_size, k_size, n_size);
-
- t_input.setRandom();
-
- std::size_t t_input_bytes = t_input.size() * sizeof(float);
- std::size_t t_result_bytes = t_result.size() * sizeof(float);
-
- float* d_t_input;
- float* d_t_result;
-
- hipMalloc((void**)(&d_t_input), t_input_bytes);
- hipMalloc((void**)(&d_t_result), t_result_bytes);
-
- hipMemcpy(d_t_input, t_input.data(), t_input_bytes, hipMemcpyHostToDevice);
-
- Eigen::HipStreamDevice stream;
- Eigen::GpuDevice gpu_device(&stream);
-
- Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> >
- gpu_t_input(d_t_input, Eigen::array<int, 3>(m_size, k_size, n_size));
- Eigen::TensorMap<Eigen::Tensor<float, 3, DataLayout> >
- gpu_t_result(d_t_result, Eigen::array<int, 3>(m_size, k_size, n_size));
-
- gpu_t_result.device(gpu_device) = gpu_t_input.cumsum(1);
- t_result = t_input.cumsum(1);
-
- hipMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, hipMemcpyDeviceToHost);
- for (DenseIndex i = 0; i < t_result.size(); i++) {
- if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) {
- continue;
- }
- if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) {
- continue;
- }
- std::cout << "mismatch detected at index " << i << ": " << t_result(i)
- << " vs " << t_result_gpu(i) << std::endl;
- assert(false);
- }
-
- hipFree((void*)d_t_input);
- hipFree((void*)d_t_result);
-}
-
-
-void test_cxx11_tensor_scan_hip()
-{
- CALL_SUBTEST(test_hip_cumsum<ColMajor>(128, 128, 128));
- CALL_SUBTEST(test_hip_cumsum<RowMajor>(128, 128, 128));
-}
