Tensor: Handle unavailable GPU device attributes
diff --git a/contrib/Eigen/src/Tensor/README.md b/contrib/Eigen/src/Tensor/README.md index ada8fd8..45348bb 100644 --- a/contrib/Eigen/src/Tensor/README.md +++ b/contrib/Eigen/src/Tensor/README.md
@@ -588,8 +588,13 @@ For HIP, replace `cuda*` calls with the corresponding `hip*` calls. -Every runtime call `GpuDevice` makes, and every kernel launch, is checked: a -failure is reported on `stderr` as `file:line: call: error name: description` +`GpuDevice` caches attributes for the device that owns its stream. When HIP +does not implement the opt-in shared-memory attribute, `sharedMemPerBlockOptin()` +returns the ordinary block limit. `memoryPoolsSupported()` returns false on CUDA +drivers older than 11.2, which do not support the memory-pool attribute query. + +Every runtime call `GpuDevice` makes, and every kernel launch, is checked: an +unexpected failure is reported on `stderr` as `file:line: call: error name: description` and stops the program, through `std::abort()` when `EIGEN_NO_DEBUG` (or `NDEBUG`) is defined and a failed `eigen_assert` otherwise. Define `EIGEN_GPU_RUNTIME_CHECK(call)` before including the Tensor header to route
diff --git a/contrib/Eigen/src/Tensor/TensorDeviceGpu.h b/contrib/Eigen/src/Tensor/TensorDeviceGpu.h index ff5c598..eaa8a81 100644 --- a/contrib/Eigen/src/Tensor/TensorDeviceGpu.h +++ b/contrib/Eigen/src/Tensor/TensorDeviceGpu.h
@@ -43,8 +43,25 @@ // aliases the enumerators but not the type. template <typename GpuDeviceAttr> inline int GetGpuDeviceAttribute(GpuDeviceAttr attribute, int device) { +#if !defined(EIGEN_USE_HIP) + if (attribute == gpuDevAttrMemoryPoolsSupported) { + int driver_version = 0; + EIGEN_GPU_RUNTIME_CHECK(cudaDriverGetVersion(&driver_version)); + // CUDA minor-version compatibility permits drivers that predate this attribute. + if (driver_version < 11020) return 0; + } +#endif int value = 0; - EIGEN_GPU_RUNTIME_CHECK(gpuDeviceGetAttribute(&value, attribute, device)); + const gpuError_t error = gpuDeviceGetAttribute(&value, attribute, device); +#if defined(EIGEN_USE_HIP) + // HIP 5.x declares the opt-in enumerator but does not implement the query. + if (attribute == gpuDevAttrMaxSharedMemoryPerBlockOptin && error == hipErrorInvalidValue) { + (void)hipGetLastError(); + EIGEN_GPU_RUNTIME_CHECK(gpuDeviceGetAttribute(&value, gpuDevAttrMaxSharedMemoryPerBlock, device)); + return value; + } +#endif + EIGEN_GPU_RUNTIME_CHECK(error); return value; } @@ -350,10 +367,10 @@ // the opt-in shared-memory limit or memory-pool support portably. Like the properties above, they describe the // device the stream and its allocations belong to, which need not be the device the calling thread is bound to. EIGEN_STRONG_INLINE int warpSize() const { return stream_->deviceAttributes().warpSize; } - // The shared memory a kernel may request with gpuFuncSetAttribute, which exceeds sharedMemPerBlock on every - // architecture since Volta. + // The shared memory a kernel may request with gpuFuncSetAttribute, or the ordinary block limit when HIP does + // not implement the opt-in attribute. EIGEN_STRONG_INLINE int sharedMemPerBlockOptin() const { return stream_->deviceAttributes().sharedMemPerBlockOptin; } - // Whether gpuMallocAsync and its pool are available on this device. + // Whether gpuMallocAsync and its pool are available on this device and driver. EIGEN_STRONG_INLINE bool memoryPoolsSupported() const { return stream_->deviceAttributes().memoryPoolsSupported != 0; }
diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 9415b14..69ec24e 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt
@@ -289,6 +289,7 @@ ei_add_test(tensor_cast_float16_gpu) ei_add_test(tensor_scan_gpu) ei_add_test(tensor_device) + ei_add_test(tensor_gpu_device_attributes) ei_add_test(tensor_gpu) ei_add_test(tensor_gpu_runtime_check) ei_add_test(tensor_gpu_runtime_check_ndebug) @@ -339,6 +340,7 @@ ei_add_test(tensor_cast_float16_gpu) ei_add_test(tensor_scan_gpu) ei_add_test(tensor_device) + ei_add_test(tensor_gpu_device_attributes) ei_add_test(tensor_gpu) ei_add_test(tensor_gpu_runtime_check) ei_add_test(tensor_gpu_runtime_check_ndebug)
diff --git a/contrib/test/tensor_device.cu b/contrib/test/tensor_device.cu index 89a5705..7128052 100644 --- a/contrib/test/tensor_device.cu +++ b/contrib/test/tensor_device.cu
@@ -459,7 +459,14 @@ // Every architecture Eigen supports can opt in to at least what a block gets by default. VERIFY(gpu_device.sharedMemPerBlockOptin() >= gpu_device.sharedMemPerBlock()); int pools = 0; - VERIFY(gpuDeviceGetAttribute(&pools, gpuDevAttrMemoryPoolsSupported, device) == gpuSuccess); +#if !defined(EIGEN_USE_HIP) + int driver_version = 0; + VERIFY(cudaDriverGetVersion(&driver_version) == cudaSuccess); + if (driver_version >= 11020) +#endif + { + VERIFY(gpuDeviceGetAttribute(&pools, gpuDevAttrMemoryPoolsSupported, device) == gpuSuccess); + } VERIFY_IS_EQUAL(gpu_device.memoryPoolsSupported(), pools != 0); }
diff --git a/contrib/test/tensor_gpu_device_attributes.cu b/contrib/test/tensor_gpu_device_attributes.cu new file mode 100644 index 0000000..aa4c983 --- /dev/null +++ b/contrib/test/tensor_gpu_device_attributes.cu
@@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#define EIGEN_TEST_NO_LONGDOUBLE +#define EIGEN_TEST_NO_COMPLEX +#define EIGEN_USE_GPU +#define EIGEN_NO_DEBUG + +#if defined(EIGEN_USE_HIP) +#include <hip/hip_runtime.h> +using TestGpuError = hipError_t; +using TestGpuAttribute = hipDeviceAttribute_t; +#else +#include <cuda_runtime.h> +using TestGpuError = cudaError_t; +using TestGpuAttribute = cudaDeviceAttr; +#endif + +static int driver_version = 11010; +static int pool_support = 1; +static int pool_queries = 0; +static int runtime_errors = 0; +static TestGpuError injected_error = static_cast<TestGpuError>(0); +static TestGpuError last_error = static_cast<TestGpuError>(0); + +static TestGpuError test_device_get_attribute(int* value, TestGpuAttribute attribute, int device); +static TestGpuError test_get_device_count(int* count) { + *count = 2; + return static_cast<TestGpuError>(0); +} +static TestGpuError test_get_last_error() { + const TestGpuError result = last_error; + last_error = static_cast<TestGpuError>(0); + return result; +} +#if defined(EIGEN_USE_HIP) +#define hipDeviceGetAttribute test_device_get_attribute +#define hipGetDeviceCount test_get_device_count +#define hipGetLastError test_get_last_error +#else +static cudaError_t test_driver_get_version(int* version) { + *version = driver_version; + return cudaSuccess; +} +#define cudaDeviceGetAttribute test_device_get_attribute +#define cudaGetDeviceCount test_get_device_count +#define cudaGetLastError test_get_last_error +#define cudaDriverGetVersion test_driver_get_version +#endif + +static void record_runtime_error(TestGpuError error) { + if (static_cast<int>(error) != 0) ++runtime_errors; +} +#define EIGEN_GPU_RUNTIME_CHECK(expr) record_runtime_error(expr) + +#include "main.h" +#include <contrib/Eigen/Tensor> + +// Inject old-runtime responses before the public umbrella is parsed, without requiring old drivers or hardware. +static TestGpuError test_device_get_attribute(int* value, TestGpuAttribute attribute, int device) { + *value = -1; + if (injected_error != gpuSuccess) return injected_error; + if (attribute == gpuDevAttrMaxSharedMemoryPerBlockOptin) { +#if defined(EIGEN_USE_HIP) + if (device == 0) return last_error = hipErrorInvalidValue; +#endif + *value = 98304; + } else if (attribute == gpuDevAttrMaxSharedMemoryPerBlock) { + *value = 49152; + } else if (attribute == gpuDevAttrMemoryPoolsSupported) { + ++pool_queries; +#if !defined(EIGEN_USE_HIP) + if (driver_version < 11020) return last_error = cudaErrorInvalidValue; +#endif + *value = pool_support; + } else if (attribute == gpuDevAttrWarpSize) { + *value = 32 * (device + 1); + } else { + *value = 1; + } + return gpuSuccess; +} + +static void test_attribute_cache() { + // numThreads() initializes every cached field, even on runtimes missing an optional attribute. + for (int device = 0; device < 2; ++device) { + Eigen::GpuStreamDevice stream(device); + Eigen::GpuDevice gpu_device(&stream); + VERIFY_IS_EQUAL(gpu_device.numThreads(), static_cast<size_t>(32 * (device + 1))); + VERIFY_IS_EQUAL(runtime_errors, 0); + VERIFY_IS_EQUAL(test_get_last_error(), gpuSuccess); +#if defined(EIGEN_USE_HIP) + VERIFY_IS_EQUAL(gpu_device.sharedMemPerBlockOptin(), device == 0 ? 49152 : 98304); + VERIFY(gpu_device.memoryPoolsSupported()); +#else + VERIFY_IS_EQUAL(gpu_device.sharedMemPerBlockOptin(), 98304); + VERIFY(!gpu_device.memoryPoolsSupported()); + VERIFY_IS_EQUAL(pool_queries, 0); +#endif + } +} + +static void test_attribute_queries() { + driver_version = 11020; + for (int supported = 0; supported <= 1; ++supported) { + pool_support = supported; + const int queries_before = pool_queries; + VERIFY_IS_EQUAL(Eigen::GetGpuDeviceAttribute(gpuDevAttrMemoryPoolsSupported, 0), supported); + VERIFY_IS_EQUAL(pool_queries, queries_before + 1); + } + VERIFY_IS_EQUAL(runtime_errors, 0); + +#if defined(EIGEN_USE_HIP) + injected_error = hipErrorInvalidValue; +#else + injected_error = cudaErrorInvalidValue; +#endif + // An invalid-value error on a required attribute or a supported pool query is still a runtime failure. + Eigen::GetGpuDeviceAttribute(gpuDevAttrWarpSize, 0); + VERIFY_IS_EQUAL(runtime_errors, 1); + Eigen::GetGpuDeviceAttribute(gpuDevAttrMemoryPoolsSupported, 0); + VERIFY_IS_EQUAL(runtime_errors, 2); +#if defined(EIGEN_USE_HIP) + injected_error = hipErrorInvalidDevice; +#else + injected_error = cudaErrorInvalidDevice; +#endif + Eigen::GetGpuDeviceAttribute(gpuDevAttrMaxSharedMemoryPerBlockOptin, 0); + VERIFY_IS_EQUAL(runtime_errors, 3); + injected_error = gpuSuccess; + runtime_errors = 0; +} + +EIGEN_DECLARE_TEST(tensor_gpu_device_attributes) { + CALL_SUBTEST(test_attribute_cache()); + CALL_SUBTEST(test_attribute_queries()); +}